mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 06:30:03 +00:00
feat(engine,producer,cli): capture failing dB/frame index on drawElement verify fallback
de_fallback_reason only told you the fallback happened (blank/psnr/oom/ capture_error), not the failing PSNR or frame index — that data existed as text inside the thrown error's message and was discarded on the way to telemetry. DrawElementVerificationError now carries structured frameIndex/failedDb/verifyThresholdDb; the orchestrator reads them via the new getDrawElementVerificationDetails helper instead of regexing message text, and both telemetry surfaces (the render_complete perfSummary path and the crash-survival RenderCaptureObservability mirror) emit de_fallback_failed_db / de_fallback_frame_index. Needed to distinguish "32dB vs the 32dB threshold, tune it" from "12dB real corruption, investigate" during the parallel-router soak — currently that distinction is invisible.
This commit is contained in:
@@ -60,6 +60,10 @@ export interface RenderCaptureObservability {
|
||||
* telemetry from one that never attempted any fallback.
|
||||
*/
|
||||
deFallbackReason?: string;
|
||||
/** The failing PSNR (dB) when `deFallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
|
||||
deFallbackFailedDb?: number;
|
||||
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
|
||||
deFallbackFrameIndex?: number;
|
||||
/** Auto-parallel inversion outcome: "inverted" (fired, held) | "reverted" (fired, self-verify retry rolled back). */
|
||||
deWorkerInversion?: "inverted" | "reverted";
|
||||
/** Worker count the resolver would have used absent the inversion; undefined if it never fired. */
|
||||
|
||||
@@ -72,6 +72,10 @@ export interface DrawElementPerfInput {
|
||||
preRouterWorkers?: number;
|
||||
selfVerifyFallback: boolean;
|
||||
fallbackReason?: string;
|
||||
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error. */
|
||||
fallbackFailedDb?: number;
|
||||
/** Frame index the verification failure was detected at; set for both "psnr" and "blank". */
|
||||
fallbackFrameIndex?: number;
|
||||
drainStats?: {
|
||||
verifyChecked: number;
|
||||
verifyMinDb?: number;
|
||||
@@ -112,6 +116,11 @@ function aggregateDrawElement(
|
||||
verifyInitMs: perfs.reduce((sum, p) => sum + (p.deVerifyInitMs ?? 0), 0),
|
||||
selfVerifyFallback: de.selfVerifyFallback,
|
||||
fallbackReason: de.fallbackReason,
|
||||
fallbackFailedDb:
|
||||
de.fallbackFailedDb === undefined
|
||||
? undefined
|
||||
: Math.round(Math.min(de.fallbackFailedDb, 999) * 10) / 10,
|
||||
fallbackFrameIndex: de.fallbackFrameIndex,
|
||||
blankSuspects: drain?.blankSuspects ?? 0,
|
||||
blankDeterministicAccepts: drain?.blankDeterministicAccepts ?? 0,
|
||||
blankRecaptures: drain?.blankRecaptures ?? 0,
|
||||
|
||||
@@ -289,6 +289,7 @@ function createDrainFrameGuard(args: {
|
||||
} catch (err) {
|
||||
throw new DrawElementVerificationError(
|
||||
`blank drawElement frame ${idx}: ${buf.length}B < floor ${Math.round(floor)}B and recapture failed (${err instanceof Error ? err.message : String(err)})`,
|
||||
{ frameIndex: idx },
|
||||
);
|
||||
}
|
||||
if (retryBuf.equals(buf)) {
|
||||
@@ -306,6 +307,7 @@ function createDrainFrameGuard(args: {
|
||||
} else if (retryBuf.length < floor) {
|
||||
throw new DrawElementVerificationError(
|
||||
`blank drawElement frame ${idx}: ${buf.length}B (retry ${retryBuf.length}B) < floor ${Math.round(floor)}B`,
|
||||
{ frameIndex: idx },
|
||||
);
|
||||
} else {
|
||||
buf = retryBuf;
|
||||
@@ -339,6 +341,7 @@ function createDrainFrameGuard(args: {
|
||||
}
|
||||
throw new DrawElementVerificationError(
|
||||
`drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
|
||||
{ frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
|
||||
);
|
||||
}
|
||||
stats.verifyChecked += 1;
|
||||
|
||||
@@ -76,6 +76,7 @@ import {
|
||||
isMemoryExhaustionError,
|
||||
isTransientBrowserError,
|
||||
isDrawElementVerificationError,
|
||||
getDrawElementVerificationDetails,
|
||||
} from "@hyperframes/engine";
|
||||
import { join, dirname, resolve } from "path";
|
||||
import { totalmem } from "node:os";
|
||||
@@ -449,6 +450,10 @@ export interface RenderPerfSummary {
|
||||
selfVerifyFallback: boolean;
|
||||
/** What tripped the fallback retry: psnr | blank | oom | capture_error. */
|
||||
fallbackReason?: string;
|
||||
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
|
||||
fallbackFailedDb?: number;
|
||||
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
|
||||
fallbackFrameIndex?: number;
|
||||
/** Blank-guard counters. */
|
||||
blankSuspects: number;
|
||||
blankDeterministicAccepts: number;
|
||||
@@ -1737,6 +1742,11 @@ export async function executeRenderJob(
|
||||
let captureParallelStreamForced = false;
|
||||
let deSelfVerifyFallback = false;
|
||||
let deFallbackReason: string | undefined;
|
||||
// Structured detail behind deFallbackReason's "blank"/"psnr" bucket — the
|
||||
// failing dB and frame index otherwise only exist as text inside the
|
||||
// thrown error's message, unavailable to telemetry.
|
||||
let deFallbackFailedDb: number | undefined;
|
||||
let deFallbackFrameIndex: number | undefined;
|
||||
let deDrainStats: import("./render/stages/captureStreamingStage.js").DeDrainStats | undefined;
|
||||
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
|
||||
observability.checkpoint("compile", "composition metadata resolved", {
|
||||
@@ -2718,6 +2728,11 @@ export async function executeRenderJob(
|
||||
: isMemoryExhaustion
|
||||
? "oom"
|
||||
: "capture_error";
|
||||
if (isVerifyError) {
|
||||
const verifyDetails = getDrawElementVerificationDetails(err);
|
||||
deFallbackFailedDb = verifyDetails?.failedDb;
|
||||
deFallbackFrameIndex = verifyDetails?.frameIndex;
|
||||
}
|
||||
log.warn(
|
||||
isVerifyError
|
||||
? "[Render] drawElement self-verification failed; re-rendering via screenshot"
|
||||
@@ -2735,6 +2750,8 @@ export async function executeRenderJob(
|
||||
forceScreenshot: true,
|
||||
deSelfVerifyFallback,
|
||||
deFallbackReason,
|
||||
deFallbackFailedDb,
|
||||
deFallbackFrameIndex,
|
||||
});
|
||||
probeSession = null;
|
||||
// Must clear BEFORE resolveParallelRouterRetryPlan recomputes
|
||||
@@ -3023,6 +3040,8 @@ export async function executeRenderJob(
|
||||
preRouterWorkers: deParallelRouter ? preRoutingWorkerCount : undefined,
|
||||
selfVerifyFallback: deSelfVerifyFallback,
|
||||
fallbackReason: deFallbackReason,
|
||||
fallbackFailedDb: deFallbackFailedDb,
|
||||
fallbackFrameIndex: deFallbackFrameIndex,
|
||||
drainStats: deDrainStats,
|
||||
},
|
||||
hdrDiagnostics,
|
||||
|
||||
Reference in New Issue
Block a user