feat(engine,producer,cli): capture failing dB/frame index on drawElement verify fallback

de_fallback_reason only told you the fallback happened (blank/psnr/oom/
capture_error), not the failing PSNR or frame index — that data existed as
text inside the thrown error's message and was discarded on the way to
telemetry. DrawElementVerificationError now carries structured
frameIndex/failedDb/verifyThresholdDb; the orchestrator reads them via the
new getDrawElementVerificationDetails helper instead of regexing message
text, and both telemetry surfaces (the render_complete perfSummary path and
the crash-survival RenderCaptureObservability mirror) emit
de_fallback_failed_db / de_fallback_frame_index.

Needed to distinguish "32dB vs the 32dB threshold, tune it" from "12dB real
corruption, investigate" during the parallel-router soak — currently that
distinction is invisible.
This commit is contained in:
Vance Ingalls
2026-07-14 23:29:49 -07:00
parent f3800f3579
commit a5cbb78ff9
12 changed files with 202 additions and 1 deletions
@@ -60,6 +60,10 @@ export interface RenderCaptureObservability {
* telemetry from one that never attempted any fallback.
*/
deFallbackReason?: string;
/** The failing PSNR (dB) when `deFallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
deFallbackFailedDb?: number;
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
deFallbackFrameIndex?: number;
/** Auto-parallel inversion outcome: "inverted" (fired, held) | "reverted" (fired, self-verify retry rolled back). */
deWorkerInversion?: "inverted" | "reverted";
/** Worker count the resolver would have used absent the inversion; undefined if it never fired. */
@@ -72,6 +72,10 @@ export interface DrawElementPerfInput {
preRouterWorkers?: number;
selfVerifyFallback: boolean;
fallbackReason?: string;
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error. */
fallbackFailedDb?: number;
/** Frame index the verification failure was detected at; set for both "psnr" and "blank". */
fallbackFrameIndex?: number;
drainStats?: {
verifyChecked: number;
verifyMinDb?: number;
@@ -112,6 +116,11 @@ function aggregateDrawElement(
verifyInitMs: perfs.reduce((sum, p) => sum + (p.deVerifyInitMs ?? 0), 0),
selfVerifyFallback: de.selfVerifyFallback,
fallbackReason: de.fallbackReason,
fallbackFailedDb:
de.fallbackFailedDb === undefined
? undefined
: Math.round(Math.min(de.fallbackFailedDb, 999) * 10) / 10,
fallbackFrameIndex: de.fallbackFrameIndex,
blankSuspects: drain?.blankSuspects ?? 0,
blankDeterministicAccepts: drain?.blankDeterministicAccepts ?? 0,
blankRecaptures: drain?.blankRecaptures ?? 0,
@@ -289,6 +289,7 @@ function createDrainFrameGuard(args: {
} catch (err) {
throw new DrawElementVerificationError(
`blank drawElement frame ${idx}: ${buf.length}B < floor ${Math.round(floor)}B and recapture failed (${err instanceof Error ? err.message : String(err)})`,
{ frameIndex: idx },
);
}
if (retryBuf.equals(buf)) {
@@ -306,6 +307,7 @@ function createDrainFrameGuard(args: {
} else if (retryBuf.length < floor) {
throw new DrawElementVerificationError(
`blank drawElement frame ${idx}: ${buf.length}B (retry ${retryBuf.length}B) < floor ${Math.round(floor)}B`,
{ frameIndex: idx },
);
} else {
buf = retryBuf;
@@ -339,6 +341,7 @@ function createDrainFrameGuard(args: {
}
throw new DrawElementVerificationError(
`drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
{ frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
);
}
stats.verifyChecked += 1;
@@ -76,6 +76,7 @@ import {
isMemoryExhaustionError,
isTransientBrowserError,
isDrawElementVerificationError,
getDrawElementVerificationDetails,
} from "@hyperframes/engine";
import { join, dirname, resolve } from "path";
import { totalmem } from "node:os";
@@ -449,6 +450,10 @@ export interface RenderPerfSummary {
selfVerifyFallback: boolean;
/** What tripped the fallback retry: psnr | blank | oom | capture_error. */
fallbackReason?: string;
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
fallbackFailedDb?: number;
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
fallbackFrameIndex?: number;
/** Blank-guard counters. */
blankSuspects: number;
blankDeterministicAccepts: number;
@@ -1737,6 +1742,11 @@ export async function executeRenderJob(
let captureParallelStreamForced = false;
let deSelfVerifyFallback = false;
let deFallbackReason: string | undefined;
// Structured detail behind deFallbackReason's "blank"/"psnr" bucket — the
// failing dB and frame index otherwise only exist as text inside the
// thrown error's message, unavailable to telemetry.
let deFallbackFailedDb: number | undefined;
let deFallbackFrameIndex: number | undefined;
let deDrainStats: import("./render/stages/captureStreamingStage.js").DeDrainStats | undefined;
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
observability.checkpoint("compile", "composition metadata resolved", {
@@ -2718,6 +2728,11 @@ export async function executeRenderJob(
: isMemoryExhaustion
? "oom"
: "capture_error";
if (isVerifyError) {
const verifyDetails = getDrawElementVerificationDetails(err);
deFallbackFailedDb = verifyDetails?.failedDb;
deFallbackFrameIndex = verifyDetails?.frameIndex;
}
log.warn(
isVerifyError
? "[Render] drawElement self-verification failed; re-rendering via screenshot"
@@ -2735,6 +2750,8 @@ export async function executeRenderJob(
forceScreenshot: true,
deSelfVerifyFallback,
deFallbackReason,
deFallbackFailedDb,
deFallbackFrameIndex,
});
probeSession = null;
// Must clear BEFORE resolveParallelRouterRetryPlan recomputes
@@ -3023,6 +3040,8 @@ export async function executeRenderJob(
preRouterWorkers: deParallelRouter ? preRoutingWorkerCount : undefined,
selfVerifyFallback: deSelfVerifyFallback,
fallbackReason: deFallbackReason,
fallbackFailedDb: deFallbackFailedDb,
fallbackFrameIndex: deFallbackFrameIndex,
drainStats: deDrainStats,
},
hdrDiagnostics,