feat(engine,producer,cli): capture failing dB/frame index on drawElement verify fallback

de_fallback_reason only told you the fallback happened (blank/psnr/oom/
capture_error), not the failing PSNR or frame index — that data existed as
text inside the thrown error's message and was discarded on the way to
telemetry. DrawElementVerificationError now carries structured
frameIndex/failedDb/verifyThresholdDb; the orchestrator reads them via the
new getDrawElementVerificationDetails helper instead of regexing message
text, and both telemetry surfaces (the render_complete perfSummary path and
the crash-survival RenderCaptureObservability mirror) emit
de_fallback_failed_db / de_fallback_frame_index.

Needed to distinguish "32dB vs the 32dB threshold, tune it" from "12dB real
corruption, investigate" during the parallel-router soak — currently that
distinction is invisible.
This commit is contained in:
Vance Ingalls
2026-07-14 23:29:49 -07:00
parent f3800f3579
commit a5cbb78ff9
12 changed files with 202 additions and 1 deletions
+2
View File
@@ -2064,6 +2064,8 @@ function trackRenderMetrics(
deVerifyInitMs: perf?.drawElement?.verifyInitMs,
deSelfVerifyFallback: perf?.drawElement?.selfVerifyFallback,
deFallbackReason: perf?.drawElement?.fallbackReason,
deFallbackFailedDb: perf?.drawElement?.fallbackFailedDb,
deFallbackFrameIndex: perf?.drawElement?.fallbackFrameIndex,
deBlankSuspects: perf?.drawElement?.blankSuspects,
deBlankDeterministicAccepts: perf?.drawElement?.blankDeterministicAccepts,
deBlankRecaptures: perf?.drawElement?.blankRecaptures,
+48
View File
@@ -252,6 +252,30 @@ describe("render telemetry events", () => {
);
});
it("carries the failing dB and frame index on render_error for a psnr fallback that failed hard afterward", () => {
trackRenderError({
fps: 30,
quality: "standard",
docker: false,
errorMessage: "worker crashed after a psnr fallback",
captureDeParallelRouter: "reverted",
captureDeSelfVerifyFallback: true,
captureDeFallbackReason: "psnr",
captureDeFallbackFailedDb: 28.4,
captureDeFallbackFrameIndex: 649,
});
expect(trackEvent).toHaveBeenCalledWith(
"render_error",
expect.objectContaining({
de_fallback_reason: "psnr",
de_fallback_failed_db: 28.4,
de_fallback_frame_index: 649,
}),
undefined,
);
});
it("prefers the explicit perfSummary-sourced de_worker_inversion over the capture-observability fallback on render_complete", () => {
trackRenderComplete({
durationMs: 1000,
@@ -272,6 +296,30 @@ describe("render telemetry events", () => {
);
});
it("carries the perfSummary-sourced failing dB and frame index on render_complete", () => {
trackRenderComplete({
durationMs: 1000,
fps: 30,
quality: "standard",
docker: false,
gpu: false,
deParallelRouter: "reverted",
deFallbackReason: "psnr",
deFallbackFailedDb: 28.4,
deFallbackFrameIndex: 649,
});
expect(trackEvent).toHaveBeenCalledWith(
"render_complete",
expect.objectContaining({
de_fallback_reason: "psnr",
de_fallback_failed_db: 28.4,
de_fallback_frame_index: 649,
}),
undefined,
);
});
it("emits render_preflight_rejected with the low-cardinality issue kind", () => {
trackRenderPreflightRejected({ kind: "aspect-mismatch" });
expect(trackEvent).toHaveBeenCalledWith("render_preflight_rejected", {
+8
View File
@@ -54,6 +54,8 @@ export interface RenderObservabilityTelemetryPayload {
captureDePreRouterWorkers?: number;
captureDeSelfVerifyFallback?: boolean;
captureDeFallbackReason?: string;
captureDeFallbackFailedDb?: number;
captureDeFallbackFrameIndex?: number;
/** Non-DE parallel-streaming router outcome ("screenshot" | "beginframe"
* routed; "eligible_off" would route but the kill switch is off). */
captureParallelStream?: string;
@@ -108,6 +110,8 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP
de_pre_router_workers: props.captureDePreRouterWorkers,
de_self_verify_fallback: props.captureDeSelfVerifyFallback,
de_fallback_reason: props.captureDeFallbackReason,
de_fallback_failed_db: props.captureDeFallbackFailedDb,
de_fallback_frame_index: props.captureDeFallbackFrameIndex,
capture_parallel_stream: props.captureParallelStream,
observability_extract_video_count: props.observabilityExtractVideoCount,
observability_extracted_video_count: props.observabilityExtractedVideoCount,
@@ -173,6 +177,8 @@ export function trackRenderComplete(
deVerifyInitMs?: number;
deSelfVerifyFallback?: boolean;
deFallbackReason?: string;
deFallbackFailedDb?: number;
deFallbackFrameIndex?: number;
deBlankSuspects?: number;
deBlankDeterministicAccepts?: number;
deBlankRecaptures?: number;
@@ -260,6 +266,8 @@ export function trackRenderComplete(
de_verify_init_ms: props.deVerifyInitMs,
de_self_verify_fallback: props.deSelfVerifyFallback,
de_fallback_reason: props.deFallbackReason,
de_fallback_failed_db: props.deFallbackFailedDb,
de_fallback_frame_index: props.deFallbackFrameIndex,
de_blank_suspects: props.deBlankSuspects,
de_blank_deterministic_accepts: props.deBlankDeterministicAccepts,
de_blank_recaptures: props.deBlankRecaptures,
@@ -81,6 +81,27 @@ describe("renderObservabilityTelemetryPayload — DE inversion/router cohort (fa
const payload = renderObservabilityTelemetryPayload(makeSummary({}));
expect(payload.captureDeFallbackReason).toBeUndefined();
});
it("carries the failing dB and frame index for a psnr fallback, still visible on a hard failure", () => {
const payload = renderObservabilityTelemetryPayload(
makeSummary({
deParallelRouter: "routed",
deFallbackReason: "psnr",
deFallbackFailedDb: 28.4,
deFallbackFrameIndex: 649,
}),
);
expect(payload.captureDeFallbackFailedDb).toBe(28.4);
expect(payload.captureDeFallbackFrameIndex).toBe(649);
});
it("leaves failedDb undefined for a blank/oom/capture_error fallback (no PSNR score exists)", () => {
const payload = renderObservabilityTelemetryPayload(
makeSummary({ deFallbackReason: "oom", deFallbackFrameIndex: undefined }),
);
expect(payload.captureDeFallbackFailedDb).toBeUndefined();
expect(payload.captureDeFallbackFrameIndex).toBeUndefined();
});
});
describe("renderObservabilityTelemetryPayload — non-DE parallel-stream router", () => {
@@ -46,6 +46,8 @@ export function renderObservabilityTelemetryPayload(
captureDePreRouterWorkers: capture.dePreRouterWorkers,
captureDeSelfVerifyFallback: capture.deSelfVerifyFallback,
captureDeFallbackReason: capture.deFallbackReason,
captureDeFallbackFailedDb: capture.deFallbackFailedDb,
captureDeFallbackFrameIndex: capture.deFallbackFrameIndex,
captureParallelStream: capture.captureParallelStream,
observabilityExtractVideoCount: extraction?.videoCount,
observabilityExtractedVideoCount: extraction?.extractedVideoCount,
+2
View File
@@ -91,6 +91,8 @@ export {
captureFramesBatchPipelined,
DrawElementVerificationError,
isDrawElementVerificationError,
getDrawElementVerificationDetails,
type DrawElementVerificationDetails,
recaptureDrawElementFrameForVerify,
completeDeferredDrawElementInit,
writeCapturedFrame,
@@ -1,11 +1,14 @@
import { describe, it, expect } from "vitest";
import {
DrawElementVerificationError,
formatHttpErrorDiagnostic,
formatConsoleDiagnostic,
formatNavigationFailureDiagnostic,
formatNavigationStartDiagnostic,
formatRequestFailureDiagnostic,
getDrawElementVerificationDetails,
isFontResourceError,
isDrawElementVerificationError,
sanitizeDiagnosticUrl,
} from "./frameCapture.js";
@@ -222,3 +225,38 @@ describe("navigation diagnostics", () => {
).toBe("[Browser:HTTP403] GET https://cdn.example.com/frame.png resource=image Forbidden");
});
});
describe("DrawElementVerificationError details", () => {
it("carries frameIndex/failedDb/verifyThresholdDb when provided", () => {
const err = new DrawElementVerificationError("drawElement self-verify failed at frame 649", {
frameIndex: 649,
failedDb: 28.4,
verifyThresholdDb: 32,
});
expect(isDrawElementVerificationError(err)).toBe(true);
expect(getDrawElementVerificationDetails(err)).toEqual({
frameIndex: 649,
failedDb: 28.4,
verifyThresholdDb: 32,
});
});
it("omits fields that weren't supplied (blank-frame throws have no dB)", () => {
const err = new DrawElementVerificationError("blank drawElement frame 12", { frameIndex: 12 });
expect(getDrawElementVerificationDetails(err)).toEqual({ frameIndex: 12 });
});
it("returns undefined for a non-verification error", () => {
expect(getDrawElementVerificationDetails(new Error("boring"))).toBeUndefined();
});
it("finds details through a wrapping cause chain (producer's CaptureStageError)", () => {
const inner = new DrawElementVerificationError("psnr breach", {
frameIndex: 5,
failedDb: 12.1,
});
const wrapper = new Error("capture stage failed", { cause: inner });
expect(isDrawElementVerificationError(wrapper)).toBe(true);
expect(getDrawElementVerificationDetails(wrapper)).toEqual({ frameIndex: 5, failedDb: 12.1 });
});
});
+46 -1
View File
@@ -201,14 +201,34 @@ export interface CaptureSession {
* forceScreenshot. Discriminant-based guard (not instanceof) so it survives
* duplicated module instances across package boundaries.
*/
/**
* Structured detail carried alongside the human-readable message lets
* telemetry report the actual failing dB / frame index instead of the
* orchestrator having to regex them back out of formatted text. All optional:
* a blank-frame trip has no PSNR score, so `failedDb`/`verifyThresholdDb`
* are omitted for that throw site.
*/
export interface DrawElementVerificationDetails {
frameIndex?: number;
failedDb?: number;
verifyThresholdDb?: number;
}
export class DrawElementVerificationError extends Error {
constructor(message: string) {
readonly frameIndex?: number;
readonly failedDb?: number;
readonly verifyThresholdDb?: number;
constructor(message: string, details?: DrawElementVerificationDetails) {
super(message);
this.name = "DrawElementVerificationError";
// Discriminant property, assigned dynamically: isDrawElementVerificationError
// reads it structurally so detection survives duplicated module instances
// across package boundaries (where instanceof fails).
(this as unknown as { deVerificationFailure: boolean }).deVerificationFailure = true;
this.frameIndex = details?.frameIndex;
this.failedDb = details?.failedDb;
this.verifyThresholdDb = details?.verifyThresholdDb;
}
}
@@ -222,6 +242,31 @@ export function isDrawElementVerificationError(err: unknown): boolean {
return false;
}
/**
* Extracts the structured details off a (possibly cause-wrapped) verification
* error same chain-walk as isDrawElementVerificationError, structural
* (not instanceof) for the same duplicated-module-instance reason. Returns
* undefined when the error isn't a verification failure at all.
*/
export function getDrawElementVerificationDetails(
err: unknown,
): DrawElementVerificationDetails | undefined {
let e: unknown = err;
for (let depth = 0; depth < 5 && typeof e === "object" && e !== null; depth++) {
const rec = e as { deVerificationFailure?: boolean } & Partial<DrawElementVerificationDetails>;
if (rec.deVerificationFailure === true) {
const details: DrawElementVerificationDetails = {};
if (typeof rec.frameIndex === "number") details.frameIndex = rec.frameIndex;
if (typeof rec.failedDb === "number") details.failedDb = rec.failedDb;
if (typeof rec.verifyThresholdDb === "number")
details.verifyThresholdDb = rec.verifyThresholdDb;
return details;
}
e = (e as { cause?: unknown }).cause;
}
return undefined;
}
/** Wait for inline CSS background images introduced by the latest seek. */
export async function decodeDynamicCssBackgroundImages(page: Page): Promise<void> {
await page.evaluate(async () => {
@@ -60,6 +60,10 @@ export interface RenderCaptureObservability {
* telemetry from one that never attempted any fallback.
*/
deFallbackReason?: string;
/** The failing PSNR (dB) when `deFallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
deFallbackFailedDb?: number;
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
deFallbackFrameIndex?: number;
/** Auto-parallel inversion outcome: "inverted" (fired, held) | "reverted" (fired, self-verify retry rolled back). */
deWorkerInversion?: "inverted" | "reverted";
/** Worker count the resolver would have used absent the inversion; undefined if it never fired. */
@@ -72,6 +72,10 @@ export interface DrawElementPerfInput {
preRouterWorkers?: number;
selfVerifyFallback: boolean;
fallbackReason?: string;
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error. */
fallbackFailedDb?: number;
/** Frame index the verification failure was detected at; set for both "psnr" and "blank". */
fallbackFrameIndex?: number;
drainStats?: {
verifyChecked: number;
verifyMinDb?: number;
@@ -112,6 +116,11 @@ function aggregateDrawElement(
verifyInitMs: perfs.reduce((sum, p) => sum + (p.deVerifyInitMs ?? 0), 0),
selfVerifyFallback: de.selfVerifyFallback,
fallbackReason: de.fallbackReason,
fallbackFailedDb:
de.fallbackFailedDb === undefined
? undefined
: Math.round(Math.min(de.fallbackFailedDb, 999) * 10) / 10,
fallbackFrameIndex: de.fallbackFrameIndex,
blankSuspects: drain?.blankSuspects ?? 0,
blankDeterministicAccepts: drain?.blankDeterministicAccepts ?? 0,
blankRecaptures: drain?.blankRecaptures ?? 0,
@@ -289,6 +289,7 @@ function createDrainFrameGuard(args: {
} catch (err) {
throw new DrawElementVerificationError(
`blank drawElement frame ${idx}: ${buf.length}B < floor ${Math.round(floor)}B and recapture failed (${err instanceof Error ? err.message : String(err)})`,
{ frameIndex: idx },
);
}
if (retryBuf.equals(buf)) {
@@ -306,6 +307,7 @@ function createDrainFrameGuard(args: {
} else if (retryBuf.length < floor) {
throw new DrawElementVerificationError(
`blank drawElement frame ${idx}: ${buf.length}B (retry ${retryBuf.length}B) < floor ${Math.round(floor)}B`,
{ frameIndex: idx },
);
} else {
buf = retryBuf;
@@ -339,6 +341,7 @@ function createDrainFrameGuard(args: {
}
throw new DrawElementVerificationError(
`drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
{ frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
);
}
stats.verifyChecked += 1;
@@ -76,6 +76,7 @@ import {
isMemoryExhaustionError,
isTransientBrowserError,
isDrawElementVerificationError,
getDrawElementVerificationDetails,
} from "@hyperframes/engine";
import { join, dirname, resolve } from "path";
import { totalmem } from "node:os";
@@ -449,6 +450,10 @@ export interface RenderPerfSummary {
selfVerifyFallback: boolean;
/** What tripped the fallback retry: psnr | blank | oom | capture_error. */
fallbackReason?: string;
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
fallbackFailedDb?: number;
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
fallbackFrameIndex?: number;
/** Blank-guard counters. */
blankSuspects: number;
blankDeterministicAccepts: number;
@@ -1737,6 +1742,11 @@ export async function executeRenderJob(
let captureParallelStreamForced = false;
let deSelfVerifyFallback = false;
let deFallbackReason: string | undefined;
// Structured detail behind deFallbackReason's "blank"/"psnr" bucket — the
// failing dB and frame index otherwise only exist as text inside the
// thrown error's message, unavailable to telemetry.
let deFallbackFailedDb: number | undefined;
let deFallbackFrameIndex: number | undefined;
let deDrainStats: import("./render/stages/captureStreamingStage.js").DeDrainStats | undefined;
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
observability.checkpoint("compile", "composition metadata resolved", {
@@ -2718,6 +2728,11 @@ export async function executeRenderJob(
: isMemoryExhaustion
? "oom"
: "capture_error";
if (isVerifyError) {
const verifyDetails = getDrawElementVerificationDetails(err);
deFallbackFailedDb = verifyDetails?.failedDb;
deFallbackFrameIndex = verifyDetails?.frameIndex;
}
log.warn(
isVerifyError
? "[Render] drawElement self-verification failed; re-rendering via screenshot"
@@ -2735,6 +2750,8 @@ export async function executeRenderJob(
forceScreenshot: true,
deSelfVerifyFallback,
deFallbackReason,
deFallbackFailedDb,
deFallbackFrameIndex,
});
probeSession = null;
// Must clear BEFORE resolveParallelRouterRetryPlan recomputes
@@ -3023,6 +3040,8 @@ export async function executeRenderJob(
preRouterWorkers: deParallelRouter ? preRoutingWorkerCount : undefined,
selfVerifyFallback: deSelfVerifyFallback,
fallbackReason: deFallbackReason,
fallbackFailedDb: deFallbackFailedDb,
fallbackFrameIndex: deFallbackFrameIndex,
drainStats: deDrainStats,
},
hdrDiagnostics,