diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index 6c7df1aed..9e5d2d702 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -2064,6 +2064,8 @@ function trackRenderMetrics( deVerifyInitMs: perf?.drawElement?.verifyInitMs, deSelfVerifyFallback: perf?.drawElement?.selfVerifyFallback, deFallbackReason: perf?.drawElement?.fallbackReason, + deFallbackFailedDb: perf?.drawElement?.fallbackFailedDb, + deFallbackFrameIndex: perf?.drawElement?.fallbackFrameIndex, deBlankSuspects: perf?.drawElement?.blankSuspects, deBlankDeterministicAccepts: perf?.drawElement?.blankDeterministicAccepts, deBlankRecaptures: perf?.drawElement?.blankRecaptures, diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/src/telemetry/events.test.ts index 17cf69a73..016a05ae2 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/src/telemetry/events.test.ts @@ -252,6 +252,30 @@ describe("render telemetry events", () => { ); }); + it("carries the failing dB and frame index on render_error for a psnr fallback that failed hard afterward", () => { + trackRenderError({ + fps: 30, + quality: "standard", + docker: false, + errorMessage: "worker crashed after a psnr fallback", + captureDeParallelRouter: "reverted", + captureDeSelfVerifyFallback: true, + captureDeFallbackReason: "psnr", + captureDeFallbackFailedDb: 28.4, + captureDeFallbackFrameIndex: 649, + }); + + expect(trackEvent).toHaveBeenCalledWith( + "render_error", + expect.objectContaining({ + de_fallback_reason: "psnr", + de_fallback_failed_db: 28.4, + de_fallback_frame_index: 649, + }), + undefined, + ); + }); + it("prefers the explicit perfSummary-sourced de_worker_inversion over the capture-observability fallback on render_complete", () => { trackRenderComplete({ durationMs: 1000, @@ -272,6 +296,30 @@ describe("render telemetry events", () => { ); }); + it("carries the perfSummary-sourced failing dB and frame index on render_complete", () => { + trackRenderComplete({ + durationMs: 1000, + fps: 30, + quality: "standard", + docker: false, + gpu: false, + deParallelRouter: "reverted", + deFallbackReason: "psnr", + deFallbackFailedDb: 28.4, + deFallbackFrameIndex: 649, + }); + + expect(trackEvent).toHaveBeenCalledWith( + "render_complete", + expect.objectContaining({ + de_fallback_reason: "psnr", + de_fallback_failed_db: 28.4, + de_fallback_frame_index: 649, + }), + undefined, + ); + }); + it("emits render_preflight_rejected with the low-cardinality issue kind", () => { trackRenderPreflightRejected({ kind: "aspect-mismatch" }); expect(trackEvent).toHaveBeenCalledWith("render_preflight_rejected", { diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index cb01725f1..3a17624df 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -54,6 +54,8 @@ export interface RenderObservabilityTelemetryPayload { captureDePreRouterWorkers?: number; captureDeSelfVerifyFallback?: boolean; captureDeFallbackReason?: string; + captureDeFallbackFailedDb?: number; + captureDeFallbackFrameIndex?: number; /** Non-DE parallel-streaming router outcome ("screenshot" | "beginframe" — * routed; "eligible_off" — would route but the kill switch is off). */ captureParallelStream?: string; @@ -108,6 +110,8 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP de_pre_router_workers: props.captureDePreRouterWorkers, de_self_verify_fallback: props.captureDeSelfVerifyFallback, de_fallback_reason: props.captureDeFallbackReason, + de_fallback_failed_db: props.captureDeFallbackFailedDb, + de_fallback_frame_index: props.captureDeFallbackFrameIndex, capture_parallel_stream: props.captureParallelStream, observability_extract_video_count: props.observabilityExtractVideoCount, observability_extracted_video_count: props.observabilityExtractedVideoCount, @@ -173,6 +177,8 @@ export function trackRenderComplete( deVerifyInitMs?: number; deSelfVerifyFallback?: boolean; deFallbackReason?: string; + deFallbackFailedDb?: number; + deFallbackFrameIndex?: number; deBlankSuspects?: number; deBlankDeterministicAccepts?: number; deBlankRecaptures?: number; @@ -260,6 +266,8 @@ export function trackRenderComplete( de_verify_init_ms: props.deVerifyInitMs, de_self_verify_fallback: props.deSelfVerifyFallback, de_fallback_reason: props.deFallbackReason, + de_fallback_failed_db: props.deFallbackFailedDb, + de_fallback_frame_index: props.deFallbackFrameIndex, de_blank_suspects: props.deBlankSuspects, de_blank_deterministic_accepts: props.deBlankDeterministicAccepts, de_blank_recaptures: props.deBlankRecaptures, diff --git a/packages/cli/src/telemetry/renderObservability.test.ts b/packages/cli/src/telemetry/renderObservability.test.ts index 0a64cc866..bca360959 100644 --- a/packages/cli/src/telemetry/renderObservability.test.ts +++ b/packages/cli/src/telemetry/renderObservability.test.ts @@ -81,6 +81,27 @@ describe("renderObservabilityTelemetryPayload — DE inversion/router cohort (fa const payload = renderObservabilityTelemetryPayload(makeSummary({})); expect(payload.captureDeFallbackReason).toBeUndefined(); }); + + it("carries the failing dB and frame index for a psnr fallback, still visible on a hard failure", () => { + const payload = renderObservabilityTelemetryPayload( + makeSummary({ + deParallelRouter: "routed", + deFallbackReason: "psnr", + deFallbackFailedDb: 28.4, + deFallbackFrameIndex: 649, + }), + ); + expect(payload.captureDeFallbackFailedDb).toBe(28.4); + expect(payload.captureDeFallbackFrameIndex).toBe(649); + }); + + it("leaves failedDb undefined for a blank/oom/capture_error fallback (no PSNR score exists)", () => { + const payload = renderObservabilityTelemetryPayload( + makeSummary({ deFallbackReason: "oom", deFallbackFrameIndex: undefined }), + ); + expect(payload.captureDeFallbackFailedDb).toBeUndefined(); + expect(payload.captureDeFallbackFrameIndex).toBeUndefined(); + }); }); describe("renderObservabilityTelemetryPayload — non-DE parallel-stream router", () => { diff --git a/packages/cli/src/telemetry/renderObservability.ts b/packages/cli/src/telemetry/renderObservability.ts index 25e17f118..c3cff4176 100644 --- a/packages/cli/src/telemetry/renderObservability.ts +++ b/packages/cli/src/telemetry/renderObservability.ts @@ -46,6 +46,8 @@ export function renderObservabilityTelemetryPayload( captureDePreRouterWorkers: capture.dePreRouterWorkers, captureDeSelfVerifyFallback: capture.deSelfVerifyFallback, captureDeFallbackReason: capture.deFallbackReason, + captureDeFallbackFailedDb: capture.deFallbackFailedDb, + captureDeFallbackFrameIndex: capture.deFallbackFrameIndex, captureParallelStream: capture.captureParallelStream, observabilityExtractVideoCount: extraction?.videoCount, observabilityExtractedVideoCount: extraction?.extractedVideoCount, diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index e5a6d426d..b539cb097 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -91,6 +91,8 @@ export { captureFramesBatchPipelined, DrawElementVerificationError, isDrawElementVerificationError, + getDrawElementVerificationDetails, + type DrawElementVerificationDetails, recaptureDrawElementFrameForVerify, completeDeferredDrawElementInit, writeCapturedFrame, diff --git a/packages/engine/src/services/frameCapture.test.ts b/packages/engine/src/services/frameCapture.test.ts index 49e39c30b..c66505791 100644 --- a/packages/engine/src/services/frameCapture.test.ts +++ b/packages/engine/src/services/frameCapture.test.ts @@ -1,11 +1,14 @@ import { describe, it, expect } from "vitest"; import { + DrawElementVerificationError, formatHttpErrorDiagnostic, formatConsoleDiagnostic, formatNavigationFailureDiagnostic, formatNavigationStartDiagnostic, formatRequestFailureDiagnostic, + getDrawElementVerificationDetails, isFontResourceError, + isDrawElementVerificationError, sanitizeDiagnosticUrl, } from "./frameCapture.js"; @@ -222,3 +225,38 @@ describe("navigation diagnostics", () => { ).toBe("[Browser:HTTP403] GET https://cdn.example.com/frame.png resource=image Forbidden"); }); }); + +describe("DrawElementVerificationError details", () => { + it("carries frameIndex/failedDb/verifyThresholdDb when provided", () => { + const err = new DrawElementVerificationError("drawElement self-verify failed at frame 649", { + frameIndex: 649, + failedDb: 28.4, + verifyThresholdDb: 32, + }); + expect(isDrawElementVerificationError(err)).toBe(true); + expect(getDrawElementVerificationDetails(err)).toEqual({ + frameIndex: 649, + failedDb: 28.4, + verifyThresholdDb: 32, + }); + }); + + it("omits fields that weren't supplied (blank-frame throws have no dB)", () => { + const err = new DrawElementVerificationError("blank drawElement frame 12", { frameIndex: 12 }); + expect(getDrawElementVerificationDetails(err)).toEqual({ frameIndex: 12 }); + }); + + it("returns undefined for a non-verification error", () => { + expect(getDrawElementVerificationDetails(new Error("boring"))).toBeUndefined(); + }); + + it("finds details through a wrapping cause chain (producer's CaptureStageError)", () => { + const inner = new DrawElementVerificationError("psnr breach", { + frameIndex: 5, + failedDb: 12.1, + }); + const wrapper = new Error("capture stage failed", { cause: inner }); + expect(isDrawElementVerificationError(wrapper)).toBe(true); + expect(getDrawElementVerificationDetails(wrapper)).toEqual({ frameIndex: 5, failedDb: 12.1 }); + }); +}); diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index 56cff28fc..1edf7b390 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -201,14 +201,34 @@ export interface CaptureSession { * forceScreenshot. Discriminant-based guard (not instanceof) so it survives * duplicated module instances across package boundaries. */ +/** + * Structured detail carried alongside the human-readable message — lets + * telemetry report the actual failing dB / frame index instead of the + * orchestrator having to regex them back out of formatted text. All optional: + * a blank-frame trip has no PSNR score, so `failedDb`/`verifyThresholdDb` + * are omitted for that throw site. + */ +export interface DrawElementVerificationDetails { + frameIndex?: number; + failedDb?: number; + verifyThresholdDb?: number; +} + export class DrawElementVerificationError extends Error { - constructor(message: string) { + readonly frameIndex?: number; + readonly failedDb?: number; + readonly verifyThresholdDb?: number; + + constructor(message: string, details?: DrawElementVerificationDetails) { super(message); this.name = "DrawElementVerificationError"; // Discriminant property, assigned dynamically: isDrawElementVerificationError // reads it structurally so detection survives duplicated module instances // across package boundaries (where instanceof fails). (this as unknown as { deVerificationFailure: boolean }).deVerificationFailure = true; + this.frameIndex = details?.frameIndex; + this.failedDb = details?.failedDb; + this.verifyThresholdDb = details?.verifyThresholdDb; } } @@ -222,6 +242,31 @@ export function isDrawElementVerificationError(err: unknown): boolean { return false; } +/** + * Extracts the structured details off a (possibly cause-wrapped) verification + * error — same chain-walk as isDrawElementVerificationError, structural + * (not instanceof) for the same duplicated-module-instance reason. Returns + * undefined when the error isn't a verification failure at all. + */ +export function getDrawElementVerificationDetails( + err: unknown, +): DrawElementVerificationDetails | undefined { + let e: unknown = err; + for (let depth = 0; depth < 5 && typeof e === "object" && e !== null; depth++) { + const rec = e as { deVerificationFailure?: boolean } & Partial; + if (rec.deVerificationFailure === true) { + const details: DrawElementVerificationDetails = {}; + if (typeof rec.frameIndex === "number") details.frameIndex = rec.frameIndex; + if (typeof rec.failedDb === "number") details.failedDb = rec.failedDb; + if (typeof rec.verifyThresholdDb === "number") + details.verifyThresholdDb = rec.verifyThresholdDb; + return details; + } + e = (e as { cause?: unknown }).cause; + } + return undefined; +} + /** Wait for inline CSS background images introduced by the latest seek. */ export async function decodeDynamicCssBackgroundImages(page: Page): Promise { await page.evaluate(async () => { diff --git a/packages/producer/src/services/render/observability.ts b/packages/producer/src/services/render/observability.ts index c180f0481..f5fa325d4 100644 --- a/packages/producer/src/services/render/observability.ts +++ b/packages/producer/src/services/render/observability.ts @@ -60,6 +60,10 @@ export interface RenderCaptureObservability { * telemetry from one that never attempted any fallback. */ deFallbackReason?: string; + /** The failing PSNR (dB) when `deFallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */ + deFallbackFailedDb?: number; + /** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */ + deFallbackFrameIndex?: number; /** Auto-parallel inversion outcome: "inverted" (fired, held) | "reverted" (fired, self-verify retry rolled back). */ deWorkerInversion?: "inverted" | "reverted"; /** Worker count the resolver would have used absent the inversion; undefined if it never fired. */ diff --git a/packages/producer/src/services/render/perfSummary.ts b/packages/producer/src/services/render/perfSummary.ts index 5fde2c880..fbecc64e1 100644 --- a/packages/producer/src/services/render/perfSummary.ts +++ b/packages/producer/src/services/render/perfSummary.ts @@ -72,6 +72,10 @@ export interface DrawElementPerfInput { preRouterWorkers?: number; selfVerifyFallback: boolean; fallbackReason?: string; + /** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error. */ + fallbackFailedDb?: number; + /** Frame index the verification failure was detected at; set for both "psnr" and "blank". */ + fallbackFrameIndex?: number; drainStats?: { verifyChecked: number; verifyMinDb?: number; @@ -112,6 +116,11 @@ function aggregateDrawElement( verifyInitMs: perfs.reduce((sum, p) => sum + (p.deVerifyInitMs ?? 0), 0), selfVerifyFallback: de.selfVerifyFallback, fallbackReason: de.fallbackReason, + fallbackFailedDb: + de.fallbackFailedDb === undefined + ? undefined + : Math.round(Math.min(de.fallbackFailedDb, 999) * 10) / 10, + fallbackFrameIndex: de.fallbackFrameIndex, blankSuspects: drain?.blankSuspects ?? 0, blankDeterministicAccepts: drain?.blankDeterministicAccepts ?? 0, blankRecaptures: drain?.blankRecaptures ?? 0, diff --git a/packages/producer/src/services/render/stages/captureStreamingStage.ts b/packages/producer/src/services/render/stages/captureStreamingStage.ts index fe81ff1f4..9fdc1dbe9 100644 --- a/packages/producer/src/services/render/stages/captureStreamingStage.ts +++ b/packages/producer/src/services/render/stages/captureStreamingStage.ts @@ -289,6 +289,7 @@ function createDrainFrameGuard(args: { } catch (err) { throw new DrawElementVerificationError( `blank drawElement frame ${idx}: ${buf.length}B < floor ${Math.round(floor)}B and recapture failed (${err instanceof Error ? err.message : String(err)})`, + { frameIndex: idx }, ); } if (retryBuf.equals(buf)) { @@ -306,6 +307,7 @@ function createDrainFrameGuard(args: { } else if (retryBuf.length < floor) { throw new DrawElementVerificationError( `blank drawElement frame ${idx}: ${buf.length}B (retry ${retryBuf.length}B) < floor ${Math.round(floor)}B`, + { frameIndex: idx }, ); } else { buf = retryBuf; @@ -339,6 +341,7 @@ function createDrainFrameGuard(args: { } throw new DrawElementVerificationError( `drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`, + { frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb }, ); } stats.verifyChecked += 1; diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index b8cfc34f0..6e7592309 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -76,6 +76,7 @@ import { isMemoryExhaustionError, isTransientBrowserError, isDrawElementVerificationError, + getDrawElementVerificationDetails, } from "@hyperframes/engine"; import { join, dirname, resolve } from "path"; import { totalmem } from "node:os"; @@ -449,6 +450,10 @@ export interface RenderPerfSummary { selfVerifyFallback: boolean; /** What tripped the fallback retry: psnr | blank | oom | capture_error. */ fallbackReason?: string; + /** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */ + fallbackFailedDb?: number; + /** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */ + fallbackFrameIndex?: number; /** Blank-guard counters. */ blankSuspects: number; blankDeterministicAccepts: number; @@ -1737,6 +1742,11 @@ export async function executeRenderJob( let captureParallelStreamForced = false; let deSelfVerifyFallback = false; let deFallbackReason: string | undefined; + // Structured detail behind deFallbackReason's "blank"/"psnr" bucket — the + // failing dB and frame index otherwise only exist as text inside the + // thrown error's message, unavailable to telemetry. + let deFallbackFailedDb: number | undefined; + let deFallbackFrameIndex: number | undefined; let deDrainStats: import("./render/stages/captureStreamingStage.js").DeDrainStats | undefined; updateCaptureObservability({ forceScreenshot: captureForceScreenshot }); observability.checkpoint("compile", "composition metadata resolved", { @@ -2718,6 +2728,11 @@ export async function executeRenderJob( : isMemoryExhaustion ? "oom" : "capture_error"; + if (isVerifyError) { + const verifyDetails = getDrawElementVerificationDetails(err); + deFallbackFailedDb = verifyDetails?.failedDb; + deFallbackFrameIndex = verifyDetails?.frameIndex; + } log.warn( isVerifyError ? "[Render] drawElement self-verification failed; re-rendering via screenshot" @@ -2735,6 +2750,8 @@ export async function executeRenderJob( forceScreenshot: true, deSelfVerifyFallback, deFallbackReason, + deFallbackFailedDb, + deFallbackFrameIndex, }); probeSession = null; // Must clear BEFORE resolveParallelRouterRetryPlan recomputes @@ -3023,6 +3040,8 @@ export async function executeRenderJob( preRouterWorkers: deParallelRouter ? preRoutingWorkerCount : undefined, selfVerifyFallback: deSelfVerifyFallback, fallbackReason: deFallbackReason, + fallbackFailedDb: deFallbackFailedDb, + fallbackFrameIndex: deFallbackFrameIndex, drainStats: deDrainStats, }, hdrDiagnostics,