mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
feat(engine,producer,cli): capture failing dB/frame index on drawElement verify fallback
de_fallback_reason only told you the fallback happened (blank/psnr/oom/ capture_error), not the failing PSNR or frame index — that data existed as text inside the thrown error's message and was discarded on the way to telemetry. DrawElementVerificationError now carries structured frameIndex/failedDb/verifyThresholdDb; the orchestrator reads them via the new getDrawElementVerificationDetails helper instead of regexing message text, and both telemetry surfaces (the render_complete perfSummary path and the crash-survival RenderCaptureObservability mirror) emit de_fallback_failed_db / de_fallback_frame_index. Needed to distinguish "32dB vs the 32dB threshold, tune it" from "12dB real corruption, investigate" during the parallel-router soak — currently that distinction is invisible.
This commit is contained in:
@@ -2064,6 +2064,8 @@ function trackRenderMetrics(
|
||||
deVerifyInitMs: perf?.drawElement?.verifyInitMs,
|
||||
deSelfVerifyFallback: perf?.drawElement?.selfVerifyFallback,
|
||||
deFallbackReason: perf?.drawElement?.fallbackReason,
|
||||
deFallbackFailedDb: perf?.drawElement?.fallbackFailedDb,
|
||||
deFallbackFrameIndex: perf?.drawElement?.fallbackFrameIndex,
|
||||
deBlankSuspects: perf?.drawElement?.blankSuspects,
|
||||
deBlankDeterministicAccepts: perf?.drawElement?.blankDeterministicAccepts,
|
||||
deBlankRecaptures: perf?.drawElement?.blankRecaptures,
|
||||
|
||||
@@ -252,6 +252,30 @@ describe("render telemetry events", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("carries the failing dB and frame index on render_error for a psnr fallback that failed hard afterward", () => {
|
||||
trackRenderError({
|
||||
fps: 30,
|
||||
quality: "standard",
|
||||
docker: false,
|
||||
errorMessage: "worker crashed after a psnr fallback",
|
||||
captureDeParallelRouter: "reverted",
|
||||
captureDeSelfVerifyFallback: true,
|
||||
captureDeFallbackReason: "psnr",
|
||||
captureDeFallbackFailedDb: 28.4,
|
||||
captureDeFallbackFrameIndex: 649,
|
||||
});
|
||||
|
||||
expect(trackEvent).toHaveBeenCalledWith(
|
||||
"render_error",
|
||||
expect.objectContaining({
|
||||
de_fallback_reason: "psnr",
|
||||
de_fallback_failed_db: 28.4,
|
||||
de_fallback_frame_index: 649,
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("prefers the explicit perfSummary-sourced de_worker_inversion over the capture-observability fallback on render_complete", () => {
|
||||
trackRenderComplete({
|
||||
durationMs: 1000,
|
||||
@@ -272,6 +296,30 @@ describe("render telemetry events", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("carries the perfSummary-sourced failing dB and frame index on render_complete", () => {
|
||||
trackRenderComplete({
|
||||
durationMs: 1000,
|
||||
fps: 30,
|
||||
quality: "standard",
|
||||
docker: false,
|
||||
gpu: false,
|
||||
deParallelRouter: "reverted",
|
||||
deFallbackReason: "psnr",
|
||||
deFallbackFailedDb: 28.4,
|
||||
deFallbackFrameIndex: 649,
|
||||
});
|
||||
|
||||
expect(trackEvent).toHaveBeenCalledWith(
|
||||
"render_complete",
|
||||
expect.objectContaining({
|
||||
de_fallback_reason: "psnr",
|
||||
de_fallback_failed_db: 28.4,
|
||||
de_fallback_frame_index: 649,
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("emits render_preflight_rejected with the low-cardinality issue kind", () => {
|
||||
trackRenderPreflightRejected({ kind: "aspect-mismatch" });
|
||||
expect(trackEvent).toHaveBeenCalledWith("render_preflight_rejected", {
|
||||
|
||||
@@ -54,6 +54,8 @@ export interface RenderObservabilityTelemetryPayload {
|
||||
captureDePreRouterWorkers?: number;
|
||||
captureDeSelfVerifyFallback?: boolean;
|
||||
captureDeFallbackReason?: string;
|
||||
captureDeFallbackFailedDb?: number;
|
||||
captureDeFallbackFrameIndex?: number;
|
||||
/** Non-DE parallel-streaming router outcome ("screenshot" | "beginframe" —
|
||||
* routed; "eligible_off" — would route but the kill switch is off). */
|
||||
captureParallelStream?: string;
|
||||
@@ -108,6 +110,8 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP
|
||||
de_pre_router_workers: props.captureDePreRouterWorkers,
|
||||
de_self_verify_fallback: props.captureDeSelfVerifyFallback,
|
||||
de_fallback_reason: props.captureDeFallbackReason,
|
||||
de_fallback_failed_db: props.captureDeFallbackFailedDb,
|
||||
de_fallback_frame_index: props.captureDeFallbackFrameIndex,
|
||||
capture_parallel_stream: props.captureParallelStream,
|
||||
observability_extract_video_count: props.observabilityExtractVideoCount,
|
||||
observability_extracted_video_count: props.observabilityExtractedVideoCount,
|
||||
@@ -173,6 +177,8 @@ export function trackRenderComplete(
|
||||
deVerifyInitMs?: number;
|
||||
deSelfVerifyFallback?: boolean;
|
||||
deFallbackReason?: string;
|
||||
deFallbackFailedDb?: number;
|
||||
deFallbackFrameIndex?: number;
|
||||
deBlankSuspects?: number;
|
||||
deBlankDeterministicAccepts?: number;
|
||||
deBlankRecaptures?: number;
|
||||
@@ -260,6 +266,8 @@ export function trackRenderComplete(
|
||||
de_verify_init_ms: props.deVerifyInitMs,
|
||||
de_self_verify_fallback: props.deSelfVerifyFallback,
|
||||
de_fallback_reason: props.deFallbackReason,
|
||||
de_fallback_failed_db: props.deFallbackFailedDb,
|
||||
de_fallback_frame_index: props.deFallbackFrameIndex,
|
||||
de_blank_suspects: props.deBlankSuspects,
|
||||
de_blank_deterministic_accepts: props.deBlankDeterministicAccepts,
|
||||
de_blank_recaptures: props.deBlankRecaptures,
|
||||
|
||||
@@ -81,6 +81,27 @@ describe("renderObservabilityTelemetryPayload — DE inversion/router cohort (fa
|
||||
const payload = renderObservabilityTelemetryPayload(makeSummary({}));
|
||||
expect(payload.captureDeFallbackReason).toBeUndefined();
|
||||
});
|
||||
|
||||
it("carries the failing dB and frame index for a psnr fallback, still visible on a hard failure", () => {
|
||||
const payload = renderObservabilityTelemetryPayload(
|
||||
makeSummary({
|
||||
deParallelRouter: "routed",
|
||||
deFallbackReason: "psnr",
|
||||
deFallbackFailedDb: 28.4,
|
||||
deFallbackFrameIndex: 649,
|
||||
}),
|
||||
);
|
||||
expect(payload.captureDeFallbackFailedDb).toBe(28.4);
|
||||
expect(payload.captureDeFallbackFrameIndex).toBe(649);
|
||||
});
|
||||
|
||||
it("leaves failedDb undefined for a blank/oom/capture_error fallback (no PSNR score exists)", () => {
|
||||
const payload = renderObservabilityTelemetryPayload(
|
||||
makeSummary({ deFallbackReason: "oom", deFallbackFrameIndex: undefined }),
|
||||
);
|
||||
expect(payload.captureDeFallbackFailedDb).toBeUndefined();
|
||||
expect(payload.captureDeFallbackFrameIndex).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderObservabilityTelemetryPayload — non-DE parallel-stream router", () => {
|
||||
|
||||
@@ -46,6 +46,8 @@ export function renderObservabilityTelemetryPayload(
|
||||
captureDePreRouterWorkers: capture.dePreRouterWorkers,
|
||||
captureDeSelfVerifyFallback: capture.deSelfVerifyFallback,
|
||||
captureDeFallbackReason: capture.deFallbackReason,
|
||||
captureDeFallbackFailedDb: capture.deFallbackFailedDb,
|
||||
captureDeFallbackFrameIndex: capture.deFallbackFrameIndex,
|
||||
captureParallelStream: capture.captureParallelStream,
|
||||
observabilityExtractVideoCount: extraction?.videoCount,
|
||||
observabilityExtractedVideoCount: extraction?.extractedVideoCount,
|
||||
|
||||
@@ -91,6 +91,8 @@ export {
|
||||
captureFramesBatchPipelined,
|
||||
DrawElementVerificationError,
|
||||
isDrawElementVerificationError,
|
||||
getDrawElementVerificationDetails,
|
||||
type DrawElementVerificationDetails,
|
||||
recaptureDrawElementFrameForVerify,
|
||||
completeDeferredDrawElementInit,
|
||||
writeCapturedFrame,
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
DrawElementVerificationError,
|
||||
formatHttpErrorDiagnostic,
|
||||
formatConsoleDiagnostic,
|
||||
formatNavigationFailureDiagnostic,
|
||||
formatNavigationStartDiagnostic,
|
||||
formatRequestFailureDiagnostic,
|
||||
getDrawElementVerificationDetails,
|
||||
isFontResourceError,
|
||||
isDrawElementVerificationError,
|
||||
sanitizeDiagnosticUrl,
|
||||
} from "./frameCapture.js";
|
||||
|
||||
@@ -222,3 +225,38 @@ describe("navigation diagnostics", () => {
|
||||
).toBe("[Browser:HTTP403] GET https://cdn.example.com/frame.png resource=image Forbidden");
|
||||
});
|
||||
});
|
||||
|
||||
describe("DrawElementVerificationError details", () => {
|
||||
it("carries frameIndex/failedDb/verifyThresholdDb when provided", () => {
|
||||
const err = new DrawElementVerificationError("drawElement self-verify failed at frame 649", {
|
||||
frameIndex: 649,
|
||||
failedDb: 28.4,
|
||||
verifyThresholdDb: 32,
|
||||
});
|
||||
expect(isDrawElementVerificationError(err)).toBe(true);
|
||||
expect(getDrawElementVerificationDetails(err)).toEqual({
|
||||
frameIndex: 649,
|
||||
failedDb: 28.4,
|
||||
verifyThresholdDb: 32,
|
||||
});
|
||||
});
|
||||
|
||||
it("omits fields that weren't supplied (blank-frame throws have no dB)", () => {
|
||||
const err = new DrawElementVerificationError("blank drawElement frame 12", { frameIndex: 12 });
|
||||
expect(getDrawElementVerificationDetails(err)).toEqual({ frameIndex: 12 });
|
||||
});
|
||||
|
||||
it("returns undefined for a non-verification error", () => {
|
||||
expect(getDrawElementVerificationDetails(new Error("boring"))).toBeUndefined();
|
||||
});
|
||||
|
||||
it("finds details through a wrapping cause chain (producer's CaptureStageError)", () => {
|
||||
const inner = new DrawElementVerificationError("psnr breach", {
|
||||
frameIndex: 5,
|
||||
failedDb: 12.1,
|
||||
});
|
||||
const wrapper = new Error("capture stage failed", { cause: inner });
|
||||
expect(isDrawElementVerificationError(wrapper)).toBe(true);
|
||||
expect(getDrawElementVerificationDetails(wrapper)).toEqual({ frameIndex: 5, failedDb: 12.1 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -201,14 +201,34 @@ export interface CaptureSession {
|
||||
* forceScreenshot. Discriminant-based guard (not instanceof) so it survives
|
||||
* duplicated module instances across package boundaries.
|
||||
*/
|
||||
/**
|
||||
* Structured detail carried alongside the human-readable message — lets
|
||||
* telemetry report the actual failing dB / frame index instead of the
|
||||
* orchestrator having to regex them back out of formatted text. All optional:
|
||||
* a blank-frame trip has no PSNR score, so `failedDb`/`verifyThresholdDb`
|
||||
* are omitted for that throw site.
|
||||
*/
|
||||
export interface DrawElementVerificationDetails {
|
||||
frameIndex?: number;
|
||||
failedDb?: number;
|
||||
verifyThresholdDb?: number;
|
||||
}
|
||||
|
||||
export class DrawElementVerificationError extends Error {
|
||||
constructor(message: string) {
|
||||
readonly frameIndex?: number;
|
||||
readonly failedDb?: number;
|
||||
readonly verifyThresholdDb?: number;
|
||||
|
||||
constructor(message: string, details?: DrawElementVerificationDetails) {
|
||||
super(message);
|
||||
this.name = "DrawElementVerificationError";
|
||||
// Discriminant property, assigned dynamically: isDrawElementVerificationError
|
||||
// reads it structurally so detection survives duplicated module instances
|
||||
// across package boundaries (where instanceof fails).
|
||||
(this as unknown as { deVerificationFailure: boolean }).deVerificationFailure = true;
|
||||
this.frameIndex = details?.frameIndex;
|
||||
this.failedDb = details?.failedDb;
|
||||
this.verifyThresholdDb = details?.verifyThresholdDb;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -222,6 +242,31 @@ export function isDrawElementVerificationError(err: unknown): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the structured details off a (possibly cause-wrapped) verification
|
||||
* error — same chain-walk as isDrawElementVerificationError, structural
|
||||
* (not instanceof) for the same duplicated-module-instance reason. Returns
|
||||
* undefined when the error isn't a verification failure at all.
|
||||
*/
|
||||
export function getDrawElementVerificationDetails(
|
||||
err: unknown,
|
||||
): DrawElementVerificationDetails | undefined {
|
||||
let e: unknown = err;
|
||||
for (let depth = 0; depth < 5 && typeof e === "object" && e !== null; depth++) {
|
||||
const rec = e as { deVerificationFailure?: boolean } & Partial<DrawElementVerificationDetails>;
|
||||
if (rec.deVerificationFailure === true) {
|
||||
const details: DrawElementVerificationDetails = {};
|
||||
if (typeof rec.frameIndex === "number") details.frameIndex = rec.frameIndex;
|
||||
if (typeof rec.failedDb === "number") details.failedDb = rec.failedDb;
|
||||
if (typeof rec.verifyThresholdDb === "number")
|
||||
details.verifyThresholdDb = rec.verifyThresholdDb;
|
||||
return details;
|
||||
}
|
||||
e = (e as { cause?: unknown }).cause;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Wait for inline CSS background images introduced by the latest seek. */
|
||||
export async function decodeDynamicCssBackgroundImages(page: Page): Promise<void> {
|
||||
await page.evaluate(async () => {
|
||||
|
||||
@@ -60,6 +60,10 @@ export interface RenderCaptureObservability {
|
||||
* telemetry from one that never attempted any fallback.
|
||||
*/
|
||||
deFallbackReason?: string;
|
||||
/** The failing PSNR (dB) when `deFallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
|
||||
deFallbackFailedDb?: number;
|
||||
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
|
||||
deFallbackFrameIndex?: number;
|
||||
/** Auto-parallel inversion outcome: "inverted" (fired, held) | "reverted" (fired, self-verify retry rolled back). */
|
||||
deWorkerInversion?: "inverted" | "reverted";
|
||||
/** Worker count the resolver would have used absent the inversion; undefined if it never fired. */
|
||||
|
||||
@@ -72,6 +72,10 @@ export interface DrawElementPerfInput {
|
||||
preRouterWorkers?: number;
|
||||
selfVerifyFallback: boolean;
|
||||
fallbackReason?: string;
|
||||
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error. */
|
||||
fallbackFailedDb?: number;
|
||||
/** Frame index the verification failure was detected at; set for both "psnr" and "blank". */
|
||||
fallbackFrameIndex?: number;
|
||||
drainStats?: {
|
||||
verifyChecked: number;
|
||||
verifyMinDb?: number;
|
||||
@@ -112,6 +116,11 @@ function aggregateDrawElement(
|
||||
verifyInitMs: perfs.reduce((sum, p) => sum + (p.deVerifyInitMs ?? 0), 0),
|
||||
selfVerifyFallback: de.selfVerifyFallback,
|
||||
fallbackReason: de.fallbackReason,
|
||||
fallbackFailedDb:
|
||||
de.fallbackFailedDb === undefined
|
||||
? undefined
|
||||
: Math.round(Math.min(de.fallbackFailedDb, 999) * 10) / 10,
|
||||
fallbackFrameIndex: de.fallbackFrameIndex,
|
||||
blankSuspects: drain?.blankSuspects ?? 0,
|
||||
blankDeterministicAccepts: drain?.blankDeterministicAccepts ?? 0,
|
||||
blankRecaptures: drain?.blankRecaptures ?? 0,
|
||||
|
||||
@@ -289,6 +289,7 @@ function createDrainFrameGuard(args: {
|
||||
} catch (err) {
|
||||
throw new DrawElementVerificationError(
|
||||
`blank drawElement frame ${idx}: ${buf.length}B < floor ${Math.round(floor)}B and recapture failed (${err instanceof Error ? err.message : String(err)})`,
|
||||
{ frameIndex: idx },
|
||||
);
|
||||
}
|
||||
if (retryBuf.equals(buf)) {
|
||||
@@ -306,6 +307,7 @@ function createDrainFrameGuard(args: {
|
||||
} else if (retryBuf.length < floor) {
|
||||
throw new DrawElementVerificationError(
|
||||
`blank drawElement frame ${idx}: ${buf.length}B (retry ${retryBuf.length}B) < floor ${Math.round(floor)}B`,
|
||||
{ frameIndex: idx },
|
||||
);
|
||||
} else {
|
||||
buf = retryBuf;
|
||||
@@ -339,6 +341,7 @@ function createDrainFrameGuard(args: {
|
||||
}
|
||||
throw new DrawElementVerificationError(
|
||||
`drawElement self-verify failed at frame ${idx}: ${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot${dumpDir ? ` (pair: ${dumpDir})` : ""}`,
|
||||
{ frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
|
||||
);
|
||||
}
|
||||
stats.verifyChecked += 1;
|
||||
|
||||
@@ -76,6 +76,7 @@ import {
|
||||
isMemoryExhaustionError,
|
||||
isTransientBrowserError,
|
||||
isDrawElementVerificationError,
|
||||
getDrawElementVerificationDetails,
|
||||
} from "@hyperframes/engine";
|
||||
import { join, dirname, resolve } from "path";
|
||||
import { totalmem } from "node:os";
|
||||
@@ -449,6 +450,10 @@ export interface RenderPerfSummary {
|
||||
selfVerifyFallback: boolean;
|
||||
/** What tripped the fallback retry: psnr | blank | oom | capture_error. */
|
||||
fallbackReason?: string;
|
||||
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
|
||||
fallbackFailedDb?: number;
|
||||
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
|
||||
fallbackFrameIndex?: number;
|
||||
/** Blank-guard counters. */
|
||||
blankSuspects: number;
|
||||
blankDeterministicAccepts: number;
|
||||
@@ -1737,6 +1742,11 @@ export async function executeRenderJob(
|
||||
let captureParallelStreamForced = false;
|
||||
let deSelfVerifyFallback = false;
|
||||
let deFallbackReason: string | undefined;
|
||||
// Structured detail behind deFallbackReason's "blank"/"psnr" bucket — the
|
||||
// failing dB and frame index otherwise only exist as text inside the
|
||||
// thrown error's message, unavailable to telemetry.
|
||||
let deFallbackFailedDb: number | undefined;
|
||||
let deFallbackFrameIndex: number | undefined;
|
||||
let deDrainStats: import("./render/stages/captureStreamingStage.js").DeDrainStats | undefined;
|
||||
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
|
||||
observability.checkpoint("compile", "composition metadata resolved", {
|
||||
@@ -2718,6 +2728,11 @@ export async function executeRenderJob(
|
||||
: isMemoryExhaustion
|
||||
? "oom"
|
||||
: "capture_error";
|
||||
if (isVerifyError) {
|
||||
const verifyDetails = getDrawElementVerificationDetails(err);
|
||||
deFallbackFailedDb = verifyDetails?.failedDb;
|
||||
deFallbackFrameIndex = verifyDetails?.frameIndex;
|
||||
}
|
||||
log.warn(
|
||||
isVerifyError
|
||||
? "[Render] drawElement self-verification failed; re-rendering via screenshot"
|
||||
@@ -2735,6 +2750,8 @@ export async function executeRenderJob(
|
||||
forceScreenshot: true,
|
||||
deSelfVerifyFallback,
|
||||
deFallbackReason,
|
||||
deFallbackFailedDb,
|
||||
deFallbackFrameIndex,
|
||||
});
|
||||
probeSession = null;
|
||||
// Must clear BEFORE resolveParallelRouterRetryPlan recomputes
|
||||
@@ -3023,6 +3040,8 @@ export async function executeRenderJob(
|
||||
preRouterWorkers: deParallelRouter ? preRoutingWorkerCount : undefined,
|
||||
selfVerifyFallback: deSelfVerifyFallback,
|
||||
fallbackReason: deFallbackReason,
|
||||
fallbackFailedDb: deFallbackFailedDb,
|
||||
fallbackFrameIndex: deFallbackFrameIndex,
|
||||
drainStats: deDrainStats,
|
||||
},
|
||||
hdrDiagnostics,
|
||||
|
||||
Reference in New Issue
Block a user