fix(engine,producer,cli): count drawElement frame timeouts where they happen

`session.deFrameTimeouts` was incremented inside `captureFrameCore`'s catch,
but the deadline races that function from OUTSIDE it — `withFrameDeadline` is
applied by `captureFrameToBuffer`, and `DeFrameTimeoutError` is constructed
nowhere else. So the branch was unreachable: on a wedged renderer the counter
stayed 0, the engine-side stall log never printed, and
`CapturePerfSummary.deFrameTimeouts` reported 0 on every affected render. The
end-to-end recovery worked throughout — `isDeRendererStallError` matches by
name at the producer — only the observability was dead.

Moved the increment and the log into an `onTimeout` hook that fires from the
deadline itself, deleted the unreachable branch and the now-unused
`isDeFrameTimeoutError`, and threaded the count out to
`render_completed.de_frame_timeouts` so the rate is graphable apart from the
other `capture_error` fallbacks. A stall always costs a whole-render re-run,
which is why it deserves its own series.

Also corrected the `fallbackReason` docstring, which still listed only
`psnr | blank | oom | capture_error` after this branch added
`de_renderer_stall`.

`withFrameDeadline` is exported for a two-case fake-timer test: the deadline
fires and calls `onTimeout` once, and a resolving inner promise passes its
value through with no late timer. The first case fails if the counter moves
back inside the work promise.
This commit is contained in:
Vance Ingalls
2026-08-20 22:48:29 -07:00
parent f7a297b35b
commit 1068fd1b4f
6 changed files with 94 additions and 25 deletions
+1
View File
@@ -1514,6 +1514,7 @@ function trackRenderMetrics(
deBlankRecaptures: perf?.drawElement?.blankRecaptures,
deBoundaryFrames: perf?.drawElement?.boundaryFrames,
deNcprFallbacks: perf?.drawElement?.ncprFallbacks,
deFrameTimeouts: perf?.drawElement?.frameTimeouts,
compositionDurationMs,
compositionWidth: perf?.resolution.width,
compositionHeight: perf?.resolution.height,
+2
View File
@@ -235,6 +235,7 @@ export function trackRenderComplete(
deBlankRecaptures?: number;
deBoundaryFrames?: number;
deNcprFallbacks?: number;
deFrameTimeouts?: number;
// "cli" when triggered by `hyperframes render` (default), "studio" when
// triggered by a studio preview-server render (POST /api/projects/:id/render).
source?: "cli" | "studio";
@@ -336,6 +337,7 @@ export function trackRenderComplete(
de_blank_recaptures: props.deBlankRecaptures,
de_boundary_frames: props.deBoundaryFrames,
de_ncpr_fallbacks: props.deNcprFallbacks,
de_frame_timeouts: props.deFrameTimeouts,
...powerStateFields(),
source: props.source ?? "cli",
composition_duration_ms: props.compositionDurationMs,
@@ -0,0 +1,45 @@
/**
* Tests for the per-frame drawElement deadline (`withFrameDeadline`, PRINFRA-488).
*
* The deadline races the capture round-trip from OUTSIDE `captureFrameCore`,
* because puppeteer cannot abort an in-flight `page.evaluate`. That is exactly
* why the stall counter has to live in the `onTimeout` hook: a wedged renderer
* never returns, so no catch block inside the work promise ever runs. The first
* shipped version incremented `session.deFrameTimeouts` in that unreachable
* catch, so the counter — and the `CapturePerfSummary` field it feeds — read 0
* on every stalled render.
*/
import { describe, expect, it, vi } from "vitest";
import { withFrameDeadline } from "./frameCapture.js";
describe("withFrameDeadline", () => {
it("rejects with DeFrameTimeoutError and fires onTimeout when work outlives the deadline", async () => {
vi.useFakeTimers();
try {
const onTimeout = vi.fn();
// Never settles — the wedged-renderer shape.
const raced = withFrameDeadline(new Promise<string>(() => {}), "frame 7", 15_000, onTimeout);
const assertion = expect(raced).rejects.toThrow(/frame 7 exceeded 15000ms/);
await vi.advanceTimersByTimeAsync(15_000);
await assertion;
expect(onTimeout).toHaveBeenCalledTimes(1);
} finally {
vi.useRealTimers();
}
});
it("passes the value through and leaves onTimeout alone when work wins", async () => {
vi.useFakeTimers();
try {
const onTimeout = vi.fn();
const raced = withFrameDeadline(Promise.resolve("buffer"), "frame 7", 15_000, onTimeout);
await expect(raced).resolves.toBe("buffer");
// Past the deadline: the cleared timer must not fire late.
await vi.advanceTimersByTimeAsync(30_000);
expect(onTimeout).not.toHaveBeenCalled();
} finally {
vi.useRealTimers();
}
});
});
+36 -23
View File
@@ -3163,21 +3163,34 @@ class DeFrameTimeoutError extends Error {
}
}
function isDeFrameTimeoutError(err: unknown): boolean {
return err instanceof DeFrameTimeoutError;
}
/**
* Race `work` against a deadline. The losing promise is NOT cancellable —
* puppeteer cannot abort an in-flight `page.evaluate` — so its rejection is
* swallowed to avoid an unhandled rejection when it eventually settles (or
* never does).
* never does). The orphaned round-trip keeps running in its Chrome worker;
* that worker is reclaimed by the outer retry rebuilding the page
* (`closeOrphanedProbeForRetry`), not by anything here.
*
* `onTimeout` fires exactly when the deadline wins, and is the ONLY place the
* stall is observable: because the deadline races `work` from outside, nothing
* inside `work` — including its own catch blocks — ever sees this error.
*
* Exported for the deadline unit test; `captureFrameToBuffer` is the only
* production caller.
*/
async function withFrameDeadline<T>(work: Promise<T>, label: string, ms: number): Promise<T> {
export async function withFrameDeadline<T>(
work: Promise<T>,
label: string,
ms: number,
onTimeout?: () => void,
): Promise<T> {
if (!(ms > 0)) return work;
let timer: ReturnType<typeof setTimeout> | undefined;
const guard = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new DeFrameTimeoutError(label, ms)), ms);
timer = setTimeout(() => {
onTimeout?.();
reject(new DeFrameTimeoutError(label, ms));
}, ms);
});
try {
return await Promise.race([work, guard]);
@@ -3316,22 +3329,6 @@ async function captureFrameCore(
// (display toggled / detached / freshly-shown at a clip-cut boundary). This
// is a per-frame condition, not a whole-comp one — fall back to screenshot
// for THIS frame instead of aborting the render. See fast-capture-limitations.md.
if (isDeFrameTimeoutError(err)) {
// Deliberately NO per-frame screenshot fallback here. When the
// renderer stops scheduling, it is wedged for EVERY subsequent
// round-trip on that page — measured: the screenshot fallback blew
// the same deadline. Fail fast instead and let the producer re-render
// the whole comp on a fresh page via the screenshot path, which is
// the only recovery that actually works.
session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1;
console.log(
`[engine] fast capture: frame ${frameIndex} — capture exceeded ` +
`${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` +
`(PRINFRA-488). Failing the drawElement attempt so the whole render ` +
`retries via screenshot.`,
);
throw err;
}
if (isNoCachedPaintRecordError(err)) {
session.deNcprFallbacks = (session.deNcprFallbacks ?? 0) + 1;
console.log(
@@ -3422,6 +3419,22 @@ export async function captureFrameToBuffer(
captureFrameCore(session, frameIndex, time),
`frame ${frameIndex}`,
DE_FRAME_TIMEOUT_MS,
() => {
// Deliberately NO per-frame screenshot fallback. When the renderer
// stops scheduling it is wedged for EVERY subsequent round-trip on
// that page — measured: the screenshot fallback blew the same
// deadline. Fail fast and let the producer re-render the whole comp
// on a fresh page via the screenshot path, the only recovery that
// works. Counted here rather than in captureFrameCore's catch: the
// deadline rejects from outside it, so that catch never runs.
session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1;
console.log(
`[engine] fast capture: frame ${frameIndex} — capture exceeded ` +
`${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` +
`(PRINFRA-488). Failing the drawElement attempt so the whole render ` +
`retries via screenshot.`,
);
},
)
: await captureFrameCore(session, frameIndex, time);
@@ -149,6 +149,7 @@ function aggregateDrawElement(
blankRecaptures: drain?.blankRecaptures ?? 0,
boundaryFrames: perfs.reduce((sum, p) => sum + (p.deBoundaryFrames ?? 0), 0),
ncprFallbacks: perfs.reduce((sum, p) => sum + (p.deNcprFallbacks ?? 0), 0),
frameTimeouts: perfs.reduce((sum, p) => sum + (p.deFrameTimeouts ?? 0), 0),
};
}
@@ -529,9 +529,9 @@ export interface RenderPerfSummary {
* `fallbackReason` being set is the "any fallback fired" signal.
*/
selfVerifyFallback: boolean;
/** What tripped the fallback retry: psnr | blank | oom | capture_error. */
/** What tripped the fallback retry: psnr | blank | oom | de_renderer_stall | capture_error. */
fallbackReason?: string;
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for every other reason (no score exists). */
fallbackFailedDb?: number;
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
fallbackFrameIndex?: number;
@@ -545,6 +545,13 @@ export interface RenderPerfSummary {
boundaryFrames: number;
/** Per-frame "No cached paint record" screenshot fallbacks. */
ncprFallbacks: number;
/**
* Frames that blew `HF_DE_FRAME_TIMEOUT_MS` — a wedged renderer
* (PRINFRA-488). Distinct from the other fallback counters: this one always
* costs a whole-render re-run via screenshot, so its rate is worth graphing
* on its own rather than inside `capture_error`.
*/
frameTimeouts: number;
};
}