diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index 06d03da67d..a939ea9f98 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -1514,6 +1514,7 @@ function trackRenderMetrics( deBlankRecaptures: perf?.drawElement?.blankRecaptures, deBoundaryFrames: perf?.drawElement?.boundaryFrames, deNcprFallbacks: perf?.drawElement?.ncprFallbacks, + deFrameTimeouts: perf?.drawElement?.frameTimeouts, compositionDurationMs, compositionWidth: perf?.resolution.width, compositionHeight: perf?.resolution.height, diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index de3e371cd5..b529dbe022 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -235,6 +235,7 @@ export function trackRenderComplete( deBlankRecaptures?: number; deBoundaryFrames?: number; deNcprFallbacks?: number; + deFrameTimeouts?: number; // "cli" when triggered by `hyperframes render` (default), "studio" when // triggered by a studio preview-server render (POST /api/projects/:id/render). source?: "cli" | "studio"; @@ -336,6 +337,7 @@ export function trackRenderComplete( de_blank_recaptures: props.deBlankRecaptures, de_boundary_frames: props.deBoundaryFrames, de_ncpr_fallbacks: props.deNcprFallbacks, + de_frame_timeouts: props.deFrameTimeouts, ...powerStateFields(), source: props.source ?? "cli", composition_duration_ms: props.compositionDurationMs, diff --git a/packages/engine/src/services/frameCapture-frameDeadline.test.ts b/packages/engine/src/services/frameCapture-frameDeadline.test.ts new file mode 100644 index 0000000000..427c09b443 --- /dev/null +++ b/packages/engine/src/services/frameCapture-frameDeadline.test.ts @@ -0,0 +1,45 @@ +/** + * Tests for the per-frame drawElement deadline (`withFrameDeadline`, PRINFRA-488). + * + * The deadline races the capture round-trip from OUTSIDE `captureFrameCore`, + * because puppeteer cannot abort an in-flight `page.evaluate`. That is exactly + * why the stall counter has to live in the `onTimeout` hook: a wedged renderer + * never returns, so no catch block inside the work promise ever runs. The first + * shipped version incremented `session.deFrameTimeouts` in that unreachable + * catch, so the counter — and the `CapturePerfSummary` field it feeds — read 0 + * on every stalled render. + */ + +import { describe, expect, it, vi } from "vitest"; +import { withFrameDeadline } from "./frameCapture.js"; + +describe("withFrameDeadline", () => { + it("rejects with DeFrameTimeoutError and fires onTimeout when work outlives the deadline", async () => { + vi.useFakeTimers(); + try { + const onTimeout = vi.fn(); + // Never settles — the wedged-renderer shape. + const raced = withFrameDeadline(new Promise(() => {}), "frame 7", 15_000, onTimeout); + const assertion = expect(raced).rejects.toThrow(/frame 7 exceeded 15000ms/); + await vi.advanceTimersByTimeAsync(15_000); + await assertion; + expect(onTimeout).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); + + it("passes the value through and leaves onTimeout alone when work wins", async () => { + vi.useFakeTimers(); + try { + const onTimeout = vi.fn(); + const raced = withFrameDeadline(Promise.resolve("buffer"), "frame 7", 15_000, onTimeout); + await expect(raced).resolves.toBe("buffer"); + // Past the deadline: the cleared timer must not fire late. + await vi.advanceTimersByTimeAsync(30_000); + expect(onTimeout).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index 1b341f8237..efc3d2092f 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -3163,21 +3163,34 @@ class DeFrameTimeoutError extends Error { } } -function isDeFrameTimeoutError(err: unknown): boolean { - return err instanceof DeFrameTimeoutError; -} - /** * Race `work` against a deadline. The losing promise is NOT cancellable — * puppeteer cannot abort an in-flight `page.evaluate` — so its rejection is * swallowed to avoid an unhandled rejection when it eventually settles (or - * never does). + * never does). The orphaned round-trip keeps running in its Chrome worker; + * that worker is reclaimed by the outer retry rebuilding the page + * (`closeOrphanedProbeForRetry`), not by anything here. + * + * `onTimeout` fires exactly when the deadline wins, and is the ONLY place the + * stall is observable: because the deadline races `work` from outside, nothing + * inside `work` — including its own catch blocks — ever sees this error. + * + * Exported for the deadline unit test; `captureFrameToBuffer` is the only + * production caller. */ -async function withFrameDeadline(work: Promise, label: string, ms: number): Promise { +export async function withFrameDeadline( + work: Promise, + label: string, + ms: number, + onTimeout?: () => void, +): Promise { if (!(ms > 0)) return work; let timer: ReturnType | undefined; const guard = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new DeFrameTimeoutError(label, ms)), ms); + timer = setTimeout(() => { + onTimeout?.(); + reject(new DeFrameTimeoutError(label, ms)); + }, ms); }); try { return await Promise.race([work, guard]); @@ -3316,22 +3329,6 @@ async function captureFrameCore( // (display toggled / detached / freshly-shown at a clip-cut boundary). This // is a per-frame condition, not a whole-comp one — fall back to screenshot // for THIS frame instead of aborting the render. See fast-capture-limitations.md. - if (isDeFrameTimeoutError(err)) { - // Deliberately NO per-frame screenshot fallback here. When the - // renderer stops scheduling, it is wedged for EVERY subsequent - // round-trip on that page — measured: the screenshot fallback blew - // the same deadline. Fail fast instead and let the producer re-render - // the whole comp on a fresh page via the screenshot path, which is - // the only recovery that actually works. - session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1; - console.log( - `[engine] fast capture: frame ${frameIndex} — capture exceeded ` + - `${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` + - `(PRINFRA-488). Failing the drawElement attempt so the whole render ` + - `retries via screenshot.`, - ); - throw err; - } if (isNoCachedPaintRecordError(err)) { session.deNcprFallbacks = (session.deNcprFallbacks ?? 0) + 1; console.log( @@ -3422,6 +3419,22 @@ export async function captureFrameToBuffer( captureFrameCore(session, frameIndex, time), `frame ${frameIndex}`, DE_FRAME_TIMEOUT_MS, + () => { + // Deliberately NO per-frame screenshot fallback. When the renderer + // stops scheduling it is wedged for EVERY subsequent round-trip on + // that page — measured: the screenshot fallback blew the same + // deadline. Fail fast and let the producer re-render the whole comp + // on a fresh page via the screenshot path, the only recovery that + // works. Counted here rather than in captureFrameCore's catch: the + // deadline rejects from outside it, so that catch never runs. + session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1; + console.log( + `[engine] fast capture: frame ${frameIndex} — capture exceeded ` + + `${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` + + `(PRINFRA-488). Failing the drawElement attempt so the whole render ` + + `retries via screenshot.`, + ); + }, ) : await captureFrameCore(session, frameIndex, time); diff --git a/packages/producer/src/services/render/perfSummary.ts b/packages/producer/src/services/render/perfSummary.ts index e4f4db3fc8..c100e3dac3 100644 --- a/packages/producer/src/services/render/perfSummary.ts +++ b/packages/producer/src/services/render/perfSummary.ts @@ -149,6 +149,7 @@ function aggregateDrawElement( blankRecaptures: drain?.blankRecaptures ?? 0, boundaryFrames: perfs.reduce((sum, p) => sum + (p.deBoundaryFrames ?? 0), 0), ncprFallbacks: perfs.reduce((sum, p) => sum + (p.deNcprFallbacks ?? 0), 0), + frameTimeouts: perfs.reduce((sum, p) => sum + (p.deFrameTimeouts ?? 0), 0), }; } diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 74c75a223e..3a0febf80c 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -529,9 +529,9 @@ export interface RenderPerfSummary { * `fallbackReason` being set is the "any fallback fired" signal. */ selfVerifyFallback: boolean; - /** What tripped the fallback retry: psnr | blank | oom | capture_error. */ + /** What tripped the fallback retry: psnr | blank | oom | de_renderer_stall | capture_error. */ fallbackReason?: string; - /** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */ + /** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for every other reason (no score exists). */ fallbackFailedDb?: number; /** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */ fallbackFrameIndex?: number; @@ -545,6 +545,13 @@ export interface RenderPerfSummary { boundaryFrames: number; /** Per-frame "No cached paint record" screenshot fallbacks. */ ncprFallbacks: number; + /** + * Frames that blew `HF_DE_FRAME_TIMEOUT_MS` — a wedged renderer + * (PRINFRA-488). Distinct from the other fallback counters: this one always + * costs a whole-render re-run via screenshot, so its rate is worth graphing + * on its own rather than inside `capture_error`. + */ + frameTimeouts: number; }; }