mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-16 06:19:21 +00:00
fix(engine,producer,cli): count drawElement frame timeouts where they happen
`session.deFrameTimeouts` was incremented inside `captureFrameCore`'s catch, but the deadline races that function from OUTSIDE it — `withFrameDeadline` is applied by `captureFrameToBuffer`, and `DeFrameTimeoutError` is constructed nowhere else. So the branch was unreachable: on a wedged renderer the counter stayed 0, the engine-side stall log never printed, and `CapturePerfSummary.deFrameTimeouts` reported 0 on every affected render. The end-to-end recovery worked throughout — `isDeRendererStallError` matches by name at the producer — only the observability was dead. Moved the increment and the log into an `onTimeout` hook that fires from the deadline itself, deleted the unreachable branch and the now-unused `isDeFrameTimeoutError`, and threaded the count out to `render_completed.de_frame_timeouts` so the rate is graphable apart from the other `capture_error` fallbacks. A stall always costs a whole-render re-run, which is why it deserves its own series. Also corrected the `fallbackReason` docstring, which still listed only `psnr | blank | oom | capture_error` after this branch added `de_renderer_stall`. `withFrameDeadline` is exported for a two-case fake-timer test: the deadline fires and calls `onTimeout` once, and a resolving inner promise passes its value through with no late timer. The first case fails if the counter moves back inside the work promise.
This commit is contained in:
@@ -1514,6 +1514,7 @@ function trackRenderMetrics(
|
||||
deBlankRecaptures: perf?.drawElement?.blankRecaptures,
|
||||
deBoundaryFrames: perf?.drawElement?.boundaryFrames,
|
||||
deNcprFallbacks: perf?.drawElement?.ncprFallbacks,
|
||||
deFrameTimeouts: perf?.drawElement?.frameTimeouts,
|
||||
compositionDurationMs,
|
||||
compositionWidth: perf?.resolution.width,
|
||||
compositionHeight: perf?.resolution.height,
|
||||
|
||||
@@ -235,6 +235,7 @@ export function trackRenderComplete(
|
||||
deBlankRecaptures?: number;
|
||||
deBoundaryFrames?: number;
|
||||
deNcprFallbacks?: number;
|
||||
deFrameTimeouts?: number;
|
||||
// "cli" when triggered by `hyperframes render` (default), "studio" when
|
||||
// triggered by a studio preview-server render (POST /api/projects/:id/render).
|
||||
source?: "cli" | "studio";
|
||||
@@ -336,6 +337,7 @@ export function trackRenderComplete(
|
||||
de_blank_recaptures: props.deBlankRecaptures,
|
||||
de_boundary_frames: props.deBoundaryFrames,
|
||||
de_ncpr_fallbacks: props.deNcprFallbacks,
|
||||
de_frame_timeouts: props.deFrameTimeouts,
|
||||
...powerStateFields(),
|
||||
source: props.source ?? "cli",
|
||||
composition_duration_ms: props.compositionDurationMs,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Tests for the per-frame drawElement deadline (`withFrameDeadline`, PRINFRA-488).
|
||||
*
|
||||
* The deadline races the capture round-trip from OUTSIDE `captureFrameCore`,
|
||||
* because puppeteer cannot abort an in-flight `page.evaluate`. That is exactly
|
||||
* why the stall counter has to live in the `onTimeout` hook: a wedged renderer
|
||||
* never returns, so no catch block inside the work promise ever runs. The first
|
||||
* shipped version incremented `session.deFrameTimeouts` in that unreachable
|
||||
* catch, so the counter — and the `CapturePerfSummary` field it feeds — read 0
|
||||
* on every stalled render.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { withFrameDeadline } from "./frameCapture.js";
|
||||
|
||||
describe("withFrameDeadline", () => {
|
||||
it("rejects with DeFrameTimeoutError and fires onTimeout when work outlives the deadline", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const onTimeout = vi.fn();
|
||||
// Never settles — the wedged-renderer shape.
|
||||
const raced = withFrameDeadline(new Promise<string>(() => {}), "frame 7", 15_000, onTimeout);
|
||||
const assertion = expect(raced).rejects.toThrow(/frame 7 exceeded 15000ms/);
|
||||
await vi.advanceTimersByTimeAsync(15_000);
|
||||
await assertion;
|
||||
expect(onTimeout).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("passes the value through and leaves onTimeout alone when work wins", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const onTimeout = vi.fn();
|
||||
const raced = withFrameDeadline(Promise.resolve("buffer"), "frame 7", 15_000, onTimeout);
|
||||
await expect(raced).resolves.toBe("buffer");
|
||||
// Past the deadline: the cleared timer must not fire late.
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(onTimeout).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -3163,21 +3163,34 @@ class DeFrameTimeoutError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
function isDeFrameTimeoutError(err: unknown): boolean {
|
||||
return err instanceof DeFrameTimeoutError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Race `work` against a deadline. The losing promise is NOT cancellable —
|
||||
* puppeteer cannot abort an in-flight `page.evaluate` — so its rejection is
|
||||
* swallowed to avoid an unhandled rejection when it eventually settles (or
|
||||
* never does).
|
||||
* never does). The orphaned round-trip keeps running in its Chrome worker;
|
||||
* that worker is reclaimed by the outer retry rebuilding the page
|
||||
* (`closeOrphanedProbeForRetry`), not by anything here.
|
||||
*
|
||||
* `onTimeout` fires exactly when the deadline wins, and is the ONLY place the
|
||||
* stall is observable: because the deadline races `work` from outside, nothing
|
||||
* inside `work` — including its own catch blocks — ever sees this error.
|
||||
*
|
||||
* Exported for the deadline unit test; `captureFrameToBuffer` is the only
|
||||
* production caller.
|
||||
*/
|
||||
async function withFrameDeadline<T>(work: Promise<T>, label: string, ms: number): Promise<T> {
|
||||
export async function withFrameDeadline<T>(
|
||||
work: Promise<T>,
|
||||
label: string,
|
||||
ms: number,
|
||||
onTimeout?: () => void,
|
||||
): Promise<T> {
|
||||
if (!(ms > 0)) return work;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const guard = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new DeFrameTimeoutError(label, ms)), ms);
|
||||
timer = setTimeout(() => {
|
||||
onTimeout?.();
|
||||
reject(new DeFrameTimeoutError(label, ms));
|
||||
}, ms);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([work, guard]);
|
||||
@@ -3316,22 +3329,6 @@ async function captureFrameCore(
|
||||
// (display toggled / detached / freshly-shown at a clip-cut boundary). This
|
||||
// is a per-frame condition, not a whole-comp one — fall back to screenshot
|
||||
// for THIS frame instead of aborting the render. See fast-capture-limitations.md.
|
||||
if (isDeFrameTimeoutError(err)) {
|
||||
// Deliberately NO per-frame screenshot fallback here. When the
|
||||
// renderer stops scheduling, it is wedged for EVERY subsequent
|
||||
// round-trip on that page — measured: the screenshot fallback blew
|
||||
// the same deadline. Fail fast instead and let the producer re-render
|
||||
// the whole comp on a fresh page via the screenshot path, which is
|
||||
// the only recovery that actually works.
|
||||
session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1;
|
||||
console.log(
|
||||
`[engine] fast capture: frame ${frameIndex} — capture exceeded ` +
|
||||
`${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` +
|
||||
`(PRINFRA-488). Failing the drawElement attempt so the whole render ` +
|
||||
`retries via screenshot.`,
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
if (isNoCachedPaintRecordError(err)) {
|
||||
session.deNcprFallbacks = (session.deNcprFallbacks ?? 0) + 1;
|
||||
console.log(
|
||||
@@ -3422,6 +3419,22 @@ export async function captureFrameToBuffer(
|
||||
captureFrameCore(session, frameIndex, time),
|
||||
`frame ${frameIndex}`,
|
||||
DE_FRAME_TIMEOUT_MS,
|
||||
() => {
|
||||
// Deliberately NO per-frame screenshot fallback. When the renderer
|
||||
// stops scheduling it is wedged for EVERY subsequent round-trip on
|
||||
// that page — measured: the screenshot fallback blew the same
|
||||
// deadline. Fail fast and let the producer re-render the whole comp
|
||||
// on a fresh page via the screenshot path, the only recovery that
|
||||
// works. Counted here rather than in captureFrameCore's catch: the
|
||||
// deadline rejects from outside it, so that catch never runs.
|
||||
session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1;
|
||||
console.log(
|
||||
`[engine] fast capture: frame ${frameIndex} — capture exceeded ` +
|
||||
`${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` +
|
||||
`(PRINFRA-488). Failing the drawElement attempt so the whole render ` +
|
||||
`retries via screenshot.`,
|
||||
);
|
||||
},
|
||||
)
|
||||
: await captureFrameCore(session, frameIndex, time);
|
||||
|
||||
|
||||
@@ -149,6 +149,7 @@ function aggregateDrawElement(
|
||||
blankRecaptures: drain?.blankRecaptures ?? 0,
|
||||
boundaryFrames: perfs.reduce((sum, p) => sum + (p.deBoundaryFrames ?? 0), 0),
|
||||
ncprFallbacks: perfs.reduce((sum, p) => sum + (p.deNcprFallbacks ?? 0), 0),
|
||||
frameTimeouts: perfs.reduce((sum, p) => sum + (p.deFrameTimeouts ?? 0), 0),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -529,9 +529,9 @@ export interface RenderPerfSummary {
|
||||
* `fallbackReason` being set is the "any fallback fired" signal.
|
||||
*/
|
||||
selfVerifyFallback: boolean;
|
||||
/** What tripped the fallback retry: psnr | blank | oom | capture_error. */
|
||||
/** What tripped the fallback retry: psnr | blank | oom | de_renderer_stall | capture_error. */
|
||||
fallbackReason?: string;
|
||||
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */
|
||||
/** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for every other reason (no score exists). */
|
||||
fallbackFailedDb?: number;
|
||||
/** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */
|
||||
fallbackFrameIndex?: number;
|
||||
@@ -545,6 +545,13 @@ export interface RenderPerfSummary {
|
||||
boundaryFrames: number;
|
||||
/** Per-frame "No cached paint record" screenshot fallbacks. */
|
||||
ncprFallbacks: number;
|
||||
/**
|
||||
* Frames that blew `HF_DE_FRAME_TIMEOUT_MS` — a wedged renderer
|
||||
* (PRINFRA-488). Distinct from the other fallback counters: this one always
|
||||
* costs a whole-render re-run via screenshot, so its rate is worth graphing
|
||||
* on its own rather than inside `capture_error`.
|
||||
*/
|
||||
frameTimeouts: number;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user