fix(producer): fail a render once when its page never became ready (#7154)

* fix(producer): fail a render once when its page never became ready

* refactor(producer): keep the page-never-ready error private to its module

* test(producer): drop a mock field the page-never-ready test never set
This commit is contained in:
Miguel Ángel
2026-11-04 11:53:21 +00:00
committed by GitHub
parent 11f43268a1
commit 5cea10e76e
6 changed files with 244 additions and 2 deletions
@@ -0,0 +1,38 @@
import { expect, it, vi } from "vitest";
import { resolveConfig } from "@hyperframes/engine";
import { runCaptureCalibration } from "./captureCost.js";
const { state } = vi.hoisted(() => ({ state: { message: "" } }));
vi.mock("@hyperframes/engine", async (importOriginal) => ({
...(await importOriginal<typeof import("@hyperframes/engine")>()),
createCaptureSession: async () => {
throw new Error(state.message);
},
closeCaptureSession: async () => {},
}));
function calibrate() {
return runCaptureCalibration({
cfg: resolveConfig({ forceScreenshot: true }),
fileServer: { url: "http://127.0.0.1:1" } as never,
workDir: "/tmp/unused",
log: { info() {}, warn() {}, error() {}, debug() {} },
job: { config: { fps: 30 } } as never,
totalFrames: 10,
forceScreenshot: true,
probeSession: null,
buildCaptureOptions: () => ({}) as never,
createRenderVideoFrameInjector: () => null,
assertNotAborted: () => {},
});
}
it("reports an evaluate timeout during calibration", async () => {
state.message = "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting";
expect((await calibrate()).calibrationEvaluateTimedOut).toBe(true);
});
it("does not report other calibration failures as an evaluate timeout", async () => {
state.message = "Navigation timeout of 60000 ms exceeded";
expect((await calibrate()).calibrationEvaluateTimedOut).toBe(false);
});
@@ -343,6 +343,8 @@ export interface CaptureCalibrationOutcome {
probeSession: CaptureSession | null;
/** Buffer of whichever session was active last; the sequencer uses it for the error-path tail. */
lastBrowserConsole: string[];
/** The final calibration failure was a `Runtime.evaluate` timeout: the page never answered. */
calibrationEvaluateTimedOut: boolean;
}
/**
@@ -381,6 +383,7 @@ export async function runCaptureCalibration(input: {
let probeSession = input.probeSession;
let forceScreenshot = input.forceScreenshot;
let lastBrowserConsole: string[] = [];
let calibrationEvaluateTimedOut = false;
const fps = fpsToNumber(job.config.fps);
// Holds whichever calibration session is currently open. The closure
@@ -440,6 +443,7 @@ export async function runCaptureCalibration(input: {
!forceScreenshot && shouldFallbackToScreenshotAfterCalibrationError(error);
if (!shouldFallback) {
calibration = createFailedCaptureCalibrationEstimate("calibration-failed");
calibrationEvaluateTimedOut = isEvaluateTimeout(error);
log.warn("[Render] Auto-worker calibration failed; using conservative worker budget.", {
protocolTimeout: calibrationCfg.protocolTimeout,
error: error instanceof Error ? error.message : String(error),
@@ -481,6 +485,7 @@ export async function runCaptureCalibration(input: {
);
} catch (fallbackError) {
calibration = createFailedCaptureCalibrationEstimate("calibration-screenshot-failed");
calibrationEvaluateTimedOut = isEvaluateTimeout(fallbackError);
log.warn(
"[Render] Screenshot auto-worker calibration failed after BeginFrame fallback; using conservative worker budget.",
{
@@ -497,7 +502,17 @@ export async function runCaptureCalibration(input: {
}
}
return { calibration, forceScreenshot, probeSession, lastBrowserConsole };
return {
calibration,
forceScreenshot,
probeSession,
lastBrowserConsole,
calibrationEvaluateTimedOut,
};
}
export function isEvaluateTimeout(error: unknown): boolean {
return /Runtime\.evaluate timed out/i.test(normalizeErrorMessage(error));
}
/**
@@ -0,0 +1,32 @@
import { isEvaluateTimeout } from "./captureCost.js";
const PAGE_NEVER_READY = "PAGE_NEVER_READY" as const;
class PageNeverReadyError extends Error {
readonly code = PAGE_NEVER_READY;
// The producer server reads these off the error for the wire envelope.
readonly owner = "user" as const;
readonly retryable = false as const;
constructor(cause: unknown) {
super(
"The composition page never became ready: it did not answer browser commands during " +
"calibration or capture, and no frame was captured. Its script may never return.",
{ cause },
);
this.name = "PageNeverReadyError";
}
}
/** Calibration and capture both timed out on `Runtime.evaluate` with no frame captured. */
export function checkPageNeverReady(input: {
calibrationEvaluateTimedOut: boolean;
framesRendered: number;
error: unknown;
}): Error | undefined {
const { calibrationEvaluateTimedOut, framesRendered, error } = input;
if (!calibrationEvaluateTimedOut || framesRendered > 0 || !isEvaluateTimeout(error)) {
return undefined;
}
return new PageNeverReadyError(error);
}
@@ -0,0 +1,148 @@
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { resolveConfig } from "@hyperframes/engine";
import { CaptureStageError } from "./render/captureStageError.js";
import { extractSafeRenderErrorMetadata } from "../server.js";
import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js";
// Entry: executeRenderJob on the streaming route; calibration and the streaming stage are scripted.
const { state } = vi.hoisted(() => ({
state: { calibrationEvaluateTimedOut: false, framesBeforeFailure: 0 },
}));
vi.mock("@hyperframes/engine", async (importOriginal) => ({
...(await importOriginal<typeof import("@hyperframes/engine")>()),
assertConfiguredFfmpegBinariesExist: () => {},
resolveBrowserGpuMode: async () => "software",
resolveHeadlessShellPath: () => "/opt/chrome-headless-shell",
createCaptureSession: async () => ({ browserConsoleBuffer: [], options: {} }),
initializeSession: async () => {},
closeCaptureSession: async () => {},
}));
vi.mock("./fileServer.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./fileServer.js")>()),
createFileServer: async () => ({
url: "http://127.0.0.1:1",
port: 1,
close: () => {},
addPreHeadScript: () => {},
}),
}));
vi.mock("./render/stages/compileStage.js", () => ({
runCompileStage: async () => ({
compiled: {
html: "<div></div>",
renderModeHints: { reasons: [] },
hasShaderTransitions: false,
unresolvedCompositions: [],
},
composition: { width: 320, height: 180, duration: 4, videos: [], images: [], audios: [] },
deviceScaleFactor: 1,
outputWidth: 320,
outputHeight: 180,
compileOnlyMs: 0,
forceScreenshot: false,
}),
}));
vi.mock("./render/stages/probeStage.js", () => ({
runProbeStage: async ({ compiled }: { compiled: object }) => ({
compiled,
fileServer: null,
probeSession: null,
lastBrowserConsole: [],
duration: 4,
totalFrames: 120,
browserProbeMs: 0,
}),
}));
vi.mock("./render/stages/extractVideosStage.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/stages/extractVideosStage.js")>()),
runExtractVideosStage: async () => ({
extractionResult: null,
frameLookup: null,
videoReadinessSkipIds: new Set<string>(),
videoMetadataHints: new Map(),
nativeHdrVideoIds: new Set<string>(),
videoTransfers: new Map(),
nativeHdrImageIds: new Set<string>(),
imageTransfers: new Map(),
hdrImageSrcPaths: new Map(),
imageColorSpaces: [],
videoExtractMs: 0,
failureToEnforce: undefined,
}),
}));
vi.mock("./render/stages/audioStage.js", () => ({
runAudioStage: async () => ({ audioOutputPath: undefined, hasAudio: false, audioProcessMs: 0 }),
}));
vi.mock("./render/captureCost.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/captureCost.js")>()),
resolveRenderWorkerCount: () => 1,
runCaptureCalibration: async (input: { forceScreenshot: boolean; probeSession: unknown }) => ({
calibration: undefined,
forceScreenshot: input.forceScreenshot,
probeSession: input.probeSession,
lastBrowserConsole: [],
calibrationEvaluateTimedOut: state.calibrationEvaluateTimedOut,
}),
}));
vi.mock("./render/stages/captureStreamingStage.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/stages/captureStreamingStage.js")>()),
runCaptureStreamingStage: async (input: { job: { framesRendered?: number } }) => {
input.job.framesRendered = state.framesBeforeFailure;
throw new CaptureStageError({ cause: new Error(EVALUATE_TIMEOUT), browserConsole: [] });
},
}));
const EVALUATE_TIMEOUT = "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting";
const dirs: string[] = [];
beforeEach(() => {
state.calibrationEvaluateTimedOut = false;
state.framesBeforeFailure = 0;
vi.stubEnv("HF_SEGMENTED_CAPTURE", "false");
});
afterEach(() => {
vi.unstubAllEnvs();
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
function render(): Promise<void> {
const root = mkdtempSync(join(tmpdir(), "hf-page-never-ready-"));
dirs.push(root);
writeFileSync(join(root, "index.html"), "<div></div>");
const job = createRenderJob({
fps: 30,
quality: "standard",
hdrMode: "force-sdr",
producerConfig: resolveConfig({ forceScreenshot: true, browserGpuMode: "software" }),
logger: { info() {}, warn() {}, error() {}, debug() {} },
});
return executeRenderJob(job, root, join(root, "output.mp4"));
}
it("fails once as non-retryable when calibration and capture both timed out with no frame", async () => {
state.calibrationEvaluateTimedOut = true;
const error = await render().then(
() => undefined,
(caught: unknown) => caught,
);
expect(error).toMatchObject({ code: "PAGE_NEVER_READY", owner: "user", retryable: false });
expect(extractSafeRenderErrorMetadata(error)).toMatchObject({
errorCode: "PAGE_NEVER_READY",
retryable: false,
});
expect((error as Error).message).toContain("never became ready");
});
it("keeps the original error when calibration did not time out", async () => {
await expect(render()).rejects.toThrow(EVALUATE_TIMEOUT);
});
it("keeps the original error once a frame was captured", async () => {
state.calibrationEvaluateTimedOut = true;
state.framesBeforeFailure = 3;
await expect(render()).rejects.toThrow(EVALUATE_TIMEOUT);
});
@@ -172,6 +172,7 @@ import {
checkRenderEta,
startRenderEtaGuard,
} from "./render/renderEtaGuard.js";
import { checkPageNeverReady } from "./render/pageNeverReady.js";
import { resolveSegmentFrames } from "./render/segmentPlan.js";
import { resolveSegmentBrowserRecycle } from "./render/segmentRecycle.js";
import {
@@ -3088,6 +3089,7 @@ async function executeRenderPipeline(input: {
const onProgress = execution.onProgress;
const tooSlow = new AbortController();
let tooSlowError: RenderTooSlowError | undefined;
let calibrationEvaluateTimedOut = false;
const executionSignal = job.config.maxRenderMs
? AbortSignal.any([tooSlow.signal, ...(execution.signal ? [execution.signal] : [])])
: execution.signal;
@@ -4169,6 +4171,7 @@ async function executeRenderPipeline(input: {
{ heartbeatMessage: "browser calibrating (frames not started)" },
);
captureCalibration = outcome.calibration;
calibrationEvaluateTimedOut = outcome.calibrationEvaluateTimedOut;
captureForceScreenshot = outcome.forceScreenshot;
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
probeSession = outcome.probeSession;
@@ -5677,7 +5680,12 @@ async function executeRenderPipeline(input: {
await eventPublisher.flush();
} catch (caught) {
await pendingAudio;
const error = surfaceScriptFailure(tooSlowError ?? caught, job);
const pageNeverReady = checkPageNeverReady({
calibrationEvaluateTimedOut,
framesRendered: job.framesRendered ?? 0,
error: caught,
});
const error = surfaceScriptFailure(tooSlowError ?? pageNeverReady ?? caught, job);
if (!tooSlowError && (error instanceof RenderCancelledError || executionSignal?.aborted)) {
job.error = error instanceof Error ? error.message : "render_cancelled";
updateJobStatus(job, "cancelled", "Render cancelled", job.progress, onProgress);
@@ -9,6 +9,7 @@ const SAFE_RENDER_ERROR_CODES = new Set<string>([
"ENCODER_INTERRUPTED",
"RENDER_TOO_SLOW",
"RENDER_STAGE_STALLED",
"PAGE_NEVER_READY",
]);
/**