From 5cea10e76e687fc381760aab73773b154577abf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 4 Nov 2026 11:53:21 +0000 Subject: [PATCH] fix(producer): fail a render once when its page never became ready (#7154) * fix(producer): fail a render once when its page never became ready * refactor(producer): keep the page-never-ready error private to its module * test(producer): drop a mock field the page-never-ready test never set --- .../captureCost.calibrationTimeout.test.ts | 38 +++++ .../src/services/render/captureCost.ts | 17 +- .../src/services/render/pageNeverReady.ts | 32 ++++ .../renderOrchestrator.pageNeverReady.test.ts | 148 ++++++++++++++++++ .../src/services/renderOrchestrator.ts | 10 +- .../src/services/safeRenderErrorCode.ts | 1 + 6 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 packages/producer/src/services/render/captureCost.calibrationTimeout.test.ts create mode 100644 packages/producer/src/services/render/pageNeverReady.ts create mode 100644 packages/producer/src/services/renderOrchestrator.pageNeverReady.test.ts diff --git a/packages/producer/src/services/render/captureCost.calibrationTimeout.test.ts b/packages/producer/src/services/render/captureCost.calibrationTimeout.test.ts new file mode 100644 index 0000000000..de581cdfe1 --- /dev/null +++ b/packages/producer/src/services/render/captureCost.calibrationTimeout.test.ts @@ -0,0 +1,38 @@ +import { expect, it, vi } from "vitest"; +import { resolveConfig } from "@hyperframes/engine"; +import { runCaptureCalibration } from "./captureCost.js"; + +const { state } = vi.hoisted(() => ({ state: { message: "" } })); +vi.mock("@hyperframes/engine", async (importOriginal) => ({ + ...(await importOriginal()), + createCaptureSession: async () => { + throw new Error(state.message); + }, + closeCaptureSession: async () => {}, +})); + +function calibrate() { + return runCaptureCalibration({ + cfg: resolveConfig({ forceScreenshot: true }), + fileServer: { url: "http://127.0.0.1:1" } as never, + workDir: "/tmp/unused", + log: { info() {}, warn() {}, error() {}, debug() {} }, + job: { config: { fps: 30 } } as never, + totalFrames: 10, + forceScreenshot: true, + probeSession: null, + buildCaptureOptions: () => ({}) as never, + createRenderVideoFrameInjector: () => null, + assertNotAborted: () => {}, + }); +} + +it("reports an evaluate timeout during calibration", async () => { + state.message = "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting"; + expect((await calibrate()).calibrationEvaluateTimedOut).toBe(true); +}); + +it("does not report other calibration failures as an evaluate timeout", async () => { + state.message = "Navigation timeout of 60000 ms exceeded"; + expect((await calibrate()).calibrationEvaluateTimedOut).toBe(false); +}); diff --git a/packages/producer/src/services/render/captureCost.ts b/packages/producer/src/services/render/captureCost.ts index 3a89b4d589..4098cfe3a6 100644 --- a/packages/producer/src/services/render/captureCost.ts +++ b/packages/producer/src/services/render/captureCost.ts @@ -343,6 +343,8 @@ export interface CaptureCalibrationOutcome { probeSession: CaptureSession | null; /** Buffer of whichever session was active last; the sequencer uses it for the error-path tail. */ lastBrowserConsole: string[]; + /** The final calibration failure was a `Runtime.evaluate` timeout: the page never answered. */ + calibrationEvaluateTimedOut: boolean; } /** @@ -381,6 +383,7 @@ export async function runCaptureCalibration(input: { let probeSession = input.probeSession; let forceScreenshot = input.forceScreenshot; let lastBrowserConsole: string[] = []; + let calibrationEvaluateTimedOut = false; const fps = fpsToNumber(job.config.fps); // Holds whichever calibration session is currently open. The closure @@ -440,6 +443,7 @@ export async function runCaptureCalibration(input: { !forceScreenshot && shouldFallbackToScreenshotAfterCalibrationError(error); if (!shouldFallback) { calibration = createFailedCaptureCalibrationEstimate("calibration-failed"); + calibrationEvaluateTimedOut = isEvaluateTimeout(error); log.warn("[Render] Auto-worker calibration failed; using conservative worker budget.", { protocolTimeout: calibrationCfg.protocolTimeout, error: error instanceof Error ? error.message : String(error), @@ -481,6 +485,7 @@ export async function runCaptureCalibration(input: { ); } catch (fallbackError) { calibration = createFailedCaptureCalibrationEstimate("calibration-screenshot-failed"); + calibrationEvaluateTimedOut = isEvaluateTimeout(fallbackError); log.warn( "[Render] Screenshot auto-worker calibration failed after BeginFrame fallback; using conservative worker budget.", { @@ -497,7 +502,17 @@ export async function runCaptureCalibration(input: { } } - return { calibration, forceScreenshot, probeSession, lastBrowserConsole }; + return { + calibration, + forceScreenshot, + probeSession, + lastBrowserConsole, + calibrationEvaluateTimedOut, + }; +} + +export function isEvaluateTimeout(error: unknown): boolean { + return /Runtime\.evaluate timed out/i.test(normalizeErrorMessage(error)); } /** diff --git a/packages/producer/src/services/render/pageNeverReady.ts b/packages/producer/src/services/render/pageNeverReady.ts new file mode 100644 index 0000000000..b544d4b6dc --- /dev/null +++ b/packages/producer/src/services/render/pageNeverReady.ts @@ -0,0 +1,32 @@ +import { isEvaluateTimeout } from "./captureCost.js"; + +const PAGE_NEVER_READY = "PAGE_NEVER_READY" as const; + +class PageNeverReadyError extends Error { + readonly code = PAGE_NEVER_READY; + // The producer server reads these off the error for the wire envelope. + readonly owner = "user" as const; + readonly retryable = false as const; + + constructor(cause: unknown) { + super( + "The composition page never became ready: it did not answer browser commands during " + + "calibration or capture, and no frame was captured. Its script may never return.", + { cause }, + ); + this.name = "PageNeverReadyError"; + } +} + +/** Calibration and capture both timed out on `Runtime.evaluate` with no frame captured. */ +export function checkPageNeverReady(input: { + calibrationEvaluateTimedOut: boolean; + framesRendered: number; + error: unknown; +}): Error | undefined { + const { calibrationEvaluateTimedOut, framesRendered, error } = input; + if (!calibrationEvaluateTimedOut || framesRendered > 0 || !isEvaluateTimeout(error)) { + return undefined; + } + return new PageNeverReadyError(error); +} diff --git a/packages/producer/src/services/renderOrchestrator.pageNeverReady.test.ts b/packages/producer/src/services/renderOrchestrator.pageNeverReady.test.ts new file mode 100644 index 0000000000..e3c5343a45 --- /dev/null +++ b/packages/producer/src/services/renderOrchestrator.pageNeverReady.test.ts @@ -0,0 +1,148 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveConfig } from "@hyperframes/engine"; +import { CaptureStageError } from "./render/captureStageError.js"; +import { extractSafeRenderErrorMetadata } from "../server.js"; +import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js"; + +// Entry: executeRenderJob on the streaming route; calibration and the streaming stage are scripted. +const { state } = vi.hoisted(() => ({ + state: { calibrationEvaluateTimedOut: false, framesBeforeFailure: 0 }, +})); + +vi.mock("@hyperframes/engine", async (importOriginal) => ({ + ...(await importOriginal()), + assertConfiguredFfmpegBinariesExist: () => {}, + resolveBrowserGpuMode: async () => "software", + resolveHeadlessShellPath: () => "/opt/chrome-headless-shell", + createCaptureSession: async () => ({ browserConsoleBuffer: [], options: {} }), + initializeSession: async () => {}, + closeCaptureSession: async () => {}, +})); +vi.mock("./fileServer.js", async (importOriginal) => ({ + ...(await importOriginal()), + createFileServer: async () => ({ + url: "http://127.0.0.1:1", + port: 1, + close: () => {}, + addPreHeadScript: () => {}, + }), +})); +vi.mock("./render/stages/compileStage.js", () => ({ + runCompileStage: async () => ({ + compiled: { + html: "
", + renderModeHints: { reasons: [] }, + hasShaderTransitions: false, + unresolvedCompositions: [], + }, + composition: { width: 320, height: 180, duration: 4, videos: [], images: [], audios: [] }, + deviceScaleFactor: 1, + outputWidth: 320, + outputHeight: 180, + compileOnlyMs: 0, + forceScreenshot: false, + }), +})); +vi.mock("./render/stages/probeStage.js", () => ({ + runProbeStage: async ({ compiled }: { compiled: object }) => ({ + compiled, + fileServer: null, + probeSession: null, + lastBrowserConsole: [], + duration: 4, + totalFrames: 120, + browserProbeMs: 0, + }), +})); +vi.mock("./render/stages/extractVideosStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runExtractVideosStage: async () => ({ + extractionResult: null, + frameLookup: null, + videoReadinessSkipIds: new Set(), + videoMetadataHints: new Map(), + nativeHdrVideoIds: new Set(), + videoTransfers: new Map(), + nativeHdrImageIds: new Set(), + imageTransfers: new Map(), + hdrImageSrcPaths: new Map(), + imageColorSpaces: [], + videoExtractMs: 0, + failureToEnforce: undefined, + }), +})); +vi.mock("./render/stages/audioStage.js", () => ({ + runAudioStage: async () => ({ audioOutputPath: undefined, hasAudio: false, audioProcessMs: 0 }), +})); +vi.mock("./render/captureCost.js", async (importOriginal) => ({ + ...(await importOriginal()), + resolveRenderWorkerCount: () => 1, + runCaptureCalibration: async (input: { forceScreenshot: boolean; probeSession: unknown }) => ({ + calibration: undefined, + forceScreenshot: input.forceScreenshot, + probeSession: input.probeSession, + lastBrowserConsole: [], + calibrationEvaluateTimedOut: state.calibrationEvaluateTimedOut, + }), +})); +vi.mock("./render/stages/captureStreamingStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCaptureStreamingStage: async (input: { job: { framesRendered?: number } }) => { + input.job.framesRendered = state.framesBeforeFailure; + throw new CaptureStageError({ cause: new Error(EVALUATE_TIMEOUT), browserConsole: [] }); + }, +})); + +const EVALUATE_TIMEOUT = "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting"; + +const dirs: string[] = []; +beforeEach(() => { + state.calibrationEvaluateTimedOut = false; + state.framesBeforeFailure = 0; + vi.stubEnv("HF_SEGMENTED_CAPTURE", "false"); +}); +afterEach(() => { + vi.unstubAllEnvs(); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function render(): Promise { + const root = mkdtempSync(join(tmpdir(), "hf-page-never-ready-")); + dirs.push(root); + writeFileSync(join(root, "index.html"), "
"); + const job = createRenderJob({ + fps: 30, + quality: "standard", + hdrMode: "force-sdr", + producerConfig: resolveConfig({ forceScreenshot: true, browserGpuMode: "software" }), + logger: { info() {}, warn() {}, error() {}, debug() {} }, + }); + return executeRenderJob(job, root, join(root, "output.mp4")); +} + +it("fails once as non-retryable when calibration and capture both timed out with no frame", async () => { + state.calibrationEvaluateTimedOut = true; + const error = await render().then( + () => undefined, + (caught: unknown) => caught, + ); + expect(error).toMatchObject({ code: "PAGE_NEVER_READY", owner: "user", retryable: false }); + expect(extractSafeRenderErrorMetadata(error)).toMatchObject({ + errorCode: "PAGE_NEVER_READY", + retryable: false, + }); + expect((error as Error).message).toContain("never became ready"); +}); + +it("keeps the original error when calibration did not time out", async () => { + await expect(render()).rejects.toThrow(EVALUATE_TIMEOUT); +}); + +it("keeps the original error once a frame was captured", async () => { + state.calibrationEvaluateTimedOut = true; + state.framesBeforeFailure = 3; + await expect(render()).rejects.toThrow(EVALUATE_TIMEOUT); +}); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 663edb364b..e31e4e87eb 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -172,6 +172,7 @@ import { checkRenderEta, startRenderEtaGuard, } from "./render/renderEtaGuard.js"; +import { checkPageNeverReady } from "./render/pageNeverReady.js"; import { resolveSegmentFrames } from "./render/segmentPlan.js"; import { resolveSegmentBrowserRecycle } from "./render/segmentRecycle.js"; import { @@ -3088,6 +3089,7 @@ async function executeRenderPipeline(input: { const onProgress = execution.onProgress; const tooSlow = new AbortController(); let tooSlowError: RenderTooSlowError | undefined; + let calibrationEvaluateTimedOut = false; const executionSignal = job.config.maxRenderMs ? AbortSignal.any([tooSlow.signal, ...(execution.signal ? [execution.signal] : [])]) : execution.signal; @@ -4169,6 +4171,7 @@ async function executeRenderPipeline(input: { { heartbeatMessage: "browser calibrating (frames not started)" }, ); captureCalibration = outcome.calibration; + calibrationEvaluateTimedOut = outcome.calibrationEvaluateTimedOut; captureForceScreenshot = outcome.forceScreenshot; updateCaptureObservability({ forceScreenshot: captureForceScreenshot }); probeSession = outcome.probeSession; @@ -5677,7 +5680,12 @@ async function executeRenderPipeline(input: { await eventPublisher.flush(); } catch (caught) { await pendingAudio; - const error = surfaceScriptFailure(tooSlowError ?? caught, job); + const pageNeverReady = checkPageNeverReady({ + calibrationEvaluateTimedOut, + framesRendered: job.framesRendered ?? 0, + error: caught, + }); + const error = surfaceScriptFailure(tooSlowError ?? pageNeverReady ?? caught, job); if (!tooSlowError && (error instanceof RenderCancelledError || executionSignal?.aborted)) { job.error = error instanceof Error ? error.message : "render_cancelled"; updateJobStatus(job, "cancelled", "Render cancelled", job.progress, onProgress); diff --git a/packages/producer/src/services/safeRenderErrorCode.ts b/packages/producer/src/services/safeRenderErrorCode.ts index 1f86b88947..dc9ac4e982 100644 --- a/packages/producer/src/services/safeRenderErrorCode.ts +++ b/packages/producer/src/services/safeRenderErrorCode.ts @@ -9,6 +9,7 @@ const SAFE_RENDER_ERROR_CODES = new Set([ "ENCODER_INTERRUPTED", "RENDER_TOO_SLOW", "RENDER_STAGE_STALLED", + "PAGE_NEVER_READY", ]); /**