mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:00:07 +00:00
fix(producer): fail a render once when its page never became ready (#7154)
* fix(producer): fail a render once when its page never became ready * refactor(producer): keep the page-never-ready error private to its module * test(producer): drop a mock field the page-never-ready test never set
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { resolveConfig } from "@hyperframes/engine";
|
||||
import { runCaptureCalibration } from "./captureCost.js";
|
||||
|
||||
const { state } = vi.hoisted(() => ({ state: { message: "" } }));
|
||||
vi.mock("@hyperframes/engine", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@hyperframes/engine")>()),
|
||||
createCaptureSession: async () => {
|
||||
throw new Error(state.message);
|
||||
},
|
||||
closeCaptureSession: async () => {},
|
||||
}));
|
||||
|
||||
function calibrate() {
|
||||
return runCaptureCalibration({
|
||||
cfg: resolveConfig({ forceScreenshot: true }),
|
||||
fileServer: { url: "http://127.0.0.1:1" } as never,
|
||||
workDir: "/tmp/unused",
|
||||
log: { info() {}, warn() {}, error() {}, debug() {} },
|
||||
job: { config: { fps: 30 } } as never,
|
||||
totalFrames: 10,
|
||||
forceScreenshot: true,
|
||||
probeSession: null,
|
||||
buildCaptureOptions: () => ({}) as never,
|
||||
createRenderVideoFrameInjector: () => null,
|
||||
assertNotAborted: () => {},
|
||||
});
|
||||
}
|
||||
|
||||
it("reports an evaluate timeout during calibration", async () => {
|
||||
state.message = "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting";
|
||||
expect((await calibrate()).calibrationEvaluateTimedOut).toBe(true);
|
||||
});
|
||||
|
||||
it("does not report other calibration failures as an evaluate timeout", async () => {
|
||||
state.message = "Navigation timeout of 60000 ms exceeded";
|
||||
expect((await calibrate()).calibrationEvaluateTimedOut).toBe(false);
|
||||
});
|
||||
@@ -343,6 +343,8 @@ export interface CaptureCalibrationOutcome {
|
||||
probeSession: CaptureSession | null;
|
||||
/** Buffer of whichever session was active last; the sequencer uses it for the error-path tail. */
|
||||
lastBrowserConsole: string[];
|
||||
/** The final calibration failure was a `Runtime.evaluate` timeout: the page never answered. */
|
||||
calibrationEvaluateTimedOut: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -381,6 +383,7 @@ export async function runCaptureCalibration(input: {
|
||||
let probeSession = input.probeSession;
|
||||
let forceScreenshot = input.forceScreenshot;
|
||||
let lastBrowserConsole: string[] = [];
|
||||
let calibrationEvaluateTimedOut = false;
|
||||
|
||||
const fps = fpsToNumber(job.config.fps);
|
||||
// Holds whichever calibration session is currently open. The closure
|
||||
@@ -440,6 +443,7 @@ export async function runCaptureCalibration(input: {
|
||||
!forceScreenshot && shouldFallbackToScreenshotAfterCalibrationError(error);
|
||||
if (!shouldFallback) {
|
||||
calibration = createFailedCaptureCalibrationEstimate("calibration-failed");
|
||||
calibrationEvaluateTimedOut = isEvaluateTimeout(error);
|
||||
log.warn("[Render] Auto-worker calibration failed; using conservative worker budget.", {
|
||||
protocolTimeout: calibrationCfg.protocolTimeout,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
@@ -481,6 +485,7 @@ export async function runCaptureCalibration(input: {
|
||||
);
|
||||
} catch (fallbackError) {
|
||||
calibration = createFailedCaptureCalibrationEstimate("calibration-screenshot-failed");
|
||||
calibrationEvaluateTimedOut = isEvaluateTimeout(fallbackError);
|
||||
log.warn(
|
||||
"[Render] Screenshot auto-worker calibration failed after BeginFrame fallback; using conservative worker budget.",
|
||||
{
|
||||
@@ -497,7 +502,17 @@ export async function runCaptureCalibration(input: {
|
||||
}
|
||||
}
|
||||
|
||||
return { calibration, forceScreenshot, probeSession, lastBrowserConsole };
|
||||
return {
|
||||
calibration,
|
||||
forceScreenshot,
|
||||
probeSession,
|
||||
lastBrowserConsole,
|
||||
calibrationEvaluateTimedOut,
|
||||
};
|
||||
}
|
||||
|
||||
export function isEvaluateTimeout(error: unknown): boolean {
|
||||
return /Runtime\.evaluate timed out/i.test(normalizeErrorMessage(error));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { isEvaluateTimeout } from "./captureCost.js";
|
||||
|
||||
const PAGE_NEVER_READY = "PAGE_NEVER_READY" as const;
|
||||
|
||||
class PageNeverReadyError extends Error {
|
||||
readonly code = PAGE_NEVER_READY;
|
||||
// The producer server reads these off the error for the wire envelope.
|
||||
readonly owner = "user" as const;
|
||||
readonly retryable = false as const;
|
||||
|
||||
constructor(cause: unknown) {
|
||||
super(
|
||||
"The composition page never became ready: it did not answer browser commands during " +
|
||||
"calibration or capture, and no frame was captured. Its script may never return.",
|
||||
{ cause },
|
||||
);
|
||||
this.name = "PageNeverReadyError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Calibration and capture both timed out on `Runtime.evaluate` with no frame captured. */
|
||||
export function checkPageNeverReady(input: {
|
||||
calibrationEvaluateTimedOut: boolean;
|
||||
framesRendered: number;
|
||||
error: unknown;
|
||||
}): Error | undefined {
|
||||
const { calibrationEvaluateTimedOut, framesRendered, error } = input;
|
||||
if (!calibrationEvaluateTimedOut || framesRendered > 0 || !isEvaluateTimeout(error)) {
|
||||
return undefined;
|
||||
}
|
||||
return new PageNeverReadyError(error);
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { resolveConfig } from "@hyperframes/engine";
|
||||
import { CaptureStageError } from "./render/captureStageError.js";
|
||||
import { extractSafeRenderErrorMetadata } from "../server.js";
|
||||
import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js";
|
||||
|
||||
// Entry: executeRenderJob on the streaming route; calibration and the streaming stage are scripted.
|
||||
const { state } = vi.hoisted(() => ({
|
||||
state: { calibrationEvaluateTimedOut: false, framesBeforeFailure: 0 },
|
||||
}));
|
||||
|
||||
vi.mock("@hyperframes/engine", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@hyperframes/engine")>()),
|
||||
assertConfiguredFfmpegBinariesExist: () => {},
|
||||
resolveBrowserGpuMode: async () => "software",
|
||||
resolveHeadlessShellPath: () => "/opt/chrome-headless-shell",
|
||||
createCaptureSession: async () => ({ browserConsoleBuffer: [], options: {} }),
|
||||
initializeSession: async () => {},
|
||||
closeCaptureSession: async () => {},
|
||||
}));
|
||||
vi.mock("./fileServer.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./fileServer.js")>()),
|
||||
createFileServer: async () => ({
|
||||
url: "http://127.0.0.1:1",
|
||||
port: 1,
|
||||
close: () => {},
|
||||
addPreHeadScript: () => {},
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/compileStage.js", () => ({
|
||||
runCompileStage: async () => ({
|
||||
compiled: {
|
||||
html: "<div></div>",
|
||||
renderModeHints: { reasons: [] },
|
||||
hasShaderTransitions: false,
|
||||
unresolvedCompositions: [],
|
||||
},
|
||||
composition: { width: 320, height: 180, duration: 4, videos: [], images: [], audios: [] },
|
||||
deviceScaleFactor: 1,
|
||||
outputWidth: 320,
|
||||
outputHeight: 180,
|
||||
compileOnlyMs: 0,
|
||||
forceScreenshot: false,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/probeStage.js", () => ({
|
||||
runProbeStage: async ({ compiled }: { compiled: object }) => ({
|
||||
compiled,
|
||||
fileServer: null,
|
||||
probeSession: null,
|
||||
lastBrowserConsole: [],
|
||||
duration: 4,
|
||||
totalFrames: 120,
|
||||
browserProbeMs: 0,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/extractVideosStage.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./render/stages/extractVideosStage.js")>()),
|
||||
runExtractVideosStage: async () => ({
|
||||
extractionResult: null,
|
||||
frameLookup: null,
|
||||
videoReadinessSkipIds: new Set<string>(),
|
||||
videoMetadataHints: new Map(),
|
||||
nativeHdrVideoIds: new Set<string>(),
|
||||
videoTransfers: new Map(),
|
||||
nativeHdrImageIds: new Set<string>(),
|
||||
imageTransfers: new Map(),
|
||||
hdrImageSrcPaths: new Map(),
|
||||
imageColorSpaces: [],
|
||||
videoExtractMs: 0,
|
||||
failureToEnforce: undefined,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/audioStage.js", () => ({
|
||||
runAudioStage: async () => ({ audioOutputPath: undefined, hasAudio: false, audioProcessMs: 0 }),
|
||||
}));
|
||||
vi.mock("./render/captureCost.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./render/captureCost.js")>()),
|
||||
resolveRenderWorkerCount: () => 1,
|
||||
runCaptureCalibration: async (input: { forceScreenshot: boolean; probeSession: unknown }) => ({
|
||||
calibration: undefined,
|
||||
forceScreenshot: input.forceScreenshot,
|
||||
probeSession: input.probeSession,
|
||||
lastBrowserConsole: [],
|
||||
calibrationEvaluateTimedOut: state.calibrationEvaluateTimedOut,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/captureStreamingStage.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./render/stages/captureStreamingStage.js")>()),
|
||||
runCaptureStreamingStage: async (input: { job: { framesRendered?: number } }) => {
|
||||
input.job.framesRendered = state.framesBeforeFailure;
|
||||
throw new CaptureStageError({ cause: new Error(EVALUATE_TIMEOUT), browserConsole: [] });
|
||||
},
|
||||
}));
|
||||
|
||||
const EVALUATE_TIMEOUT = "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting";
|
||||
|
||||
const dirs: string[] = [];
|
||||
beforeEach(() => {
|
||||
state.calibrationEvaluateTimedOut = false;
|
||||
state.framesBeforeFailure = 0;
|
||||
vi.stubEnv("HF_SEGMENTED_CAPTURE", "false");
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function render(): Promise<void> {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-page-never-ready-"));
|
||||
dirs.push(root);
|
||||
writeFileSync(join(root, "index.html"), "<div></div>");
|
||||
const job = createRenderJob({
|
||||
fps: 30,
|
||||
quality: "standard",
|
||||
hdrMode: "force-sdr",
|
||||
producerConfig: resolveConfig({ forceScreenshot: true, browserGpuMode: "software" }),
|
||||
logger: { info() {}, warn() {}, error() {}, debug() {} },
|
||||
});
|
||||
return executeRenderJob(job, root, join(root, "output.mp4"));
|
||||
}
|
||||
|
||||
it("fails once as non-retryable when calibration and capture both timed out with no frame", async () => {
|
||||
state.calibrationEvaluateTimedOut = true;
|
||||
const error = await render().then(
|
||||
() => undefined,
|
||||
(caught: unknown) => caught,
|
||||
);
|
||||
expect(error).toMatchObject({ code: "PAGE_NEVER_READY", owner: "user", retryable: false });
|
||||
expect(extractSafeRenderErrorMetadata(error)).toMatchObject({
|
||||
errorCode: "PAGE_NEVER_READY",
|
||||
retryable: false,
|
||||
});
|
||||
expect((error as Error).message).toContain("never became ready");
|
||||
});
|
||||
|
||||
it("keeps the original error when calibration did not time out", async () => {
|
||||
await expect(render()).rejects.toThrow(EVALUATE_TIMEOUT);
|
||||
});
|
||||
|
||||
it("keeps the original error once a frame was captured", async () => {
|
||||
state.calibrationEvaluateTimedOut = true;
|
||||
state.framesBeforeFailure = 3;
|
||||
await expect(render()).rejects.toThrow(EVALUATE_TIMEOUT);
|
||||
});
|
||||
@@ -172,6 +172,7 @@ import {
|
||||
checkRenderEta,
|
||||
startRenderEtaGuard,
|
||||
} from "./render/renderEtaGuard.js";
|
||||
import { checkPageNeverReady } from "./render/pageNeverReady.js";
|
||||
import { resolveSegmentFrames } from "./render/segmentPlan.js";
|
||||
import { resolveSegmentBrowserRecycle } from "./render/segmentRecycle.js";
|
||||
import {
|
||||
@@ -3088,6 +3089,7 @@ async function executeRenderPipeline(input: {
|
||||
const onProgress = execution.onProgress;
|
||||
const tooSlow = new AbortController();
|
||||
let tooSlowError: RenderTooSlowError | undefined;
|
||||
let calibrationEvaluateTimedOut = false;
|
||||
const executionSignal = job.config.maxRenderMs
|
||||
? AbortSignal.any([tooSlow.signal, ...(execution.signal ? [execution.signal] : [])])
|
||||
: execution.signal;
|
||||
@@ -4169,6 +4171,7 @@ async function executeRenderPipeline(input: {
|
||||
{ heartbeatMessage: "browser calibrating (frames not started)" },
|
||||
);
|
||||
captureCalibration = outcome.calibration;
|
||||
calibrationEvaluateTimedOut = outcome.calibrationEvaluateTimedOut;
|
||||
captureForceScreenshot = outcome.forceScreenshot;
|
||||
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
|
||||
probeSession = outcome.probeSession;
|
||||
@@ -5677,7 +5680,12 @@ async function executeRenderPipeline(input: {
|
||||
await eventPublisher.flush();
|
||||
} catch (caught) {
|
||||
await pendingAudio;
|
||||
const error = surfaceScriptFailure(tooSlowError ?? caught, job);
|
||||
const pageNeverReady = checkPageNeverReady({
|
||||
calibrationEvaluateTimedOut,
|
||||
framesRendered: job.framesRendered ?? 0,
|
||||
error: caught,
|
||||
});
|
||||
const error = surfaceScriptFailure(tooSlowError ?? pageNeverReady ?? caught, job);
|
||||
if (!tooSlowError && (error instanceof RenderCancelledError || executionSignal?.aborted)) {
|
||||
job.error = error instanceof Error ? error.message : "render_cancelled";
|
||||
updateJobStatus(job, "cancelled", "Render cancelled", job.progress, onProgress);
|
||||
|
||||
@@ -9,6 +9,7 @@ const SAFE_RENDER_ERROR_CODES = new Set<string>([
|
||||
"ENCODER_INTERRUPTED",
|
||||
"RENDER_TOO_SLOW",
|
||||
"RENDER_STAGE_STALLED",
|
||||
"PAGE_NEVER_READY",
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user