ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-10-30 09:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(render): retry timed-out browser initialization once

Browse Source
This commit is contained in:
Vance Ingalls
2026-09-08 23:40:09 -07:00
parent 3a7fcd10e0
commit d0ee207b06
4 changed files with 143 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/engine/src/services/captureFailure.test.ts
+2
View File
@@ -6,6 +6,8 @@ describe("classifyCaptureFailure", () => {
["Target closed", "transient_browser"],
["Runtime.callFunctionOn timed out after 30000ms", "protocol_timeout"],
["Runtime.evaluate timed out", "protocol_timeout"],
["Network.enable timed out. Increase the protocolTimeout setting.", "protocol_timeout"],
["[Parallel] Capture failed: Worker 0: Network.enable timed out", "protocol_timeout"],
[
"Page.captureScreenshot timed out. Increase the 'protocolTimeout' setting in launch/connect calls for a higher timeout if needed.",
"protocol_timeout",
packages/engine/src/services/captureFailure.ts
+1
View File
@@ -55,6 +55,7 @@ const TRANSIENT_BROWSER_ERROR_PATTERNS = [
];
const PROTOCOL_TIMEOUT_PATTERNS = [
/Network\.enable timed out/i,
/Runtime\.callFunctionOn timed out/i,
/Runtime\.evaluate timed out/i,
/Page\.captureScreenshot timed out/i,
packages/producer/src/services/renderOrchestrator.test.ts
+113 -63
View File
@@ -3,7 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync
import { join, win32 } from "node:path";
import { tmpdir } from "node:os";
import type { CaptureOptions, EngineConfig, ExtractedFrames } from "@hyperframes/engine";
import { executeParallelCapture, mergeWorkerFrames } from "@hyperframes/engine";
import { DEFAULT_CONFIG, executeParallelCapture, mergeWorkerFrames } from "@hyperframes/engine";
import type { CompiledComposition } from "./htmlCompiler.js";
// Replace only the two engine functions the adaptive-retry loop uses to touch
@@ -312,43 +312,43 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
}
});
it("does NOT retry a transient error when the render was aborted", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-frames-"));
it("retries Network.enable startup timeout once with fewer workers and zero progress", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-frames-"));
const log = makeLog();
const controller = new AbortController();
// Cancellation tears the browser down, surfacing as a transient-looking
// "Target closed" — but an aborted render must fail immediately, not retry.
let call = 0;
vi.mocked(executeParallelCapture).mockImplementation(async () => {
controller.abort();
throw new Error("Target closed");
call++;
if (call === 1) {
throw new Error("[Parallel] Capture failed: Worker 0: Network.enable timed out");
}
writeAllFrames(framesDir, 4);
return [];
});
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 2,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
abortSignal: controller.signal,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(/Target closed/);
const attempts = await executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 4,
allowRetry: true,
frameExt: "jpg",
captureOptions: { width: 64, height: 64, fps: { num: 30, den: 1 } },
createBeforeCaptureHook: () => null,
cfg: DEFAULT_CONFIG,
log,
dedupPerfs: [],
});
// Exactly one attempt — no transient retry burned on a cancelled render.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(1);
expect(log.warn).not.toHaveBeenCalledWith(
expect.stringContaining("Transient browser failure"),
expect.anything(),
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(2);
expect(attempts.map((a) => a.workers)).toEqual([4, 2]);
expect(attempts.map((a) => a.reason)).toEqual(["initial", "retry"]);
expect(log.warn).toHaveBeenCalledWith(
expect.stringContaining("Browser initialization timed out"),
expect.objectContaining({ fromWorkers: 4, toWorkers: 2 }),
);
} finally {
rmSync(workDir, { recursive: true, force: true });
@@ -356,40 +356,90 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
}
});
it("gives up after MAX_TRANSIENT_CAPTURE_RETRIES when the tab keeps dying", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient2-frames-"));
const log = makeLog();
vi.mocked(executeParallelCapture).mockRejectedValue(new Error("Session closed"));
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
it.each(["Target closed", "Network.enable timed out"])(
"does NOT retry %s after cancellation",
async (message) => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-frames-"));
const log = makeLog();
const controller = new AbortController();
// Cancellation tears the browser down, surfacing as a transient-looking
// "Target closed" — but an aborted render must fail immediately, not retry.
vi.mocked(executeParallelCapture).mockImplementation(async () => {
controller.abort();
throw new Error(message);
});
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 1,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(/Session closed/);
try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 2,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
abortSignal: controller.signal,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(message);
// 1 initial attempt + exactly MAX_TRANSIENT_CAPTURE_RETRIES retries.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(
1 + MAX_TRANSIENT_CAPTURE_RETRIES,
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
});
// Exactly one attempt — no transient retry burned on a cancelled render.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(1);
expect(log.warn).not.toHaveBeenCalledWith(
expect.stringContaining("Transient browser failure"),
expect.anything(),
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
},
);
it.each(["Session closed", "Network.enable timed out"])(
"bounds repeated %s failures",
async (message) => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient2-frames-"));
const log = makeLog();
vi.mocked(executeParallelCapture).mockRejectedValue(new Error(message));
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 1,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(message);
// 1 initial attempt + exactly MAX_TRANSIENT_CAPTURE_RETRIES retries.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(
1 + MAX_TRANSIENT_CAPTURE_RETRIES,
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
},
);
});
describe("describeMemoryExhaustion", () => {
packages/producer/src/services/renderOrchestrator.ts
+27
View File
@@ -1044,6 +1044,7 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
let missingRanges: FrameRange[] | null = null;
let attempt = 0;
let transientRetriesUsed = 0;
let initializationRetriesUsed = 0;
// Set when the *previous* iteration retried after a transient browser death,
// so the attempt it spawns is tagged `"transient-retry"` (vs the worker-halving
// `"retry"`) for telemetry. Reset after each attempt is recorded.
@@ -1210,6 +1211,32 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
continue;
}
// CDP initialization can time out before any frame exists. Give that
// specific startup failure one fresh attempt with less concurrency;
// arbitrary zero-progress authoring/capture errors still fail below.
if (
options.allowRetry &&
!madeProgress &&
initializationRetriesUsed === 0 &&
failure.kind === "protocol_timeout" &&
/\bNetwork\.enable timed out/i.test(failure.message)
) {
initializationRetriesUsed++;
const nextWorkers = getNextRetryWorkerCount(currentWorkers);
options.log.warn(
"[Render] Browser initialization timed out; retrying once with fresh sessions.",
{
fromWorkers: currentWorkers,
toWorkers: nextWorkers,
error: failure.message,
},
);
currentWorkers = nextWorkers;
missingRanges = remaining;
attempt++;
continue;
}
if (!madeProgress) {
options.log.warn(
"[Render] Capture attempt made no forward progress; composition is likely structurally broken — not retrying.",