diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index 8901773ff2..d2e2de64f0 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -428,6 +428,46 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single } }); + it("halves the workers when every page times out loading, instead of reopening as many", async () => { + const workDir = mkdtempSync(join(tmpdir(), "hf-navtimeout-work-")); + const framesDir = mkdtempSync(join(tmpdir(), "hf-navtimeout-frames-")); + const log = makeLog(); + let call = 0; + vi.mocked(executeParallelCapture).mockImplementation(async () => { + call++; + if (call === 1) { + throw new Error( + "[Parallel] Capture failed: Worker 0: Navigation timeout of 60000 ms exceeded", + ); + } + writeAllFrames(framesDir, 16); + return []; + }); + vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined); + + try { + const attempts = await executeDiskCaptureWithAdaptiveRetry({ + serverUrl: "http://localhost:0", + workDir, + framesDir, + totalFrames: 16, + initialWorkerCount: 8, + allowRetry: true, + frameExt: "jpg", + captureOptions: { width: 64, height: 64, fps: { num: 30, den: 1 } }, + createBeforeCaptureHook: () => null, + cfg: DEFAULT_CONFIG, + log, + dedupPerfs: [], + }); + + expect(attempts.map((a) => a.workers)).toEqual([8, 4]); + } finally { + rmSync(workDir, { recursive: true, force: true }); + rmSync(framesDir, { recursive: true, force: true }); + } + }); + it.each(["Target closed", "Network.enable timed out"])( "does NOT retry %s after cancellation", async (message) => { @@ -475,7 +515,7 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single }, ); - it.each(["Session closed", "Network.enable timed out"])( + it.each(["Session closed", "Network.enable timed out", "Navigation timeout of 9 ms exceeded"])( "bounds repeated %s failures", async (message) => { const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-")); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index e31e4e87eb..3f3f7e2ab0 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1472,6 +1472,32 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: { const remainingCount = countFrameRanges(remaining); const madeProgress = captureAttemptMadeProgress(frameCount, remainingCount); + // A browser that times out starting up or loading its page before any frame exists had too many pages + // opening at once: retry with half the workers, down to one. Other zero-progress errors fail below. + // A lone worker gets one retry: here for Network.enable, as the transient retry below for a page load. + if ( + options.allowRetry && + !madeProgress && + (currentWorkers > 1 || + (initializationRetriesUsed === 0 && failure.kind === "protocol_timeout")) && + /\bNetwork\.enable timed out|Navigation timeout of \d+ ms exceeded/i.test(failure.message) + ) { + initializationRetriesUsed++; + const nextWorkers = getNextRetryWorkerCount(currentWorkers); + options.log.warn( + "[Render] Browser initialization timed out; retrying with fewer workers.", + { + fromWorkers: currentWorkers, + toWorkers: nextWorkers, + error: failure.message, + }, + ); + currentWorkers = nextWorkers; + missingRanges = remaining; + attempt++; + continue; + } + // Single bounded retry for a transient browser death (`Target closed` / // `Page crashed` / `Session closed`): the tab died mid-capture, not the // composition. Unlike the worker-halving retry below, this keeps the same @@ -1500,32 +1526,6 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: { continue; } - // CDP initialization can time out before any frame exists. Give that - // specific startup failure one fresh attempt with less concurrency; - // arbitrary zero-progress authoring/capture errors still fail below. - if ( - options.allowRetry && - !madeProgress && - initializationRetriesUsed === 0 && - failure.kind === "protocol_timeout" && - /\bNetwork\.enable timed out/i.test(failure.message) - ) { - initializationRetriesUsed++; - const nextWorkers = getNextRetryWorkerCount(currentWorkers); - options.log.warn( - "[Render] Browser initialization timed out; retrying once with fresh sessions.", - { - fromWorkers: currentWorkers, - toWorkers: nextWorkers, - error: failure.message, - }, - ); - currentWorkers = nextWorkers; - missingRanges = remaining; - attempt++; - continue; - } - if (!madeProgress) { options.log.warn( "[Render] Capture attempt made no forward progress; composition is likely structurally broken, not retrying.",