fix(producer): retry with fewer workers when every page times out loading (#7171)

This commit is contained in:
Miguel Ángel
2026-11-04 17:29:28 +00:00
committed by GitHub
parent cd7b7383cb
commit 04bc4737ad
2 changed files with 67 additions and 27 deletions
@@ -428,6 +428,46 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
}
});
it("halves the workers when every page times out loading, instead of reopening as many", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-navtimeout-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-navtimeout-frames-"));
const log = makeLog();
let call = 0;
vi.mocked(executeParallelCapture).mockImplementation(async () => {
call++;
if (call === 1) {
throw new Error(
"[Parallel] Capture failed: Worker 0: Navigation timeout of 60000 ms exceeded",
);
}
writeAllFrames(framesDir, 16);
return [];
});
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
try {
const attempts = await executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 16,
initialWorkerCount: 8,
allowRetry: true,
frameExt: "jpg",
captureOptions: { width: 64, height: 64, fps: { num: 30, den: 1 } },
createBeforeCaptureHook: () => null,
cfg: DEFAULT_CONFIG,
log,
dedupPerfs: [],
});
expect(attempts.map((a) => a.workers)).toEqual([8, 4]);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
});
it.each(["Target closed", "Network.enable timed out"])(
"does NOT retry %s after cancellation",
async (message) => {
@@ -475,7 +515,7 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
},
);
it.each(["Session closed", "Network.enable timed out"])(
it.each(["Session closed", "Network.enable timed out", "Navigation timeout of 9 ms exceeded"])(
"bounds repeated %s failures",
async (message) => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-"));
@@ -1472,6 +1472,32 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
const remainingCount = countFrameRanges(remaining);
const madeProgress = captureAttemptMadeProgress(frameCount, remainingCount);
// A browser that times out starting up or loading its page before any frame exists had too many pages
// opening at once: retry with half the workers, down to one. Other zero-progress errors fail below.
// A lone worker gets one retry: here for Network.enable, as the transient retry below for a page load.
if (
options.allowRetry &&
!madeProgress &&
(currentWorkers > 1 ||
(initializationRetriesUsed === 0 && failure.kind === "protocol_timeout")) &&
/\bNetwork\.enable timed out|Navigation timeout of \d+ ms exceeded/i.test(failure.message)
) {
initializationRetriesUsed++;
const nextWorkers = getNextRetryWorkerCount(currentWorkers);
options.log.warn(
"[Render] Browser initialization timed out; retrying with fewer workers.",
{
fromWorkers: currentWorkers,
toWorkers: nextWorkers,
error: failure.message,
},
);
currentWorkers = nextWorkers;
missingRanges = remaining;
attempt++;
continue;
}
// Single bounded retry for a transient browser death (`Target closed` /
// `Page crashed` / `Session closed`): the tab died mid-capture, not the
// composition. Unlike the worker-halving retry below, this keeps the same
@@ -1500,32 +1526,6 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
continue;
}
// CDP initialization can time out before any frame exists. Give that
// specific startup failure one fresh attempt with less concurrency;
// arbitrary zero-progress authoring/capture errors still fail below.
if (
options.allowRetry &&
!madeProgress &&
initializationRetriesUsed === 0 &&
failure.kind === "protocol_timeout" &&
/\bNetwork\.enable timed out/i.test(failure.message)
) {
initializationRetriesUsed++;
const nextWorkers = getNextRetryWorkerCount(currentWorkers);
options.log.warn(
"[Render] Browser initialization timed out; retrying once with fresh sessions.",
{
fromWorkers: currentWorkers,
toWorkers: nextWorkers,
error: failure.message,
},
);
currentWorkers = nextWorkers;
missingRanges = remaining;
attempt++;
continue;
}
if (!madeProgress) {
options.log.warn(
"[Render] Capture attempt made no forward progress; composition is likely structurally broken, not retrying.",