mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(producer): retry with fewer workers when every page times out loading (#7171)
This commit is contained in:
@@ -428,6 +428,46 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
|
||||
}
|
||||
});
|
||||
|
||||
it("halves the workers when every page times out loading, instead of reopening as many", async () => {
|
||||
const workDir = mkdtempSync(join(tmpdir(), "hf-navtimeout-work-"));
|
||||
const framesDir = mkdtempSync(join(tmpdir(), "hf-navtimeout-frames-"));
|
||||
const log = makeLog();
|
||||
let call = 0;
|
||||
vi.mocked(executeParallelCapture).mockImplementation(async () => {
|
||||
call++;
|
||||
if (call === 1) {
|
||||
throw new Error(
|
||||
"[Parallel] Capture failed: Worker 0: Navigation timeout of 60000 ms exceeded",
|
||||
);
|
||||
}
|
||||
writeAllFrames(framesDir, 16);
|
||||
return [];
|
||||
});
|
||||
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
|
||||
|
||||
try {
|
||||
const attempts = await executeDiskCaptureWithAdaptiveRetry({
|
||||
serverUrl: "http://localhost:0",
|
||||
workDir,
|
||||
framesDir,
|
||||
totalFrames: 16,
|
||||
initialWorkerCount: 8,
|
||||
allowRetry: true,
|
||||
frameExt: "jpg",
|
||||
captureOptions: { width: 64, height: 64, fps: { num: 30, den: 1 } },
|
||||
createBeforeCaptureHook: () => null,
|
||||
cfg: DEFAULT_CONFIG,
|
||||
log,
|
||||
dedupPerfs: [],
|
||||
});
|
||||
|
||||
expect(attempts.map((a) => a.workers)).toEqual([8, 4]);
|
||||
} finally {
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
rmSync(framesDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["Target closed", "Network.enable timed out"])(
|
||||
"does NOT retry %s after cancellation",
|
||||
async (message) => {
|
||||
@@ -475,7 +515,7 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["Session closed", "Network.enable timed out"])(
|
||||
it.each(["Session closed", "Network.enable timed out", "Navigation timeout of 9 ms exceeded"])(
|
||||
"bounds repeated %s failures",
|
||||
async (message) => {
|
||||
const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-"));
|
||||
|
||||
@@ -1472,6 +1472,32 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
|
||||
const remainingCount = countFrameRanges(remaining);
|
||||
const madeProgress = captureAttemptMadeProgress(frameCount, remainingCount);
|
||||
|
||||
// A browser that times out starting up or loading its page before any frame exists had too many pages
|
||||
// opening at once: retry with half the workers, down to one. Other zero-progress errors fail below.
|
||||
// A lone worker gets one retry: here for Network.enable, as the transient retry below for a page load.
|
||||
if (
|
||||
options.allowRetry &&
|
||||
!madeProgress &&
|
||||
(currentWorkers > 1 ||
|
||||
(initializationRetriesUsed === 0 && failure.kind === "protocol_timeout")) &&
|
||||
/\bNetwork\.enable timed out|Navigation timeout of \d+ ms exceeded/i.test(failure.message)
|
||||
) {
|
||||
initializationRetriesUsed++;
|
||||
const nextWorkers = getNextRetryWorkerCount(currentWorkers);
|
||||
options.log.warn(
|
||||
"[Render] Browser initialization timed out; retrying with fewer workers.",
|
||||
{
|
||||
fromWorkers: currentWorkers,
|
||||
toWorkers: nextWorkers,
|
||||
error: failure.message,
|
||||
},
|
||||
);
|
||||
currentWorkers = nextWorkers;
|
||||
missingRanges = remaining;
|
||||
attempt++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Single bounded retry for a transient browser death (`Target closed` /
|
||||
// `Page crashed` / `Session closed`): the tab died mid-capture, not the
|
||||
// composition. Unlike the worker-halving retry below, this keeps the same
|
||||
@@ -1500,32 +1526,6 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
|
||||
continue;
|
||||
}
|
||||
|
||||
// CDP initialization can time out before any frame exists. Give that
|
||||
// specific startup failure one fresh attempt with less concurrency;
|
||||
// arbitrary zero-progress authoring/capture errors still fail below.
|
||||
if (
|
||||
options.allowRetry &&
|
||||
!madeProgress &&
|
||||
initializationRetriesUsed === 0 &&
|
||||
failure.kind === "protocol_timeout" &&
|
||||
/\bNetwork\.enable timed out/i.test(failure.message)
|
||||
) {
|
||||
initializationRetriesUsed++;
|
||||
const nextWorkers = getNextRetryWorkerCount(currentWorkers);
|
||||
options.log.warn(
|
||||
"[Render] Browser initialization timed out; retrying once with fresh sessions.",
|
||||
{
|
||||
fromWorkers: currentWorkers,
|
||||
toWorkers: nextWorkers,
|
||||
error: failure.message,
|
||||
},
|
||||
);
|
||||
currentWorkers = nextWorkers;
|
||||
missingRanges = remaining;
|
||||
attempt++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!madeProgress) {
|
||||
options.log.warn(
|
||||
"[Render] Capture attempt made no forward progress; composition is likely structurally broken, not retrying.",
|
||||
|
||||
Reference in New Issue
Block a user