diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.ts index f6e2ccfbf7..9b44b1e936 100644 --- a/packages/producer/src/services/distributed/renderChunk.ts +++ b/packages/producer/src/services/distributed/renderChunk.ts @@ -1062,7 +1062,7 @@ export async function renderChunk( verifyFirstSession: forceScreenshot || !verifyInWorker ? undefined : verifyFirstWorker, }), - { stall: frameStallRule(job) }, + { stall: frameStallRule(job, framesInChunk) }, ).catch(async (captureError: unknown) => { await overlap.current?.abort(); overlap.current = null; diff --git a/packages/producer/src/services/render/observability.ts b/packages/producer/src/services/render/observability.ts index 51b61a3c66..fad56b6e53 100644 --- a/packages/producer/src/services/render/observability.ts +++ b/packages/producer/src/services/render/observability.ts @@ -609,6 +609,10 @@ export interface StageStallRule { noProgressMs: number; /** Allowed silence before the current attempt's first change. */ firstProgressMs: number; + /** True once every unit of work has landed: nothing more can change `progress()`, only the tail work runs. */ + complete?: () => boolean; + /** Allowed silence once `complete()` holds; defaults to `noProgressMs`. */ + tailMs?: number; } /** @@ -642,6 +646,7 @@ function watchStall(phase: string, rule: StageStallRule) { let lastChangeAt = Date.now(); let attempt: AbortController | undefined; let timer: ReturnType | undefined; + let inTail = false; const restartWindow = (ms: number) => { last = rule.progress(); budgetMs = ms; @@ -651,6 +656,10 @@ function watchStall(phase: string, rule: StageStallRule) { timer = setInterval( () => { const current = rule.progress(); + if (rule.complete?.() && !inTail) { + inTail = true; + return restartWindow(rule.tailMs ?? rule.noProgressMs); + } if (current !== last) return restartWindow(rule.noProgressMs); if (Date.now() - lastChangeAt < budgetMs) return; const error = new RenderStageStalledError(phase, budgetMs, current); @@ -666,6 +675,7 @@ function watchStall(phase: string, rule: StageStallRule) { }); const openAttempt: OpenStallAttempt = (parent) => { attempt = new AbortController(); + inTail = false; restartWindow(rule.firstProgressMs); return parent ? AbortSignal.any([parent, attempt.signal]) : attempt.signal; }; diff --git a/packages/producer/src/services/renderOrchestrator.stall.test.ts b/packages/producer/src/services/renderOrchestrator.stall.test.ts index 5d4bb1f4e0..cac8f7de97 100644 --- a/packages/producer/src/services/renderOrchestrator.stall.test.ts +++ b/packages/producer/src/services/renderOrchestrator.stall.test.ts @@ -9,7 +9,10 @@ import { OrderedRenderEventPublisher } from "./render/renderEventPublisher.js"; // Entry: executeRenderJob with the real disk capture stage and retry ladder. Each parallel attempt is // scripted: "stall" lands three frames then hangs until its signal aborts; a pace lands every frame. -type AttemptScript = "stall" | "spread" | { firstFrameAfterMs: number; frameEveryMs: number }; +type AttemptScript = + | "stall" + | "spread" + | { firstFrameAfterMs: number; frameEveryMs: number; tailMs?: number }; const capture = vi.hoisted(() => ({ scripts: [] as AttemptScript[], calls: 0, @@ -90,6 +93,11 @@ async function landFrame( writeFrame(frame, extension); } +/** Work after the last frame (closing workers, flushing) does not watch the abort. */ +async function holdTail(script: AttemptScript): Promise { + if (typeof script === "object" && script.tailMs) await wait(script.tailMs); +} + async function runScriptedAttempt( tasks: WorkerTask[], extension: string, @@ -109,6 +117,7 @@ async function runScriptedAttempt( if (pace) await wait(pace.frameEveryMs - 1_000); } if (!pace) await neverSettlesUntilAborted(signal); + await holdTail(script); return []; } @@ -398,6 +407,21 @@ it("fails an HDR capture that stops, the only attempt it has, and aborts that at expect(outcome()).toMatchObject({ code: "RENDER_STAGE_STALLED" }); }); +it("does not fail a capture that landed every frame for the work that follows its last frame", async () => { + capture.scripts = [{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000, tailMs: 130_000 }]; + const outcome = startRender(); + await runFor(40 * 1_000 + 130_000 + 10_000, outcome); + expect(capture.retriedWithWorkers).toEqual([]); + expect(await settled(outcome)).toBe("completed"); +}); + +it("still fails a capture whose tail outlasts the tail budget", async () => { + capture.scripts = [{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000, tailMs: 700_000 }]; + const outcome = startRender(); + await runFor(40 * 1_000 + 2 * 300_000 + 20_000, outcome); + expect(outcome()).toMatchObject({ code: "RENDER_STAGE_STALLED", retryable: false }); +}); + it("completes even when the progress sink never drains", async () => { capture.scripts = [{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000 }]; const sinkNeverDrains: ProgressCallback = () => new Promise(() => {}); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 6c0dbe44c4..78a5618c9f 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1171,14 +1171,21 @@ export function resetCaptureAttemptProgress(job: { framesRendered?: number }): v } const FIRST_FRAME_STALL_MS = 300_000; +const TAIL_STALL_MS = FIRST_FRAME_STALL_MS; -/** The no-progress rule for capture stages that have no watchdog of their own: disk, HDR, chunk. */ -export function frameStallRule(job: { framesRendered?: number }): StageStallRule { +/** The no-progress rule for capture stages that have no watchdog of their own: disk, HDR, chunk. Once + * `totalFrames` have landed the frame count can no longer change, so the stage is judged on the tail budget. */ +export function frameStallRule( + job: { framesRendered?: number }, + totalFrames: number, +): StageStallRule { const noProgressMs = resolveCaptureStallTimeoutMs(); return { progress: () => job.framesRendered ?? 0, noProgressMs, firstProgressMs: Math.max(FIRST_FRAME_STALL_MS, noProgressMs), + complete: () => (job.framesRendered ?? 0) >= totalFrames, + tailMs: Math.max(TAIL_STALL_MS, noProgressMs), }; } @@ -4680,7 +4687,7 @@ async function executeRenderPipeline(input: { onProgress, }); }, - { stall: frameStallRule(job) }, + { stall: frameStallRule(job, totalFrames) }, ); lastBrowserConsole = hdrRes.lastBrowserConsole; layeredCaptureWarnings.push(...hdrRes.warnings); @@ -5247,7 +5254,7 @@ async function executeRenderPipeline(input: { interleaveFrames: overlap !== null, openAttempt, }), - { stall: frameStallRule(job) }, + { stall: frameStallRule(job, totalFrames) }, ); let captureRes; try {