fix(producer): a capture that landed every frame is not failed for the work after its last frame (#6674)

* fix(producer): a capture that landed every frame is not failed for the work after its last frame

* chore(producer): drop a comment the ratchet counts

packages/producer/src/services/distributed/renderChunk.ts

@@ -1062,7 +1062,7 @@ export async function renderChunk(
                 verifyFirstSession:
                   forceScreenshot || !verifyInWorker ? undefined : verifyFirstWorker,
               }),
-            { stall: frameStallRule(job) },
+            { stall: frameStallRule(job, framesInChunk) },
           ).catch(async (captureError: unknown) => {
             await overlap.current?.abort();
             overlap.current = null;

packages/producer/src/services/render/observability.ts

@@ -609,6 +609,10 @@ export interface StageStallRule {
   noProgressMs: number;
   /** Allowed silence before the current attempt's first change. */
   firstProgressMs: number;
+  /** True once every unit of work has landed: nothing more can change `progress()`, only the tail work runs. */
+  complete?: () => boolean;
+  /** Allowed silence once `complete()` holds; defaults to `noProgressMs`. */
+  tailMs?: number;
 }
 
 /**
@@ -642,6 +646,7 @@ function watchStall(phase: string, rule: StageStallRule) {
   let lastChangeAt = Date.now();
   let attempt: AbortController | undefined;
   let timer: ReturnType<typeof setInterval> | undefined;
+  let inTail = false;
   const restartWindow = (ms: number) => {
     last = rule.progress();
     budgetMs = ms;
@@ -651,6 +656,10 @@ function watchStall(phase: string, rule: StageStallRule) {
     timer = setInterval(
       () => {
         const current = rule.progress();
+        if (rule.complete?.() && !inTail) {
+          inTail = true;
+          return restartWindow(rule.tailMs ?? rule.noProgressMs);
+        }
         if (current !== last) return restartWindow(rule.noProgressMs);
         if (Date.now() - lastChangeAt < budgetMs) return;
         const error = new RenderStageStalledError(phase, budgetMs, current);
@@ -666,6 +675,7 @@ function watchStall(phase: string, rule: StageStallRule) {
   });
   const openAttempt: OpenStallAttempt = (parent) => {
     attempt = new AbortController();
+    inTail = false;
     restartWindow(rule.firstProgressMs);
     return parent ? AbortSignal.any([parent, attempt.signal]) : attempt.signal;
   };

packages/producer/src/services/renderOrchestrator.stall.test.ts

@@ -9,7 +9,10 @@ import { OrderedRenderEventPublisher } from "./render/renderEventPublisher.js";
 
 // Entry: executeRenderJob with the real disk capture stage and retry ladder. Each parallel attempt is
 // scripted: "stall" lands three frames then hangs until its signal aborts; a pace lands every frame.
-type AttemptScript = "stall" | "spread" | { firstFrameAfterMs: number; frameEveryMs: number };
+type AttemptScript =
+  | "stall"
+  | "spread"
+  | { firstFrameAfterMs: number; frameEveryMs: number; tailMs?: number };
 const capture = vi.hoisted(() => ({
   scripts: [] as AttemptScript[],
   calls: 0,
@@ -90,6 +93,11 @@ async function landFrame(
   writeFrame(frame, extension);
 }
 
+/** Work after the last frame (closing workers, flushing) does not watch the abort. */
+async function holdTail(script: AttemptScript): Promise<void> {
+  if (typeof script === "object" && script.tailMs) await wait(script.tailMs);
+}
+
 async function runScriptedAttempt(
   tasks: WorkerTask[],
   extension: string,
@@ -109,6 +117,7 @@ async function runScriptedAttempt(
     if (pace) await wait(pace.frameEveryMs - 1_000);
   }
   if (!pace) await neverSettlesUntilAborted(signal);
+  await holdTail(script);
   return [];
 }
 
@@ -398,6 +407,21 @@ it("fails an HDR capture that stops, the only attempt it has, and aborts that at
   expect(outcome()).toMatchObject({ code: "RENDER_STAGE_STALLED" });
 });
 
+it("does not fail a capture that landed every frame for the work that follows its last frame", async () => {
+  capture.scripts = [{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000, tailMs: 130_000 }];
+  const outcome = startRender();
+  await runFor(40 * 1_000 + 130_000 + 10_000, outcome);
+  expect(capture.retriedWithWorkers).toEqual([]);
+  expect(await settled(outcome)).toBe("completed");
+});
+
+it("still fails a capture whose tail outlasts the tail budget", async () => {
+  capture.scripts = [{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000, tailMs: 700_000 }];
+  const outcome = startRender();
+  await runFor(40 * 1_000 + 2 * 300_000 + 20_000, outcome);
+  expect(outcome()).toMatchObject({ code: "RENDER_STAGE_STALLED", retryable: false });
+});
+
 it("completes even when the progress sink never drains", async () => {
   capture.scripts = [{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000 }];
   const sinkNeverDrains: ProgressCallback = () => new Promise<void>(() => {});

packages/producer/src/services/renderOrchestrator.ts

@@ -1171,14 +1171,21 @@ export function resetCaptureAttemptProgress(job: { framesRendered?: number }): v
 }
 
 const FIRST_FRAME_STALL_MS = 300_000;
+const TAIL_STALL_MS = FIRST_FRAME_STALL_MS;
 
-/** The no-progress rule for capture stages that have no watchdog of their own: disk, HDR, chunk. */
-export function frameStallRule(job: { framesRendered?: number }): StageStallRule {
+/** The no-progress rule for capture stages that have no watchdog of their own: disk, HDR, chunk. Once
+ * `totalFrames` have landed the frame count can no longer change, so the stage is judged on the tail budget. */
+export function frameStallRule(
+  job: { framesRendered?: number },
+  totalFrames: number,
+): StageStallRule {
   const noProgressMs = resolveCaptureStallTimeoutMs();
   return {
     progress: () => job.framesRendered ?? 0,
     noProgressMs,
     firstProgressMs: Math.max(FIRST_FRAME_STALL_MS, noProgressMs),
+    complete: () => (job.framesRendered ?? 0) >= totalFrames,
+    tailMs: Math.max(TAIL_STALL_MS, noProgressMs),
   };
 }
 
@@ -4680,7 +4687,7 @@ async function executeRenderPipeline(input: {
             onProgress,
           });
         },
-        { stall: frameStallRule(job) },
+        { stall: frameStallRule(job, totalFrames) },
       );
       lastBrowserConsole = hdrRes.lastBrowserConsole;
       layeredCaptureWarnings.push(...hdrRes.warnings);
@@ -5247,7 +5254,7 @@ async function executeRenderPipeline(input: {
                 interleaveFrames: overlap !== null,
                 openAttempt,
               }),
-            { stall: frameStallRule(job) },
+            { stall: frameStallRule(job, totalFrames) },
           );
         let captureRes;
         try {