From 7ff8ece461e87282e17f3da14f0b83d0cbe9d38f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 10 Nov 2026 22:46:45 +0000 Subject: [PATCH] fix(producer): size the first-frame stall window from the probe's browser start-up (#7517) * fix(producer): size the first-frame stall window from the probe's browser start-up * fix(producer): keep the chunk's fixed first-frame window without a comment --- .../src/services/distributed/renderChunk.ts | 2 +- .../services/renderOrchestrator.stall.test.ts | 58 ++++++++++++++++++- .../src/services/renderOrchestrator.ts | 17 +++++- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.ts index a25fc55148..06a79bb17c 100644 --- a/packages/producer/src/services/distributed/renderChunk.ts +++ b/packages/producer/src/services/distributed/renderChunk.ts @@ -1074,7 +1074,7 @@ export async function renderChunk( verifyFirstSession: forceScreenshot || !verifyInWorker ? undefined : verifyFirstWorker, }), - { stall: frameStallRule(job, framesInChunk) }, + { stall: frameStallRule(job, framesInChunk, undefined) }, ).catch(async (captureError: unknown) => { await overlap.current?.abort(); overlap.current = null; diff --git a/packages/producer/src/services/renderOrchestrator.stall.test.ts b/packages/producer/src/services/renderOrchestrator.stall.test.ts index 8f7dc5c401..a9f7599bf2 100644 --- a/packages/producer/src/services/renderOrchestrator.stall.test.ts +++ b/packages/producer/src/services/renderOrchestrator.stall.test.ts @@ -4,7 +4,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { resolveConfig, type WorkerTask } from "@hyperframes/engine"; import { formatCaptureFrameName } from "../utils/paths.js"; -import { createRenderJob, executeRenderJob, type ProgressCallback } from "./renderOrchestrator.js"; +import { + createRenderJob, + executeRenderJob, + firstFrameWindowMs, + type ProgressCallback, +} from "./renderOrchestrator.js"; import { OrderedRenderEventPublisher } from "./render/renderEventPublisher.js"; // Entry: executeRenderJob with the real disk capture stage and retry ladder. Each parallel attempt is @@ -21,6 +26,7 @@ const capture = vi.hoisted(() => ({ workerCount: 4, shaderTransitions: false, blocked: 0, + probeMs: 0, })); // The 250 ms memory sampler is a faked interval: left running it is the next timer on every jump. @@ -177,7 +183,7 @@ vi.mock("./render/stages/probeStage.js", () => ({ lastBrowserConsole: [], duration: 40 / 30, totalFrames: 40, - browserProbeMs: 0, + browserProbeMs: capture.probeMs, beginFrameStalled: false, }), })); @@ -258,6 +264,7 @@ beforeEach(() => { capture.shaderTransitions = false; capture.workerCount = 4; capture.blocked = 0; + capture.probeMs = 0; vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], }); @@ -425,6 +432,53 @@ it("still fails a capture whose tail outlasts the tail budget", async () => { expect(outcome()).toMatchObject({ code: "RENDER_STAGE_STALLED", retryable: false }); }); +it("waits for a first frame that comes after a start-up as slow as the probe's", async () => { + capture.probeMs = 304_000; + capture.scripts = [{ firstFrameAfterMs: 304_000, frameEveryMs: 1_000 }]; + const outcome = startRender(); + await runFor(304_000 + 40 * 1_000 + 10_000, outcome); + expect(capture.retriedWithWorkers).toEqual([]); + expect(await settled(outcome)).toBe("completed"); +}); + +it("after a slow start, still retries a capture that stops once frames flow, on the same window", async () => { + capture.probeMs = 304_000; + capture.scripts = ["stall", { firstFrameAfterMs: 1_000, frameEveryMs: 1_000 }]; + const outcome = startRender(); + await runFor(3_000 + 60_000 + 10_000, outcome); + expect(capture.retriedWithWorkers).toEqual([2]); +}); + +it("with no probe start-up, treats a first frame later than 300 s as a stall", async () => { + capture.scripts = [{ firstFrameAfterMs: 400_000, frameEveryMs: 1_000 }]; + const outcome = startRender(); + await runFor(290_000, outcome); + expect(outcome()).toBe(PENDING); + await runFor(20_000 + 60_000 + 10_000, outcome); + expect(await settled(outcome)).toMatchObject({ code: "RENDER_STAGE_STALLED" }); +}); + +it("never waits past the ceiling for a first frame, however slow the probe was", async () => { + capture.probeMs = 600_000; + capture.scripts = [{ firstFrameAfterMs: 1_300_000, frameEveryMs: 1_000 }]; + const outcome = startRender(); + await runFor(880_000, outcome); + expect(outcome()).toBe(PENDING); + await runFor(30_000 + 60_000 + 10_000, outcome); + expect(await settled(outcome)).toMatchObject({ code: "RENDER_STAGE_STALLED" }); +}); + +it.each([ + [undefined, 300_000], + [0, 300_000], + [Number.NaN, 300_000], + [100_000, 300_000], + [304_000, 608_000], + [600_000, 900_000], +])("gives a probe start-up of %s ms a first-frame window of %s ms", (probeMs, windowMs) => { + expect(firstFrameWindowMs(probeMs)).toBe(windowMs); +}); + it("completes even when the progress sink never drains", async () => { capture.scripts = [{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000 }]; const sinkNeverDrains: ProgressCallback = () => new Promise(() => {}); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 2badbe7288..42915112f5 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1219,19 +1219,30 @@ export function resetCaptureAttemptProgress(job: { framesRendered?: number }): v } const FIRST_FRAME_STALL_MS = 300_000; +// Capture browsers start like the probe's did: allow twice its start-up before the first frame, up to the ceiling. +const FIRST_FRAME_STARTUP_FACTOR = 2; +const FIRST_FRAME_STALL_CEILING_MS = 900_000; const TAIL_STALL_MS = FIRST_FRAME_STALL_MS; +/** Allowed wait for the first frame, given how long the probe's browser took to start; 0 or unknown keeps 300 s. */ +export function firstFrameWindowMs(probeStartupMs: number | undefined): number { + const fromStartup = (probeStartupMs ?? 0) * FIRST_FRAME_STARTUP_FACTOR; + if (!Number.isFinite(fromStartup)) return FIRST_FRAME_STALL_MS; + return Math.max(FIRST_FRAME_STALL_MS, Math.min(FIRST_FRAME_STALL_CEILING_MS, fromStartup)); +} + /** The no-progress rule for capture stages that have no watchdog of their own: disk, HDR, chunk. Once * `totalFrames` have landed the frame count can no longer change, so the stage is judged on the tail budget. */ export function frameStallRule( job: { framesRendered?: number }, totalFrames: number, + probeStartupMs: number | undefined, ): StageStallRule { const noProgressMs = resolveCaptureStallTimeoutMs(); return { progress: () => job.framesRendered ?? 0, noProgressMs, - firstProgressMs: Math.max(FIRST_FRAME_STALL_MS, noProgressMs), + firstProgressMs: Math.max(firstFrameWindowMs(probeStartupMs), noProgressMs), complete: () => (job.framesRendered ?? 0) >= totalFrames, tailMs: Math.max(TAIL_STALL_MS, noProgressMs), detail: () => describeWorkerPhases(job), @@ -4899,7 +4910,7 @@ async function executeRenderPipeline(input: { onProgress, }); }, - { stall: frameStallRule(job, totalFrames) }, + { stall: frameStallRule(job, totalFrames, perfStages.browserProbeMs) }, ); lastBrowserConsole = hdrRes.lastBrowserConsole; layeredCaptureWarnings.push(...hdrRes.warnings); @@ -5476,7 +5487,7 @@ async function executeRenderPipeline(input: { interleaveFrames: overlap !== null, openAttempt, }), - { stall: frameStallRule(job, totalFrames) }, + { stall: frameStallRule(job, totalFrames, perfStages.browserProbeMs) }, ); let captureRes; try {