fix(producer): size the first-frame stall window from the probe's browser start-up (#7517)

* fix(producer): size the first-frame stall window from the probe's browser start-up

* fix(producer): keep the chunk's fixed first-frame window without a comment
This commit is contained in:
Miguel Ángel
2026-11-10 22:46:45 +00:00
committed by GitHub
parent 124591a9d5
commit 7ff8ece461
3 changed files with 71 additions and 6 deletions
@@ -1074,7 +1074,7 @@ export async function renderChunk(
verifyFirstSession:
forceScreenshot || !verifyInWorker ? undefined : verifyFirstWorker,
}),
{ stall: frameStallRule(job, framesInChunk) },
{ stall: frameStallRule(job, framesInChunk, undefined) },
).catch(async (captureError: unknown) => {
await overlap.current?.abort();
overlap.current = null;
@@ -4,7 +4,12 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { resolveConfig, type WorkerTask } from "@hyperframes/engine";
import { formatCaptureFrameName } from "../utils/paths.js";
import { createRenderJob, executeRenderJob, type ProgressCallback } from "./renderOrchestrator.js";
import {
createRenderJob,
executeRenderJob,
firstFrameWindowMs,
type ProgressCallback,
} from "./renderOrchestrator.js";
import { OrderedRenderEventPublisher } from "./render/renderEventPublisher.js";
// Entry: executeRenderJob with the real disk capture stage and retry ladder. Each parallel attempt is
@@ -21,6 +26,7 @@ const capture = vi.hoisted(() => ({
workerCount: 4,
shaderTransitions: false,
blocked: 0,
probeMs: 0,
}));
// The 250 ms memory sampler is a faked interval: left running it is the next timer on every jump.
@@ -177,7 +183,7 @@ vi.mock("./render/stages/probeStage.js", () => ({
lastBrowserConsole: [],
duration: 40 / 30,
totalFrames: 40,
browserProbeMs: 0,
browserProbeMs: capture.probeMs,
beginFrameStalled: false,
}),
}));
@@ -258,6 +264,7 @@ beforeEach(() => {
capture.shaderTransitions = false;
capture.workerCount = 4;
capture.blocked = 0;
capture.probeMs = 0;
vi.useFakeTimers({
toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"],
});
@@ -425,6 +432,53 @@ it("still fails a capture whose tail outlasts the tail budget", async () => {
expect(outcome()).toMatchObject({ code: "RENDER_STAGE_STALLED", retryable: false });
});
it("waits for a first frame that comes after a start-up as slow as the probe's", async () => {
capture.probeMs = 304_000;
capture.scripts = [{ firstFrameAfterMs: 304_000, frameEveryMs: 1_000 }];
const outcome = startRender();
await runFor(304_000 + 40 * 1_000 + 10_000, outcome);
expect(capture.retriedWithWorkers).toEqual([]);
expect(await settled(outcome)).toBe("completed");
});
it("after a slow start, still retries a capture that stops once frames flow, on the same window", async () => {
capture.probeMs = 304_000;
capture.scripts = ["stall", { firstFrameAfterMs: 1_000, frameEveryMs: 1_000 }];
const outcome = startRender();
await runFor(3_000 + 60_000 + 10_000, outcome);
expect(capture.retriedWithWorkers).toEqual([2]);
});
it("with no probe start-up, treats a first frame later than 300 s as a stall", async () => {
capture.scripts = [{ firstFrameAfterMs: 400_000, frameEveryMs: 1_000 }];
const outcome = startRender();
await runFor(290_000, outcome);
expect(outcome()).toBe(PENDING);
await runFor(20_000 + 60_000 + 10_000, outcome);
expect(await settled(outcome)).toMatchObject({ code: "RENDER_STAGE_STALLED" });
});
it("never waits past the ceiling for a first frame, however slow the probe was", async () => {
capture.probeMs = 600_000;
capture.scripts = [{ firstFrameAfterMs: 1_300_000, frameEveryMs: 1_000 }];
const outcome = startRender();
await runFor(880_000, outcome);
expect(outcome()).toBe(PENDING);
await runFor(30_000 + 60_000 + 10_000, outcome);
expect(await settled(outcome)).toMatchObject({ code: "RENDER_STAGE_STALLED" });
});
it.each([
[undefined, 300_000],
[0, 300_000],
[Number.NaN, 300_000],
[100_000, 300_000],
[304_000, 608_000],
[600_000, 900_000],
])("gives a probe start-up of %s ms a first-frame window of %s ms", (probeMs, windowMs) => {
expect(firstFrameWindowMs(probeMs)).toBe(windowMs);
});
it("completes even when the progress sink never drains", async () => {
capture.scripts = [{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000 }];
const sinkNeverDrains: ProgressCallback = () => new Promise<void>(() => {});
@@ -1219,19 +1219,30 @@ export function resetCaptureAttemptProgress(job: { framesRendered?: number }): v
}
const FIRST_FRAME_STALL_MS = 300_000;
// Capture browsers start like the probe's did: allow twice its start-up before the first frame, up to the ceiling.
const FIRST_FRAME_STARTUP_FACTOR = 2;
const FIRST_FRAME_STALL_CEILING_MS = 900_000;
const TAIL_STALL_MS = FIRST_FRAME_STALL_MS;
/** Allowed wait for the first frame, given how long the probe's browser took to start; 0 or unknown keeps 300 s. */
export function firstFrameWindowMs(probeStartupMs: number | undefined): number {
const fromStartup = (probeStartupMs ?? 0) * FIRST_FRAME_STARTUP_FACTOR;
if (!Number.isFinite(fromStartup)) return FIRST_FRAME_STALL_MS;
return Math.max(FIRST_FRAME_STALL_MS, Math.min(FIRST_FRAME_STALL_CEILING_MS, fromStartup));
}
/** The no-progress rule for capture stages that have no watchdog of their own: disk, HDR, chunk. Once
* `totalFrames` have landed the frame count can no longer change, so the stage is judged on the tail budget. */
export function frameStallRule(
job: { framesRendered?: number },
totalFrames: number,
probeStartupMs: number | undefined,
): StageStallRule {
const noProgressMs = resolveCaptureStallTimeoutMs();
return {
progress: () => job.framesRendered ?? 0,
noProgressMs,
firstProgressMs: Math.max(FIRST_FRAME_STALL_MS, noProgressMs),
firstProgressMs: Math.max(firstFrameWindowMs(probeStartupMs), noProgressMs),
complete: () => (job.framesRendered ?? 0) >= totalFrames,
tailMs: Math.max(TAIL_STALL_MS, noProgressMs),
detail: () => describeWorkerPhases(job),
@@ -4899,7 +4910,7 @@ async function executeRenderPipeline(input: {
onProgress,
});
},
{ stall: frameStallRule(job, totalFrames) },
{ stall: frameStallRule(job, totalFrames, perfStages.browserProbeMs) },
);
lastBrowserConsole = hdrRes.lastBrowserConsole;
layeredCaptureWarnings.push(...hdrRes.warnings);
@@ -5476,7 +5487,7 @@ async function executeRenderPipeline(input: {
interleaveFrames: overlap !== null,
openAttempt,
}),
{ stall: frameStallRule(job, totalFrames) },
{ stall: frameStallRule(job, totalFrames, perfStages.browserProbeMs) },
);
let captureRes;
try {