mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(producer): size the first-frame stall window from the probe's browser start-up (#7517)
* fix(producer): size the first-frame stall window from the probe's browser start-up * fix(producer): keep the chunk's fixed first-frame window without a comment
This commit is contained in:
@@ -1074,7 +1074,7 @@ export async function renderChunk(
|
||||
verifyFirstSession:
|
||||
forceScreenshot || !verifyInWorker ? undefined : verifyFirstWorker,
|
||||
}),
|
||||
{ stall: frameStallRule(job, framesInChunk) },
|
||||
{ stall: frameStallRule(job, framesInChunk, undefined) },
|
||||
).catch(async (captureError: unknown) => {
|
||||
await overlap.current?.abort();
|
||||
overlap.current = null;
|
||||
|
||||
@@ -4,7 +4,12 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { resolveConfig, type WorkerTask } from "@hyperframes/engine";
|
||||
import { formatCaptureFrameName } from "../utils/paths.js";
|
||||
import { createRenderJob, executeRenderJob, type ProgressCallback } from "./renderOrchestrator.js";
|
||||
import {
|
||||
createRenderJob,
|
||||
executeRenderJob,
|
||||
firstFrameWindowMs,
|
||||
type ProgressCallback,
|
||||
} from "./renderOrchestrator.js";
|
||||
import { OrderedRenderEventPublisher } from "./render/renderEventPublisher.js";
|
||||
|
||||
// Entry: executeRenderJob with the real disk capture stage and retry ladder. Each parallel attempt is
|
||||
@@ -21,6 +26,7 @@ const capture = vi.hoisted(() => ({
|
||||
workerCount: 4,
|
||||
shaderTransitions: false,
|
||||
blocked: 0,
|
||||
probeMs: 0,
|
||||
}));
|
||||
|
||||
// The 250 ms memory sampler is a faked interval: left running it is the next timer on every jump.
|
||||
@@ -177,7 +183,7 @@ vi.mock("./render/stages/probeStage.js", () => ({
|
||||
lastBrowserConsole: [],
|
||||
duration: 40 / 30,
|
||||
totalFrames: 40,
|
||||
browserProbeMs: 0,
|
||||
browserProbeMs: capture.probeMs,
|
||||
beginFrameStalled: false,
|
||||
}),
|
||||
}));
|
||||
@@ -258,6 +264,7 @@ beforeEach(() => {
|
||||
capture.shaderTransitions = false;
|
||||
capture.workerCount = 4;
|
||||
capture.blocked = 0;
|
||||
capture.probeMs = 0;
|
||||
vi.useFakeTimers({
|
||||
toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"],
|
||||
});
|
||||
@@ -425,6 +432,53 @@ it("still fails a capture whose tail outlasts the tail budget", async () => {
|
||||
expect(outcome()).toMatchObject({ code: "RENDER_STAGE_STALLED", retryable: false });
|
||||
});
|
||||
|
||||
it("waits for a first frame that comes after a start-up as slow as the probe's", async () => {
|
||||
capture.probeMs = 304_000;
|
||||
capture.scripts = [{ firstFrameAfterMs: 304_000, frameEveryMs: 1_000 }];
|
||||
const outcome = startRender();
|
||||
await runFor(304_000 + 40 * 1_000 + 10_000, outcome);
|
||||
expect(capture.retriedWithWorkers).toEqual([]);
|
||||
expect(await settled(outcome)).toBe("completed");
|
||||
});
|
||||
|
||||
it("after a slow start, still retries a capture that stops once frames flow, on the same window", async () => {
|
||||
capture.probeMs = 304_000;
|
||||
capture.scripts = ["stall", { firstFrameAfterMs: 1_000, frameEveryMs: 1_000 }];
|
||||
const outcome = startRender();
|
||||
await runFor(3_000 + 60_000 + 10_000, outcome);
|
||||
expect(capture.retriedWithWorkers).toEqual([2]);
|
||||
});
|
||||
|
||||
it("with no probe start-up, treats a first frame later than 300 s as a stall", async () => {
|
||||
capture.scripts = [{ firstFrameAfterMs: 400_000, frameEveryMs: 1_000 }];
|
||||
const outcome = startRender();
|
||||
await runFor(290_000, outcome);
|
||||
expect(outcome()).toBe(PENDING);
|
||||
await runFor(20_000 + 60_000 + 10_000, outcome);
|
||||
expect(await settled(outcome)).toMatchObject({ code: "RENDER_STAGE_STALLED" });
|
||||
});
|
||||
|
||||
it("never waits past the ceiling for a first frame, however slow the probe was", async () => {
|
||||
capture.probeMs = 600_000;
|
||||
capture.scripts = [{ firstFrameAfterMs: 1_300_000, frameEveryMs: 1_000 }];
|
||||
const outcome = startRender();
|
||||
await runFor(880_000, outcome);
|
||||
expect(outcome()).toBe(PENDING);
|
||||
await runFor(30_000 + 60_000 + 10_000, outcome);
|
||||
expect(await settled(outcome)).toMatchObject({ code: "RENDER_STAGE_STALLED" });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[undefined, 300_000],
|
||||
[0, 300_000],
|
||||
[Number.NaN, 300_000],
|
||||
[100_000, 300_000],
|
||||
[304_000, 608_000],
|
||||
[600_000, 900_000],
|
||||
])("gives a probe start-up of %s ms a first-frame window of %s ms", (probeMs, windowMs) => {
|
||||
expect(firstFrameWindowMs(probeMs)).toBe(windowMs);
|
||||
});
|
||||
|
||||
it("completes even when the progress sink never drains", async () => {
|
||||
capture.scripts = [{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000 }];
|
||||
const sinkNeverDrains: ProgressCallback = () => new Promise<void>(() => {});
|
||||
|
||||
@@ -1219,19 +1219,30 @@ export function resetCaptureAttemptProgress(job: { framesRendered?: number }): v
|
||||
}
|
||||
|
||||
const FIRST_FRAME_STALL_MS = 300_000;
|
||||
// Capture browsers start like the probe's did: allow twice its start-up before the first frame, up to the ceiling.
|
||||
const FIRST_FRAME_STARTUP_FACTOR = 2;
|
||||
const FIRST_FRAME_STALL_CEILING_MS = 900_000;
|
||||
const TAIL_STALL_MS = FIRST_FRAME_STALL_MS;
|
||||
|
||||
/** Allowed wait for the first frame, given how long the probe's browser took to start; 0 or unknown keeps 300 s. */
|
||||
export function firstFrameWindowMs(probeStartupMs: number | undefined): number {
|
||||
const fromStartup = (probeStartupMs ?? 0) * FIRST_FRAME_STARTUP_FACTOR;
|
||||
if (!Number.isFinite(fromStartup)) return FIRST_FRAME_STALL_MS;
|
||||
return Math.max(FIRST_FRAME_STALL_MS, Math.min(FIRST_FRAME_STALL_CEILING_MS, fromStartup));
|
||||
}
|
||||
|
||||
/** The no-progress rule for capture stages that have no watchdog of their own: disk, HDR, chunk. Once
|
||||
* `totalFrames` have landed the frame count can no longer change, so the stage is judged on the tail budget. */
|
||||
export function frameStallRule(
|
||||
job: { framesRendered?: number },
|
||||
totalFrames: number,
|
||||
probeStartupMs: number | undefined,
|
||||
): StageStallRule {
|
||||
const noProgressMs = resolveCaptureStallTimeoutMs();
|
||||
return {
|
||||
progress: () => job.framesRendered ?? 0,
|
||||
noProgressMs,
|
||||
firstProgressMs: Math.max(FIRST_FRAME_STALL_MS, noProgressMs),
|
||||
firstProgressMs: Math.max(firstFrameWindowMs(probeStartupMs), noProgressMs),
|
||||
complete: () => (job.framesRendered ?? 0) >= totalFrames,
|
||||
tailMs: Math.max(TAIL_STALL_MS, noProgressMs),
|
||||
detail: () => describeWorkerPhases(job),
|
||||
@@ -4899,7 +4910,7 @@ async function executeRenderPipeline(input: {
|
||||
onProgress,
|
||||
});
|
||||
},
|
||||
{ stall: frameStallRule(job, totalFrames) },
|
||||
{ stall: frameStallRule(job, totalFrames, perfStages.browserProbeMs) },
|
||||
);
|
||||
lastBrowserConsole = hdrRes.lastBrowserConsole;
|
||||
layeredCaptureWarnings.push(...hdrRes.warnings);
|
||||
@@ -5476,7 +5487,7 @@ async function executeRenderPipeline(input: {
|
||||
interleaveFrames: overlap !== null,
|
||||
openAttempt,
|
||||
}),
|
||||
{ stall: frameStallRule(job, totalFrames) },
|
||||
{ stall: frameStallRule(job, totalFrames, perfStages.browserProbeMs) },
|
||||
);
|
||||
let captureRes;
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user