fix(producer): segmented capture gives up on a session stuck opening or initializing (#7329)

* fix(producer): segmented capture gives up on a session stuck opening or initializing

* fix(producer): sequential disk capture bounds the opt-in drawElement setup the same way
This commit is contained in:
Miguel Ángel
2026-11-06 18:54:32 +00:00
committed by GitHub
parent 18fc75dcd7
commit 6fab219142
5 changed files with 193 additions and 19 deletions
@@ -1,5 +1,5 @@
// fallow-ignore-file code-duplication
import { afterAll, describe, expect, it, mock } from "bun:test";
import { afterAll, describe, expect, it, mock, vi } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -26,6 +26,9 @@ let sessionWorkerEncodeEnabled = false;
let captureSessionMode: "drawelement" | "screenshot" = "drawelement";
let failPrepareCaptureSessionForReuse = false;
let initializeSessionErrorMessage = "initialize failed";
let hangDeferredDrawElementInit = false;
/** While set, opening a session waits for this promise instead of returning a fresh one. */
let pendingOpen: Promise<unknown> | null = null;
const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"];
const closeCaptureSession = mock(async () => {});
const createdSessionConfigs: Array<{ enableBrowserPool?: boolean }> = [];
@@ -62,10 +65,12 @@ mock.module("@hyperframes/engine", () => ({
return { buffer: Buffer.from("frame"), captureTimeMs: 1 };
},
closeCaptureSession,
completeDeferredDrawElementInit: async () => {},
completeDeferredDrawElementInit: () =>
hangDeferredDrawElementInit ? new Promise<void>(() => {}) : Promise.resolve(),
createCaptureSession: async (...args: unknown[]) => {
createdSessionConfigs.push(args[4] as { enableBrowserPool?: boolean });
createdSessionHooks.push(args[3]);
if (pendingOpen) return pendingOpen;
return {
isInitialized: false,
browserConsoleBuffer,
@@ -451,6 +456,81 @@ describe("runCaptureSegmentedStage", () => {
expect(result).toEqual({ success: false });
});
/** Runs the stage on a fake clock, a second at a time, until it settles or 30 minutes pass. */
const settleOnFakeClock = async (run: Promise<unknown>) => {
const state = { done: false, error: undefined as unknown, elapsedMs: 0 };
run.then(
() => (state.done = true),
(error: unknown) => Object.assign(state, { done: true, error }),
);
const flush = async () => {
for (let i = 0; i < 200; i++) await Promise.resolve();
};
await flush();
while (!state.done && state.elapsedMs < 30 * 60_000) {
vi.advanceTimersByTime(1_000);
state.elapsedMs += 1_000;
await flush();
}
return state;
};
it("gives up on a deferred drawElement init that never finishes, within the session-init budget", async () => {
hangDeferredDrawElementInit = true;
closeCaptureSession.mockClear();
const prev = process.env.HF_DE_STALL_MS;
delete process.env.HF_DE_STALL_MS;
vi.useFakeTimers();
try {
const input = fakeStageInput({ totalFrames: 3 });
const state = await settleOnFakeClock(
runCaptureSegmentedStage({ ...input, segmentFrames: 3 }),
);
expect(state.done).toBe(true);
expect(state.elapsedMs).toBeLessThanOrEqual(305_000);
expect((state.error as Error).message).toContain(
"no frame progress for 300000ms (stuck at frame 0/3)",
);
expect(closeCaptureSession.mock.calls.map((c) => c[0])).toContain(input.probeSession);
} finally {
vi.useRealTimers();
hangDeferredDrawElementInit = false;
if (prev !== undefined) process.env.HF_DE_STALL_MS = prev;
}
});
it("gives up on a session that never finishes opening, and closes it when it opens late", async () => {
let openLate: (session: unknown) => void = () => {};
pendingOpen = new Promise((resolve) => (openLate = resolve));
closeCaptureSession.mockClear();
const prev = process.env.HF_DE_STALL_MS;
delete process.env.HF_DE_STALL_MS;
vi.useFakeTimers();
try {
const state = await settleOnFakeClock(
runCaptureSegmentedStage({
...fakeStageInput({ totalFrames: 3 }),
probeSession: undefined,
segmentFrames: 3,
}),
);
expect(state.done).toBe(true);
expect(state.elapsedMs).toBeLessThanOrEqual(305_000);
expect((state.error as Error).message).toContain(
"no frame progress for 300000ms (stuck at frame 0/3 while opening the capture session)",
);
const late = { browserConsoleBuffer: [] };
openLate(late);
for (let i = 0; i < 20; i++) await Promise.resolve();
expect(closeCaptureSession).toHaveBeenCalledWith(late);
} finally {
vi.useRealTimers();
pendingOpen = null;
if (prev !== undefined) process.env.HF_DE_STALL_MS = prev;
}
});
it("returns success:false when the first encoder cannot spawn, and has closed the probe", async () => {
const spawnEncoder = mock(async () => {
throw new Error("ffmpeg missing");
@@ -47,6 +47,8 @@ import type { SegmentVideoFrames, SegmentWorkerFrames } from "../segmentVideoFra
import { createSegmentQueue } from "../segmentQueue.js";
import { isTargetLossError } from "../segmentRecycle.js";
import {
OPENING_CAPTURE_SESSION,
createSessionInitBudget,
raceAgainstStall,
resolveCaptureStallTimeoutMs,
verifyDrawElementSample,
@@ -337,14 +339,28 @@ function defaultSessionFactory(input: CaptureSegmentedStageInput): SessionFactor
await closeCaptureSession(probeSession).catch(() => {});
probeSession = null;
}
const session = probeSession ?? (await openSegmentedSession(input, injector));
const withinInitBudget = createSessionInitBudget({
frameIndex: input.job.framesRendered ?? 0,
totalFrames: input.totalFrames,
abortSignal: input.abortSignal,
});
const openSession = async (): Promise<CaptureSession> => {
const opening = openSegmentedSession(input, injector);
try {
return await withinInitBudget(opening, OPENING_CAPTURE_SESSION);
} catch (error) {
void opening.then(closeCaptureSession, () => {});
throw error;
}
};
const session = probeSession ?? (await openSession());
if (probeSession) {
openSegmentedSessionReuse(input, probeSession, injector);
probeSession = null;
}
try {
if (!session.isInitialized) await initializeSession(session);
await completeDeferredDrawElementInit(session);
if (!session.isInitialized) await withinInitBudget(initializeSession(session), session);
await withinInitBudget(completeDeferredDrawElementInit(session), session);
} catch (error) {
await closeCaptureSession(session).catch(() => {});
throw error;
@@ -15,6 +15,7 @@ const initializeSession = vi.fn(async () => {});
const captureFrame = vi.fn();
const closeCaptureSession = vi.fn(async () => {});
const verifyDiskDrawElementSamples = vi.fn(async () => {});
const completeDeferredDrawElementInit = vi.fn(async () => {});
const getCapturePerfSummary = vi.fn(() => ({}));
const executeParallelCapture = vi.fn();
@@ -27,6 +28,8 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
captureFrame: (...args: unknown[]) => captureFrame(...args),
closeCaptureSession: (...args: unknown[]) => closeCaptureSession(...args),
verifyDiskDrawElementSamples: (...args: unknown[]) => verifyDiskDrawElementSamples(...args),
completeDeferredDrawElementInit: (...args: unknown[]) =>
completeDeferredDrawElementInit(...args),
getCapturePerfSummary: (...args: unknown[]) => getCapturePerfSummary(...args),
executeParallelCapture: (...args: unknown[]) => executeParallelCapture(...args),
mergeWorkerFrames: async () => {},
@@ -265,6 +268,52 @@ describe("runCaptureStage measured disk projection (both branches)", () => {
closeCaptureSession.mockReset().mockResolvedValue(undefined);
});
it("sequential: gives up on a deferred drawElement init that never finishes, within the session-init budget", async () => {
const { workDir, framesDir } = makeDirs();
freeBytes.mockReturnValue({ bavail: 10_000_000_000, bsize: 1 });
completeDeferredDrawElementInit.mockImplementation(() => new Promise<void>(() => {}));
const probeSession = {
isInitialized: true,
browserConsoleBuffer: [],
options: { captureBeyondViewport: false, format: "jpeg" },
workerEncodeEnabled: false,
outputDir: framesDir,
};
const prevOptIn = process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = "true";
vi.useFakeTimers();
try {
const state = { done: false, error: undefined as unknown, elapsedMs: 0 };
runCaptureStage({
...makeInput(1, workDir, framesDir),
probeSession: probeSession as never,
}).then(
() => (state.done = true),
(error: unknown) => Object.assign(state, { done: true, error }),
);
const flush = async () => {
for (let i = 0; i < 200; i++) await Promise.resolve();
};
await flush();
while (!state.done && state.elapsedMs < 30 * 60_000) {
vi.advanceTimersByTime(1_000);
state.elapsedMs += 1_000;
await flush();
}
expect(state.done).toBe(true);
expect(state.elapsedMs).toBeLessThanOrEqual(305_000);
expect((state.error as Error).message).toContain(
"no frame progress for 300000ms (stuck at frame 0/20)",
);
expect(closeCaptureSession).toHaveBeenCalledWith(probeSession);
} finally {
vi.useRealTimers();
completeDeferredDrawElementInit.mockReset().mockResolvedValue(undefined);
if (prevOptIn === undefined) delete process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
else process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = prevOptIn;
}
});
it("sequential: stops after ten frames when the measured remainder cannot fit", async () => {
const { workDir, framesDir } = makeDirs();
// 8 MB free: passes the static jpeg estimate, fails the 10 MB measured remainder.
@@ -71,6 +71,7 @@ import { reportFrameProgress, reportWorkerStartup } from "../shared.js";
import { withWorkerPageErrors } from "../perfSummary.js";
import type { SdrDiskCapturePlan } from "../capturePlan.js";
import type { OpenStallAttempt } from "../observability.js";
import { createSessionInitBudget } from "./captureStreamingStage.js";
export interface CaptureStageInput {
fileServer: FileServerHandle;
@@ -501,7 +502,11 @@ async function runSequentialDiskCapture(
prepareCaptureSessionForReuse(session, framesDir, videoInjector);
probeSession = null;
}
await ensureSessionReady(session);
await ensureSessionReady(session, {
frameIndex: resumeFrom,
totalFrames,
abortSignal: input.abortSignal,
});
assertAttemptActive();
lastBrowserConsole = session.browserConsoleBuffer;
@@ -576,7 +581,10 @@ function planTransientResume(
: null;
}
async function ensureSessionReady(session: CaptureSession): Promise<void> {
async function ensureSessionReady(
session: CaptureSession,
budget: Parameters<typeof createSessionInitBudget>[0],
): Promise<void> {
if (!session.isInitialized) {
await initializeSession(session);
return;
@@ -584,7 +592,7 @@ async function ensureSessionReady(session: CaptureSession): Promise<void> {
// Deferred drawElement init (probe-initialized video comps). The disk path has no
// drain-time self-verification, so only the explicit fast-capture opt-in completes it.
if (process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE === "true") {
await completeDeferredDrawElementInit(session);
await createSessionInitBudget(budget)(completeDeferredDrawElementInit(session), session);
}
}
@@ -189,9 +189,9 @@ function captureModeLabel(mode: CaptureSession["captureMode"]): string {
return "screenshot";
}
type StalledSession = { captureMode: CaptureSession["captureMode"]; captureStep?: string };
export type StalledSession = { captureMode: CaptureSession["captureMode"]; captureStep?: string };
const OPENING_CAPTURE_SESSION: StalledSession = {
export const OPENING_CAPTURE_SESSION: StalledSession = {
captureMode: "screenshot",
captureStep: "opening the capture session",
};
@@ -271,6 +271,31 @@ export function raceAgainstStall<T>(
});
}
/**
* One budget for everything before a session's first frame: opening it, initializing it, and the
* deferred drawElement init. Each call races against what is left of the same clock.
*/
export function createSessionInitBudget(input: {
frameIndex: number;
totalFrames: number;
abortSignal: AbortSignal | undefined;
}): <T>(work: Promise<T>, stalled: StalledSession) => Promise<T> {
const initStallMs = Math.max(SESSION_INIT_STALL_MS, resolveCaptureStallTimeoutMs());
const setup = createStallClock();
return (work, stalled) =>
raceAgainstStall(
work,
initStallMs - setup.idle(0),
{
session: stalled,
frameIndex: input.frameIndex,
totalFrames: input.totalFrames,
stallTimeoutMs: initStallMs,
},
input.abortSignal,
);
}
/**
* Pre-built ffmpeg streaming-encoder options, exactly matching the
* second argument to `spawnStreamingEncoder`. The sequencer constructs
@@ -1037,15 +1062,11 @@ export async function runCaptureStreamingStage(
let resumes = 0;
for (;;) {
const videoInjector = createRenderVideoFrameInjector();
const initStallMs = Math.max(SESSION_INIT_STALL_MS, resolveCaptureStallTimeoutMs());
const setup = createStallClock();
const withinInitBudget = <T>(work: Promise<T>, stalled: StalledSession) =>
raceAgainstStall(
work,
initStallMs - setup.idle(0),
{ session: stalled, frameIndex: resume.from, totalFrames, stallTimeoutMs: initStallMs },
abortSignal,
);
const withinInitBudget = createSessionInitBudget({
frameIndex: resume.from,
totalFrames,
abortSignal,
});
const openSession = async (): Promise<CaptureSession> => {
const opening = createCaptureSession(
fileServer.url,