mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(producer): segmented capture gives up on a session stuck opening or initializing (#7329)
* fix(producer): segmented capture gives up on a session stuck opening or initializing * fix(producer): sequential disk capture bounds the opt-in drawElement setup the same way
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { afterAll, describe, expect, it, mock } from "bun:test";
|
||||
import { afterAll, describe, expect, it, mock, vi } from "bun:test";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
@@ -26,6 +26,9 @@ let sessionWorkerEncodeEnabled = false;
|
||||
let captureSessionMode: "drawelement" | "screenshot" = "drawelement";
|
||||
let failPrepareCaptureSessionForReuse = false;
|
||||
let initializeSessionErrorMessage = "initialize failed";
|
||||
let hangDeferredDrawElementInit = false;
|
||||
/** While set, opening a session waits for this promise instead of returning a fresh one. */
|
||||
let pendingOpen: Promise<unknown> | null = null;
|
||||
const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"];
|
||||
const closeCaptureSession = mock(async () => {});
|
||||
const createdSessionConfigs: Array<{ enableBrowserPool?: boolean }> = [];
|
||||
@@ -62,10 +65,12 @@ mock.module("@hyperframes/engine", () => ({
|
||||
return { buffer: Buffer.from("frame"), captureTimeMs: 1 };
|
||||
},
|
||||
closeCaptureSession,
|
||||
completeDeferredDrawElementInit: async () => {},
|
||||
completeDeferredDrawElementInit: () =>
|
||||
hangDeferredDrawElementInit ? new Promise<void>(() => {}) : Promise.resolve(),
|
||||
createCaptureSession: async (...args: unknown[]) => {
|
||||
createdSessionConfigs.push(args[4] as { enableBrowserPool?: boolean });
|
||||
createdSessionHooks.push(args[3]);
|
||||
if (pendingOpen) return pendingOpen;
|
||||
return {
|
||||
isInitialized: false,
|
||||
browserConsoleBuffer,
|
||||
@@ -451,6 +456,81 @@ describe("runCaptureSegmentedStage", () => {
|
||||
expect(result).toEqual({ success: false });
|
||||
});
|
||||
|
||||
/** Runs the stage on a fake clock, a second at a time, until it settles or 30 minutes pass. */
|
||||
const settleOnFakeClock = async (run: Promise<unknown>) => {
|
||||
const state = { done: false, error: undefined as unknown, elapsedMs: 0 };
|
||||
run.then(
|
||||
() => (state.done = true),
|
||||
(error: unknown) => Object.assign(state, { done: true, error }),
|
||||
);
|
||||
const flush = async () => {
|
||||
for (let i = 0; i < 200; i++) await Promise.resolve();
|
||||
};
|
||||
await flush();
|
||||
while (!state.done && state.elapsedMs < 30 * 60_000) {
|
||||
vi.advanceTimersByTime(1_000);
|
||||
state.elapsedMs += 1_000;
|
||||
await flush();
|
||||
}
|
||||
return state;
|
||||
};
|
||||
|
||||
it("gives up on a deferred drawElement init that never finishes, within the session-init budget", async () => {
|
||||
hangDeferredDrawElementInit = true;
|
||||
closeCaptureSession.mockClear();
|
||||
const prev = process.env.HF_DE_STALL_MS;
|
||||
delete process.env.HF_DE_STALL_MS;
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const input = fakeStageInput({ totalFrames: 3 });
|
||||
const state = await settleOnFakeClock(
|
||||
runCaptureSegmentedStage({ ...input, segmentFrames: 3 }),
|
||||
);
|
||||
expect(state.done).toBe(true);
|
||||
expect(state.elapsedMs).toBeLessThanOrEqual(305_000);
|
||||
expect((state.error as Error).message).toContain(
|
||||
"no frame progress for 300000ms (stuck at frame 0/3)",
|
||||
);
|
||||
expect(closeCaptureSession.mock.calls.map((c) => c[0])).toContain(input.probeSession);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
hangDeferredDrawElementInit = false;
|
||||
if (prev !== undefined) process.env.HF_DE_STALL_MS = prev;
|
||||
}
|
||||
});
|
||||
|
||||
it("gives up on a session that never finishes opening, and closes it when it opens late", async () => {
|
||||
let openLate: (session: unknown) => void = () => {};
|
||||
pendingOpen = new Promise((resolve) => (openLate = resolve));
|
||||
closeCaptureSession.mockClear();
|
||||
const prev = process.env.HF_DE_STALL_MS;
|
||||
delete process.env.HF_DE_STALL_MS;
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const state = await settleOnFakeClock(
|
||||
runCaptureSegmentedStage({
|
||||
...fakeStageInput({ totalFrames: 3 }),
|
||||
probeSession: undefined,
|
||||
segmentFrames: 3,
|
||||
}),
|
||||
);
|
||||
expect(state.done).toBe(true);
|
||||
expect(state.elapsedMs).toBeLessThanOrEqual(305_000);
|
||||
expect((state.error as Error).message).toContain(
|
||||
"no frame progress for 300000ms (stuck at frame 0/3 while opening the capture session)",
|
||||
);
|
||||
|
||||
const late = { browserConsoleBuffer: [] };
|
||||
openLate(late);
|
||||
for (let i = 0; i < 20; i++) await Promise.resolve();
|
||||
expect(closeCaptureSession).toHaveBeenCalledWith(late);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
pendingOpen = null;
|
||||
if (prev !== undefined) process.env.HF_DE_STALL_MS = prev;
|
||||
}
|
||||
});
|
||||
|
||||
it("returns success:false when the first encoder cannot spawn, and has closed the probe", async () => {
|
||||
const spawnEncoder = mock(async () => {
|
||||
throw new Error("ffmpeg missing");
|
||||
|
||||
@@ -47,6 +47,8 @@ import type { SegmentVideoFrames, SegmentWorkerFrames } from "../segmentVideoFra
|
||||
import { createSegmentQueue } from "../segmentQueue.js";
|
||||
import { isTargetLossError } from "../segmentRecycle.js";
|
||||
import {
|
||||
OPENING_CAPTURE_SESSION,
|
||||
createSessionInitBudget,
|
||||
raceAgainstStall,
|
||||
resolveCaptureStallTimeoutMs,
|
||||
verifyDrawElementSample,
|
||||
@@ -337,14 +339,28 @@ function defaultSessionFactory(input: CaptureSegmentedStageInput): SessionFactor
|
||||
await closeCaptureSession(probeSession).catch(() => {});
|
||||
probeSession = null;
|
||||
}
|
||||
const session = probeSession ?? (await openSegmentedSession(input, injector));
|
||||
const withinInitBudget = createSessionInitBudget({
|
||||
frameIndex: input.job.framesRendered ?? 0,
|
||||
totalFrames: input.totalFrames,
|
||||
abortSignal: input.abortSignal,
|
||||
});
|
||||
const openSession = async (): Promise<CaptureSession> => {
|
||||
const opening = openSegmentedSession(input, injector);
|
||||
try {
|
||||
return await withinInitBudget(opening, OPENING_CAPTURE_SESSION);
|
||||
} catch (error) {
|
||||
void opening.then(closeCaptureSession, () => {});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const session = probeSession ?? (await openSession());
|
||||
if (probeSession) {
|
||||
openSegmentedSessionReuse(input, probeSession, injector);
|
||||
probeSession = null;
|
||||
}
|
||||
try {
|
||||
if (!session.isInitialized) await initializeSession(session);
|
||||
await completeDeferredDrawElementInit(session);
|
||||
if (!session.isInitialized) await withinInitBudget(initializeSession(session), session);
|
||||
await withinInitBudget(completeDeferredDrawElementInit(session), session);
|
||||
} catch (error) {
|
||||
await closeCaptureSession(session).catch(() => {});
|
||||
throw error;
|
||||
|
||||
@@ -15,6 +15,7 @@ const initializeSession = vi.fn(async () => {});
|
||||
const captureFrame = vi.fn();
|
||||
const closeCaptureSession = vi.fn(async () => {});
|
||||
const verifyDiskDrawElementSamples = vi.fn(async () => {});
|
||||
const completeDeferredDrawElementInit = vi.fn(async () => {});
|
||||
const getCapturePerfSummary = vi.fn(() => ({}));
|
||||
const executeParallelCapture = vi.fn();
|
||||
|
||||
@@ -27,6 +28,8 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
|
||||
captureFrame: (...args: unknown[]) => captureFrame(...args),
|
||||
closeCaptureSession: (...args: unknown[]) => closeCaptureSession(...args),
|
||||
verifyDiskDrawElementSamples: (...args: unknown[]) => verifyDiskDrawElementSamples(...args),
|
||||
completeDeferredDrawElementInit: (...args: unknown[]) =>
|
||||
completeDeferredDrawElementInit(...args),
|
||||
getCapturePerfSummary: (...args: unknown[]) => getCapturePerfSummary(...args),
|
||||
executeParallelCapture: (...args: unknown[]) => executeParallelCapture(...args),
|
||||
mergeWorkerFrames: async () => {},
|
||||
@@ -265,6 +268,52 @@ describe("runCaptureStage measured disk projection (both branches)", () => {
|
||||
closeCaptureSession.mockReset().mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
it("sequential: gives up on a deferred drawElement init that never finishes, within the session-init budget", async () => {
|
||||
const { workDir, framesDir } = makeDirs();
|
||||
freeBytes.mockReturnValue({ bavail: 10_000_000_000, bsize: 1 });
|
||||
completeDeferredDrawElementInit.mockImplementation(() => new Promise<void>(() => {}));
|
||||
const probeSession = {
|
||||
isInitialized: true,
|
||||
browserConsoleBuffer: [],
|
||||
options: { captureBeyondViewport: false, format: "jpeg" },
|
||||
workerEncodeEnabled: false,
|
||||
outputDir: framesDir,
|
||||
};
|
||||
const prevOptIn = process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
|
||||
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = "true";
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const state = { done: false, error: undefined as unknown, elapsedMs: 0 };
|
||||
runCaptureStage({
|
||||
...makeInput(1, workDir, framesDir),
|
||||
probeSession: probeSession as never,
|
||||
}).then(
|
||||
() => (state.done = true),
|
||||
(error: unknown) => Object.assign(state, { done: true, error }),
|
||||
);
|
||||
const flush = async () => {
|
||||
for (let i = 0; i < 200; i++) await Promise.resolve();
|
||||
};
|
||||
await flush();
|
||||
while (!state.done && state.elapsedMs < 30 * 60_000) {
|
||||
vi.advanceTimersByTime(1_000);
|
||||
state.elapsedMs += 1_000;
|
||||
await flush();
|
||||
}
|
||||
expect(state.done).toBe(true);
|
||||
expect(state.elapsedMs).toBeLessThanOrEqual(305_000);
|
||||
expect((state.error as Error).message).toContain(
|
||||
"no frame progress for 300000ms (stuck at frame 0/20)",
|
||||
);
|
||||
expect(closeCaptureSession).toHaveBeenCalledWith(probeSession);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
completeDeferredDrawElementInit.mockReset().mockResolvedValue(undefined);
|
||||
if (prevOptIn === undefined) delete process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
|
||||
else process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = prevOptIn;
|
||||
}
|
||||
});
|
||||
|
||||
it("sequential: stops after ten frames when the measured remainder cannot fit", async () => {
|
||||
const { workDir, framesDir } = makeDirs();
|
||||
// 8 MB free: passes the static jpeg estimate, fails the 10 MB measured remainder.
|
||||
|
||||
@@ -71,6 +71,7 @@ import { reportFrameProgress, reportWorkerStartup } from "../shared.js";
|
||||
import { withWorkerPageErrors } from "../perfSummary.js";
|
||||
import type { SdrDiskCapturePlan } from "../capturePlan.js";
|
||||
import type { OpenStallAttempt } from "../observability.js";
|
||||
import { createSessionInitBudget } from "./captureStreamingStage.js";
|
||||
|
||||
export interface CaptureStageInput {
|
||||
fileServer: FileServerHandle;
|
||||
@@ -501,7 +502,11 @@ async function runSequentialDiskCapture(
|
||||
prepareCaptureSessionForReuse(session, framesDir, videoInjector);
|
||||
probeSession = null;
|
||||
}
|
||||
await ensureSessionReady(session);
|
||||
await ensureSessionReady(session, {
|
||||
frameIndex: resumeFrom,
|
||||
totalFrames,
|
||||
abortSignal: input.abortSignal,
|
||||
});
|
||||
assertAttemptActive();
|
||||
lastBrowserConsole = session.browserConsoleBuffer;
|
||||
|
||||
@@ -576,7 +581,10 @@ function planTransientResume(
|
||||
: null;
|
||||
}
|
||||
|
||||
async function ensureSessionReady(session: CaptureSession): Promise<void> {
|
||||
async function ensureSessionReady(
|
||||
session: CaptureSession,
|
||||
budget: Parameters<typeof createSessionInitBudget>[0],
|
||||
): Promise<void> {
|
||||
if (!session.isInitialized) {
|
||||
await initializeSession(session);
|
||||
return;
|
||||
@@ -584,7 +592,7 @@ async function ensureSessionReady(session: CaptureSession): Promise<void> {
|
||||
// Deferred drawElement init (probe-initialized video comps). The disk path has no
|
||||
// drain-time self-verification, so only the explicit fast-capture opt-in completes it.
|
||||
if (process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE === "true") {
|
||||
await completeDeferredDrawElementInit(session);
|
||||
await createSessionInitBudget(budget)(completeDeferredDrawElementInit(session), session);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -189,9 +189,9 @@ function captureModeLabel(mode: CaptureSession["captureMode"]): string {
|
||||
return "screenshot";
|
||||
}
|
||||
|
||||
type StalledSession = { captureMode: CaptureSession["captureMode"]; captureStep?: string };
|
||||
export type StalledSession = { captureMode: CaptureSession["captureMode"]; captureStep?: string };
|
||||
|
||||
const OPENING_CAPTURE_SESSION: StalledSession = {
|
||||
export const OPENING_CAPTURE_SESSION: StalledSession = {
|
||||
captureMode: "screenshot",
|
||||
captureStep: "opening the capture session",
|
||||
};
|
||||
@@ -271,6 +271,31 @@ export function raceAgainstStall<T>(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* One budget for everything before a session's first frame: opening it, initializing it, and the
|
||||
* deferred drawElement init. Each call races against what is left of the same clock.
|
||||
*/
|
||||
export function createSessionInitBudget(input: {
|
||||
frameIndex: number;
|
||||
totalFrames: number;
|
||||
abortSignal: AbortSignal | undefined;
|
||||
}): <T>(work: Promise<T>, stalled: StalledSession) => Promise<T> {
|
||||
const initStallMs = Math.max(SESSION_INIT_STALL_MS, resolveCaptureStallTimeoutMs());
|
||||
const setup = createStallClock();
|
||||
return (work, stalled) =>
|
||||
raceAgainstStall(
|
||||
work,
|
||||
initStallMs - setup.idle(0),
|
||||
{
|
||||
session: stalled,
|
||||
frameIndex: input.frameIndex,
|
||||
totalFrames: input.totalFrames,
|
||||
stallTimeoutMs: initStallMs,
|
||||
},
|
||||
input.abortSignal,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-built ffmpeg streaming-encoder options, exactly matching the
|
||||
* second argument to `spawnStreamingEncoder`. The sequencer constructs
|
||||
@@ -1037,15 +1062,11 @@ export async function runCaptureStreamingStage(
|
||||
let resumes = 0;
|
||||
for (;;) {
|
||||
const videoInjector = createRenderVideoFrameInjector();
|
||||
const initStallMs = Math.max(SESSION_INIT_STALL_MS, resolveCaptureStallTimeoutMs());
|
||||
const setup = createStallClock();
|
||||
const withinInitBudget = <T>(work: Promise<T>, stalled: StalledSession) =>
|
||||
raceAgainstStall(
|
||||
work,
|
||||
initStallMs - setup.idle(0),
|
||||
{ session: stalled, frameIndex: resume.from, totalFrames, stallTimeoutMs: initStallMs },
|
||||
abortSignal,
|
||||
);
|
||||
const withinInitBudget = createSessionInitBudget({
|
||||
frameIndex: resume.from,
|
||||
totalFrames,
|
||||
abortSignal,
|
||||
});
|
||||
const openSession = async (): Promise<CaptureSession> => {
|
||||
const opening = createCaptureSession(
|
||||
fileServer.url,
|
||||
|
||||
Reference in New Issue
Block a user