mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
* fix(engine): render work is sized by the CPUs the process may use * fix(engine): keep the usable CPU count comment under the line limit
669 lines
25 KiB
TypeScript
669 lines
25 KiB
TypeScript
import { availableParallelism, totalmem } from "os";
|
|
import { getHeapStatistics } from "v8";
|
|
import { afterEach, describe, it, expect, vi } from "vitest";
|
|
import {
|
|
calculateOptimalWorkers,
|
|
computeWorkerSizing,
|
|
createPoolFailureHandler,
|
|
distributeFrames,
|
|
expectedFramesForTask,
|
|
flagSilentWorkerExits,
|
|
formatWorkerFailure,
|
|
isFfmpegInfrastructureFailure,
|
|
isPoolFatalWorkerFailure,
|
|
selectVerifySampleIndicesForTask,
|
|
selectWorkerDiagnostics,
|
|
shouldDisableBrowserPoolForParallelWorker,
|
|
shouldVerifyWorkerGpu,
|
|
synthesizeSilentWorkerExitError,
|
|
withParallelWorkerDeadline,
|
|
resolveParallelDeVerifySamples,
|
|
type WorkerResult,
|
|
} from "./parallelCoordinator.js";
|
|
import type { EngineConfig } from "../config.js";
|
|
import { CaptureFailure } from "./captureFailure.js";
|
|
|
|
describe("parallel worker phase deadline", () => {
|
|
it("fails a wedged operation with phase and browser diagnostics before the aggregate watchdog", async () => {
|
|
vi.useFakeTimers();
|
|
try {
|
|
const raced = withParallelWorkerDeadline(
|
|
new Promise<void>(() => {}),
|
|
{
|
|
workerId: 2,
|
|
phase: "frame_capture",
|
|
frameIndex: 0,
|
|
browserExecutable: "C:/Chrome/chrome.exe",
|
|
browserVersion: "Chrome/152.0.7977.30",
|
|
canvasDrawElement: true,
|
|
gpuBackend: "d3d11/nvidia",
|
|
},
|
|
30_000,
|
|
);
|
|
const assertion = expect(raced).rejects.toThrow(
|
|
/worker=2.*phase=frame_capture.*frame=0.*Chrome\/152.*CanvasDrawElement=true.*gpu=d3d11\/nvidia/,
|
|
);
|
|
await vi.advanceTimersByTimeAsync(30_000);
|
|
await assertion;
|
|
} finally {
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("distributeFrames", () => {
|
|
it("distributes frames evenly across workers", () => {
|
|
const tasks = distributeFrames(100, 4, "/tmp/work");
|
|
expect(tasks).toHaveLength(4);
|
|
|
|
// First worker: frames 0-24
|
|
expect(tasks[0]?.startFrame).toBe(0);
|
|
expect(tasks[0]?.endFrame).toBe(25);
|
|
|
|
// Last worker: frames 75-99
|
|
expect(tasks[3]?.startFrame).toBe(75);
|
|
expect(tasks[3]?.endFrame).toBe(100);
|
|
});
|
|
|
|
it("handles single worker", () => {
|
|
const tasks = distributeFrames(50, 1, "/tmp/work");
|
|
expect(tasks).toHaveLength(1);
|
|
expect(tasks[0]?.startFrame).toBe(0);
|
|
expect(tasks[0]?.endFrame).toBe(50);
|
|
});
|
|
|
|
it("does not create empty tasks when workers exceed frames", () => {
|
|
const tasks = distributeFrames(3, 10, "/tmp/work");
|
|
// Can't have more tasks than frames
|
|
expect(tasks.length).toBeLessThanOrEqual(3);
|
|
// All frames are covered
|
|
const totalFrames = tasks.reduce((sum, t) => sum + (t.endFrame - t.startFrame), 0);
|
|
expect(totalFrames).toBe(3);
|
|
});
|
|
|
|
it("assigns worker output directories", () => {
|
|
const tasks = distributeFrames(60, 2, "/tmp/my-work");
|
|
expect(tasks[0]?.outputDir).toContain("worker-0");
|
|
expect(tasks[1]?.outputDir).toContain("worker-1");
|
|
});
|
|
|
|
it("assigns sequential worker IDs", () => {
|
|
const tasks = distributeFrames(100, 3, "/tmp/work");
|
|
expect(tasks.map((t) => t.workerId)).toEqual([0, 1, 2]);
|
|
});
|
|
});
|
|
|
|
describe("calculateOptimalWorkers", () => {
|
|
it("lets high-cost auto renders fall back to one worker when CPU budget requires it", () => {
|
|
const workers = calculateOptimalWorkers(180, undefined, {
|
|
concurrency: 6,
|
|
coresPerWorker: 100,
|
|
minParallelFrames: 120,
|
|
largeRenderThreshold: 1000,
|
|
captureCostMultiplier: 4,
|
|
});
|
|
|
|
expect(workers).toBe(1);
|
|
});
|
|
|
|
it("does not apply capture cost to explicit worker requests", () => {
|
|
const workers = calculateOptimalWorkers(180, 4, {
|
|
concurrency: 6,
|
|
coresPerWorker: 100,
|
|
minParallelFrames: 120,
|
|
largeRenderThreshold: 1000,
|
|
captureCostMultiplier: 4,
|
|
});
|
|
|
|
expect(workers).toBe(4);
|
|
});
|
|
|
|
it.each([12, 16])(
|
|
"honors an explicit %i-worker request above the auto safe maximum",
|
|
(requested) => {
|
|
expect(calculateOptimalWorkers(900, requested, { concurrency: "auto" })).toBe(requested);
|
|
},
|
|
);
|
|
|
|
it("caps explicit worker requests at the documented 24-worker ceiling", () => {
|
|
expect(calculateOptimalWorkers(900, 25, { concurrency: "auto" })).toBe(24);
|
|
});
|
|
});
|
|
|
|
vi.mock("os", async (importOriginal) => {
|
|
const os = await importOriginal<typeof import("os")>();
|
|
return {
|
|
...os,
|
|
default: os,
|
|
availableParallelism: vi.fn(os.availableParallelism),
|
|
totalmem: vi.fn(os.totalmem),
|
|
};
|
|
});
|
|
vi.mock("v8", async (importOriginal) => {
|
|
const v8 = await importOriginal<typeof import("v8")>();
|
|
return { ...v8, default: v8, getHeapStatistics: vi.fn(v8.getHeapStatistics) };
|
|
});
|
|
|
|
describe("computeWorkerSizing", () => {
|
|
afterEach(() => {
|
|
vi.mocked(availableParallelism).mockRestore();
|
|
vi.mocked(totalmem).mockRestore();
|
|
vi.mocked(getHeapStatistics).mockRestore();
|
|
});
|
|
|
|
// The field case: a 24GB 14-core Mac, Node's default ~4GB heap, a long 1080x1920 project.
|
|
it("never auto-picks more workers than the V8 heap can feed", () => {
|
|
vi.mocked(availableParallelism).mockReturnValue(14);
|
|
vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
|
|
vi.mocked(getHeapStatistics).mockReturnValue({
|
|
heap_size_limit: 2096 * 1024 ** 2,
|
|
} as ReturnType<typeof getHeapStatistics>);
|
|
const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" });
|
|
expect(sizing.heapBasedWorkers).toBe(3);
|
|
expect(sizing.workers).toBe(3);
|
|
expect(sizing.boundBy).toBe("heap");
|
|
expect(sizing.exceedsHeapAdvisory).toBe(false);
|
|
expect(computeWorkerSizing(1300, 6, { concurrency: "auto" }).workers).toBe(6);
|
|
vi.mocked(getHeapStatistics).mockReturnValue({
|
|
heap_size_limit: 8192 * 1024 ** 2,
|
|
} as ReturnType<typeof getHeapStatistics>);
|
|
expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).workers).toBe(5);
|
|
});
|
|
|
|
it("keeps the desktop app's 4,192 MB heap at the old per-worker figure (#5149)", () => {
|
|
vi.mocked(availableParallelism).mockReturnValue(14);
|
|
vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
|
|
vi.mocked(getHeapStatistics).mockReturnValue({
|
|
heap_size_limit: 4192 * 1024 ** 2,
|
|
} as ReturnType<typeof getHeapStatistics>);
|
|
expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).heapBasedWorkers).toBe(9);
|
|
Object.defineProperty(process.versions, "electron", { value: "44.2.0", configurable: true });
|
|
try {
|
|
const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" });
|
|
expect(sizing.heapBasedWorkers).toBe(4);
|
|
expect(sizing.workers).toBe(4);
|
|
expect(sizing.boundBy).toBe("heap");
|
|
} finally {
|
|
delete (process.versions as Record<string, string | undefined>).electron;
|
|
}
|
|
// A packaged app runs the CLI on its own plain Node, tagged as the desktop client.
|
|
vi.stubEnv("HYPERFRAMES_CLIENT", "desktop/1.4.0/stable");
|
|
try {
|
|
expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).heapBasedWorkers).toBe(
|
|
4,
|
|
);
|
|
} finally {
|
|
vi.unstubAllEnvs();
|
|
}
|
|
});
|
|
|
|
it("sizes Apple-GPU disk capture by its own heap and core budget", () => {
|
|
vi.mocked(availableParallelism).mockReturnValue(14);
|
|
vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
|
|
vi.mocked(getHeapStatistics).mockReturnValue({
|
|
heap_size_limit: 4192 * 1024 ** 2,
|
|
} as ReturnType<typeof getHeapStatistics>);
|
|
const apple = { concurrency: "auto" as const, appleGpuDiskCapture: true };
|
|
const sizing = computeWorkerSizing(1300, undefined, apple);
|
|
expect(sizing.workers).toBe(8);
|
|
expect(sizing.boundBy).toBe("memory");
|
|
expect(sizing.exceedsHeapAdvisory).toBe(false);
|
|
expect(computeWorkerSizing(1300, undefined, { ...apple, coresPerWorker: 4 }).workers).toBe(3);
|
|
});
|
|
|
|
it("matches calculateOptimalWorkers and reports every constraint", () => {
|
|
const config = { concurrency: "auto" as const };
|
|
const sizing = computeWorkerSizing(900, undefined, config);
|
|
expect(sizing.workers).toBe(calculateOptimalWorkers(900, undefined, config));
|
|
expect(sizing.cpuBasedWorkers).toBeGreaterThanOrEqual(1);
|
|
expect(sizing.memoryBasedWorkers).toBeGreaterThanOrEqual(1);
|
|
expect(sizing.heapBasedWorkers).toBeGreaterThanOrEqual(1);
|
|
expect(sizing.frameBasedWorkers).toBe(30); // 900 / MIN_FRAMES_PER_WORKER(30)
|
|
expect(sizing.heapLimitMb).toBeGreaterThan(0);
|
|
expect(sizing.totalMemoryMb).toBeGreaterThan(0);
|
|
expect(sizing.cpuCount).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("labels explicit requests and clamps them like the legacy wrapper", () => {
|
|
const sizing = computeWorkerSizing(900, 25, { concurrency: "auto" });
|
|
expect(sizing.workers).toBe(24);
|
|
expect(sizing.boundBy).toBe("explicit");
|
|
});
|
|
|
|
it("labels tiny renders as too_few_frames", () => {
|
|
const sizing = computeWorkerSizing(30, undefined, { concurrency: "auto" });
|
|
expect(sizing.workers).toBe(1);
|
|
expect(sizing.boundBy).toBe("too_few_frames");
|
|
});
|
|
|
|
it("labels the contention cap when high capture cost binds", () => {
|
|
const sizing = computeWorkerSizing(180, undefined, {
|
|
concurrency: 6,
|
|
coresPerWorker: 100,
|
|
minParallelFrames: 120,
|
|
largeRenderThreshold: 1000,
|
|
captureCostMultiplier: 4,
|
|
});
|
|
expect(sizing.workers).toBe(1);
|
|
expect(sizing.boundBy).toBe("contention");
|
|
});
|
|
|
|
it("flags exceedsHeapAdvisory exactly when workers exceed the heap budget", () => {
|
|
const sizing = computeWorkerSizing(900, 24, { concurrency: "auto" });
|
|
expect(sizing.exceedsHeapAdvisory).toBe(sizing.workers > sizing.heapBasedWorkers);
|
|
});
|
|
});
|
|
|
|
describe("shouldDisableBrowserPoolForParallelWorker", () => {
|
|
// heygen-com/hyperframes#5548: pages sharing one browser share its
|
|
// connections to the file server, so every platform isolates workers.
|
|
it("gives every parallel worker its own browser", () => {
|
|
expect(shouldDisableBrowserPoolForParallelWorker({ parallel: true })).toBe(true);
|
|
});
|
|
|
|
it("keeps the shared pool for a single worker", () => {
|
|
expect(shouldDisableBrowserPoolForParallelWorker({ parallel: false })).toBe(false);
|
|
expect(shouldDisableBrowserPoolForParallelWorker({})).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("worker failure diagnostics", () => {
|
|
it("keeps only actionable worker diagnostics and caps the tail", () => {
|
|
const diagnostics = selectWorkerDiagnostics(
|
|
[
|
|
"[Browser] harmless log",
|
|
"[Browser:WARN] noisy warning",
|
|
"[Browser:REQUESTFAILED] GET https://cdn.example.com/a.mp4 resource=media error=net::ERR_FAILED",
|
|
"[Browser:HTTP404] GET https://cdn.example.com/missing.png resource=image Not Found",
|
|
"[FrameCapture:ERROR] page.goto failed mode=screenshot timeoutMs=60000 elapsedMs=60001 url=http://127.0.0.1:4173/index.html error=timeout",
|
|
],
|
|
2,
|
|
);
|
|
|
|
expect(diagnostics).toEqual([
|
|
"[Browser:HTTP404] GET https://cdn.example.com/missing.png resource=image Not Found",
|
|
"[FrameCapture:ERROR] page.goto failed mode=screenshot timeoutMs=60000 elapsedMs=60001 url=http://127.0.0.1:4173/index.html error=timeout",
|
|
]);
|
|
});
|
|
|
|
it("adds compact diagnostics to the worker failure message", () => {
|
|
expect(
|
|
formatWorkerFailure({
|
|
workerId: 1,
|
|
framesCaptured: 0,
|
|
startFrame: 0,
|
|
endFrame: 30,
|
|
durationMs: 60_100,
|
|
error: "Navigation timeout of 60000 ms exceeded",
|
|
diagnostics: ["[FrameCapture:ERROR] page.goto failed\n mode=screenshot timeoutMs=60000"],
|
|
}),
|
|
).toBe(
|
|
"Worker 1: Navigation timeout of 60000 ms exceeded; diagnostics: [FrameCapture:ERROR] page.goto failed mode=screenshot timeoutMs=60000",
|
|
);
|
|
});
|
|
|
|
// Field signal ts=1784042064: a Windows render hard-exited during video
|
|
// frame extraction without emitting a terminal error string; the parent
|
|
// treated the result as success because `error` was falsy. The synthesized
|
|
// string surfaces the shortfall so downstream telemetry can classify the
|
|
// failure instead of it disappearing silently.
|
|
it("synthesizes a terminal error when a worker exits with no error string", () => {
|
|
const message = formatWorkerFailure({
|
|
workerId: 3,
|
|
framesCaptured: 12,
|
|
startFrame: 0,
|
|
endFrame: 60,
|
|
durationMs: 180_000,
|
|
});
|
|
expect(message).toContain("Worker 3: worker 3 exited without terminal error string");
|
|
expect(message).toContain("framesCaptured=12");
|
|
expect(message).toContain("expected=60");
|
|
expect(message).toContain("range=[0, 60)");
|
|
expect(message).toContain("ts=1784042064");
|
|
expect(message).toContain("--workers=1");
|
|
});
|
|
|
|
it("prefers an explicit error string over the synthesized silent-exit message", () => {
|
|
const message = formatWorkerFailure({
|
|
workerId: 3,
|
|
framesCaptured: 12,
|
|
startFrame: 0,
|
|
endFrame: 60,
|
|
durationMs: 180_000,
|
|
error: "Target closed",
|
|
});
|
|
expect(message).toBe("Worker 3: Target closed");
|
|
expect(message).not.toContain("ts=1784042064");
|
|
expect(message).not.toContain("exited without terminal error string");
|
|
});
|
|
|
|
it("treats an empty error string as a silent exit for the message contract", () => {
|
|
const message = formatWorkerFailure({
|
|
workerId: 3,
|
|
framesCaptured: 12,
|
|
startFrame: 0,
|
|
endFrame: 60,
|
|
durationMs: 180_000,
|
|
error: "",
|
|
});
|
|
expect(message).toContain("exited without terminal error string");
|
|
expect(message).toContain("ts=1784042064");
|
|
});
|
|
});
|
|
|
|
describe("expectedFramesForTask", () => {
|
|
it("returns the range length for contiguous tasks", () => {
|
|
expect(expectedFramesForTask({ startFrame: 0, endFrame: 30 })).toBe(30);
|
|
expect(expectedFramesForTask({ startFrame: 10, endFrame: 25 })).toBe(15);
|
|
});
|
|
|
|
it("divides by stride for interleaved tasks", () => {
|
|
// Worker 0 in a 3-way interleave over 30 frames captures 0, 3, 6, ..., 27 → 10 frames.
|
|
expect(expectedFramesForTask({ startFrame: 0, endFrame: 30, frameStride: 3 })).toBe(10);
|
|
});
|
|
|
|
it("rounds up so the last-stride-remainder frame is expected", () => {
|
|
// Worker 0 in a 3-way interleave over 31 frames captures 0, 3, ..., 30 → 11 frames.
|
|
expect(expectedFramesForTask({ startFrame: 0, endFrame: 31, frameStride: 3 })).toBe(11);
|
|
});
|
|
|
|
it("returns zero for empty ranges", () => {
|
|
expect(expectedFramesForTask({ startFrame: 10, endFrame: 10 })).toBe(0);
|
|
expect(expectedFramesForTask({ startFrame: 30, endFrame: 10 })).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe("synthesizeSilentWorkerExitError", () => {
|
|
it("names the field signal and reruns hint so operators can classify the failure", () => {
|
|
const message = synthesizeSilentWorkerExitError(
|
|
{ workerId: 7, framesCaptured: 40, startFrame: 60, endFrame: 120 },
|
|
60,
|
|
);
|
|
expect(message).toContain("worker 7 exited without terminal error string");
|
|
expect(message).toContain("framesCaptured=40");
|
|
expect(message).toContain("expected=60");
|
|
expect(message).toContain("range=[60, 120)");
|
|
expect(message).toContain("Field signal ts=1784042064");
|
|
expect(message).toContain("--workers=1");
|
|
});
|
|
});
|
|
|
|
describe("flagSilentWorkerExits", () => {
|
|
it("does not flag a fully-successful interleaved worker (regression: PRINFRA-300)", () => {
|
|
// 3-way interleave over 150 frames: worker 0 captures 0, 3, ..., 147 → 50
|
|
// frames, which is its FULL expected count at stride=3. Without
|
|
// `frameStride` on the result, expectedFramesForTask would fall back to
|
|
// stride=1 (150) and false-positive this as a silent death.
|
|
const results: WorkerResult[] = [
|
|
{
|
|
workerId: 0,
|
|
framesCaptured: 50,
|
|
startFrame: 0,
|
|
endFrame: 150,
|
|
frameStride: 3,
|
|
durationMs: 1000,
|
|
},
|
|
];
|
|
flagSilentWorkerExits(results);
|
|
expect(results[0]?.error).toBeUndefined();
|
|
});
|
|
|
|
it("still flags a genuinely under-captured interleaved worker", () => {
|
|
const results: WorkerResult[] = [
|
|
{
|
|
workerId: 1,
|
|
framesCaptured: 20, // expected 50 at stride=3
|
|
startFrame: 1,
|
|
endFrame: 150,
|
|
frameStride: 3,
|
|
durationMs: 1000,
|
|
},
|
|
];
|
|
flagSilentWorkerExits(results);
|
|
expect(results[0]?.error).toContain("worker 1 exited without terminal error string");
|
|
expect(results[0]?.error).toContain("expected=50");
|
|
});
|
|
|
|
it("does not overwrite an existing error", () => {
|
|
const results: WorkerResult[] = [
|
|
{
|
|
workerId: 2,
|
|
framesCaptured: 0,
|
|
startFrame: 0,
|
|
endFrame: 10,
|
|
durationMs: 1000,
|
|
error: "Protocol error (Page.captureScreenshot): Target closed",
|
|
},
|
|
];
|
|
flagSilentWorkerExits(results);
|
|
expect(results[0]?.error).toBe("Protocol error (Page.captureScreenshot): Target closed");
|
|
});
|
|
});
|
|
|
|
describe("selectVerifySampleIndicesForTask", () => {
|
|
it("keeps only samples inside the task's contiguous range, sorted", () => {
|
|
// 2-worker split of 3032 frames: worker 1 owns [1516, 3032).
|
|
expect(
|
|
selectVerifySampleIndicesForTask([2274, 758, 1516, 3031, 3032], {
|
|
startFrame: 1516,
|
|
endFrame: 3032,
|
|
}),
|
|
).toEqual([1516, 2274, 3031]);
|
|
});
|
|
|
|
it("respects the stride lattice for interleaved tasks", () => {
|
|
// Worker 1 of a 3-way interleave over [1, 30): captures 1, 4, 7, ...
|
|
expect(
|
|
selectVerifySampleIndicesForTask([1, 2, 4, 6, 7, 28, 29], {
|
|
startFrame: 1,
|
|
endFrame: 30,
|
|
frameStride: 3,
|
|
}),
|
|
).toEqual([1, 4, 7, 28]);
|
|
});
|
|
|
|
it("returns empty when no samples fall in the range", () => {
|
|
expect(
|
|
selectVerifySampleIndicesForTask([0, 10, 20], { startFrame: 100, endFrame: 200 }),
|
|
).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe("shouldVerifyWorkerGpu", () => {
|
|
const softwareConfig: Partial<EngineConfig> = { browserGpuMode: "software" };
|
|
|
|
it("returns true for worker 0 when GPU mode is software", () => {
|
|
expect(shouldVerifyWorkerGpu(0, softwareConfig)).toBe(true);
|
|
});
|
|
|
|
it("returns false for non-zero workers when GPU mode is software", () => {
|
|
expect(shouldVerifyWorkerGpu(1, softwareConfig)).toBe(false);
|
|
expect(shouldVerifyWorkerGpu(5, softwareConfig)).toBe(false);
|
|
expect(shouldVerifyWorkerGpu(17, softwareConfig)).toBe(false);
|
|
});
|
|
|
|
it("returns false for any worker when GPU mode is not software", () => {
|
|
expect(shouldVerifyWorkerGpu(0, { browserGpuMode: "hardware" } as Partial<EngineConfig>)).toBe(
|
|
false,
|
|
);
|
|
expect(shouldVerifyWorkerGpu(0, {})).toBe(false);
|
|
});
|
|
|
|
it("returns false when config is undefined", () => {
|
|
expect(shouldVerifyWorkerGpu(0, undefined)).toBe(false);
|
|
expect(shouldVerifyWorkerGpu(3, undefined)).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("resolveParallelDeVerifySamples", () => {
|
|
it("densifies with worker count: 4 base + 2 per extra worker", () => {
|
|
expect(resolveParallelDeVerifySamples(undefined, 2)).toBe(6);
|
|
expect(resolveParallelDeVerifySamples(undefined, 3)).toBe(8);
|
|
});
|
|
|
|
it("clamps at the verify path's max of 8", () => {
|
|
expect(resolveParallelDeVerifySamples(undefined, 5)).toBe(8);
|
|
expect(resolveParallelDeVerifySamples(undefined, 16)).toBe(8);
|
|
});
|
|
|
|
it("leaves single-worker capture on the session default", () => {
|
|
expect(resolveParallelDeVerifySamples(undefined, 1)).toBeUndefined();
|
|
expect(resolveParallelDeVerifySamples(undefined, 0)).toBeUndefined();
|
|
});
|
|
|
|
it("passes a caller-set value through untouched", () => {
|
|
expect(resolveParallelDeVerifySamples(2, 3)).toBe(2);
|
|
});
|
|
});
|
|
|
|
describe("isFfmpegInfrastructureFailure", () => {
|
|
it("matches an execFile ENOENT (missing ffmpeg binary)", () => {
|
|
const err = Object.assign(new Error("spawn ffmpeg ENOENT"), { code: "ENOENT" });
|
|
expect(isFfmpegInfrastructureFailure(err)).toBe(true);
|
|
});
|
|
|
|
it('matches ffmpeg\'s "No such filter" stderr (libpostproc-less build)', () => {
|
|
const err = Object.assign(new Error("Command failed"), {
|
|
stderr:
|
|
"Error initializing filter 'psnr' with args ''\n" +
|
|
" No such filter: 'psnr'\n" +
|
|
"Error opening filters!",
|
|
});
|
|
expect(isFfmpegInfrastructureFailure(err)).toBe(true);
|
|
});
|
|
|
|
it("matches the older `Unknown filter 'psnr'` wording (ffmpeg <=5)", () => {
|
|
const err = Object.assign(new Error("Command failed"), {
|
|
stderr: "Unknown filter 'psnr'",
|
|
});
|
|
expect(isFfmpegInfrastructureFailure(err)).toBe(true);
|
|
});
|
|
|
|
it("does not match per-sample noise (readFile race, transient EPERM)", () => {
|
|
const eperm = Object.assign(new Error("EACCES: permission denied, open '/tmp/…'"), {
|
|
code: "EACCES",
|
|
});
|
|
expect(isFfmpegInfrastructureFailure(eperm)).toBe(false);
|
|
|
|
const parseErr = new Error("psnr parse failed: average=<truncated>");
|
|
expect(isFfmpegInfrastructureFailure(parseErr)).toBe(false);
|
|
|
|
const enoentFile = Object.assign(
|
|
new Error("ENOENT: no such file or directory, open '/tmp/frame_000042.jpg'"),
|
|
{
|
|
code: "ENOENT",
|
|
},
|
|
);
|
|
// ⚠ known aliasing edge: an execFile ENOENT and an fs ENOENT reading the
|
|
// sample frame share the same code. The discriminator errs toward the
|
|
// infrastructure classification — a spurious per-sample fs ENOENT
|
|
// (impossible for a frame that was just written by the worker before this
|
|
// verify call) would abort the render, which is acceptable given how
|
|
// rarely that shape appears vs. how important the infra-fail signal is.
|
|
// Documented here so a future maintainer sees why the assertion below
|
|
// reads "true": this is the deliberate false-positive on collision.
|
|
expect(isFfmpegInfrastructureFailure(enoentFile)).toBe(true);
|
|
});
|
|
|
|
it("returns false for null / non-object errors", () => {
|
|
expect(isFfmpegInfrastructureFailure(null)).toBe(false);
|
|
expect(isFfmpegInfrastructureFailure(undefined)).toBe(false);
|
|
expect(isFfmpegInfrastructureFailure("psnr broken")).toBe(false);
|
|
expect(isFfmpegInfrastructureFailure(42)).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("isPoolFatalWorkerFailure", () => {
|
|
const failure = (kind: CaptureFailure["kind"]) => new CaptureFailure({ kind, message: kind });
|
|
|
|
it("keeps the disk path's tolerance: transient deaths are retried per worker, not pool-fatal", () => {
|
|
expect(isPoolFatalWorkerFailure(failure("transient_browser"), false)).toBe(false);
|
|
expect(isPoolFatalWorkerFailure(failure("protocol_timeout"), false)).toBe(false);
|
|
expect(isPoolFatalWorkerFailure(failure("cancelled"), false)).toBe(false);
|
|
expect(isPoolFatalWorkerFailure(failure("authoring"), false)).toBe(true);
|
|
expect(isPoolFatalWorkerFailure(failure("io"), false)).toBe(true);
|
|
});
|
|
|
|
it("makes every non-cancelled failure pool-fatal on the streaming path", () => {
|
|
// No per-worker retry exists there; the dead worker's frames are gone and
|
|
// its peers would otherwise park until the watchdog relabels the death.
|
|
expect(isPoolFatalWorkerFailure(failure("transient_browser"), true)).toBe(true);
|
|
expect(isPoolFatalWorkerFailure(failure("protocol_timeout"), true)).toBe(true);
|
|
expect(isPoolFatalWorkerFailure(failure("authoring"), true)).toBe(true);
|
|
// Cancelled means the pool is already aborting: nothing to propagate.
|
|
expect(isPoolFatalWorkerFailure(failure("cancelled"), true)).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("createPoolFailureHandler", () => {
|
|
const fatal = () => new CaptureFailure({ kind: "transient_browser", message: "Target closed" });
|
|
|
|
it("delivers the original failure to the hook before the peers are aborted", () => {
|
|
const peerController = new AbortController();
|
|
const seen: Array<{ failure: CaptureFailure; abortedYet: boolean }> = [];
|
|
const handler = createPoolFailureHandler({
|
|
streaming: true,
|
|
peerController,
|
|
hooks: {
|
|
onWorkerFailure: (failure) =>
|
|
seen.push({ failure, abortedYet: peerController.signal.aborted }),
|
|
},
|
|
});
|
|
const failure = fatal();
|
|
handler.onFailure(failure);
|
|
expect(seen).toHaveLength(1);
|
|
expect(seen[0]?.failure).toBe(failure);
|
|
expect(seen[0]?.abortedYet).toBe(false);
|
|
expect(peerController.signal.aborted).toBe(true);
|
|
expect(peerController.signal.reason).toBe(failure);
|
|
expect(handler.firstFatalFailure()).toBe(failure);
|
|
});
|
|
|
|
it("still aborts the peers when the hook throws, and keeps the classified failure", () => {
|
|
const peerController = new AbortController();
|
|
const handler = createPoolFailureHandler({
|
|
streaming: true,
|
|
peerController,
|
|
hooks: {
|
|
onWorkerFailure: () => {
|
|
throw new Error("hook exploded");
|
|
},
|
|
},
|
|
});
|
|
const failure = fatal();
|
|
expect(() => handler.onFailure(failure)).not.toThrow();
|
|
expect(peerController.signal.aborted).toBe(true);
|
|
expect(peerController.signal.reason).toBe(failure);
|
|
expect(handler.firstFatalFailure()).toBe(failure);
|
|
});
|
|
|
|
it("ignores every failure after the first, which owns the abort reason", () => {
|
|
const peerController = new AbortController();
|
|
const hook = vi.fn();
|
|
const handler = createPoolFailureHandler({
|
|
streaming: true,
|
|
peerController,
|
|
hooks: { onWorkerFailure: hook },
|
|
});
|
|
const first = fatal();
|
|
handler.onFailure(first);
|
|
handler.onFailure(fatal());
|
|
expect(hook).toHaveBeenCalledTimes(1);
|
|
expect(peerController.signal.reason).toBe(first);
|
|
});
|
|
|
|
it("leaves the peers running for a cancellation on the streaming path", () => {
|
|
const peerController = new AbortController();
|
|
const hook = vi.fn();
|
|
const handler = createPoolFailureHandler({
|
|
streaming: true,
|
|
peerController,
|
|
hooks: { onWorkerFailure: hook },
|
|
});
|
|
handler.onFailure(new CaptureFailure({ kind: "cancelled", message: "aborted" }));
|
|
expect(hook).not.toHaveBeenCalled();
|
|
expect(peerController.signal.aborted).toBe(false);
|
|
expect(handler.firstFatalFailure()).toBeUndefined();
|
|
});
|
|
});
|