Files
hyperframes/packages/engine/src/services/parallelCoordinator.ts
T
Miguel Ángel 5073100b7d fix(producer): long, mostly still films reach their first frame before the stall check stops them (#7521)
* perf(engine): check still frames once per page and share the result with later sessions

* perf(producer): the probe skips the still-frame check unless its session captures

* chore(engine): keep the comment share of the deferred still-frame check files at their base

* fix(producer): the probe session's deferred still-frame check counts as stall-watcher activity

* docs(engine): the still-frame spot-check range is the task's span, not only its own frames

* fix(engine): a reused fast-capture probe session keeps still-frame reuse
2026-11-11 05:40:33 +00:00

1306 lines
50 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Parallel Coordinator Service
*
* Coordinates parallel frame capture across multiple Puppeteer sessions.
* Auto-detects optimal worker count based on CPU/memory.
*/
import { cpus, freemem } from "os";
import { existsSync, mkdirSync, readdirSync } from "fs";
import { copyFile, readFile } from "fs/promises";
import { renameWithRetryAsync } from "@hyperframes/core/rename-retry";
import { join } from "path";
import { getHeapStatistics } from "v8";
import {
createCaptureSession,
initializeSession,
closeCaptureSession,
captureFrame,
captureFrameToBufferPipelined,
captureFrameToBuffer,
getCapturePerfSummary,
DrawElementVerificationError,
type CaptureSession,
type CaptureOptions,
type CapturePerfSummary,
type BeforeCaptureHook,
} from "./frameCapture.js";
import { psnrDb, resolveDeVerifyMinDb } from "../utils/psnr.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { assertSwiftShader } from "../utils/assertSwiftShader.js";
import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js";
import { resolveHeadlessShellPath } from "./browserManager.js";
import { getCgroupCpuLimit, getSystemTotalMb } from "./systemMemory.js";
import {
CaptureFailure,
classifyCaptureFailure,
isFatalCaptureFailure,
type CaptureWorkerDiagnostic,
} from "./captureFailure.js";
export interface WorkerTask {
workerId: number;
startFrame: number;
endFrame: number;
outputDir: string;
/**
* Offset subtracted from the absolute frame index when naming the captured
* file (`frame_<i - outputFrameOffset>.{ext}`). Default 0. Distributed
* chunks set this to the chunk's absolute startFrame so file names land
* 0-indexed within the chunk's range — the encoder reads frames
* sequentially without an `-start_number` override. The per-frame TIME
* calculation still uses the absolute frame index.
*/
outputFrameOffset?: number;
/**
* Frame stride for interleaved distribution (HF_DE_PARALLEL_STREAM spike):
* the worker captures startFrame, startFrame+stride, … < endFrame. Default 1
* (contiguous range). Interleaving keeps the ordered streaming writer's
* reorder window at O(workerCount) frames instead of O(totalFrames/N).
*/
frameStride?: number;
}
export interface WorkerResult {
workerId: number;
framesCaptured: number;
startFrame: number;
endFrame: number;
/**
* Mirrors the originating `WorkerTask.frameStride` (default 1). Required by
* `expectedFramesForTask` — without it, every interleaved (stride > 1)
* worker's expected count is computed as the full contiguous range instead
* of range/stride, so a fully-successful worker looks like it under-captured
* and gets misclassified as a silent death (see `synthesizeSilentWorkerExitError`).
*/
frameStride?: number;
durationMs: number;
perf?: CapturePerfSummary;
error?: string;
diagnostics?: string[];
/** The worker session's `[Browser:PAGEERROR]` console lines, kept so the render's error count sees them. */
pageErrorLines?: string[];
failure?: CaptureFailure;
}
export interface ParallelProgress {
totalFrames: number;
capturedFrames: number;
activeWorkers: number;
workerProgress: Map<number, number>;
/** Latest lifecycle transition; absent on ordinary completed-frame updates. */
latestWorkerPhase?: ParallelWorkerPhaseDiagnostic;
/** Disk capture only: the frame file this update finished writing, readable from now on. */
landedFrame?: LandedFrame;
}
/** A captured frame file: `index` is its file index (`frame_<index>`), `path` where it was written. */
export interface LandedFrame {
index: number;
path: string;
}
export type ParallelWorkerPhase =
| "browser_launch"
| "browser_probe"
| "session_init"
| "frame_capture"
| "frame_encode";
export interface ParallelWorkerPhaseDiagnostic {
workerId: number;
phase: ParallelWorkerPhase;
frameIndex?: number;
browserExecutable: string;
browserVersion: string;
canvasDrawElement: boolean | "unknown";
gpuBackend: string;
readStep?: () => string | undefined;
}
export interface WorkerSizingConfig extends Partial<
Pick<
EngineConfig,
"concurrency" | "coresPerWorker" | "minParallelFrames" | "largeRenderThreshold"
>
> {
/**
* Relative per-frame capture cost for auto worker sizing. Values above 1
* represent compositions that put more CPU pressure on each Chrome worker
* than a plain DOM screenshot. Explicit --workers requests ignore this hint.
*/
captureCostMultiplier?: number;
/** macOS, hardware GPU, screenshot frames written to disk (not streamed through this process). */
appleGpuDiskCapture?: boolean;
}
type WorkerBrowserPoolDecision = {
parallel?: boolean;
};
// System-memory budget per parallel worker. Each worker is a full Chrome
// process (SwiftShader compositor + raster threads) plus its share of parent-
// process frame buffering — the old 256MB figure was ~6× under a measured
// Chrome-under-capture RSS and let memory-constrained hosts overcommit (wild
// 16GB black-slab report; 0.7.66 heap-OOM report on 24GB with 6 auto workers).
const MEMORY_PER_WORKER_MB = 1536;
// Parent-process V8 heap the coordinator itself needs regardless of worker
// count (compile artifacts, puppeteer sessions, encoder state).
const HEAP_RESERVED_MB = 1024;
// Parent heap per worker, by --trace-gc in a 10 GB container (10-28): 21 MB on a short film, 228 MB on a 1080×1920
// video-heavy one (427 → 1,110 MB from 1 to 4 workers). 320 keeps 1.4× over that; `--workers N` still overrides.
const HEAP_PER_WORKER_MB = 320;
// The desktop app keeps the old figure: its 4,192 MB heap is where 5 workers ran out (#5149). A packaged app runs the
// CLI on its own plain Node, so it is known by the client tag it sets on every CLI call, or by Electron itself.
const DESKTOP_HEAP_PER_WORKER_MB = 640;
const inDesktopApp = () =>
Boolean(process.versions.electron) || /^desktop\//.test(process.env.HYPERFRAMES_CLIENT ?? "");
// Apple-GPU disk capture measured ~15-25 MB parent heap per worker; 14 cores kept gaining to 8.
const APPLE_DISK_HEAP_PER_WORKER_MB = 128;
const APPLE_DISK_CORES_PER_WORKER = 1.75;
const APPLE_DISK_MAX_WORKERS = 8;
const MIN_WORKERS = 1;
const MAX_WORKER_DIAGNOSTIC_LINES = 8;
// Hard ceiling on explicit `--workers N` requests. Above this, the cost of
// CDP-protocol dispatch through Node's main event loop and OS scheduling
// noise overwhelms any further parallelism. Bumped from 10 → 24 in hf#732
// follow-up so high-core hosts (32-96+ cores) can actually surface the
// hardware to renders that are CPU-bound on DOM capture.
const ABSOLUTE_MAX_WORKERS = 24;
// `auto` concurrency picks this many workers as the upper bound. Bumped
// from a hardcoded 6 → CPU-scaled value (floor(cpuCount/8), floor at 6,
// ceiling at 16) in hf#732 follow-up. Rationale: the prior fixed cap of 6
// left ~90 cores idle on the validation host and forced users to pass
// `--workers N` to opt in. Now `auto` matches what a thoughtful operator
// would pick by hand. The /8 divisor leaves headroom for each Chrome
// worker's SwiftShader compositor + the shader-blend thread pool, both of
// which are themselves CPU-heavy.
function defaultSafeMaxWorkers(): number {
return Math.max(6, Math.min(16, Math.floor(cpus().length / 8)));
}
const MIN_FRAMES_PER_WORKER = 30;
// Every parallel worker gets its own browser: pages in one browser share six
// connections to the file server, which a media-heavy page holds (#5548), and
// sharing also crashes BeginFrame and serializes screenshots.
export function shouldDisableBrowserPoolForParallelWorker({
parallel,
}: WorkerBrowserPoolDecision): boolean {
return Boolean(parallel);
}
export function selectWorkerDiagnostics(
lines: readonly string[],
maxLines: number = MAX_WORKER_DIAGNOSTIC_LINES,
): string[] {
return lines
.filter((line) =>
/\[(FrameCapture:ERROR|Browser:ERROR|Browser:PAGEERROR|Browser:REQUESTFAILED|Browser:HTTP\d{3})\]/.test(
line,
),
)
.slice(-maxLines);
}
function compactDiagnosticLine(line: string): string {
return line.replace(/\s+/g, " ").trim();
}
/**
* Expected frame count for a worker task, honoring its stride. Contiguous
* tasks (stride 1) expect `endFrame - startFrame`; interleaved tasks
* (stride > 1) expect `ceil((endFrame - startFrame) / stride)`, matching
* the loop shape in `captureFrameRange`.
*/
export function expectedFramesForTask(task: {
startFrame: number;
endFrame: number;
frameStride?: number;
}): number {
const stride = task.frameStride ?? 1;
return Math.max(0, Math.ceil((task.endFrame - task.startFrame) / stride));
}
/**
* Synthetic terminal-error message for a worker whose exit didn't produce
* an explicit error string but under-captured its expected frame range.
* Field signal ts=1784042064: a 1292s Windows render hard-exited during
* capture with no final error string, leaving the operator with no
* actionable trace. This message surfaces the shortfall + reruns hint
* so downstream telemetry (and operators grepping logs) can classify the
* failure instead of it disappearing silently.
*/
export function synthesizeSilentWorkerExitError(
result: Pick<WorkerResult, "workerId" | "framesCaptured" | "startFrame" | "endFrame">,
expectedFrames: number,
): string {
return (
`worker ${result.workerId} exited without terminal error string ` +
`(framesCaptured=${result.framesCaptured}, expected=${expectedFrames}, ` +
`range=[${result.startFrame}, ${result.endFrame})). ` +
`Field signal ts=1784042064: this class of failure has been reported; ` +
`consider re-run with --workers=1 to isolate.`
);
}
/**
* A worker may return without an error string yet with `framesCaptured`
* below its task's expected count — the silent-exit shape field signal
* ts=1784042064 called out. Synthesize a terminal error string in-place so
* the caller's failure filter treats it as a failure (and so the caller's
* failure message actually names what went wrong). Requires each result to
* carry `frameStride` (see `WorkerResult`) — without it, an interleaved
* worker's true `framesCaptured` (range/stride) is compared against the full
* contiguous range and every successful interleaved worker false-positives.
*/
export function flagSilentWorkerExits(results: WorkerResult[]): void {
for (const r of results) {
if (!r.error && r.framesCaptured < expectedFramesForTask(r)) {
r.error = synthesizeSilentWorkerExitError(r, expectedFramesForTask(r));
}
}
}
export function formatWorkerFailure(result: WorkerResult): string {
const errorText =
result.error && result.error.length > 0
? result.error
: synthesizeSilentWorkerExitError(result, expectedFramesForTask(result));
const base = `Worker ${result.workerId}: ${errorText}`;
if (!result.diagnostics || result.diagnostics.length === 0) return base;
const diagnostics = result.diagnostics.map(compactDiagnosticLine).join(" | ");
return `${base}; diagnostics: ${diagnostics}`;
}
/** Which constraint produced the final auto-sized worker count. */
export type WorkerSizingBound =
| "explicit"
| "too_few_frames"
| "cpu"
| "memory"
| "frames"
| "max_workers"
| "min_parallel_floor"
| "contention"
| "heap";
/**
* Full provenance of a worker-sizing decision. Threaded into render
* observability/telemetry so fleet data can answer "why N workers?" and
* "would the heap budget have prevented this OOM?" without a repro.
*/
export interface WorkerSizing {
workers: number;
boundBy: WorkerSizingBound;
cpuBasedWorkers: number;
memoryBasedWorkers: number;
frameBasedWorkers: number;
effectiveMaxWorkers: number;
/**
* How many workers the parent process's V8 heap can feed (see
* HEAP_PER_WORKER_MB); auto sizing never exceeds it.
*/
heapBasedWorkers: number;
/** V8 `heap_size_limit` for the parent process, MB. */
heapLimitMb: number;
totalMemoryMb: number;
cpuCount: number;
captureCostMultiplier: number;
/** true when the chosen count exceeds the heap budget (only an explicit request can). */
exceedsHeapAdvisory: boolean;
}
/**
* Compute the auto worker count together with the full sizing provenance.
* `calculateOptimalWorkers` is the thin legacy wrapper returning `.workers`.
*/
// The branch count is the decision provenance itself — each if records WHICH
// constraint bound, which is the entire point of the function.
// fallow-ignore-next-line complexity
export function computeWorkerSizing(
totalFrames: number,
requested?: number,
config?: WorkerSizingConfig,
): WorkerSizing {
const cpuCount = cpus().length;
// Use total memory instead of free memory — macOS reports misleadingly low
// freemem() because it aggressively caches files in "inactive" memory that
// is immediately reclaimable.
const totalMemoryMb = getSystemTotalMb();
const heapLimitMb = Math.round(getHeapStatistics().heap_size_limit / (1024 * 1024));
const appleDisk = config?.appleGpuDiskCapture === true;
const heapBasedWorkers = Math.max(
1,
Math.floor(
(heapLimitMb - HEAP_RESERVED_MB) /
(appleDisk
? APPLE_DISK_HEAP_PER_WORKER_MB
: inDesktopApp()
? DESKTOP_HEAP_PER_WORKER_MB
: HEAP_PER_WORKER_MB),
),
);
// A CPU quota (docker --cpus) bounds workers at its whole CPUs: at 3.5, 3 beat 2 and 4 (johnny, 10-28).
const cpuQuota = getCgroupCpuLimit();
const cpuBasedWorkers = Math.max(
1,
cpuQuota === null ? cpuCount - 2 : Math.min(cpuCount - 2, Math.floor(cpuQuota)),
);
const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMb * 0.5) / MEMORY_PER_WORKER_MB));
const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER);
const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1);
const base = {
cpuBasedWorkers,
memoryBasedWorkers,
frameBasedWorkers,
heapBasedWorkers,
heapLimitMb,
totalMemoryMb,
cpuCount,
captureCostMultiplier,
};
const finish = (workers: number, boundBy: WorkerSizingBound, effectiveMaxWorkers: number) => ({
workers,
boundBy,
effectiveMaxWorkers,
...base,
exceedsHeapAdvisory: workers > heapBasedWorkers,
});
if (requested !== undefined) {
return finish(
Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, requested)),
"explicit",
ABSOLUTE_MAX_WORKERS,
);
}
// Resolve effective values: config overrides → DEFAULT_CONFIG fallback.
const effectiveMaxWorkers = (() => {
const concurrency = config?.concurrency ?? DEFAULT_CONFIG.concurrency;
if (concurrency !== "auto") {
return Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, Math.floor(concurrency)));
}
return appleDisk
? Math.max(defaultSafeMaxWorkers(), APPLE_DISK_MAX_WORKERS)
: defaultSafeMaxWorkers();
})();
const configuredCoresPerWorker = config?.coresPerWorker ?? DEFAULT_CONFIG.coresPerWorker;
const effectiveCoresPerWorker =
appleDisk && configuredCoresPerWorker === DEFAULT_CONFIG.coresPerWorker
? APPLE_DISK_CORES_PER_WORKER
: configuredCoresPerWorker;
const effectiveMinParallelFrames = config?.minParallelFrames ?? DEFAULT_CONFIG.minParallelFrames;
const effectiveLargeRenderThreshold =
config?.largeRenderThreshold ?? DEFAULT_CONFIG.largeRenderThreshold;
if (totalFrames < MIN_FRAMES_PER_WORKER * 2) {
return finish(1, "too_few_frames", effectiveMaxWorkers);
}
const optimal = Math.min(cpuBasedWorkers, memoryBasedWorkers, frameBasedWorkers);
const optimalBound: WorkerSizingBound =
optimal === cpuBasedWorkers ? "cpu" : optimal === memoryBasedWorkers ? "memory" : "frames";
const minWorkersForJob = totalFrames >= effectiveMinParallelFrames ? 2 : MIN_WORKERS;
let finalWorkers = Math.max(minWorkersForJob, Math.min(effectiveMaxWorkers, optimal));
let boundBy: WorkerSizingBound =
finalWorkers === optimal
? optimalBound
: finalWorkers === effectiveMaxWorkers && effectiveMaxWorkers < optimal
? "max_workers"
: "min_parallel_floor";
// Adaptive scaling: cap workers for large or expensive renders to prevent
// CPU contention. Each Chrome process (with SwiftShader) is CPU-heavy; too
// many concurrent captures can starve the compositor and surface as CDP
// protocol timeouts. Scale proportionally to CPU count and composition cost:
// 8 cores → 2 workers, 16 cores → 5 workers, 32 cores → 10 workers.
const weightedFrames = totalFrames * captureCostMultiplier;
const contentionThreshold = Math.max(
effectiveMinParallelFrames,
Math.floor(effectiveLargeRenderThreshold / 3),
);
if (totalFrames >= effectiveLargeRenderThreshold || weightedFrames >= contentionThreshold) {
const weightedCoresPerWorker = effectiveCoresPerWorker * captureCostMultiplier;
const cpuScaledMax = Math.max(MIN_WORKERS, Math.floor(cpuCount / weightedCoresPerWorker));
if (finalWorkers > cpuScaledMax) {
finalWorkers = cpuScaledMax;
boundBy = "contention";
}
}
if (finalWorkers > heapBasedWorkers) {
finalWorkers = heapBasedWorkers;
boundBy = "heap";
}
return finish(finalWorkers, boundBy, effectiveMaxWorkers);
}
export function calculateOptimalWorkers(
totalFrames: number,
requested?: number,
config?: WorkerSizingConfig,
): number {
return computeWorkerSizing(totalFrames, requested, config).workers;
}
export function distributeFrames(
totalFrames: number,
workerCount: number,
workDir: string,
rangeStart: number = 0,
): WorkerTask[] {
const tasks: WorkerTask[] = [];
const framesPerWorker = Math.ceil(totalFrames / workerCount);
for (let i = 0; i < workerCount; i++) {
const startFrame = rangeStart + i * framesPerWorker;
const endFrame = Math.min(rangeStart + (i + 1) * framesPerWorker, rangeStart + totalFrames);
if (startFrame >= rangeStart + totalFrames) break;
tasks.push({
workerId: i,
startFrame,
endFrame,
outputDir: join(workDir, `worker-${i}`),
outputFrameOffset: rangeStart,
});
}
return tasks;
}
/**
* Interleaved (round-robin) distribution: worker i captures frames
* i, i+N, i+2N, …. Seek-based capture makes stride access free (every frame
* is an absolute seek), and the streaming reorder window shrinks from
* totalFrames/N to N — contiguous chunks serialize workers behind the
* ordered writer (worker 1's first frame waits for ALL of worker 0's).
* HF_DE_PARALLEL_STREAM spike; disk-path capture keeps contiguous chunks.
*/
export function distributeFramesInterleaved(
totalFrames: number,
workerCount: number,
workDir: string,
rangeStart: number = 0,
): WorkerTask[] {
const tasks: WorkerTask[] = [];
for (let i = 0; i < workerCount && i < totalFrames; i++) {
tasks.push({
workerId: i,
startFrame: rangeStart + i,
endFrame: rangeStart + totalFrames,
frameStride: workerCount,
outputDir: join(workDir, `worker-${i}`),
outputFrameOffset: rangeStart,
});
}
return tasks;
}
/**
* Decide whether a parallel worker should run the per-worker SwiftShader
* assertion. Gated to worker 0 only: workers within a chunk share the same
* Chrome binary, flags, and OS/driver state, so one verification per chunk
* is sufficient. See `heygen-com/hyperframes#955`.
*/
export function shouldVerifyWorkerGpu(workerId: number, config?: Partial<EngineConfig>): boolean {
return config?.browserGpuMode === "software" && workerId === 0;
}
/**
* Race a single in-flight capture call against `signal` actually firing.
*
* `captureFrame`/`captureFrameToBuffer`/`captureFrameToBufferPipelined` take
* no abort signal of their own — a native browser call wedged inside one of
* them (WSL2 hangs the very first drawElement/BeginFrame capture at frame 0
* with no error, heygen-com/hyperframes#3441) cannot be cancelled, only
* raced. Without this, the `signal?.aborted` checks at the top of the
* `captureFrameRange` loop are a no-op the moment a worker is already
* awaiting a hung call: nothing revisits that check until the await settles,
* which on a genuine hang is never. The DE parallel-router's stall watchdog
* (`captureStreamingStage.ts`) does fire `stallController.abort()` after
* `HF_DE_STALL_MS`, but until this call actually observes the signal, that
* abort has no effect on an already-wedged worker — `executeParallelCapture`'s
* `Promise.all` waits forever, so the render hangs indefinitely and the CLI's
* circuit breaker (which only runs after `executeRenderJob` settles) never
* gets a chance to trip.
*/
export function withParallelWorkerDeadline<T>(
promise: Promise<T>,
diagnostic: ParallelWorkerPhaseDiagnostic,
timeoutMs: number,
signal?: AbortSignal,
): Promise<T> {
if (signal?.aborted) return Promise.reject(new Error("Parallel worker cancelled"));
return new Promise<T>((resolve, reject) => {
let timer: ReturnType<typeof setTimeout> | undefined;
const cleanup = () => {
if (timer) clearTimeout(timer);
signal?.removeEventListener("abort", onAbort);
};
const onAbort = () => {
cleanup();
reject(new Error("Parallel worker cancelled"));
};
signal?.addEventListener("abort", onAbort, { once: true });
if (timeoutMs > 0) {
timer = setTimeout(() => {
cleanup();
const error = new Error(
`Parallel worker operation timeout exceeded after ${timeoutMs}ms: ` +
`worker=${diagnostic.workerId} phase=${diagnostic.phase} ` +
`frame=${diagnostic.frameIndex ?? "n/a"} ` +
`browser=${diagnostic.browserExecutable} version=${diagnostic.browserVersion} ` +
`CanvasDrawElement=${diagnostic.canvasDrawElement} gpu=${diagnostic.gpuBackend}`,
);
error.name = "ParallelWorkerPhaseTimeoutError";
reject(error);
}, timeoutMs);
timer.unref?.();
}
promise.then(
(value) => {
cleanup();
resolve(value);
},
(error) => {
cleanup();
reject(error);
},
);
});
}
const LATE_LAUNCH_CLOSE_WAIT_MS = 10_000;
function resolveParallelWorkerTimeoutMs(enabled: boolean): number {
if (!enabled) return 0;
const raw = process.env.HF_DE_PARALLEL_PHASE_TIMEOUT_MS;
if (raw === "0") return 0;
const parsed = raw ? Number(raw) : Number.NaN;
return Number.isFinite(parsed) && parsed > 0 ? parsed : 30_000;
}
// fallow-ignore-next-line complexity
async function captureFrameRange(
session: CaptureSession,
task: WorkerTask,
captureOptions: CaptureOptions,
signal: AbortSignal | undefined,
onFrameCaptured:
| ((workerId: number, frameIndex: number, landedFrame?: LandedFrame) => void)
| undefined,
onFrameBuffer:
| ((frameIndex: number, buffer: Buffer, session: CaptureSession) => Promise<void>)
| undefined,
phaseTimeoutMs: number,
diagnosticFor: (phase: ParallelWorkerPhase, frameIndex?: number) => ParallelWorkerPhaseDiagnostic,
onWorkerPhase: ((diagnostic: ParallelWorkerPhaseDiagnostic) => void) | undefined,
onWorkerOperation?: (diagnostic: ParallelWorkerPhaseDiagnostic) => void,
): Promise<number> {
let framesCaptured = 0;
const outputOffset = task.outputFrameOffset ?? 0;
const stride = task.frameStride ?? 1;
const runCaptureOperation = <T>(
phase: ParallelWorkerPhase,
frameIndex: number,
operation: () => Promise<T>,
): Promise<T> => {
const diagnostic = { ...diagnosticFor(phase, frameIndex), readStep: () => session.captureStep };
if (framesCaptured === 0) onWorkerPhase?.(diagnostic);
onWorkerOperation?.(diagnostic);
return withParallelWorkerDeadline(operation(), diagnostic, phaseTimeoutMs, signal);
};
const awaitEncode = (frameIndex: number, encodeResult: Promise<Buffer>): Promise<Buffer> => {
const diagnostic = diagnosticFor("frame_encode", frameIndex);
if (framesCaptured === 0) onWorkerPhase?.(diagnostic);
onWorkerOperation?.(diagnostic);
return withParallelWorkerDeadline(encodeResult, diagnostic, phaseTimeoutMs, signal);
};
// Depth-2 pipelined drawElement produce (HF_DE_PARALLEL_STREAM spike): frame
// k's in-page worker encode overlaps frame k+stride's produce phase — the
// same shape as the sequential worker-encode loop. Only engaged when the
// session's encode worker initialized (drawElement mode) and frames stream
// back via onFrameBuffer; the ordered writer's waitForFrame provides the
// cross-worker backpressure (each worker runs at most `stride` frames ahead).
// NOTE: this branch fires for any stride, but production only ever reaches
// it via HF_DE_PARALLEL_STREAM, which always uses interleaved distribution
// (stride = workerCount). The stride=1 (contiguous) path through here is
// validation-only — exercised by tests wiring onFrameBuffer with a
// contiguous multi-worker task, not a shape real renders take. Don't
// "simplify" the flag checks around this without accounting for that.
if (onFrameBuffer && session.workerEncodeEnabled) {
const dbg = process.env.HF_DE_PAR_DEBUG === "1";
const dbgT0 = Date.now();
const dbgWin = 40 * stride;
let prev: { idx: number; encodeResult: Promise<Buffer> } | null = null;
for (let i = task.startFrame; i < task.endFrame; i += stride) {
if (signal?.aborted) throw new Error("Parallel worker cancelled");
const time = (i * captureOptions.fps.den) / captureOptions.fps.num;
if (dbg && i < task.startFrame + dbgWin) {
console.error(`[par:w${task.workerId}] +${Date.now() - dbgT0}ms produce ${i} start`);
}
const { encodeResult } = await runCaptureOperation("frame_capture", i, () =>
captureFrameToBufferPipelined(session, i - outputOffset, time),
);
// Marks the promise "handled" for Node's unhandled-rejection detector
// without affecting the real `await prev.encodeResult` below — if a
// later iteration throws (abort, downstream writeFrame failure) before
// this frame's encode is drained, it's abandoned rather than awaited,
// and would otherwise surface as an unhandled rejection during teardown.
encodeResult.catch(() => {});
if (dbg && i < task.startFrame + dbgWin) {
console.error(`[par:w${task.workerId}] +${Date.now() - dbgT0}ms produce ${i} kicked`);
}
if (prev) {
if (dbg && prev.idx < task.startFrame + dbgWin) {
console.error(
`[par:w${task.workerId}] +${Date.now() - dbgT0}ms drain ${prev.idx} await-encode`,
);
}
const buf = await awaitEncode(prev.idx, prev.encodeResult);
if (dbg && prev.idx < task.startFrame + dbgWin) {
console.error(
`[par:w${task.workerId}] +${Date.now() - dbgT0}ms drain ${prev.idx} encoded ${buf.length}B`,
);
}
await onFrameBuffer(prev.idx, buf, session);
if (dbg && prev.idx < task.startFrame + dbgWin) {
console.error(
`[par:w${task.workerId}] +${Date.now() - dbgT0}ms drain ${prev.idx} written`,
);
}
framesCaptured++;
if (onFrameCaptured) onFrameCaptured(task.workerId, prev.idx);
}
prev = { idx: i, encodeResult };
}
if (prev) {
await onFrameBuffer(prev.idx, await awaitEncode(prev.idx, prev.encodeResult), session);
framesCaptured++;
if (onFrameCaptured) onFrameCaptured(task.workerId, prev.idx);
}
return framesCaptured;
}
for (let i = task.startFrame; i < task.endFrame; i += stride) {
if (signal?.aborted) throw new Error("Parallel worker cancelled");
const time = (i * captureOptions.fps.den) / captureOptions.fps.num;
const fileFrameIdx = i - outputOffset;
let landedFrame: LandedFrame | undefined;
if (onFrameBuffer) {
const { buffer } = await runCaptureOperation("frame_capture", i, () =>
captureFrameToBuffer(session, fileFrameIdx, time),
);
await onFrameBuffer(i, buffer, session);
} else {
const { path } = await runCaptureOperation("frame_capture", i, () =>
captureFrame(session, fileFrameIdx, time),
);
landedFrame = { index: fileFrameIdx, path };
}
framesCaptured++;
if (onFrameCaptured) onFrameCaptured(task.workerId, i, landedFrame);
}
return framesCaptured;
}
/**
* The armed self-verify sample indices this task actually captured: inside
* `[startFrame, endFrame)` and on the task's stride lattice. Mirrors the
* capture loop in `captureFrameRange` (`i += stride` from `startFrame`).
*/
export function selectVerifySampleIndicesForTask(
sampleIndices: Iterable<number>,
task: Pick<WorkerTask, "startFrame" | "endFrame" | "frameStride">,
): number[] {
const stride = task.frameStride ?? 1;
const selected: number[] = [];
for (const idx of sampleIndices) {
if (idx < task.startFrame || idx >= task.endFrame) continue;
if ((idx - task.startFrame) % stride !== 0) continue;
selected.push(idx);
}
return selected.sort((a, b) => a - b);
}
/**
* Disk-path drawElement self-verification (PRINFRA-352). Parallel DISK
* workers arm the same pre-injection ground-truth samples as the streaming
* path (`resolveParallelDeVerifySamples` even raises the density for
* multi-worker capture) — but only the streaming drain ever CHECKED them,
* so an explicit `--experimental-fast-capture --workers N` render shipped
* unverified drawElement frames. On a 16GB host, two concurrent
* hardware-GPU Chrome instances hit the documented compositor-tile-eviction
* damage class (frames displaced into vertical strips for one worker's
* whole range — reads as "corruption from the exact worker boundary").
*
* After a worker's range completes, re-read its captured files for the
* sampled indices and PSNR-compare against the session's ground truth.
* A breach throws `DrawElementVerificationError`, which the orchestrator's
* existing pinned-fallback retry converts into a screenshot re-render —
* the same recovery the streaming drain gets.
*/
/** HF_DE_PAR_DEBUG=1 gated per-worker trace line (message built lazily). */
function logParDebug(message: () => string): void {
if (process.env.HF_DE_PAR_DEBUG === "1") console.error(message());
}
/**
* Throw the verification error for a sample below the PSNR floor; log the
* pass otherwise. Split from the sampling loop for the complexity gate.
*/
function assertDiskSampleAboveFloor(
db: number,
verifyMinDb: number,
idx: number,
workerId: number,
): void {
if (db < verifyMinDb) {
// Message keeps the contiguous "drawElement self-verify" phrase —
// captureFailure's VERIFICATION_ERROR_PATTERNS classifies on it.
throw new DrawElementVerificationError(
`drawElement self-verify failed at frame ${idx} (disk path, worker ${workerId}): ` +
`${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot`,
{ kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
);
}
console.error(
`[Parallel] drawElement disk self-verify passed (worker ${workerId}, frame ${idx}, ` +
`${db === Infinity ? "inf" : db.toFixed(1)}dB)`,
);
}
/**
* Distinguishes infrastructure-class ffmpeg failures (spawn ENOENT, missing
* `psnr` filter) from per-sample noise (readFile races, transient tmpdir
* EPERM). Only the infrastructure class should terminate the render — the
* ffmpeg preflight in `initDrawElementOrTransparentBackground` catches these
* at bootstrap, so surfacing them here means the preflight was bypassed or
* the host ffmpeg changed mid-render.
*
* Exported for testing; the discriminator is a pure error-shape read.
*/
export function isFfmpegInfrastructureFailure(err: unknown): boolean {
if (!err || typeof err !== "object") return false;
const record = err as { code?: unknown; message?: unknown; stderr?: unknown };
if (record.code === "ENOENT") return true;
const message = typeof record.message === "string" ? record.message : "";
const stderr = typeof record.stderr === "string" ? record.stderr : "";
const text = `${message}\n${stderr}`;
// Spawn-side failures ("spawn ffmpeg ENOENT") and filter-side failures
// ("No such filter: 'psnr'", ffmpeg <=5 emits "Unknown filter 'psnr'").
return /\bENOENT\b|No such filter|Unknown filter/i.test(text);
}
/**
* Compare one captured frame file against its ground truth. Returns the
* PSNR, or null on per-sample noise (readFile races, transient EPERM,
* unparseable ffmpeg output on a single sample) — a skipped sample is not
* damage evidence and must not fail the capture. Re-throws when the error
* shape indicates the ffmpeg install itself is broken (missing binary or
* missing `psnr` filter): the drawElement self-verify safety net cannot
* possibly run in that state, and continuing would silently ship every
* remaining frame unverified.
*/
async function psnrForDiskSample(
framePath: string,
truth: Buffer,
workerId: number,
idx: number,
): Promise<number | null> {
const skipSample = (err: unknown): null => {
console.warn(
`[Parallel] drawElement disk self-verify sample skipped (worker ${workerId}, ` +
`frame ${idx}): ${err instanceof Error ? err.message : String(err)}`,
);
return null;
};
// Read apart from the compare: a frame file's ENOENT says nothing about ffmpeg.
let frame: Buffer;
try {
frame = await readFile(framePath);
} catch (err) {
return skipSample(err);
}
try {
return await psnrDb(frame, truth);
} catch (err) {
if (isFfmpegInfrastructureFailure(err)) {
const detail = err instanceof Error ? err.message : String(err);
throw new Error(
`[Parallel] drawElement disk self-verify aborted (worker ${workerId}, frame ${idx}): ` +
`ffmpeg or the \`psnr\` filter is unavailable (${detail}). The preflight in ` +
"initDrawElementOrTransparentBackground normally catches this at bootstrap; if you " +
"hit this after a successful preflight, ffmpeg was replaced mid-render or " +
"HYPERFRAMES_FFMPEG_PATH now points at a different binary.",
{ cause: err },
);
}
return skipSample(err);
}
}
// Branches are the gate conditions themselves (mode/armed/streaming guards +
// per-sample skip/breach) — already decomposed into psnrForDiskSample +
// assertDiskSampleAboveFloor; further splitting obscures the check.
// fallow-ignore-next-line complexity
export async function verifyDiskDrawElementSamples(
session: CaptureSession,
task: WorkerTask,
streaming: boolean,
): Promise<void> {
// Streaming capture verifies every sampled frame in the drain guard already.
if (streaming || session.captureMode !== "drawelement") return;
const truths = session.deVerifyFrames;
if (!truths || truths.size === 0) return;
const verifyMinDb = resolveDeVerifyMinDb();
const ext = session.options.format === "png" ? "png" : "jpg";
const offset = task.outputFrameOffset ?? 0;
for (const idx of selectVerifySampleIndicesForTask(truths.keys(), task)) {
const truth = truths.get(idx);
if (!truth) continue;
const framePath = join(task.outputDir, `frame_${String(idx - offset).padStart(6, "0")}.${ext}`);
const db = await psnrForDiskSample(framePath, truth, task.workerId, idx);
if (db === null) continue;
assertDiskSampleAboveFloor(db, verifyMinDb, idx, task.workerId);
}
}
// Inherited worker-lifecycle shape (session create → verify GPU → init →
// capture → self-verify → perf, with a classifying catch + closing finally);
// flagged only because the disk self-verify call shifted its line range into
// the changed-code audit. Not restructured by this PR.
// fallow-ignore-next-line complexity
async function executeWorkerTask(
task: WorkerTask,
serverUrl: string,
captureOptions: CaptureOptions,
createBeforeCaptureHook: () => BeforeCaptureHook | null,
signal?: AbortSignal,
onFrameCaptured?: (workerId: number, frameIndex: number, landedFrame?: LandedFrame) => void,
onFrameBuffer?: (frameIndex: number, buffer: Buffer, session: CaptureSession) => Promise<void>,
config?: Partial<EngineConfig>,
parallel?: boolean,
onFailure?: (failure: CaptureFailure) => void,
onWorkerPhase?: (diagnostic: ParallelWorkerPhaseDiagnostic) => void,
onWorkerOperation?: (diagnostic: ParallelWorkerPhaseDiagnostic) => void,
gate?: FirstSessionGate,
): Promise<WorkerResult> {
const startTime = Date.now();
let framesCaptured = 0;
if (!existsSync(task.outputDir)) mkdirSync(task.outputDir, { recursive: true });
let session: CaptureSession | null = null;
let perf: CapturePerfSummary | undefined;
const needsSeparateBrowsers = shouldDisableBrowserPoolForParallelWorker({ parallel });
const workerConfig: Partial<EngineConfig> | undefined = needsSeparateBrowsers
? { ...config, enableBrowserPool: false }
: config;
const phaseTimeoutMs = resolveParallelWorkerTimeoutMs(
Boolean(parallel && onFrameBuffer && workerConfig?.useDrawElement),
);
let browserVersion = "unknown";
let canvasDrawElement: boolean | "unknown" = "unknown";
let gpuBackend = "unknown";
let browserExecutable = resolveHeadlessShellPath(workerConfig) ?? "system/default";
const diagnosticFor = (
phase: ParallelWorkerPhase,
frameIndex?: number,
): ParallelWorkerPhaseDiagnostic => ({
workerId: task.workerId,
phase,
frameIndex,
browserExecutable,
browserVersion,
canvasDrawElement,
gpuBackend,
});
const runPhase = <T>(phase: ParallelWorkerPhase, operation: () => Promise<T>): Promise<T> => {
const diagnostic = diagnosticFor(phase);
onWorkerPhase?.(diagnostic);
return withParallelWorkerDeadline(operation(), diagnostic, phaseTimeoutMs, signal);
};
try {
const launch = createCaptureSession(
serverUrl,
task.outputDir,
{ ...captureOptions, staticDedupFrameRange: { start: task.startFrame, end: task.endFrame } },
createBeforeCaptureHook(),
workerConfig,
);
session = await runPhase("browser_launch", () => launch).catch(async (error: unknown) => {
// Close a launch that lost to cancel or its deadline before settling: Puppeteer's exit hook skips browsers.
const closing = launch.then(closeCaptureSession).catch(() => {});
await Promise.race([
closing,
new Promise((settle) => setTimeout(settle, LATE_LAUNCH_CLOSE_WAIT_MS).unref()),
]);
throw error;
});
const activeSession = session;
browserExecutable = activeSession.browser?.process?.()?.spawnfile || browserExecutable;
logParDebug(() => `[par:w${task.workerId}] session created`);
browserVersion = await runPhase(
"browser_probe",
async () => (await activeSession.browser?.version?.().catch(() => "unknown")) ?? "unknown",
);
await runPhase("session_init", async () => {
// Worker-0-only SwiftShader assertion — see `shouldVerifyWorkerGpu` and #955.
if (shouldVerifyWorkerGpu(task.workerId, workerConfig)) {
await assertSwiftShader(activeSession.page, readWebGlVendorInfoFromCanvas);
}
await initializeSession(activeSession);
if (gate) await (task.workerId === 0 ? gate.verify(activeSession) : gate.wait());
});
canvasDrawElement = activeSession.captureMode === "drawelement";
gpuBackend = activeSession.gpuRenderer ?? "unknown";
logParDebug(
() =>
`[par:w${task.workerId}] init done (mode=${session?.captureMode} workerEncode=${session?.workerEncodeEnabled === true})`,
);
framesCaptured = await captureFrameRange(
activeSession,
task,
captureOptions,
signal,
onFrameCaptured,
onFrameBuffer,
phaseTimeoutMs,
diagnosticFor,
onWorkerPhase,
onWorkerOperation,
);
await verifyDiskDrawElementSamples(activeSession, task, Boolean(onFrameBuffer));
perf = getCapturePerfSummary(activeSession);
return {
workerId: task.workerId,
framesCaptured,
startFrame: task.startFrame,
endFrame: task.endFrame,
frameStride: task.frameStride,
durationMs: Date.now() - startTime,
perf,
pageErrorLines: pageErrorLinesOf(activeSession),
};
} catch (error) {
if (task.workerId === 0) gate?.fail(error);
const diagnostics = session ? selectWorkerDiagnostics(session.browserConsoleBuffer) : [];
const workerDiagnostic: CaptureWorkerDiagnostic = {
workerId: task.workerId,
framesCaptured,
startFrame: task.startFrame,
endFrame: task.endFrame,
lines: diagnostics,
};
const failure = classifyCaptureFailure(error, {
signal,
workerDiagnostics: [workerDiagnostic],
pageErrorLines: session ? pageErrorLinesOf(session) : undefined,
});
onFailure?.(failure);
return {
workerId: task.workerId,
framesCaptured,
startFrame: task.startFrame,
endFrame: task.endFrame,
frameStride: task.frameStride,
durationMs: Date.now() - startTime,
perf,
error: failure.message,
diagnostics: diagnostics.length > 0 ? diagnostics : undefined,
pageErrorLines: session ? pageErrorLinesOf(session) : undefined,
failure,
};
} finally {
if (session) await closeCaptureSession(session).catch(() => {});
}
}
/**
* drawElement self-verify sample count for multi-worker capture. Each worker
* arms the same shared sample grid but drains only ~1/N of it, and N
* concurrent hardware-GPU browsers are exactly where compositor-tile damage
* shows up (wild 0.7.52 black-slab report) — so density rises with worker
* count: 4 base + 2 per extra worker, clamped to the verify path's max of 8.
* A caller-set value passes through untouched, and explicit HF_DE_VERIFY
* still overrides inside the session.
*/
export function resolveParallelDeVerifySamples(
callerValue: number | undefined,
workerCount: number,
): number | undefined {
if (callerValue !== undefined) return callerValue;
if (workerCount <= 1) return undefined;
return Math.min(8, 4 + 2 * (workerCount - 1));
}
/**
* Whether one worker's failure ends the whole pool. On the disk path a
* transient death (Target closed, Page crashed) is not fatal: the
* orchestrator's adaptive retry re-captures that worker's missing frames. On
* the streaming path (`onFrameBuffer` present) there is no per-worker retry
* and the dead worker's frames are gone, so its peers would park in the
* ordered writer waiting for a frame that never comes until the producer's
* no-progress watchdog relabelled the death as a stall a minute later.
* Every non-cancelled failure is therefore pool-fatal there; `cancelled`
* means the pool was already aborted and there is nothing left to propagate.
*/
export function isPoolFatalWorkerFailure(failure: CaptureFailure, streaming: boolean): boolean {
if (streaming) return failure.kind !== "cancelled";
return isFatalCaptureFailure(failure);
}
export interface ParallelCaptureHooks {
/**
* The first pool-fatal worker failure, delivered BEFORE peers are aborted
* so a streaming caller can release anything parked on the dead worker's
* frame (the ordered writer) with the original error, not a stall.
*/
onWorkerFailure?: (failure: CaptureFailure) => void;
onWorkerOperation?: (diagnostic: ParallelWorkerPhaseDiagnostic) => void;
/** Worker 0 runs this on its initialised session; the other workers wait for it. A rejection fails the pool. */
verifyFirstSession?: (session: CaptureSession) => Promise<void>;
}
interface FirstSessionGate {
verify: (session: CaptureSession) => Promise<void>;
wait: () => Promise<void>;
fail: (error: unknown) => void;
}
function createFirstSessionGate(
verify: (session: CaptureSession) => Promise<void>,
): FirstSessionGate {
let fail!: (error: unknown) => void;
let pass!: () => void;
const settled = new Promise<void>((resolve, reject) => {
pass = resolve;
fail = reject;
});
settled.catch(() => {});
return {
verify: (session) =>
verify(session).then(pass, (error: unknown) => {
fail(error);
throw error;
}),
wait: () => settled,
fail,
};
}
function pageErrorLinesOf(session: CaptureSession): string[] {
return session.browserConsoleBuffer.filter((line) => line.includes("PAGEERROR"));
}
/**
* The pool's single failure gate. Both halves of the contract the streaming
* stage relies on live here so they can be pinned without a browser: the
* first pool-fatal failure reaches `hooks.onWorkerFailure` with the ORIGINAL
* `CaptureFailure` before the peers are aborted, and a hook that throws cannot
* skip that abort (the pool still rejects with its own classified failure,
* not the hook's error). Later failures are ignored: the first one owns the
* abort reason. `onFailure` runs synchronously at the tail of a worker's
* catch, before that worker's promise settles, so the hook always precedes
* the pool's rejection. Exported for tests.
*/
export function createPoolFailureHandler(args: {
streaming: boolean;
peerController: AbortController;
hooks?: ParallelCaptureHooks;
}): {
onFailure: (failure: CaptureFailure) => void;
firstFatalFailure: () => CaptureFailure | undefined;
} {
let firstFatalFailure: CaptureFailure | undefined;
return {
firstFatalFailure: () => firstFatalFailure,
onFailure: (failure) => {
if (firstFatalFailure || !isPoolFatalWorkerFailure(failure, args.streaming)) return;
firstFatalFailure = failure;
try {
args.hooks?.onWorkerFailure?.(failure);
} catch {
// A caller-supplied hook must not be able to disable the pool abort;
// the worker's classified failure is what the pool reports.
} finally {
args.peerController.abort(failure);
}
},
};
}
export async function executeParallelCapture(
serverUrl: string,
workDir: string,
tasks: WorkerTask[],
captureOptions: CaptureOptions,
createBeforeCaptureHook: () => BeforeCaptureHook | null,
signal?: AbortSignal,
onProgress?: (progress: ParallelProgress) => void,
onFrameBuffer?: (frameIndex: number, buffer: Buffer, session: CaptureSession) => Promise<void>,
config?: Partial<EngineConfig>,
hooks?: ParallelCaptureHooks,
): Promise<WorkerResult[]> {
// `endFrame - startFrame` is the correct per-task frame count for contiguous
// tasks (stride 1), but for interleaved tasks (stride = workerCount) each
// task spans nearly the full range while only actually capturing 1/stride
// of it — dividing by stride here matches the loop in `captureFrameRange`
// (`i += stride`) so progress doesn't plateau at ~1/workerCount.
const totalFrames = tasks.reduce(
(sum, t) => sum + Math.ceil((t.endFrame - t.startFrame) / (t.frameStride ?? 1)),
0,
);
const workerProgress = new Map<number, number>();
const workerPhases = new Map<number, ParallelWorkerPhaseDiagnostic>();
for (const task of tasks) workerProgress.set(task.workerId, 0);
const onFrameCaptured = (workerId: number, _frameIndex: number, landedFrame?: LandedFrame) => {
const current = workerProgress.get(workerId) || 0;
workerProgress.set(workerId, current + 1);
if (onProgress) {
const capturedFrames = Array.from(workerProgress.values()).reduce((a, b) => a + b, 0);
onProgress({
totalFrames,
capturedFrames,
activeWorkers: tasks.length,
workerProgress: new Map(workerProgress),
...(landedFrame && { landedFrame }),
});
}
};
const onWorkerPhase = (diagnostic: ParallelWorkerPhaseDiagnostic) => {
workerPhases.set(diagnostic.workerId, diagnostic);
if (!onProgress) return;
const capturedFrames = Array.from(workerProgress.values()).reduce((a, b) => a + b, 0);
onProgress({
totalFrames,
capturedFrames,
activeWorkers: tasks.length,
workerProgress: new Map(workerProgress),
latestWorkerPhase: diagnostic,
});
};
const parallel = tasks.length > 1;
const deVerifySamples = resolveParallelDeVerifySamples(
captureOptions.deVerifySamples,
tasks.length,
);
const workerCaptureOptions: CaptureOptions = {
...captureOptions,
deVerifySamples,
deFrameFallback: false,
};
const peerController = new AbortController();
const workerSignal = signal
? AbortSignal.any([signal, peerController.signal])
: peerController.signal;
const failureHandler = createPoolFailureHandler({
streaming: Boolean(onFrameBuffer),
peerController,
hooks,
});
const onFailure = failureHandler.onFailure;
const gate = hooks?.verifyFirstSession && createFirstSessionGate(hooks.verifyFirstSession);
const results = await Promise.all(
tasks.map((task) =>
executeWorkerTask(
task,
serverUrl,
workerCaptureOptions,
createBeforeCaptureHook,
workerSignal,
onFrameCaptured,
onFrameBuffer,
config,
parallel,
onFailure,
onWorkerPhase,
hooks?.onWorkerOperation,
gate,
),
),
);
flagSilentWorkerExits(results);
const errors = results.filter((r) => r.failure || r.error);
if (errors.length > 0) {
const errorMessages = errors.map(formatWorkerFailure).join("; ");
const representative =
failureHandler.firstFatalFailure() ?? errors.find((result) => result.failure)?.failure;
const workerDiagnostics = errors.flatMap((result) => result.failure?.workerDiagnostics ?? []);
throw new CaptureFailure({
kind: representative?.kind ?? "io",
message: `[Parallel] Capture failed: ${errorMessages}`,
cause: representative,
workerDiagnostics,
pageErrorLines: [...new Set(results.flatMap((result) => result.pageErrorLines ?? []))],
});
}
return results;
}
export async function mergeWorkerFrames(
workDir: string,
tasks: WorkerTask[],
outputDir: string,
): Promise<number> {
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
let totalFrames = 0;
const sortedTasks = [...tasks].sort((a, b) => a.startFrame - b.startFrame);
for (const task of sortedTasks) {
if (!existsSync(task.outputDir)) {
continue;
}
const files = readdirSync(task.outputDir)
.filter((f) => f.startsWith("frame_") && (f.endsWith(".jpg") || f.endsWith(".png")))
.sort();
const copyTasks = files.map(async (file) => {
const sourcePath = join(task.outputDir, file);
const targetPath = join(outputDir, file);
try {
await renameWithRetryAsync(sourcePath, targetPath);
} catch {
await copyFile(sourcePath, targetPath);
}
});
await Promise.all(copyTasks);
totalFrames += files.length;
}
return totalFrames;
}
export function getSystemResources(): {
cpuCores: number;
totalMemoryMB: number;
freeMemoryMB: number;
recommendedWorkers: number;
} {
return {
cpuCores: cpus().length,
totalMemoryMB: getSystemTotalMb(),
freeMemoryMB: Math.round(freemem() / (1024 * 1024)),
recommendedWorkers: calculateOptimalWorkers(1000),
};
}