refactor(engine): manage child process lifecycles (#2160)

* refactor(engine): manage child process lifecycles

* fix(engine): preserve child reaping after runtime errors

* fix(engine): untrack child processes on exit
This commit is contained in:
James Russo
2026-07-17 01:17:53 -04:00
committed by GitHub
parent 8e162921bc
commit 57d3bf4960
17 changed files with 697 additions and 537 deletions
+44 -44
View File
@@ -3,42 +3,40 @@ import { spawn } from "child_process";
import { readFileSync } from "fs";
import { extname } from "path";
import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js";
import { ManagedChildProcess } from "./managedChildProcess.js";
import { trackChildProcess } from "./processTracker.js";
/** Spawn ffprobe with given args, return stdout. Throws on non-zero exit or missing binary. */
function runFfprobe(args: string[]): Promise<string> {
return new Promise((resolve, reject) => {
const command = getFfprobeBinary();
const proc = spawn(command, args);
let stdout = "";
let stderr = "";
proc.stdout.on("data", (data) => {
stdout += data.toString();
});
proc.stderr.on("data", (data) => {
stderr += data.toString();
});
proc.on("close", (code) => {
if (code !== 0) {
reject(new Error(`[FFmpeg] ffprobe exited with code ${code}: ${stderr}`));
} else {
resolve(stdout);
}
});
proc.on("error", (err) => {
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
const configured = process.env[FFPROBE_PATH_ENV]?.trim();
reject(
new Error(
configured
? `[FFmpeg] ffprobe not found at ${FFPROBE_PATH_ENV}="${configured}". Please install FFmpeg.`
: "[FFmpeg] ffprobe not found. Please install FFmpeg.",
),
);
} else {
reject(err);
}
});
async function runFfprobe(args: string[], signal?: AbortSignal): Promise<string> {
const command = getFfprobeBinary();
const proc = spawn(command, args);
trackChildProcess(proc);
let stdout = "";
proc.stdout.on("data", (data) => {
stdout += data.toString();
});
const managed = new ManagedChildProcess(proc, {
signal,
deadlineAtMs: Date.now() + 30_000,
});
const outcome = await managed.wait();
if (outcome.reason === "spawn_error") {
if ((outcome.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT") {
const configured = process.env[FFPROBE_PATH_ENV]?.trim();
throw new Error(
configured
? `[FFmpeg] ffprobe not found at ${FFPROBE_PATH_ENV}="${configured}". Please install FFmpeg.`
: "[FFmpeg] ffprobe not found. Please install FFmpeg.",
);
}
throw outcome.error ?? new Error(outcome.stderr);
}
if (outcome.reason !== "exit" || outcome.exitCode !== 0) {
throw new Error(
`[FFmpeg] ffprobe ${outcome.reason} with code ${outcome.exitCode}: ${outcome.stderr}`,
);
}
return stdout;
}
function parseProbeJson(stdout: string): FFProbeOutput {
@@ -351,20 +349,21 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
*/
export const extractVideoMetadata = extractMediaMetadata;
export async function extractAudioMetadata(filePath: string): Promise<AudioMetadata> {
const cached = audioMetadataCache.get(filePath);
export async function extractAudioMetadata(
filePath: string,
options?: { signal?: AbortSignal },
): Promise<AudioMetadata> {
// A caller-owned abort signal cannot safely share a cached in-flight probe:
// cancelling one consumer would also cancel unrelated consumers. Signal-bound
// probes therefore bypass the process-promise cache.
const cached = options?.signal ? undefined : audioMetadataCache.get(filePath);
if (cached) return cached;
const probePromise = (async (): Promise<AudioMetadata> => {
const stdout = await runFfprobe([
"-v",
"quiet",
"-print_format",
"json",
"-show_format",
"-show_streams",
filePath,
]);
const stdout = await runFfprobe(
["-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", filePath],
options?.signal,
);
const output = parseProbeJson(stdout);
const audioStream = output.streams.find((s) => s.codec_type === "audio");
if (!audioStream) throw new Error("[FFmpeg] No audio stream found");
@@ -403,6 +402,7 @@ export async function extractAudioMetadata(filePath: string): Promise<AudioMetad
};
})();
if (options?.signal) return probePromise;
audioMetadataCache.set(filePath, probePromise);
probePromise.catch(() => {
if (audioMetadataCache.get(filePath) === probePromise) {
+31 -57
View File
@@ -8,6 +8,8 @@
import { spawn } from "child_process";
import { getFfmpegBinary } from "./ffmpegBinaries.js";
import { ManagedChildProcess } from "./managedChildProcess.js";
import { trackChildProcess } from "./processTracker.js";
export type ConcreteGpuEncoder = "nvenc" | "videotoolbox" | "vaapi" | "qsv" | "amf";
export type GpuEncoder = ConcreteGpuEncoder | null;
@@ -60,25 +62,20 @@ export async function selectUsableGpuEncoder(
}
export async function detectGpuEncoder(): Promise<GpuEncoder> {
return new Promise((resolve) => {
const ffmpeg = spawn(getFfmpegBinary(), ["-encoders"], {
stdio: ["pipe", "pipe", "pipe"],
});
let stdout = "";
ffmpeg.stdout.on("data", (data) => {
stdout += data.toString();
});
ffmpeg.on("close", () => {
const candidates = getCompiledGpuEncoders(stdout);
void selectUsableGpuEncoder(candidates, canUseGpuEncoder)
.then(resolve)
.catch(() => resolve(null));
});
ffmpeg.on("error", () => resolve(null));
const ffmpeg = spawn(getFfmpegBinary(), ["-encoders"], {
stdio: ["pipe", "pipe", "pipe"],
});
trackChildProcess(ffmpeg);
let stdout = "";
ffmpeg.stdout.on("data", (data) => {
stdout += data.toString();
});
const outcome = await new ManagedChildProcess(ffmpeg, {
deadlineAtMs: Date.now() + 30_000,
}).wait();
if (outcome.reason !== "exit" || outcome.exitCode !== 0) return null;
const candidates = getCompiledGpuEncoders(stdout);
return selectUsableGpuEncoder(candidates, canUseGpuEncoder).catch(() => null);
}
let cachedGpuEncoder: GpuEncoder | undefined = undefined;
@@ -147,46 +144,23 @@ export function getProbeArgs(encoder: ConcreteGpuEncoder): string[] {
}
async function canUseGpuEncoder(encoder: ConcreteGpuEncoder): Promise<boolean> {
return new Promise((resolve) => {
let settled = false;
let timedOut = false;
let killTimer: ReturnType<typeof setTimeout> | undefined;
let stderr = "";
const finish = (usable: boolean) => {
if (settled) return;
settled = true;
clearTimeout(timer);
if (killTimer) clearTimeout(killTimer);
resolve(usable);
};
const ffmpeg = spawn(getFfmpegBinary(), getProbeArgs(encoder), {
stdio: ["ignore", "ignore", "pipe"],
});
ffmpeg.stderr?.on("data", (data) => {
stderr += data.toString();
});
const timer = setTimeout(() => {
timedOut = true;
ffmpeg.kill("SIGTERM");
killTimer = setTimeout(() => {
ffmpeg.kill("SIGKILL");
finish(false);
}, GPU_PROBE_KILL_GRACE_MS);
}, GPU_PROBE_TIMEOUT_MS);
ffmpeg.on("close", (code, signal) => {
const usable = code === 0;
logGpuProbeFailure(encoder, { code, signal, stderr, timedOut });
finish(usable);
});
ffmpeg.on("error", (error) => {
logGpuProbeFailure(encoder, { error, timedOut });
finish(false);
});
const ffmpeg = spawn(getFfmpegBinary(), getProbeArgs(encoder), {
stdio: ["ignore", "ignore", "pipe"],
});
trackChildProcess(ffmpeg);
const outcome = await new ManagedChildProcess(ffmpeg, {
deadlineAtMs: Date.now() + GPU_PROBE_TIMEOUT_MS,
terminationGraceMs: GPU_PROBE_KILL_GRACE_MS,
}).wait();
const usable = outcome.reason === "exit" && outcome.exitCode === 0;
logGpuProbeFailure(encoder, {
code: outcome.exitCode,
signal: outcome.signal,
stderr: outcome.stderr,
error: outcome.error,
timedOut: outcome.reason === "deadline",
});
return usable;
}
function logGpuProbeFailure(
@@ -0,0 +1,132 @@
import { EventEmitter } from "node:events";
import type { ChildProcess } from "node:child_process";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ManagedChildProcess } from "./managedChildProcess.js";
function childProcess() {
const child = new EventEmitter() as EventEmitter & {
stderr: EventEmitter;
kill: ReturnType<typeof vi.fn>;
};
child.stderr = new EventEmitter();
child.kill = vi.fn().mockReturnValue(true);
return child as unknown as ChildProcess & {
stderr: EventEmitter;
kill: ReturnType<typeof vi.fn>;
};
}
describe("ManagedChildProcess", () => {
afterEach(() => {
vi.useRealTimers();
});
it("returns a typed natural exit and bounded stderr tail", async () => {
const child = childProcess();
const managed = new ManagedChildProcess(child, { stderrMaxBytes: 5 });
child.stderr.emit("data", Buffer.from("123456789"));
child.emit("close", 0, null);
await expect(managed.wait()).resolves.toMatchObject({
reason: "exit",
exitCode: 0,
stderr: "56789",
});
});
it("escalates abort from SIGTERM to SIGKILL and resolves only after close", async () => {
vi.useFakeTimers();
const child = childProcess();
const controller = new AbortController();
const managed = new ManagedChildProcess(child, {
signal: controller.signal,
terminationGraceMs: 50,
});
controller.abort();
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
await vi.advanceTimersByTimeAsync(50);
expect(child.kill).toHaveBeenCalledWith("SIGKILL");
let reaped = false;
void managed.wait().then(() => {
reaped = true;
});
await Promise.resolve();
expect(reaped).toBe(false);
child.emit("close", null, "SIGKILL");
await expect(managed.wait()).resolves.toMatchObject({ reason: "abort", signal: "SIGKILL" });
});
it("keeps escalation and reaping active after a post-spawn error", async () => {
vi.useFakeTimers();
const child = childProcess();
const controller = new AbortController();
const managed = new ManagedChildProcess(child, {
signal: controller.signal,
terminationGraceMs: 50,
});
child.emit("spawn");
controller.abort();
child.emit("error", new Error("kill EPERM"));
let reaped = false;
void managed.wait().then(() => {
reaped = true;
});
await Promise.resolve();
expect(reaped).toBe(false);
await vi.advanceTimersByTimeAsync(50);
expect(child.kill).toHaveBeenNthCalledWith(1, "SIGTERM");
expect(child.kill).toHaveBeenNthCalledWith(2, "SIGKILL");
child.emit("error", new Error("kill EPERM"));
expect(reaped).toBe(false);
child.emit("close", null, "SIGKILL");
await expect(managed.wait()).resolves.toMatchObject({ reason: "abort", signal: "SIGKILL" });
});
it("distinguishes deadline from inactivity and refreshes activity", async () => {
vi.useFakeTimers();
const deadlineChild = childProcess();
const deadline = new ManagedChildProcess(deadlineChild, {
deadlineAtMs: Date.now() + 100,
terminationGraceMs: 1_000,
});
await vi.advanceTimersByTimeAsync(100);
expect(deadlineChild.kill).toHaveBeenCalledWith("SIGTERM");
deadlineChild.emit("close", null, "SIGTERM");
await expect(deadline.wait()).resolves.toMatchObject({ reason: "deadline" });
const inactiveChild = childProcess();
const inactive = new ManagedChildProcess(inactiveChild, {
inactivityTimeoutMs: 100,
terminationGraceMs: 1_000,
});
await vi.advanceTimersByTimeAsync(75);
inactive.markActivity();
await vi.advanceTimersByTimeAsync(75);
expect(inactiveChild.kill).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(25);
expect(inactiveChild.kill).toHaveBeenCalledWith("SIGTERM");
inactiveChild.emit("close", null, "SIGTERM");
await expect(inactive.wait()).resolves.toMatchObject({ reason: "inactivity" });
});
it("settles a spawn failure and removes cancellation listeners", async () => {
const child = childProcess();
const controller = new AbortController();
const managed = new ManagedChildProcess(child, { signal: controller.signal });
child.emit("error", new Error("spawn ENOENT"));
controller.abort();
await expect(managed.wait()).resolves.toMatchObject({
reason: "spawn_error",
exitCode: null,
stderr: "spawn ENOENT",
});
expect(child.kill).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,186 @@
import type { ChildProcess } from "node:child_process";
export type ManagedProcessTerminationReason =
| "exit"
| "abort"
| "deadline"
| "inactivity"
| "spawn_error";
export interface ManagedChildProcessOutcome {
reason: ManagedProcessTerminationReason;
exitCode: number | null;
signal: NodeJS.Signals | null;
stderr: string;
durationMs: number;
error?: Error;
}
export interface ManagedChildProcessOptions {
signal?: AbortSignal;
deadlineAtMs?: number;
inactivityTimeoutMs?: number;
terminationGraceMs?: number;
stderrMaxBytes?: number;
onStderr?: (chunk: string) => void;
now?: () => number;
}
const DEFAULT_TERMINATION_GRACE_MS = 2_000;
const DEFAULT_STDERR_MAX_BYTES = 64 * 1024;
/** Owns cancellation, escalation, stderr and reaping for one child process. */
export class ManagedChildProcess {
private readonly startedAtMs: number;
private readonly now: () => number;
private readonly outcomePromise: Promise<ManagedChildProcessOutcome>;
private resolveOutcome!: (outcome: ManagedChildProcessOutcome) => void;
private requestedReason: Exclude<ManagedProcessTerminationReason, "exit" | "spawn_error"> | null =
null;
private stderrTail = Buffer.alloc(0);
private spawned = false;
private settled = false;
private deadlineTimer: NodeJS.Timeout | null = null;
private inactivityTimer: NodeJS.Timeout | null = null;
private escalationTimer: NodeJS.Timeout | null = null;
constructor(
readonly child: ChildProcess,
private readonly options: ManagedChildProcessOptions = {},
) {
this.now = options.now ?? Date.now;
this.startedAtMs = this.now();
this.outcomePromise = new Promise((resolve) => {
this.resolveOutcome = resolve;
});
child.stderr?.on("data", this.onStderr);
child.once("spawn", this.onSpawn);
child.once("close", this.onClose);
child.on("error", this.onError);
this.installAbort();
this.installDeadline();
this.markActivity();
}
wait(): Promise<ManagedChildProcessOutcome> {
return this.outcomePromise;
}
get isSettled(): boolean {
return this.settled;
}
markActivity(): void {
if (this.settled || this.options.inactivityTimeoutMs === undefined) return;
if (this.inactivityTimer) clearTimeout(this.inactivityTimer);
this.inactivityTimer = setTimeout(
() => this.requestTermination("inactivity"),
Math.max(0, this.options.inactivityTimeoutMs),
);
this.inactivityTimer.unref?.();
}
private readonly onStderr = (data: Buffer | string): void => {
const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data);
const maxBytes = this.options.stderrMaxBytes ?? DEFAULT_STDERR_MAX_BYTES;
this.stderrTail = Buffer.concat([this.stderrTail, chunk]);
if (this.stderrTail.byteLength > maxBytes) {
this.stderrTail = this.stderrTail.subarray(this.stderrTail.byteLength - maxBytes);
}
this.options.onStderr?.(chunk.toString());
};
private readonly onSpawn = (): void => {
this.spawned = true;
};
private readonly onClose = (exitCode: number | null, signal: NodeJS.Signals | null): void => {
this.settle({
reason: this.requestedReason ?? "exit",
exitCode,
signal,
stderr: this.stderrTail.toString(),
durationMs: this.now() - this.startedAtMs,
});
};
private readonly onError = (error: Error): void => {
if (this.spawned) return;
this.settle({
reason: "spawn_error",
exitCode: null,
signal: null,
stderr: this.stderrTail.length > 0 ? this.stderrTail.toString() : error.message,
durationMs: this.now() - this.startedAtMs,
error,
});
};
private installAbort(): void {
const signal = this.options.signal;
if (!signal) return;
if (signal.aborted) {
this.requestTermination("abort");
return;
}
signal.addEventListener("abort", this.onAbort, { once: true });
}
private readonly onAbort = (): void => {
this.requestTermination("abort");
};
private installDeadline(): void {
if (this.options.deadlineAtMs === undefined) return;
const remainingMs = Math.max(0, this.options.deadlineAtMs - this.now());
this.deadlineTimer = setTimeout(() => this.requestTermination("deadline"), remainingMs);
this.deadlineTimer.unref?.();
}
private requestTermination(
reason: Exclude<ManagedProcessTerminationReason, "exit" | "spawn_error">,
): void {
if (this.settled || this.requestedReason) return;
this.requestedReason = reason;
try {
this.child.kill("SIGTERM");
} catch {
// A close/error event owns settlement; escalation remains the backstop.
}
const graceMs = this.options.terminationGraceMs ?? DEFAULT_TERMINATION_GRACE_MS;
this.escalationTimer = setTimeout(
() => {
if (this.settled) return;
try {
this.child.kill("SIGKILL");
} catch {
// The child may have exited between the settled check and kill.
}
},
Math.max(0, graceMs),
);
this.escalationTimer.unref?.();
}
private settle(outcome: ManagedChildProcessOutcome): void {
if (this.settled) return;
this.settled = true;
this.clearTimers();
this.options.signal?.removeEventListener("abort", this.onAbort);
this.child.stderr?.off("data", this.onStderr);
this.child.off("spawn", this.onSpawn);
this.child.off("close", this.onClose);
this.child.off("error", this.onError);
this.resolveOutcome(outcome);
}
private clearTimers(): void {
if (this.deadlineTimer) clearTimeout(this.deadlineTimer);
if (this.inactivityTimer) clearTimeout(this.inactivityTimer);
if (this.escalationTimer) clearTimeout(this.escalationTimer);
this.deadlineTimer = null;
this.inactivityTimer = null;
this.escalationTimer = null;
}
}
@@ -1,4 +1,4 @@
import { describe, it, expect, beforeEach } from "vitest";
import { describe, it, expect, beforeEach, vi } from "vitest";
import { spawn } from "node:child_process";
import { trackChildProcess, killTrackedProcesses } from "./processTracker.js";
@@ -18,14 +18,45 @@ describe("trackChildProcess", () => {
killTrackedProcesses();
});
it("removes the process on spawn error", async () => {
const proc = spawn("/nonexistent-binary-that-does-not-exist", { stdio: "ignore" });
it("removes an exited process before its stdio closes", async () => {
const proc = spawn("sleep", ["60"], { stdio: "ignore" });
const closePromise = new Promise<void>((resolve) => proc.on("close", resolve));
const kill = vi.spyOn(proc, "kill");
trackChildProcess(proc);
await new Promise<void>((resolve) => proc.on("error", () => resolve()));
try {
proc.emit("exit", 0, null);
killTrackedProcesses();
expect(kill).not.toHaveBeenCalled();
} finally {
kill.mockRestore();
proc.kill("SIGKILL");
await closePromise;
}
});
it("removes the process on spawn error", async () => {
const proc = spawn("/nonexistent-binary-that-does-not-exist", { stdio: "ignore" });
proc.on("error", () => undefined);
trackChildProcess(proc);
await new Promise<void>((resolve) => proc.on("close", () => resolve()));
killTrackedProcesses();
});
it("keeps a process tracked after a post-spawn error", () => {
const proc = spawn("sleep", ["60"], { stdio: "ignore" });
const kill = vi.spyOn(proc, "kill");
proc.on("error", () => undefined);
trackChildProcess(proc);
proc.emit("error", new Error("kill EPERM"));
killTrackedProcesses();
expect(kill).toHaveBeenCalledWith("SIGTERM");
});
});
describe("killTrackedProcesses", () => {
+1 -1
View File
@@ -6,7 +6,7 @@ export function trackChildProcess(proc: ChildProcess): void {
tracked.add(proc);
const remove = () => tracked.delete(proc);
proc.once("exit", remove);
proc.once("error", remove);
proc.once("close", remove);
}
/**
+21 -54
View File
@@ -9,6 +9,10 @@
import { spawn } from "child_process";
import { getFfmpegBinary } from "./ffmpegBinaries.js";
import { trackChildProcess } from "./processTracker.js";
import {
ManagedChildProcess,
type ManagedProcessTerminationReason,
} from "./managedChildProcess.js";
export interface RunFfmpegOptions {
signal?: AbortSignal;
@@ -21,6 +25,8 @@ export interface RunFfmpegResult {
exitCode: number | null;
stderr: string;
durationMs: number;
terminationReason: ManagedProcessTerminationReason;
error?: Error;
}
const DEFAULT_TIMEOUT = 300_000;
@@ -85,60 +91,21 @@ export function formatFfmpegError(
}
export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promise<RunFfmpegResult> {
const startMs = Date.now();
const signal = opts?.signal;
const timeout = opts?.timeout ?? DEFAULT_TIMEOUT;
const onStderr = opts?.onStderr;
return new Promise<RunFfmpegResult>((resolve) => {
const ffmpeg = spawn(getFfmpegBinary(), args);
trackChildProcess(ffmpeg);
let stderr = "";
const onAbort = () => {
ffmpeg.kill("SIGTERM");
};
if (signal) {
if (signal.aborted) {
ffmpeg.kill("SIGTERM");
} else {
signal.addEventListener("abort", onAbort, { once: true });
}
}
const timer = setTimeout(() => {
ffmpeg.kill("SIGTERM");
}, timeout);
ffmpeg.stderr.on("data", (data: Buffer) => {
const chunk = data.toString();
stderr += chunk;
if (onStderr) {
onStderr(chunk);
}
});
ffmpeg.on("close", (code) => {
clearTimeout(timer);
if (signal) signal.removeEventListener("abort", onAbort);
resolve({
success: !signal?.aborted && code === 0,
exitCode: code,
stderr,
durationMs: Date.now() - startMs,
});
});
ffmpeg.on("error", (err) => {
clearTimeout(timer);
if (signal) signal.removeEventListener("abort", onAbort);
resolve({
success: false,
exitCode: null,
stderr: err.message,
durationMs: Date.now() - startMs,
});
});
const ffmpeg = spawn(getFfmpegBinary(), args);
trackChildProcess(ffmpeg);
const managed = new ManagedChildProcess(ffmpeg, {
signal: opts?.signal,
deadlineAtMs: Date.now() + timeout,
onStderr: opts?.onStderr,
});
const outcome = await managed.wait();
return {
success: outcome.reason === "exit" && outcome.exitCode === 0,
exitCode: outcome.exitCode,
stderr: outcome.stderr,
durationMs: outcome.durationMs,
terminationReason: outcome.reason,
error: outcome.error,
};
}