Merge pull request #1039 from heygen-com/fix/orphaned-child-processes

fix: clean up orphaned Chrome/ffmpeg on preview exit

packages/cli/src/commands/preview.ts

@@ -28,6 +28,7 @@ import {
   killActiveServers,
   type FindPortResult,
 } from "../server/portUtils.js";
+import { killOrphanedProcesses, killProcessTree } from "../utils/orphanCleanup.js";
 
 export default defineCommand({
   meta: { name: "preview", description: "Start the studio for previewing compositions" },
@@ -96,6 +97,14 @@ export default defineCommand({
       return;
     }
 
+    // Kill orphaned chrome-headless-shell processes from previous crashed sessions.
+    const orphansKilled = killOrphanedProcesses();
+    if (orphansKilled > 0) {
+      console.log(
+        `  ${c.dim(`Cleaned up ${orphansKilled} orphaned process${orphansKilled === 1 ? "" : "es"} from a previous session.`)}`,
+      );
+    }
+
     const rawArg = args.dir;
     const dir = resolve(rawArg ?? ".");
 
@@ -249,8 +258,18 @@ async function runDevMode(
     });
   }
 
-  // Wait for child to exit. Ctrl+C sends SIGINT to the entire process group,
-  // so the child (Vite) receives it directly — no need to intercept or forward.
+  // Kill the child's entire process tree on SIGTERM/SIGINT. Ctrl+C sends
+  // SIGINT to the foreground process group (covers the common case), but
+  // `kill <pid>` only targets this process — the child tree (Vite + Chrome)
+  // would survive without explicit cleanup.
+  // On Windows, killProcessTree is a no-op (pgrep/ps unavailable); Ctrl+C
+  // propagates via the console process group instead.
+  const shutdown = () => {
+    if (child.pid) killProcessTree(child.pid);
+  };
+  process.once("SIGINT", shutdown);
+  process.once("SIGTERM", shutdown);
+
   return new Promise<void>((resolve) => {
     child.on("close", () => resolve());
   });
@@ -349,6 +368,13 @@ async function runLocalStudioMode(
     });
   }
 
+  // Same tree-kill handler as dev mode. No-op on Windows (see comment above).
+  const shutdown = () => {
+    if (child.pid) killProcessTree(child.pid);
+  };
+  process.once("SIGINT", shutdown);
+  process.once("SIGTERM", shutdown);
+
   return new Promise<void>((resolve) => {
     child.on("close", () => resolve());
   });
@@ -477,21 +503,42 @@ async function runEmbeddedMode(
       shuttingDown = true;
       process.off("SIGINT", shutdown);
       process.off("SIGTERM", shutdown);
-      // Close the readline interface so a second Ctrl+C during the grace
-      // period below doesn't re-emit SIGINT and trigger Node's default
-      // exit-130 behaviour, contradicting our intent to exit cleanly.
       rl?.close();
-      // `server.close()` can take a second or two to drain keep-alive
-      // connections; surface progress so the terminal doesn't look frozen.
       console.log();
       console.log(`  ${c.dim("Shutting down studio...")}`);
-      result.server.close(() => resolveRun());
-      // If close() hangs on an open connection, force exit after a short
-      // grace period. Exit 0 because user-initiated Ctrl+C isn't an error
-      // — a non-zero code makes pnpm / npm print ELIFECYCLE.
-      setTimeout(() => process.exit(0), 2000).unref();
+
+      // Hard deadline: if cleanup hangs (e.g. dead Chrome never responds to
+      // browser.close()), force exit. Armed before awaiting cleanup so it
+      // can't be blocked by a stuck drainBrowserPool().
+      setTimeout(() => process.exit(0), 3000).unref();
+
+      // Kill ffmpeg first (sync, fast), then drain browsers (async, slower).
+      const cleanup = async () => {
+        const { closeThumbnailBrowser } = await import("../server/studioServer.js");
+        const { drainBrowserPool, killTrackedProcesses } = await import("@hyperframes/engine");
+        killTrackedProcesses();
+        await closeThumbnailBrowser().catch(() => {});
+        await drainBrowserPool().catch(() => {});
+      };
+
+      cleanup()
+        .catch(() => {})
+        .finally(() => {
+          result.server.close(() => resolveRun());
+        });
     };
     process.once("SIGINT", shutdown);
     process.once("SIGTERM", shutdown);
+
+    // Last-resort cleanup for crash paths (unhandled exceptions/rejections)
+    // that bypass the signal handlers. Eagerly resolve the sync killer so
+    // the 'exit' handler (which is synchronous) can call it directly.
+    import("@hyperframes/engine")
+      .then(({ killTrackedProcesses }) => {
+        process.once("exit", () => {
+          if (!shuttingDown) killTrackedProcesses();
+        });
+      })
+      .catch(() => {});
   });
 }

packages/cli/src/server/studioServer.ts

@@ -148,16 +148,6 @@ async function getThumbnailBrowser(): Promise<import("puppeteer-core").Browser |
         _thumbnailBrowser = null;
         _thumbnailBrowserInitializing = null;
       });
-      // Release the pool ref on process exit so the browser closes cleanly.
-      const onExit = async () => {
-        const { releaseBrowser } = await import("@hyperframes/engine");
-        if (_thumbnailBrowser) {
-          await releaseBrowser(_thumbnailBrowser).catch(() => {});
-          _thumbnailBrowser = null;
-        }
-      };
-      process.once("SIGTERM", () => void onExit());
-      process.once("SIGINT", () => void onExit());
       return _thumbnailBrowser;
     } catch (err) {
       console.warn(
@@ -172,6 +162,15 @@ async function getThumbnailBrowser(): Promise<import("puppeteer-core").Browser |
   return _thumbnailBrowserInitializing;
 }
 
+export async function closeThumbnailBrowser(): Promise<void> {
+  if (!_thumbnailBrowser) return;
+  const browser = _thumbnailBrowser;
+  _thumbnailBrowser = null;
+  _thumbnailBrowserInitializing = null;
+  const { releaseBrowser } = await import("@hyperframes/engine");
+  await releaseBrowser(browser).catch(() => {});
+}
+
 // ── Server factory ──────────────────────────────────────────────────────────
 
 export interface StudioServerOptions {

packages/cli/src/utils/orphanCleanup.test.ts

@@ -0,0 +1,55 @@
+import { describe, it, expect } from "vitest";
+import { spawn } from "node:child_process";
+import { killProcessTree, killOrphanedProcesses } from "./orphanCleanup.js";
+
+const IS_UNIX = process.platform !== "win32";
+
+describe.skipIf(!IS_UNIX)("killProcessTree", () => {
+  it("kills a process and all its children", async () => {
+    // Spawn a parent that spawns two sleeping children
+    const parent = spawn("bash", ["-c", "sleep 60 & sleep 60 & wait"], { stdio: "ignore" });
+    // Let children spawn
+    await new Promise((r) => setTimeout(r, 200));
+
+    const exitPromise = new Promise<void>((resolve) => parent.on("close", resolve));
+    killProcessTree(parent.pid!);
+
+    await exitPromise;
+
+    // Verify parent is dead
+    expect(() => process.kill(parent.pid!, 0)).toThrow();
+  }, 5000);
+
+  it("handles non-existent PID gracefully", () => {
+    // Should not throw for a PID that doesn't exist
+    killProcessTree(999999999);
+  });
+
+  it("escalates to SIGKILL after grace period", async () => {
+    // Spawn a process that traps SIGTERM
+    const proc = spawn("bash", ["-c", "trap '' TERM; sleep 60"], { stdio: "ignore" });
+    await new Promise((r) => setTimeout(r, 100));
+
+    const exitPromise = new Promise<void>((resolve) => proc.on("close", resolve));
+    killProcessTree(proc.pid!);
+
+    // Should die within 1s (500ms SIGKILL grace + buffer)
+    await exitPromise;
+    expect(() => process.kill(proc.pid!, 0)).toThrow();
+  }, 5000);
+});
+
+describe.skipIf(!IS_UNIX)("killOrphanedProcesses", () => {
+  it("returns 0 when no orphans exist", () => {
+    const killed = killOrphanedProcesses();
+    expect(killed).toBe(0);
+  });
+
+  it("does not kill non-orphaned Chrome processes", () => {
+    // Our current process is not an orphan (PPID !== 1), so any
+    // chrome-headless-shell processes we'd find with our PID as
+    // ancestor wouldn't be killed.
+    const killed = killOrphanedProcesses();
+    expect(killed).toBe(0);
+  });
+});

packages/cli/src/utils/orphanCleanup.ts

@@ -0,0 +1,137 @@
+import { execSync } from "node:child_process";
+
+/**
+ * Find and kill orphaned Chrome processes from previous crashed sessions.
+ * Targets both chrome-headless-shell (production/CI) and Google Chrome
+ * launched by Puppeteer (dev mode). Puppeteer Chrome is identified by the
+ * `puppeteer_dev_chrome_profile` marker in its user-data-dir argument.
+ *
+ * An orphan is a process whose PPID=1 (reparented to init/launchd after
+ * its parent died). We kill the orphan's entire subtree so child helper
+ * processes (GPU, renderer, network, etc.) are also cleaned up.
+ *
+ * Scoped to the current user via `pgrep -u` to avoid touching other
+ * users' processes on shared machines.
+ *
+ * Returns the count of killed process trees.
+ */
+export function killOrphanedProcesses(): number {
+  if (process.platform === "win32") return 0;
+
+  let killed = 0;
+
+  for (const name of ["chrome-headless-shell", "chrome_headless_shell"]) {
+    killed += killOrphansByName(name);
+  }
+
+  killed += killOrphansByName("puppeteer_dev_chrome_profile");
+
+  return killed;
+}
+
+/**
+ * Kill an entire process tree rooted at `pid`. Walks descendants
+ * depth-first so children are killed before parents, preventing
+ * re-adoption races.
+ *
+ * No-op on Windows — process groups are managed differently and
+ * the pgrep/ps utilities are not available.
+ */
+export function killProcessTree(pid: number, signal: NodeJS.Signals = "SIGTERM"): void {
+  if (process.platform === "win32") return;
+
+  const descendants = getDescendants(pid);
+  const allPids = [...descendants.reverse(), pid];
+
+  for (const p of allPids) {
+    try {
+      process.kill(p, signal);
+    } catch {
+      // Already exited.
+    }
+  }
+
+  // Escalate to SIGKILL after a short grace period for any survivors.
+  if (signal !== "SIGKILL") {
+    setTimeout(() => {
+      for (const p of allPids) {
+        try {
+          process.kill(p, "SIGKILL");
+        } catch {
+          // Already exited.
+        }
+      }
+    }, 500).unref();
+  }
+}
+
+function getDescendants(pid: number): number[] {
+  let children: number[];
+  try {
+    const raw = execSync(`pgrep -P ${pid}`, { encoding: "utf-8", timeout: 2000 }).trim();
+    if (!raw) return [];
+    children = raw
+      .split("\n")
+      .map((s) => parseInt(s, 10))
+      .filter((n) => !isNaN(n) && n > 0);
+  } catch {
+    return [];
+  }
+  const all: number[] = [];
+  for (const child of children) {
+    all.push(child);
+    all.push(...getDescendants(child));
+  }
+  return all;
+}
+
+function killOrphansByName(processName: string): number {
+  const uid = getUid();
+  const userFlag = uid !== null ? `-u ${uid} ` : "";
+  let pids: number[];
+  try {
+    const raw = execSync(`pgrep ${userFlag}-f ${processName}`, {
+      encoding: "utf-8",
+      timeout: 3000,
+    }).trim();
+    if (!raw) return 0;
+    pids = raw
+      .split("\n")
+      .map((s) => parseInt(s, 10))
+      .filter((n) => !isNaN(n) && n > 0);
+  } catch {
+    return 0;
+  }
+
+  let killed = 0;
+  for (const pid of pids) {
+    if (!isOrphan(pid)) continue;
+    killProcessTree(pid);
+    killed++;
+  }
+  return killed;
+}
+
+let _cachedUid: string | null | undefined;
+
+function getUid(): string | null {
+  if (_cachedUid !== undefined) return _cachedUid;
+  try {
+    _cachedUid = execSync("id -u", { encoding: "utf-8", timeout: 1000 }).trim();
+  } catch {
+    _cachedUid = null;
+  }
+  return _cachedUid;
+}
+
+function isOrphan(pid: number): boolean {
+  try {
+    const ppid = execSync(`ps -p ${pid} -o ppid=`, {
+      encoding: "utf-8",
+      timeout: 2000,
+    }).trim();
+    return ppid === "1";
+  } catch {
+    return false;
+  }
+}

packages/engine/src/index.ts

@@ -186,6 +186,8 @@ export {
   type RunFfmpegResult,
 } from "./utils/runFfmpeg.js";
 
+export { trackChildProcess, killTrackedProcesses } from "./utils/processTracker.js";
+
 export {
   decodePng,
   decodePngToRgb48le,

packages/engine/src/services/chunkEncoder.ts

@@ -8,6 +8,7 @@
 import { spawn } from "child_process";
 import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from "fs";
 import { join, dirname } from "path";
+import { trackChildProcess } from "../utils/processTracker.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 import {
   type GpuEncoder,
@@ -404,6 +405,7 @@ export async function encodeFramesFromDir(
 
   return new Promise((resolve) => {
     const ffmpeg = spawn("ffmpeg", args);
+    trackChildProcess(ffmpeg);
     let stderr = "";
     const onAbort = () => {
       ffmpeg.kill("SIGTERM");
@@ -535,6 +537,7 @@ export async function encodeFramesChunkedConcat(
     const args = buildEncoderArgs(options, inputArgs, chunkPath, gpuEncoder);
     const chunkResult = await new Promise<{ success: boolean; error?: string }>((resolve) => {
       const ffmpeg = spawn("ffmpeg", args);
+      trackChildProcess(ffmpeg);
       let stderr = "";
       ffmpeg.stderr.on("data", (d) => {
         stderr += d.toString();
@@ -578,6 +581,7 @@ export async function encodeFramesChunkedConcat(
   ];
   const concatResult = await new Promise<{ success: boolean; error?: string }>((resolve) => {
     const ffmpeg = spawn("ffmpeg", concatArgs);
+    trackChildProcess(ffmpeg);
     let stderr = "";
     ffmpeg.stderr.on("data", (d) => {
       stderr += d.toString();

packages/engine/src/services/streamingEncoder.ts

@@ -13,6 +13,7 @@
  */
 
 import { spawn, type ChildProcess } from "child_process";
+import { trackChildProcess } from "../utils/processTracker.js";
 import { existsSync, mkdirSync, statSync } from "fs";
 import { dirname } from "path";
 
@@ -375,6 +376,7 @@ export async function spawnStreamingEncoder(
   const ffmpeg: ChildProcess = spawn("ffmpeg", args, {
     stdio: ["pipe", "pipe", "pipe"],
   });
+  trackChildProcess(ffmpeg);
 
   let exitStatus: "running" | "success" | "error" = "running";
   let stderr = "";

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -395,8 +395,10 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
     expect(result.extracted).toHaveLength(1);
     const frames = readdirSync(join(outputDir, "v1")).filter((f) => f.endsWith(".jpg"));
     // Pre-fix behavior produced ~90 frames (a 25% shortfall).
-    expect(frames.length).toBeGreaterThanOrEqual(119);
-    expect(frames.length).toBeLessThanOrEqual(121);
+    // ±3 tolerance: FFmpeg's VFR→CFR normalization yields slightly different
+    // frame counts across versions (timestamp rounding in the fps filter).
+    expect(frames.length).toBeGreaterThanOrEqual(117);
+    expect(frames.length).toBeLessThanOrEqual(123);
 
     expect(result.phaseBreakdown).toBeDefined();
     expect(result.phaseBreakdown.extractMs).toBeGreaterThan(0);
@@ -656,8 +658,9 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
     const frames = readdirSync(frameDir)
       .filter((f) => f.endsWith(".jpg"))
       .sort();
-    expect(frames.length).toBeGreaterThanOrEqual(299);
-    expect(frames.length).toBeLessThanOrEqual(301);
+    // ±3 tolerance: same FFmpeg VFR→CFR rounding variance as the mid-segment test.
+    expect(frames.length).toBeGreaterThanOrEqual(297);
+    expect(frames.length).toBeLessThanOrEqual(303);
 
     let prevHash: string | null = null;
     let duplicates = 0;

packages/engine/src/services/videoFrameExtractor.ts

@@ -9,6 +9,7 @@ import { spawn } from "child_process";
 import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
 import { isAbsolute, join, posix, resolve, sep } from "path";
 import { parseHTML } from "linkedom";
+import { trackChildProcess } from "../utils/processTracker.js";
 import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js";
 import {
   analyzeCompositionHdr,
@@ -258,6 +259,7 @@ export async function extractVideoFramesRange(
 
   return new Promise((resolve, reject) => {
     const ffmpeg = spawn("ffmpeg", args);
+    trackChildProcess(ffmpeg);
     let stderr = "";
     const onAbort = () => {
       ffmpeg.kill("SIGTERM");

packages/engine/src/utils/processTracker.test.ts

@@ -0,0 +1,74 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { spawn } from "node:child_process";
+import { trackChildProcess, killTrackedProcesses } from "./processTracker.js";
+
+// Reset tracked set between tests by killing everything
+beforeEach(() => {
+  killTrackedProcesses();
+});
+
+describe("trackChildProcess", () => {
+  it("tracks a spawned process and removes it after exit", async () => {
+    const proc = spawn("echo", ["hello"], { stdio: "ignore" });
+    trackChildProcess(proc);
+
+    await new Promise<void>((resolve) => proc.on("close", resolve));
+
+    // After exit, killTrackedProcesses should be a no-op (nothing to kill)
+    killTrackedProcesses();
+  });
+
+  it("removes the process on spawn error", async () => {
+    const proc = spawn("/nonexistent-binary-that-does-not-exist", { stdio: "ignore" });
+    trackChildProcess(proc);
+
+    await new Promise<void>((resolve) => proc.on("error", () => resolve()));
+
+    killTrackedProcesses();
+  });
+});
+
+describe("killTrackedProcesses", () => {
+  it("kills a running process", async () => {
+    const proc = spawn("sleep", ["60"], { stdio: "ignore" });
+    trackChildProcess(proc);
+
+    const exitPromise = new Promise<number | null>((resolve) => proc.on("close", resolve));
+    killTrackedProcesses();
+
+    const code = await exitPromise;
+    // SIGTERM exit: code is null (killed by signal)
+    expect(code).toBeNull();
+  });
+
+  it("handles already-exited processes gracefully", async () => {
+    const proc = spawn("true", { stdio: "ignore" });
+    trackChildProcess(proc);
+
+    await new Promise<void>((resolve) => proc.on("close", resolve));
+
+    // Should not throw even though process already exited
+    killTrackedProcesses();
+  });
+
+  it("escalates to SIGKILL for processes that ignore SIGTERM", async () => {
+    // Spawn a process that traps SIGTERM (bash ignoring it)
+    const proc = spawn("bash", ["-c", "trap '' TERM; sleep 60"], { stdio: "ignore" });
+    trackChildProcess(proc);
+
+    const exitPromise = new Promise<void>((resolve) => proc.on("close", resolve));
+    killTrackedProcesses();
+
+    // The 500ms SIGKILL escalation should kill it
+    await exitPromise;
+    expect(proc.killed).toBe(true);
+  }, 5000);
+
+  it("is idempotent — second call is a no-op", () => {
+    const proc = spawn("sleep", ["60"], { stdio: "ignore" });
+    trackChildProcess(proc);
+
+    killTrackedProcesses();
+    killTrackedProcesses();
+  });
+});

packages/engine/src/utils/processTracker.ts

@@ -0,0 +1,41 @@
+import type { ChildProcess } from "node:child_process";
+
+const tracked = new Set<ChildProcess>();
+
+export function trackChildProcess(proc: ChildProcess): void {
+  tracked.add(proc);
+  const remove = () => tracked.delete(proc);
+  proc.once("exit", remove);
+  proc.once("error", remove);
+}
+
+/**
+ * SIGTERM all tracked child processes, then SIGKILL any that survive
+ * after a short grace period.
+ */
+export function killTrackedProcesses(): void {
+  const alive: ChildProcess[] = [];
+  for (const proc of tracked) {
+    if (!proc.killed) {
+      try {
+        proc.kill("SIGTERM");
+        alive.push(proc);
+      } catch {
+        // Already exited between the check and the kill.
+      }
+    }
+  }
+  tracked.clear();
+
+  if (alive.length === 0) return;
+
+  setTimeout(() => {
+    for (const proc of alive) {
+      try {
+        proc.kill("SIGKILL");
+      } catch {
+        // Already exited.
+      }
+    }
+  }, 500).unref();
+}

packages/engine/src/utils/runFfmpeg.ts

@@ -6,6 +6,7 @@
  */
 
 import { spawn } from "child_process";
+import { trackChildProcess } from "./processTracker.js";
 
 export interface RunFfmpegOptions {
   signal?: AbortSignal;
@@ -60,6 +61,7 @@ export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promis
 
   return new Promise<RunFfmpegResult>((resolve) => {
     const ffmpeg = spawn("ffmpeg", args);
+    trackChildProcess(ffmpeg);
     let stderr = "";
 
     const onAbort = () => {

packages/producer/src/services/audioExtractor.ts

@@ -8,6 +8,7 @@
 import { spawn } from "node:child_process";
 import { existsSync, mkdirSync, rmSync, readFileSync } from "node:fs";
 import { join, dirname } from "node:path";
+import { trackChildProcess } from "@hyperframes/engine";
 
 export interface AudioElement {
   id: string;
@@ -82,6 +83,7 @@ export function parseAudioElements(html: string): AudioElement[] {
 function runFFmpeg(args: string[]): Promise<void> {
   return new Promise((resolve, reject) => {
     const ffmpeg = spawn("ffmpeg", args);
+    trackChildProcess(ffmpeg);
     let stderr = "";
 
     ffmpeg.stderr.on("data", (data) => {