perf(engine): size render workers from measured heap and the container's CPU quota (#6641)

* perf(cli): give render the heap its auto workers need in small containers

Auto worker sizing caps capture workers at (V8 heap - 1024 MB) / 640 MB.
Node sizes its default heap from visible memory, so in a 10 GB container
the heap is 2,096 MB and every render captures with one worker, whatever
the core count.

When no --workers is given, no --docker, no heap was set by hand, and the
heap is the only bound holding workers back, render relaunches itself once
with --max-old-space-size set to what CPU and memory allow (capped at half
of memory). The engine exposes heapMbForAutoWorkers() for this.

* fix(cli): count a heap-relaunched render once, keep detached renders alive

The relaunched child skips the CLI's command telemetry, update and skills
notices, and app-history notice; the parent already records and prints
them. A detached render no longer gets SIGHUP forwarded, which its
default handler would treat as a kill. A heap set as --max_old_space_size
counts as set by hand.

* fix(engine): keep the desktop app's per-worker heap figure at 640 MB

Inside Electron the heap is a hard 4,192 MB cage, and that is where the
field OOM behind the old figure happened (#5149: 5 auto workers on a heavy
export). The new 320 MB figure is measured only in a system Node, so the
desktop app keeps 640 MB (4 workers there) until a heavy desktop export is
measured. Containers and plain Node keep 320.

* fix(engine): know the packaged desktop app by its client tag, which runs the CLI on plain Node

packages/engine/src/services/parallelCoordinator.test.ts

@@ -153,11 +153,11 @@ describe("computeWorkerSizing", () => {
     );
     vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
     vi.mocked(getHeapStatistics).mockReturnValue({
-      heap_size_limit: 4192 * 1024 ** 2,
+      heap_size_limit: 2096 * 1024 ** 2,
     } as ReturnType<typeof getHeapStatistics>);
     const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" });
-    expect(sizing.heapBasedWorkers).toBe(4);
-    expect(sizing.workers).toBe(4);
+    expect(sizing.heapBasedWorkers).toBe(3);
+    expect(sizing.workers).toBe(3);
     expect(sizing.boundBy).toBe("heap");
     expect(sizing.exceedsHeapAdvisory).toBe(false);
     expect(computeWorkerSizing(1300, 6, { concurrency: "auto" }).workers).toBe(6);
@@ -167,6 +167,35 @@ describe("computeWorkerSizing", () => {
     expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).workers).toBe(5);
   });
 
+  it("keeps the desktop app's 4,192 MB heap at the old per-worker figure (#5149)", () => {
+    vi.mocked(cpus).mockReturnValue(
+      Array.from({ length: 14 }, () => ({}) as ReturnType<typeof cpus>[number]),
+    );
+    vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
+    vi.mocked(getHeapStatistics).mockReturnValue({
+      heap_size_limit: 4192 * 1024 ** 2,
+    } as ReturnType<typeof getHeapStatistics>);
+    expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).heapBasedWorkers).toBe(9);
+    Object.defineProperty(process.versions, "electron", { value: "44.2.0", configurable: true });
+    try {
+      const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" });
+      expect(sizing.heapBasedWorkers).toBe(4);
+      expect(sizing.workers).toBe(4);
+      expect(sizing.boundBy).toBe("heap");
+    } finally {
+      delete (process.versions as Record<string, string | undefined>).electron;
+    }
+    // A packaged app runs the CLI on its own plain Node, tagged as the desktop client.
+    vi.stubEnv("HYPERFRAMES_CLIENT", "desktop/1.4.0/stable");
+    try {
+      expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).heapBasedWorkers).toBe(
+        4,
+      );
+    } finally {
+      vi.unstubAllEnvs();
+    }
+  });
+
   it("sizes Apple-GPU disk capture by its own heap and core budget", () => {
     vi.mocked(cpus).mockReturnValue(
       Array.from({ length: 14 }, () => ({}) as ReturnType<typeof cpus>[number]),

packages/engine/src/services/parallelCoordinator.ts

@@ -31,7 +31,7 @@ import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 import { assertSwiftShader } from "../utils/assertSwiftShader.js";
 import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js";
 import { resolveHeadlessShellPath } from "./browserManager.js";
-import { getSystemTotalMb } from "./systemMemory.js";
+import { getCgroupCpuLimit, getSystemTotalMb } from "./systemMemory.js";
 import {
   CaptureFailure,
   classifyCaptureFailure,
@@ -148,11 +148,14 @@ const MEMORY_PER_WORKER_MB = 1536;
 // Parent-process V8 heap the coordinator itself needs regardless of worker
 // count (compile artifacts, puppeteer sessions, encoder state).
 const HEAP_RESERVED_MB = 1024;
-// Parent-process V8 heap consumed per worker (protocol buffers + in-flight
-// frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB
-// default heap ⇒ >~500MB/worker + base. Caps auto sizing; an explicit
-// `--workers N` is still the operator's call.
-const HEAP_PER_WORKER_MB = 640;
+// Parent heap per worker, by --trace-gc in a 10 GB container (10-28): 21 MB on a short film, 228 MB on a 1080×1920
+// video-heavy one (427 → 1,110 MB from 1 to 4 workers). 320 keeps 1.4× over that; `--workers N` still overrides.
+const HEAP_PER_WORKER_MB = 320;
+// The desktop app keeps the old figure: its 4,192 MB heap is where 5 workers ran out (#5149). A packaged app runs the
+// CLI on its own plain Node, so it is known by the client tag it sets on every CLI call, or by Electron itself.
+const DESKTOP_HEAP_PER_WORKER_MB = 640;
+const inDesktopApp = () =>
+  Boolean(process.versions.electron) || /^desktop\//.test(process.env.HYPERFRAMES_CLIENT ?? "");
 // Apple-GPU disk capture measured ~15-25 MB parent heap per worker; 14 cores kept gaining to 8.
 const APPLE_DISK_HEAP_PER_WORKER_MB = 128;
 const APPLE_DISK_CORES_PER_WORKER = 1.75;
@@ -332,10 +335,19 @@ export function computeWorkerSizing(
     1,
     Math.floor(
       (heapLimitMb - HEAP_RESERVED_MB) /
-        (appleDisk ? APPLE_DISK_HEAP_PER_WORKER_MB : HEAP_PER_WORKER_MB),
+        (appleDisk
+          ? APPLE_DISK_HEAP_PER_WORKER_MB
+          : inDesktopApp()
+            ? DESKTOP_HEAP_PER_WORKER_MB
+            : HEAP_PER_WORKER_MB),
     ),
   );
-  const cpuBasedWorkers = Math.max(1, cpuCount - 2);
+  // A CPU quota (docker --cpus) bounds workers at its whole CPUs: at 3.5, 3 beat 2 and 4 (johnny, 10-28).
+  const cpuQuota = getCgroupCpuLimit();
+  const cpuBasedWorkers = Math.max(
+    1,
+    cpuQuota === null ? cpuCount - 2 : Math.min(cpuCount - 2, Math.floor(cpuQuota)),
+  );
   const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMb * 0.5) / MEMORY_PER_WORKER_MB));
   const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER);
   const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1);

packages/engine/src/services/systemMemory.test.ts

@@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
 import {
   _resetCgroupLimitCacheForTests,
   isLowMemorySystem,
+  parseCgroupCpus,
   LOW_MEMORY_TOTAL_MB_THRESHOLD,
   parseCgroupLimitMb,
 } from "./systemMemory.js";
@@ -155,6 +156,40 @@ describe("parseCgroupLimitMb", () => {
   });
 });
 
+describe("parseCgroupCpus", () => {
+  it("reads a v2 cpu.max quota as CPUs", () => {
+    expect(parseCgroupCpus("350000 100000\n", null, null)).toBe(3.5);
+    expect(parseCgroupCpus("max 100000", null, null)).toBeNull();
+  });
+
+  it("reads a v1 quota over its period, and -1 as no quota", () => {
+    expect(parseCgroupCpus(null, "400000\n", "100000\n")).toBe(4);
+    expect(parseCgroupCpus(null, "-1", "100000")).toBeNull();
+  });
+
+  it("ignores absent and malformed quotas", () => {
+    expect(parseCgroupCpus(null, null, null)).toBeNull();
+    for (const content of ["", "garbage", "0 100000", "100000 0"])
+      expect(parseCgroupCpus(content, null, null)).toBeNull();
+  });
+});
+
+describe("getCgroupCpuLimit", () => {
+  it("reads the quota from cpu.max, and none from a host without one", async () => {
+    await withSystemMemoryMocks(
+      { files: { "/sys/fs/cgroup/cpu.max": "350000 100000" } },
+      ({ getCgroupCpuLimit }) => expect(getCgroupCpuLimit()).toBe(3.5),
+    );
+    await withSystemMemoryMocks(
+      { files: { "/sys/fs/cgroup/cpu.max": "max 100000" } },
+      ({ getCgroupCpuLimit }) => expect(getCgroupCpuLimit()).toBeNull(),
+    );
+    await withSystemMemoryMocks({ platform: "darwin" }, ({ getCgroupCpuLimit }) =>
+      expect(getCgroupCpuLimit()).toBeNull(),
+    );
+  });
+});
+
 describe("getCgroupMemoryLimitMb", () => {
   it("returns only an actual cgroup limit and never host RAM", async () => {
     await withSystemMemoryMocks(

packages/engine/src/services/systemMemory.ts

@@ -26,7 +26,12 @@ const CGROUP_V1_MEMORY_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes
 // Kernel no-limit sentinel is page-rounded 2^63-1 (~9223372036854771712); >= 2^60 is implausible as a real limit.
 const CGROUP_V1_NO_LIMIT_CUTOFF_BYTES = 2n ** 60n;
 
+const CGROUP_V2_CPU_MAX_PATH = "/sys/fs/cgroup/cpu.max";
+const CGROUP_V1_CPU_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us";
+const CGROUP_V1_CPU_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us";
+
 let _cachedCgroupLimitMb: number | null | undefined;
+let _cachedCgroupCpus: number | null | undefined;
 let _warnedCgroupReadFailure = false;
 
 /** Parse cgroup v2/v1 memory limits from sysfs file contents into MiB. */
@@ -75,12 +80,48 @@ function parsePositiveByteLimitMb(content: string, noLimitCutoffBytes?: bigint):
   return Number(bytes / BYTES_PER_MIB_BIGINT);
 }
 
+/** The CPUs a cgroup quota allows (v2 `cpu.max`, else v1 quota over period), or null when it sets none. */
+export function parseCgroupCpus(
+  v2Content: string | null,
+  v1Quota: string | null,
+  v1Period: string | null,
+): number | null {
+  if (v2Content !== null) {
+    const [quota, period] = v2Content.trim().split(/\s+/);
+    return quotaCpus(quota, period);
+  }
+  return quotaCpus(v1Quota?.trim(), v1Period?.trim());
+}
+
+function quotaCpus(quota: string | undefined, period: string | undefined): number | null {
+  if (!quota || !period || !/^\d+$/.test(quota) || !/^\d+$/.test(period)) return null;
+  const [q, p] = [Number(quota), Number(period)];
+  return q > 0 && p > 0 ? q / p : null;
+}
+
 /** Test-only: reset the cached cgroup memory probe. */
 export function _resetCgroupLimitCacheForTests(): void {
   _cachedCgroupLimitMb = undefined;
+  _cachedCgroupCpus = undefined;
   _warnedCgroupReadFailure = false;
 }
 
+/** The CPUs this process's cgroup may use (docker `--cpus`), or null off Linux or without a quota. */
+export function getCgroupCpuLimit(): number | null {
+  if (_cachedCgroupCpus !== undefined) return _cachedCgroupCpus;
+  if (process.platform !== "linux") return (_cachedCgroupCpus = null);
+  const v2 = readCgroupFile(CGROUP_V2_CPU_MAX_PATH);
+  _cachedCgroupCpus =
+    v2 !== null
+      ? parseCgroupCpus(v2, null, null)
+      : parseCgroupCpus(
+          null,
+          readCgroupFile(CGROUP_V1_CPU_QUOTA_PATH),
+          readCgroupFile(CGROUP_V1_CPU_PERIOD_PATH),
+        );
+  return _cachedCgroupCpus;
+}
+
 /**
  * Actual Linux cgroup memory ceiling in MiB, or null when the process is not
  * cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM.