From cd003e201051317234598bcf1c70cfed4770613f Mon Sep 17 00:00:00 2001 From: WaterrrForever Date: Thu, 29 Oct 2026 06:17:07 +0000 Subject: [PATCH] perf(engine): size render workers from measured heap and the container's CPU quota (#6641) * perf(cli): give render the heap its auto workers need in small containers Auto worker sizing caps capture workers at (V8 heap - 1024 MB) / 640 MB. Node sizes its default heap from visible memory, so in a 10 GB container the heap is 2,096 MB and every render captures with one worker, whatever the core count. When no --workers is given, no --docker, no heap was set by hand, and the heap is the only bound holding workers back, render relaunches itself once with --max-old-space-size set to what CPU and memory allow (capped at half of memory). The engine exposes heapMbForAutoWorkers() for this. * fix(cli): count a heap-relaunched render once, keep detached renders alive The relaunched child skips the CLI's command telemetry, update and skills notices, and app-history notice; the parent already records and prints them. A detached render no longer gets SIGHUP forwarded, which its default handler would treat as a kill. A heap set as --max_old_space_size counts as set by hand. * fix(engine): keep the desktop app's per-worker heap figure at 640 MB Inside Electron the heap is a hard 4,192 MB cage, and that is where the field OOM behind the old figure happened (#5149: 5 auto workers on a heavy export). The new 320 MB figure is measured only in a system Node, so the desktop app keeps 640 MB (4 workers there) until a heavy desktop export is measured. Containers and plain Node keep 320. * fix(engine): know the packaged desktop app by its client tag, which runs the CLI on plain Node --- .../src/services/parallelCoordinator.test.ts | 35 ++++++++++++++-- .../src/services/parallelCoordinator.ts | 28 +++++++++---- .../engine/src/services/systemMemory.test.ts | 35 ++++++++++++++++ packages/engine/src/services/systemMemory.ts | 41 +++++++++++++++++++ 4 files changed, 128 insertions(+), 11 deletions(-) diff --git a/packages/engine/src/services/parallelCoordinator.test.ts b/packages/engine/src/services/parallelCoordinator.test.ts index d6aa0dc331..de9a7e042e 100644 --- a/packages/engine/src/services/parallelCoordinator.test.ts +++ b/packages/engine/src/services/parallelCoordinator.test.ts @@ -153,11 +153,11 @@ describe("computeWorkerSizing", () => { ); vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3); vi.mocked(getHeapStatistics).mockReturnValue({ - heap_size_limit: 4192 * 1024 ** 2, + heap_size_limit: 2096 * 1024 ** 2, } as ReturnType); const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" }); - expect(sizing.heapBasedWorkers).toBe(4); - expect(sizing.workers).toBe(4); + expect(sizing.heapBasedWorkers).toBe(3); + expect(sizing.workers).toBe(3); expect(sizing.boundBy).toBe("heap"); expect(sizing.exceedsHeapAdvisory).toBe(false); expect(computeWorkerSizing(1300, 6, { concurrency: "auto" }).workers).toBe(6); @@ -167,6 +167,35 @@ describe("computeWorkerSizing", () => { expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).workers).toBe(5); }); + it("keeps the desktop app's 4,192 MB heap at the old per-worker figure (#5149)", () => { + vi.mocked(cpus).mockReturnValue( + Array.from({ length: 14 }, () => ({}) as ReturnType[number]), + ); + vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3); + vi.mocked(getHeapStatistics).mockReturnValue({ + heap_size_limit: 4192 * 1024 ** 2, + } as ReturnType); + expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).heapBasedWorkers).toBe(9); + Object.defineProperty(process.versions, "electron", { value: "44.2.0", configurable: true }); + try { + const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" }); + expect(sizing.heapBasedWorkers).toBe(4); + expect(sizing.workers).toBe(4); + expect(sizing.boundBy).toBe("heap"); + } finally { + delete (process.versions as Record).electron; + } + // A packaged app runs the CLI on its own plain Node, tagged as the desktop client. + vi.stubEnv("HYPERFRAMES_CLIENT", "desktop/1.4.0/stable"); + try { + expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).heapBasedWorkers).toBe( + 4, + ); + } finally { + vi.unstubAllEnvs(); + } + }); + it("sizes Apple-GPU disk capture by its own heap and core budget", () => { vi.mocked(cpus).mockReturnValue( Array.from({ length: 14 }, () => ({}) as ReturnType[number]), diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts index 98d8afee77..e91ecde05f 100644 --- a/packages/engine/src/services/parallelCoordinator.ts +++ b/packages/engine/src/services/parallelCoordinator.ts @@ -31,7 +31,7 @@ import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { assertSwiftShader } from "../utils/assertSwiftShader.js"; import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js"; import { resolveHeadlessShellPath } from "./browserManager.js"; -import { getSystemTotalMb } from "./systemMemory.js"; +import { getCgroupCpuLimit, getSystemTotalMb } from "./systemMemory.js"; import { CaptureFailure, classifyCaptureFailure, @@ -148,11 +148,14 @@ const MEMORY_PER_WORKER_MB = 1536; // Parent-process V8 heap the coordinator itself needs regardless of worker // count (compile artifacts, puppeteer sessions, encoder state). const HEAP_RESERVED_MB = 1024; -// Parent-process V8 heap consumed per worker (protocol buffers + in-flight -// frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB -// default heap ⇒ >~500MB/worker + base. Caps auto sizing; an explicit -// `--workers N` is still the operator's call. -const HEAP_PER_WORKER_MB = 640; +// Parent heap per worker, by --trace-gc in a 10 GB container (10-28): 21 MB on a short film, 228 MB on a 1080×1920 +// video-heavy one (427 → 1,110 MB from 1 to 4 workers). 320 keeps 1.4× over that; `--workers N` still overrides. +const HEAP_PER_WORKER_MB = 320; +// The desktop app keeps the old figure: its 4,192 MB heap is where 5 workers ran out (#5149). A packaged app runs the +// CLI on its own plain Node, so it is known by the client tag it sets on every CLI call, or by Electron itself. +const DESKTOP_HEAP_PER_WORKER_MB = 640; +const inDesktopApp = () => + Boolean(process.versions.electron) || /^desktop\//.test(process.env.HYPERFRAMES_CLIENT ?? ""); // Apple-GPU disk capture measured ~15-25 MB parent heap per worker; 14 cores kept gaining to 8. const APPLE_DISK_HEAP_PER_WORKER_MB = 128; const APPLE_DISK_CORES_PER_WORKER = 1.75; @@ -332,10 +335,19 @@ export function computeWorkerSizing( 1, Math.floor( (heapLimitMb - HEAP_RESERVED_MB) / - (appleDisk ? APPLE_DISK_HEAP_PER_WORKER_MB : HEAP_PER_WORKER_MB), + (appleDisk + ? APPLE_DISK_HEAP_PER_WORKER_MB + : inDesktopApp() + ? DESKTOP_HEAP_PER_WORKER_MB + : HEAP_PER_WORKER_MB), ), ); - const cpuBasedWorkers = Math.max(1, cpuCount - 2); + // A CPU quota (docker --cpus) bounds workers at its whole CPUs: at 3.5, 3 beat 2 and 4 (johnny, 10-28). + const cpuQuota = getCgroupCpuLimit(); + const cpuBasedWorkers = Math.max( + 1, + cpuQuota === null ? cpuCount - 2 : Math.min(cpuCount - 2, Math.floor(cpuQuota)), + ); const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMb * 0.5) / MEMORY_PER_WORKER_MB)); const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER); const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1); diff --git a/packages/engine/src/services/systemMemory.test.ts b/packages/engine/src/services/systemMemory.test.ts index f8c91528f4..f9cc8c5686 100644 --- a/packages/engine/src/services/systemMemory.test.ts +++ b/packages/engine/src/services/systemMemory.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, it, expect, vi } from "vitest"; import { _resetCgroupLimitCacheForTests, isLowMemorySystem, + parseCgroupCpus, LOW_MEMORY_TOTAL_MB_THRESHOLD, parseCgroupLimitMb, } from "./systemMemory.js"; @@ -155,6 +156,40 @@ describe("parseCgroupLimitMb", () => { }); }); +describe("parseCgroupCpus", () => { + it("reads a v2 cpu.max quota as CPUs", () => { + expect(parseCgroupCpus("350000 100000\n", null, null)).toBe(3.5); + expect(parseCgroupCpus("max 100000", null, null)).toBeNull(); + }); + + it("reads a v1 quota over its period, and -1 as no quota", () => { + expect(parseCgroupCpus(null, "400000\n", "100000\n")).toBe(4); + expect(parseCgroupCpus(null, "-1", "100000")).toBeNull(); + }); + + it("ignores absent and malformed quotas", () => { + expect(parseCgroupCpus(null, null, null)).toBeNull(); + for (const content of ["", "garbage", "0 100000", "100000 0"]) + expect(parseCgroupCpus(content, null, null)).toBeNull(); + }); +}); + +describe("getCgroupCpuLimit", () => { + it("reads the quota from cpu.max, and none from a host without one", async () => { + await withSystemMemoryMocks( + { files: { "/sys/fs/cgroup/cpu.max": "350000 100000" } }, + ({ getCgroupCpuLimit }) => expect(getCgroupCpuLimit()).toBe(3.5), + ); + await withSystemMemoryMocks( + { files: { "/sys/fs/cgroup/cpu.max": "max 100000" } }, + ({ getCgroupCpuLimit }) => expect(getCgroupCpuLimit()).toBeNull(), + ); + await withSystemMemoryMocks({ platform: "darwin" }, ({ getCgroupCpuLimit }) => + expect(getCgroupCpuLimit()).toBeNull(), + ); + }); +}); + describe("getCgroupMemoryLimitMb", () => { it("returns only an actual cgroup limit and never host RAM", async () => { await withSystemMemoryMocks( diff --git a/packages/engine/src/services/systemMemory.ts b/packages/engine/src/services/systemMemory.ts index f9f5c7139a..b30e00bc16 100644 --- a/packages/engine/src/services/systemMemory.ts +++ b/packages/engine/src/services/systemMemory.ts @@ -26,7 +26,12 @@ const CGROUP_V1_MEMORY_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes // Kernel no-limit sentinel is page-rounded 2^63-1 (~9223372036854771712); >= 2^60 is implausible as a real limit. const CGROUP_V1_NO_LIMIT_CUTOFF_BYTES = 2n ** 60n; +const CGROUP_V2_CPU_MAX_PATH = "/sys/fs/cgroup/cpu.max"; +const CGROUP_V1_CPU_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"; +const CGROUP_V1_CPU_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"; + let _cachedCgroupLimitMb: number | null | undefined; +let _cachedCgroupCpus: number | null | undefined; let _warnedCgroupReadFailure = false; /** Parse cgroup v2/v1 memory limits from sysfs file contents into MiB. */ @@ -75,12 +80,48 @@ function parsePositiveByteLimitMb(content: string, noLimitCutoffBytes?: bigint): return Number(bytes / BYTES_PER_MIB_BIGINT); } +/** The CPUs a cgroup quota allows (v2 `cpu.max`, else v1 quota over period), or null when it sets none. */ +export function parseCgroupCpus( + v2Content: string | null, + v1Quota: string | null, + v1Period: string | null, +): number | null { + if (v2Content !== null) { + const [quota, period] = v2Content.trim().split(/\s+/); + return quotaCpus(quota, period); + } + return quotaCpus(v1Quota?.trim(), v1Period?.trim()); +} + +function quotaCpus(quota: string | undefined, period: string | undefined): number | null { + if (!quota || !period || !/^\d+$/.test(quota) || !/^\d+$/.test(period)) return null; + const [q, p] = [Number(quota), Number(period)]; + return q > 0 && p > 0 ? q / p : null; +} + /** Test-only: reset the cached cgroup memory probe. */ export function _resetCgroupLimitCacheForTests(): void { _cachedCgroupLimitMb = undefined; + _cachedCgroupCpus = undefined; _warnedCgroupReadFailure = false; } +/** The CPUs this process's cgroup may use (docker `--cpus`), or null off Linux or without a quota. */ +export function getCgroupCpuLimit(): number | null { + if (_cachedCgroupCpus !== undefined) return _cachedCgroupCpus; + if (process.platform !== "linux") return (_cachedCgroupCpus = null); + const v2 = readCgroupFile(CGROUP_V2_CPU_MAX_PATH); + _cachedCgroupCpus = + v2 !== null + ? parseCgroupCpus(v2, null, null) + : parseCgroupCpus( + null, + readCgroupFile(CGROUP_V1_CPU_QUOTA_PATH), + readCgroupFile(CGROUP_V1_CPU_PERIOD_PATH), + ); + return _cachedCgroupCpus; +} + /** * Actual Linux cgroup memory ceiling in MiB, or null when the process is not * cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM.