mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(engine): render work is sized by the CPUs the process may use (#7612)
* fix(engine): render work is sized by the CPUs the process may use * fix(engine): keep the usable CPU count comment under the line limit
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { cpus } from "node:os";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { availableParallelism } from "node:os";
|
||||
import { fpsToFfmpegArg } from "@hyperframes/core";
|
||||
import { c } from "../../ui/colors.js";
|
||||
import type { RenderPlan } from "./plan.js";
|
||||
@@ -30,7 +30,7 @@ function presentRenderSummary(plan: RenderPlan): void {
|
||||
const workerLabel =
|
||||
plan.workers != null
|
||||
? `${plan.workers} workers`
|
||||
: `auto workers (${cpus().length} cores detected)`;
|
||||
: `auto workers (${availableParallelism()} cores usable)`;
|
||||
const nameLabel = plan.entryFile ? `${plan.project.name}/${plan.entryFile}` : plan.project.name;
|
||||
console.log("");
|
||||
console.log(
|
||||
|
||||
@@ -85,6 +85,8 @@ export {
|
||||
export {
|
||||
getCgroupMemoryLimitMb,
|
||||
getSystemTotalMb,
|
||||
hostCpuCount,
|
||||
usableCpuCount,
|
||||
isLowMemorySystem,
|
||||
LOW_MEMORY_TOTAL_MB_THRESHOLD,
|
||||
} from "./services/systemMemory.js";
|
||||
|
||||
@@ -13,6 +13,7 @@ vi.mock("../utils/runFfmpeg.js", async (importOriginal) => ({
|
||||
// Enough cores that every preparation starts at once, so they can finish in reverse.
|
||||
vi.mock("node:os", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("node:os")>()),
|
||||
availableParallelism: () => 16,
|
||||
cpus: () =>
|
||||
Array.from({ length: 16 }, () => ({}) as ReturnType<typeof import("node:os").cpus>[number]),
|
||||
}));
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { copyFileSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { cpus, tmpdir } from "node:os";
|
||||
import { availableParallelism, tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
|
||||
@@ -52,7 +52,7 @@ describe.skipIf(!HAS_FFMPEG)("preparing the clips of a long mix", () => {
|
||||
{ encoding: "utf-8" },
|
||||
);
|
||||
expect(tone.status).toBe(0);
|
||||
const slots = Math.max(2, Math.floor(cpus().length / 2));
|
||||
const slots = Math.max(2, Math.floor(availableParallelism() / 2));
|
||||
const count = Math.max(40, slots * 3);
|
||||
const clips = Array.from({ length: count }, (_, i) => {
|
||||
copyFileSync(join(projectDir, "a.wav"), join(projectDir, `a${i}.wav`));
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// fallow-ignore-file complexity code-duplication
|
||||
import { usableCpuCount } from "./systemMemory.js";
|
||||
import { isSelfOrAncestorHidden, memberGroupKey, isMemberGroupHidden } from "./mediaHidden.js";
|
||||
import { closeSync, existsSync, mkdirSync, openSync, rmSync, writeFileSync } from "fs";
|
||||
import { renameWithRetry } from "@hyperframes/core/rename-retry";
|
||||
@@ -23,7 +24,6 @@ import {
|
||||
} from "../utils/urlDownloader.js";
|
||||
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
|
||||
import { describeFfmpegFailure, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
|
||||
import { cpus } from "node:os";
|
||||
import { createConcurrencyLimit } from "../utils/concurrencyLimit.js";
|
||||
import { encodeAac } from "./aacEncode.js";
|
||||
import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js";
|
||||
@@ -210,7 +210,7 @@ function legacyFilterScriptOptionIsUnsupported(stderr: string): boolean {
|
||||
* than a piecewise-linear volume envelope needs.
|
||||
*/
|
||||
// Caps the ffmpeg runs that prepare clips; all of a long mix's clips at once starved the host (200 clips, 200 ffmpeg).
|
||||
const inPrepareSlot = createConcurrencyLimit(Math.max(2, Math.floor(cpus().length / 2)));
|
||||
const inPrepareSlot = createConcurrencyLimit(Math.max(2, Math.floor(usableCpuCount() / 2)));
|
||||
// A clip read from a URL spends its time waiting on the network, so it needs no CPU-sized limit.
|
||||
const inRemotePrepareSlot = createConcurrencyLimit(16);
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import { processCompositionAudio } from "./audioMixer.js";
|
||||
|
||||
vi.mock("node:os", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("node:os")>()),
|
||||
availableParallelism: () => 8,
|
||||
cpus: () =>
|
||||
Array.from({ length: 8 }, () => ({}) as ReturnType<typeof import("node:os").cpus>[number]),
|
||||
}));
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { cpus, totalmem } from "os";
|
||||
import { availableParallelism, totalmem } from "os";
|
||||
import { getHeapStatistics } from "v8";
|
||||
import { afterEach, describe, it, expect, vi } from "vitest";
|
||||
import {
|
||||
@@ -132,7 +132,12 @@ describe("calculateOptimalWorkers", () => {
|
||||
|
||||
vi.mock("os", async (importOriginal) => {
|
||||
const os = await importOriginal<typeof import("os")>();
|
||||
return { ...os, default: os, cpus: vi.fn(os.cpus), totalmem: vi.fn(os.totalmem) };
|
||||
return {
|
||||
...os,
|
||||
default: os,
|
||||
availableParallelism: vi.fn(os.availableParallelism),
|
||||
totalmem: vi.fn(os.totalmem),
|
||||
};
|
||||
});
|
||||
vi.mock("v8", async (importOriginal) => {
|
||||
const v8 = await importOriginal<typeof import("v8")>();
|
||||
@@ -141,16 +146,14 @@ vi.mock("v8", async (importOriginal) => {
|
||||
|
||||
describe("computeWorkerSizing", () => {
|
||||
afterEach(() => {
|
||||
vi.mocked(cpus).mockRestore();
|
||||
vi.mocked(availableParallelism).mockRestore();
|
||||
vi.mocked(totalmem).mockRestore();
|
||||
vi.mocked(getHeapStatistics).mockRestore();
|
||||
});
|
||||
|
||||
// The field case: a 24GB 14-core Mac, Node's default ~4GB heap, a long 1080x1920 project.
|
||||
it("never auto-picks more workers than the V8 heap can feed", () => {
|
||||
vi.mocked(cpus).mockReturnValue(
|
||||
Array.from({ length: 14 }, () => ({}) as ReturnType<typeof cpus>[number]),
|
||||
);
|
||||
vi.mocked(availableParallelism).mockReturnValue(14);
|
||||
vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
|
||||
vi.mocked(getHeapStatistics).mockReturnValue({
|
||||
heap_size_limit: 2096 * 1024 ** 2,
|
||||
@@ -168,9 +171,7 @@ describe("computeWorkerSizing", () => {
|
||||
});
|
||||
|
||||
it("keeps the desktop app's 4,192 MB heap at the old per-worker figure (#5149)", () => {
|
||||
vi.mocked(cpus).mockReturnValue(
|
||||
Array.from({ length: 14 }, () => ({}) as ReturnType<typeof cpus>[number]),
|
||||
);
|
||||
vi.mocked(availableParallelism).mockReturnValue(14);
|
||||
vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
|
||||
vi.mocked(getHeapStatistics).mockReturnValue({
|
||||
heap_size_limit: 4192 * 1024 ** 2,
|
||||
@@ -197,9 +198,7 @@ describe("computeWorkerSizing", () => {
|
||||
});
|
||||
|
||||
it("sizes Apple-GPU disk capture by its own heap and core budget", () => {
|
||||
vi.mocked(cpus).mockReturnValue(
|
||||
Array.from({ length: 14 }, () => ({}) as ReturnType<typeof cpus>[number]),
|
||||
);
|
||||
vi.mocked(availableParallelism).mockReturnValue(14);
|
||||
vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
|
||||
vi.mocked(getHeapStatistics).mockReturnValue({
|
||||
heap_size_limit: 4192 * 1024 ** 2,
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* Auto-detects optimal worker count based on CPU/memory.
|
||||
*/
|
||||
|
||||
import { cpus, freemem } from "os";
|
||||
import { freemem } from "os";
|
||||
import { existsSync, mkdirSync, readdirSync } from "fs";
|
||||
import { copyFile, readFile } from "fs/promises";
|
||||
import { renameWithRetryAsync } from "@hyperframes/core/rename-retry";
|
||||
@@ -31,7 +31,12 @@ import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
|
||||
import { assertSwiftShader } from "../utils/assertSwiftShader.js";
|
||||
import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js";
|
||||
import { resolveHeadlessShellPath } from "./browserManager.js";
|
||||
import { getCgroupCpuLimit, getSystemTotalMb } from "./systemMemory.js";
|
||||
import {
|
||||
getCgroupCpuLimit,
|
||||
getSystemTotalMb,
|
||||
hostCpuCount,
|
||||
usableCpuCount,
|
||||
} from "./systemMemory.js";
|
||||
import {
|
||||
CaptureFailure,
|
||||
classifyCaptureFailure,
|
||||
@@ -177,7 +182,7 @@ const ABSOLUTE_MAX_WORKERS = 24;
|
||||
// worker's SwiftShader compositor + the shader-blend thread pool, both of
|
||||
// which are themselves CPU-heavy.
|
||||
function defaultSafeMaxWorkers(): number {
|
||||
return Math.max(6, Math.min(16, Math.floor(cpus().length / 8)));
|
||||
return Math.max(6, Math.min(16, Math.floor(usableCpuCount() / 8)));
|
||||
}
|
||||
const MIN_FRAMES_PER_WORKER = 30;
|
||||
|
||||
@@ -324,7 +329,7 @@ export function computeWorkerSizing(
|
||||
requested?: number,
|
||||
config?: WorkerSizingConfig,
|
||||
): WorkerSizing {
|
||||
const cpuCount = cpus().length;
|
||||
const cpuCount = usableCpuCount();
|
||||
// Use total memory instead of free memory — macOS reports misleadingly low
|
||||
// freemem() because it aggressively caches files in "inactive" memory that
|
||||
// is immediately reclaimable.
|
||||
@@ -1297,7 +1302,7 @@ export function getSystemResources(): {
|
||||
recommendedWorkers: number;
|
||||
} {
|
||||
return {
|
||||
cpuCores: cpus().length,
|
||||
cpuCores: hostCpuCount(),
|
||||
totalMemoryMB: getSystemTotalMb(),
|
||||
freeMemoryMB: Math.round(freemem() / (1024 * 1024)),
|
||||
recommendedWorkers: calculateOptimalWorkers(1000),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { availableParallelism } from "node:os";
|
||||
import { usableCpuCount } from "./systemMemory.js";
|
||||
|
||||
export const X265_MAX_THREADS = 16;
|
||||
|
||||
@@ -6,7 +6,7 @@ export const X265_MAX_THREADS = 16;
|
||||
export function appendSoftwareEncoderThreadsArg(
|
||||
args: string[],
|
||||
encoder: "libx264" | "libx265",
|
||||
parallelism: number = availableParallelism(),
|
||||
parallelism: number = usableCpuCount(),
|
||||
): void {
|
||||
const cap = encoder === "libx265" ? X265_MAX_THREADS : Infinity;
|
||||
args.push("-threads", String(Math.min(cap, Math.max(1, Math.floor(parallelism)))));
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { cpus } from "node:os";
|
||||
import { join, sep } from "node:path";
|
||||
import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
|
||||
import {
|
||||
_resetCgroupLimitCacheForTests,
|
||||
@@ -403,3 +407,46 @@ describe("getSystemTotalMb", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
const hasTaskset = process.platform === "linux" && spawnSync("taskset", ["-V"]).status === 0;
|
||||
|
||||
describe.skipIf(!hasTaskset)("usableCpuCount under a cpuset", () => {
|
||||
it("is the CPUs the process may run on, while hostCpuCount stays the machine's", () => {
|
||||
const run = spawnSync(
|
||||
"taskset",
|
||||
[
|
||||
...["-c", "0", process.execPath, "--experimental-strip-types", "--no-warnings", "-e"],
|
||||
`import(${JSON.stringify(join(import.meta.dirname, "systemMemory.ts"))})` +
|
||||
".then((m) => console.log(m.usableCpuCount(), m.hostCpuCount()))",
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
expect(run.stderr).toBe("");
|
||||
expect(run.stdout.trim()).toBe(`1 ${cpus().length}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("work is sized by usableCpuCount", () => {
|
||||
it("leaves os.cpus() to systemMemory.ts in engine and producer sources", () => {
|
||||
const packages = join(import.meta.dirname, "../../..");
|
||||
const callers: string[] = [];
|
||||
const walk = (dir: string): void => {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(path);
|
||||
else if (/\.ts$/.test(entry.name) && !/\.test\.ts$/.test(entry.name)) {
|
||||
if (/\bcpus\(\)/.test(readFileSync(path, "utf8")))
|
||||
callers.push(
|
||||
path
|
||||
.slice(packages.length + 1)
|
||||
.split(sep)
|
||||
.join("/"),
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(join(packages, "engine/src"));
|
||||
walk(join(packages, "producer/src"));
|
||||
expect(callers).toEqual(["engine/src/services/systemMemory.ts"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
import { execSync } from "child_process";
|
||||
import { readFileSync } from "fs";
|
||||
import { freemem, platform, totalmem } from "os";
|
||||
import { availableParallelism, cpus, freemem, platform, totalmem } from "os";
|
||||
|
||||
const BYTES_PER_MIB = 1024 * 1024;
|
||||
const BYTES_PER_MIB_BIGINT = BigInt(BYTES_PER_MIB);
|
||||
@@ -269,3 +269,12 @@ export function getAvailableMemoryMb(): number {
|
||||
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/** CPUs this process may run on: a cpuset (taskset, --cpuset-cpus) lowers it, a CPU quota does not. */
|
||||
export function usableCpuCount(): number {
|
||||
return availableParallelism();
|
||||
}
|
||||
|
||||
export function hostCpuCount(): number {
|
||||
return cpus().length;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import { runFfmpeg } from "../utils/runFfmpeg.js";
|
||||
// Two CPUs: the extractor runs at most one ffmpeg at a time.
|
||||
vi.mock("os", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("os")>()),
|
||||
availableParallelism: () => 2,
|
||||
cpus: () => [{}, {}],
|
||||
}));
|
||||
|
||||
|
||||
@@ -2,7 +2,8 @@ import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { cpus, tmpdir } from "node:os";
|
||||
import { tmpdir } from "node:os";
|
||||
import { usableCpuCount } from "./systemMemory.js";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { VideoElement } from "../types.js";
|
||||
@@ -89,7 +90,7 @@ describe.skipIf(!HAS_FFMPEG)("deferred windows of a remote source", { timeout: 1
|
||||
const result = await defer((i) => 5 + i * 0.05);
|
||||
|
||||
expect(result.extracted.length).toBe(24);
|
||||
expect(peak).toBeGreaterThan(Math.max(1, Math.floor(cpus().length / 2)));
|
||||
expect(peak).toBeGreaterThan(Math.max(1, Math.floor(usableCpuCount() / 2)));
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import { runFfmpeg } from "../utils/runFfmpeg.js";
|
||||
// Four CPUs: two extraction slots.
|
||||
vi.mock("os", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("os")>()),
|
||||
availableParallelism: () => 4,
|
||||
cpus: () => [{}, {}, {}, {}],
|
||||
}));
|
||||
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
* Videos are replaced with <img> elements during capture.
|
||||
*/
|
||||
|
||||
import { usableCpuCount } from "./systemMemory.js";
|
||||
import { audioGroupsById, isMemberGroupHidden, isSelfOrAncestorHidden } from "./mediaHidden.js";
|
||||
import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync } from "fs";
|
||||
import { basename, isAbsolute, join, normalize, relative, win32 } from "path";
|
||||
import { cpus } from "os";
|
||||
import { createConcurrencyLimit } from "../utils/concurrencyLimit.js";
|
||||
import { parseHTML } from "linkedom";
|
||||
import { HF_COLOR_GRADING_ATTR } from "@hyperframes/core";
|
||||
@@ -1051,7 +1051,7 @@ function undoTickRoundingOnTheSourceFrameGrid(
|
||||
}
|
||||
|
||||
// Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts.
|
||||
const EXTRACTION_SLOTS = Math.max(1, Math.floor(cpus().length / 2));
|
||||
const EXTRACTION_SLOTS = Math.max(1, Math.floor(usableCpuCount() / 2));
|
||||
const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS);
|
||||
const TAIL_PROBE_SLOTS = Math.max(1, Number(process.env.HF_TAIL_PROBE_SLOTS) || 16);
|
||||
const inTailProbeSlot = createConcurrencyLimit(TAIL_PROBE_SLOTS);
|
||||
|
||||
@@ -36,7 +36,6 @@
|
||||
import { chunkGopFrames } from "../render/deliveryEncoder.js";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { cpus } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
assertSwiftShader,
|
||||
@@ -59,6 +58,7 @@ import {
|
||||
readWebGlVendorInfoFromCanvas,
|
||||
resolveConfig,
|
||||
type ShownBox,
|
||||
usableCpuCount,
|
||||
} from "@hyperframes/engine";
|
||||
import { defaultLogger } from "../../logger.js";
|
||||
import {
|
||||
@@ -1043,7 +1043,7 @@ export async function renderChunk(
|
||||
const captureSession = session;
|
||||
session = null;
|
||||
verifyStarted = Date.now();
|
||||
overlap.current = canEncodeWhileCapturing(encodeInput, chunkWorkerCount, cpus().length)
|
||||
overlap.current = canEncodeWhileCapturing(encodeInput, chunkWorkerCount, usableCpuCount())
|
||||
? encodeWhileCapturing(encodeInput, framesInChunk)
|
||||
: null;
|
||||
await observeRenderStage(
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
* the `perf-summary.json` debug artifact.
|
||||
*/
|
||||
|
||||
import { arch, cpus, platform, totalmem } from "node:os";
|
||||
import { arch, platform, totalmem } from "node:os";
|
||||
import { fpsToNumber, type HfVfxCapture } from "@hyperframes/core";
|
||||
import { hostCpuCount } from "@hyperframes/engine";
|
||||
import type {
|
||||
CapturePerfSummary,
|
||||
StaticVerificationOutcome,
|
||||
@@ -469,7 +470,7 @@ export function buildRenderPerfSummary(input: {
|
||||
host: {
|
||||
platform: platform(),
|
||||
arch: arch(),
|
||||
cpuCount: cpus().length,
|
||||
cpuCount: hostCpuCount(),
|
||||
totalMemMb: Math.round(totalmem() / (1024 * 1024)),
|
||||
nodeVersion: process.version,
|
||||
gpuDisabled: input.gpuDisabled,
|
||||
|
||||
@@ -39,8 +39,8 @@ import { Worker } from "node:worker_threads";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
import { usableCpuCount } from "@hyperframes/engine";
|
||||
import { existsSync } from "node:fs";
|
||||
import { cpus } from "node:os";
|
||||
|
||||
interface PoolLogger {
|
||||
info?: (msg: string, meta?: Record<string, unknown>) => void;
|
||||
@@ -49,7 +49,7 @@ interface PoolLogger {
|
||||
}
|
||||
|
||||
export interface ShaderTransitionPoolOptions {
|
||||
/** Number of worker threads. Clamped to [1, cpus().length]. */
|
||||
/** Number of worker threads. Clamped to [1, usableCpuCount()]. */
|
||||
size: number;
|
||||
/** Optional logger; falls back to no-op. */
|
||||
log?: PoolLogger;
|
||||
@@ -197,7 +197,7 @@ function buildExecArgv(entryIsTs: boolean): string[] {
|
||||
export async function createShaderTransitionWorkerPool(
|
||||
opts: ShaderTransitionPoolOptions,
|
||||
): Promise<ShaderTransitionWorkerPool> {
|
||||
const cpuCount = Math.max(1, cpus().length);
|
||||
const cpuCount = Math.max(1, usableCpuCount());
|
||||
const size = Math.max(1, Math.min(opts.size, cpuCount));
|
||||
const log = opts.log ?? {};
|
||||
const { path: entry, isTs: entryIsTs } = resolveWorkerEntry(opts.workerEntryPath);
|
||||
|
||||
Reference in New Issue
Block a user