mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
* fix(engine): start nvidia-smi, vm_stat, ps and tasklist from trusted folders only The renderer resolves these helpers through the shared trusted lookup and runs the absolute path, so a program with the same name in a project folder is never started in their place. * test(engine): memory and import tests follow helpers started by resolved path
284 lines
10 KiB
TypeScript
284 lines
10 KiB
TypeScript
/**
|
|
* System-memory probing for memory-adaptive render behaviour.
|
|
*
|
|
* The render pipeline tunes itself to the host's RAM in several places —
|
|
* frame-cache sizes (`config.ts`), Chrome heap + GPU budget flags
|
|
* (`browserManager.ts`), and worker count (`parallelCoordinator.ts`).
|
|
* They all need the same "how much memory does this box have" reading, so
|
|
* it lives here once instead of being re-derived inline.
|
|
*/
|
|
|
|
import { execFileSync } from "child_process";
|
|
import { findTrustedExecutable } from "@hyperframes/parsers/ff-binaries";
|
|
import { readFileSync } from "fs";
|
|
import { availableParallelism, cpus, freemem, platform, totalmem } from "os";
|
|
|
|
const BYTES_PER_MIB = 1024 * 1024;
|
|
const BYTES_PER_MIB_BIGINT = BigInt(BYTES_PER_MIB);
|
|
// These are the paths as seen from INSIDE a container, where the runtime
|
|
// mounts the container's own cgroup at the namespace root — the case this
|
|
// probe exists for. They are deliberately not resolved via /proc/self/cgroup:
|
|
// on a bare host under systemd the process's real limit may live in a nested
|
|
// slice (e.g. /sys/fs/cgroup/user.slice/.../memory.max) that these root paths
|
|
// don't see, and that's acceptable — bare hosts are covered by total-RAM
|
|
// detection, and chasing nested slices adds fragility for no container gain.
|
|
const CGROUP_V2_MEMORY_MAX_PATH = "/sys/fs/cgroup/memory.max";
|
|
const CGROUP_V1_MEMORY_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes";
|
|
// Kernel no-limit sentinel is page-rounded 2^63-1 (~9223372036854771712); >= 2^60 is implausible as a real limit.
|
|
const CGROUP_V1_NO_LIMIT_CUTOFF_BYTES = 2n ** 60n;
|
|
|
|
const CGROUP_V2_CPU_MAX_PATH = "/sys/fs/cgroup/cpu.max";
|
|
const CGROUP_V1_CPU_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us";
|
|
const CGROUP_V1_CPU_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us";
|
|
|
|
let _cachedCgroupLimitMb: number | null | undefined;
|
|
let _cachedCgroupCpus: number | null | undefined;
|
|
let _warnedCgroupReadFailure = false;
|
|
|
|
/** Parse cgroup v2/v1 memory limits from sysfs file contents into MiB. */
|
|
export function parseCgroupLimitMb(
|
|
v2Content: string | null,
|
|
v1Content: string | null,
|
|
): number | null {
|
|
if (v2Content !== null) {
|
|
return parseCgroupV2LimitMb(v2Content);
|
|
}
|
|
|
|
return parseCgroupV1LimitMb(v1Content);
|
|
}
|
|
|
|
function parseCgroupV2LimitMb(content: string): number | null {
|
|
const trimmed = content.trim();
|
|
if (trimmed === "max") {
|
|
return null;
|
|
}
|
|
|
|
return parsePositiveByteLimitMb(trimmed);
|
|
}
|
|
|
|
function parseCgroupV1LimitMb(content: string | null): number | null {
|
|
if (content === null) {
|
|
return null;
|
|
}
|
|
|
|
return parsePositiveByteLimitMb(content.trim(), CGROUP_V1_NO_LIMIT_CUTOFF_BYTES);
|
|
}
|
|
|
|
function parsePositiveByteLimitMb(content: string, noLimitCutoffBytes?: bigint): number | null {
|
|
if (!/^\d+$/.test(content)) {
|
|
return null;
|
|
}
|
|
|
|
const bytes = BigInt(content);
|
|
if (bytes <= 0n) {
|
|
return null;
|
|
}
|
|
|
|
if (noLimitCutoffBytes !== undefined && bytes >= noLimitCutoffBytes) {
|
|
return null;
|
|
}
|
|
|
|
return Number(bytes / BYTES_PER_MIB_BIGINT);
|
|
}
|
|
|
|
/** The CPUs a cgroup quota allows (v2 `cpu.max`, else v1 quota over period), or null when it sets none. */
|
|
export function parseCgroupCpus(
|
|
v2Content: string | null,
|
|
v1Quota: string | null,
|
|
v1Period: string | null,
|
|
): number | null {
|
|
if (v2Content !== null) {
|
|
const [quota, period] = v2Content.trim().split(/\s+/);
|
|
return quotaCpus(quota, period);
|
|
}
|
|
return quotaCpus(v1Quota?.trim(), v1Period?.trim());
|
|
}
|
|
|
|
function quotaCpus(quota: string | undefined, period: string | undefined): number | null {
|
|
if (!quota || !period || !/^\d+$/.test(quota) || !/^\d+$/.test(period)) return null;
|
|
const [q, p] = [Number(quota), Number(period)];
|
|
return q > 0 && p > 0 ? q / p : null;
|
|
}
|
|
|
|
/** Test-only: reset the cached cgroup memory probe. */
|
|
export function _resetCgroupLimitCacheForTests(): void {
|
|
_cachedCgroupLimitMb = undefined;
|
|
_cachedCgroupCpus = undefined;
|
|
_warnedCgroupReadFailure = false;
|
|
}
|
|
|
|
/** The CPUs this process's cgroup may use (docker `--cpus`), or null off Linux or without a quota. */
|
|
export function getCgroupCpuLimit(): number | null {
|
|
if (_cachedCgroupCpus !== undefined) return _cachedCgroupCpus;
|
|
if (process.platform !== "linux") return (_cachedCgroupCpus = null);
|
|
const v2 = readCgroupFile(CGROUP_V2_CPU_MAX_PATH);
|
|
_cachedCgroupCpus =
|
|
v2 !== null
|
|
? parseCgroupCpus(v2, null, null)
|
|
: parseCgroupCpus(
|
|
null,
|
|
readCgroupFile(CGROUP_V1_CPU_QUOTA_PATH),
|
|
readCgroupFile(CGROUP_V1_CPU_PERIOD_PATH),
|
|
);
|
|
return _cachedCgroupCpus;
|
|
}
|
|
|
|
/**
|
|
* Actual Linux cgroup memory ceiling in MiB, or null when the process is not
|
|
* cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM.
|
|
*/
|
|
export function getCgroupMemoryLimitMb(): number | null {
|
|
if (_cachedCgroupLimitMb !== undefined) return _cachedCgroupLimitMb;
|
|
|
|
if (process.platform !== "linux") {
|
|
_cachedCgroupLimitMb = null;
|
|
return null;
|
|
}
|
|
|
|
const v2Content = readCgroupFile(CGROUP_V2_MEMORY_MAX_PATH);
|
|
const v1Content = v2Content === null ? readCgroupFile(CGROUP_V1_MEMORY_LIMIT_PATH) : null;
|
|
|
|
_cachedCgroupLimitMb = parseCgroupLimitMb(v2Content, v1Content);
|
|
if (_cachedCgroupLimitMb !== null) {
|
|
// stderr, not stdout: this is a diagnostic notice, and commands like `check --json`
|
|
// write their machine-readable payload to stdout. A banner on stdout corrupts that
|
|
// payload for any JSON consumer (it broke the Video Agent's hyperframes check parse).
|
|
console.warn(
|
|
`[SystemMemory] cgroup memory limit detected: ${_cachedCgroupLimitMb} MiB. ` +
|
|
`It governs memory-adaptive render behaviour instead of host RAM.`,
|
|
);
|
|
}
|
|
return _cachedCgroupLimitMb;
|
|
}
|
|
|
|
function readCgroupFile(path: string): string | null {
|
|
try {
|
|
return readFileSync(path, "utf8");
|
|
} catch (error) {
|
|
const code = getErrorCode(error);
|
|
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
warnCgroupReadFailure(path, error);
|
|
}
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function getErrorCode(error: unknown): string | undefined {
|
|
if (typeof error !== "object" || error === null || !("code" in error)) {
|
|
return undefined;
|
|
}
|
|
|
|
return typeof error.code === "string" ? error.code : undefined;
|
|
}
|
|
|
|
function formatCgroupReadError(error: unknown): string {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
const code = getErrorCode(error);
|
|
return code ? `${code}: ${message}` : message;
|
|
}
|
|
|
|
function warnCgroupReadFailure(path: string, error: unknown): void {
|
|
if (_warnedCgroupReadFailure) return;
|
|
_warnedCgroupReadFailure = true;
|
|
console.warn(
|
|
`[SystemMemory] Unable to read cgroup memory limit at ${path} ` +
|
|
`(${formatCgroupReadError(error)}); falling back to host RAM.`,
|
|
);
|
|
}
|
|
|
|
/** Total physical RAM in MiB. */
|
|
export function getSystemTotalMb(): number {
|
|
const hostTotalMb = Math.floor(totalmem() / BYTES_PER_MIB);
|
|
const cgroupLimitMb = getCgroupMemoryLimitMb();
|
|
|
|
return cgroupLimitMb === null ? hostTotalMb : Math.min(hostTotalMb, cgroupLimitMb);
|
|
}
|
|
|
|
/**
|
|
* Total-RAM ceiling (MiB) at or below which the host is treated as
|
|
* memory-constrained. Tuned to the 8 GB laptops in
|
|
* heygen-com/hyperframes#1218 / #1219: on those boxes the default render
|
|
* shape (probe Chrome + a throwaway calibration Chrome + N capture
|
|
* workers) thrashes, so the pipeline collapses to its cheapest form.
|
|
*
|
|
* `<=` deliberately includes machines that report exactly 8192 MiB —
|
|
* real "8 GB" hardware reports anywhere from ~7600 to 8192 MiB once
|
|
* firmware/integrated-GPU reservations are subtracted, and a strict `<`
|
|
* would skip the optimisation on the very hardware that needs it.
|
|
*/
|
|
export const LOW_MEMORY_TOTAL_MB_THRESHOLD = 8192;
|
|
|
|
/**
|
|
* True when the host should run the low-memory render profile.
|
|
*
|
|
* Keyed on total physical RAM, not free memory: free memory swings
|
|
* moment to moment and is underreported on macOS, whereas total RAM is a
|
|
* stable proxy for "how many concurrent Chrome instances can this box
|
|
* survive". Accepts an explicit `totalMb` so callers (and tests) can pass
|
|
* a known value instead of re-probing.
|
|
*
|
|
* Caveat: Linux cgroup v1/v2 memory limits are consulted when readable, so
|
|
* Docker and serverless runtimes, including Lambda tiers with readable cgroup
|
|
* ceilings, inherit the tighter container limit instead of the host's physical
|
|
* RAM. Environments that hide cgroup files should set
|
|
* `PRODUCER_LOW_MEMORY_MODE` explicitly rather than relying on auto-detection.
|
|
* Hosts whose *effective* total RAM is genuinely <= the threshold (laptops,
|
|
* small VMs, small Lambda tiers, small containers) are detected correctly.
|
|
*/
|
|
export function isLowMemorySystem(totalMb: number = getSystemTotalMb()): boolean {
|
|
return totalMb <= LOW_MEMORY_TOTAL_MB_THRESHOLD;
|
|
}
|
|
|
|
/** Memory available in MiB: on macOS and Linux, counting pages the OS reclaims on demand. */
|
|
export function getAvailableMemoryMb(): number {
|
|
const fallback = Math.trunc(freemem() / BYTES_PER_MIB);
|
|
|
|
if (platform() === "darwin") {
|
|
try {
|
|
const vmStat = findTrustedExecutable("vm_stat");
|
|
if (!vmStat) return fallback;
|
|
const raw = execFileSync(vmStat, [], { encoding: "utf-8", timeout: 5000 });
|
|
const pageSize = parseInt(raw.match(/page size of (\d+)/)?.[1] ?? "0", 10);
|
|
if (!pageSize) return fallback;
|
|
|
|
const pages = (key: string) =>
|
|
parseInt(raw.match(new RegExp(`${key}:\\s+(\\d+)`))?.[1] ?? "0", 10);
|
|
|
|
const available =
|
|
(pages("Pages free") +
|
|
pages("Pages inactive") +
|
|
pages("Pages purgeable") +
|
|
pages("Pages speculative")) *
|
|
pageSize;
|
|
|
|
return available > 0 ? Math.trunc(available / BYTES_PER_MIB) : fallback;
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
if (platform() === "linux") {
|
|
try {
|
|
const meminfo = readFileSync("/proc/meminfo", "utf-8");
|
|
const match = meminfo.match(/MemAvailable:\s+(\d+)\s+kB/);
|
|
if (match) {
|
|
return Math.trunc(parseInt(match[1]!, 10) / 1024);
|
|
}
|
|
return fallback;
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
return fallback;
|
|
}
|
|
|
|
/** CPUs this process may run on: a cpuset (taskset, --cpuset-cpus) lowers it, a CPU quota does not. */
|
|
export function usableCpuCount(): number {
|
|
return availableParallelism();
|
|
}
|
|
|
|
export function hostCpuCount(): number {
|
|
return cpus().length;
|
|
}
|