Merge pull request #2723 from heygen-com/07-21-fix_engine_worker_autoscaler_memory_budget

fix(engine): realistic worker memory budget + sizing/feedback telemetry
This commit is contained in:
Vance Ingalls
2026-07-21 21:44:35 -07:00
committed by GitHub
13 changed files with 510 additions and 23 deletions
@@ -0,0 +1,39 @@
import { describe, expect, it } from "vitest";
import { buildTelemetryJoinKeys } from "./feedback.js";
describe("buildTelemetryJoinKeys", () => {
it("emits fid + tid and omits renders when the ring is empty", () => {
const keys = buildTelemetryJoinKeys({
feedbackId: "feedback-uuid",
anonymousId: "install-uuid",
});
expect(keys).toBe("fid=feedback-uuid tid=install-uuid");
});
it("appends recent render ids newest-last with a ! marking failed renders", () => {
const keys = buildTelemetryJoinKeys({
feedbackId: "f",
anonymousId: "t",
recentRenders: [
{ id: "render-a", at: "2026-07-21T00:00:00Z", ok: true },
{ id: "render-b", at: "2026-07-21T01:00:00Z", ok: false },
],
});
expect(keys).toBe("fid=f tid=t renders=render-a,render-b!");
});
it("stays within the backend env cap for a full ring of uuid render ids", () => {
const uuid = "01234567-89ab-cdef-0123-456789abcdef";
const keys = buildTelemetryJoinKeys({
feedbackId: uuid,
anonymousId: uuid,
recentRenders: Array.from({ length: 5 }, (_, i) => ({
id: uuid,
at: "2026-07-21T00:00:00Z",
ok: i % 2 === 0,
})),
});
// submitFeedback caps env at 500 chars; the doctor summary consumes ~100.
expect(keys.length).toBeLessThan(400);
});
});
+43 -2
View File
@@ -1,4 +1,5 @@
import { failCommand } from "../utils/commandResult.js";
import { randomUUID } from "node:crypto";
import { resolve } from "node:path";
import { defineCommand } from "citty";
import * as clack from "@clack/prompts";
@@ -7,6 +8,7 @@ import type { Example } from "./_examples.js";
import { trackRenderFeedback } from "../telemetry/events.js";
import { shouldTrack, flush } from "../telemetry/client.js";
import { getDoctorSummary } from "../telemetry/feedback.js";
import { readConfig, type RecentRenderRecord } from "../telemetry/config.js";
import { publishProjectArchive } from "../utils/publishProject.js";
import { submitFeedback } from "../utils/submitFeedback.js";
import { buildIssueUrl, HYPERFRAMES_REPO_URL } from "../utils/feedbackIssue.js";
@@ -28,6 +30,27 @@ function normalizeComment(raw?: string): string | undefined {
return raw || undefined;
}
/**
* Compact PostHog join keys appended to the environment string that rides
* along with the forwarded report (and therefore lands verbatim in the wild
* feedback channel): `fid` = this submission's PostHog `survey sent`
* `feedback_id`; `tid` = the install's telemetry distinct_id; `renders` =
* recent `render_job_id`s (newest last, `!` suffix = the render failed).
* Together they turn a wild report into an exact telemetry lookup instead of
* a hardware-fingerprint hunt.
*/
export function buildTelemetryJoinKeys(input: {
feedbackId: string;
anonymousId: string;
recentRenders?: RecentRenderRecord[];
}): string {
const parts = [`fid=${input.feedbackId}`, `tid=${input.anonymousId}`];
if (input.recentRenders?.length) {
parts.push(`renders=${input.recentRenders.map((r) => `${r.id}${r.ok ? "" : "!"}`).join(",")}`);
}
return parts.join(" ");
}
function printIssueConsent(dir: string): void {
console.log();
console.log(
@@ -170,6 +193,18 @@ export default defineCommand({
const comment = normalizeComment(args.comment);
const doctorSummary = await getDoctorSummary();
// Join keys tying this report to the install's PostHog rows — see
// buildTelemetryJoinKeys. Appended to the env string so they surface in
// the forwarded report; mirrored as structured props on the PostHog event.
const feedbackId = randomUUID();
const config = readConfig();
const joinKeys = buildTelemetryJoinKeys({
feedbackId,
anonymousId: config.anonymousId,
recentRenders: config.recentRenders,
});
const envWithJoinKeys = doctorSummary ? `${doctorSummary} ${joinKeys}` : joinKeys;
// Soft-warn (never blocks) when the comment for a non-clean report is
// missing the mandated reproduction-packet markers. Prints before the
// submission ack so the reporter sees the nudge while their run is fresh.
@@ -177,13 +212,19 @@ export default defineCommand({
// The standalone command runs separately from `render`, so it has no real
// elapsed time to report. Omit it rather than recording a fake duration.
trackRenderFeedback({ rating, comment, doctorSummary });
trackRenderFeedback({
rating,
comment,
doctorSummary,
feedbackId,
recentRenderIds: config.recentRenders?.map((r) => r.id),
});
await flush();
// Ack first so the user isn't kept waiting on the best-effort forward (which
// is bounded to a few seconds and never surfaces an error either way).
console.log(c.dim("Thanks for the feedback!"));
await submitFeedback({ rating, comment, cliVersion: VERSION, env: doctorSummary });
await submitFeedback({ rating, comment, cliVersion: VERSION, env: envWithJoinKeys });
if (args["file-issue"] === true) {
await fileGithubIssue({
+19 -1
View File
@@ -61,7 +61,12 @@ import {
trackRenderObservation,
} from "../telemetry/events.js";
import { maybePromptRenderFeedback } from "../telemetry/feedback.js";
import { readConfigFresh, writeConfig, type HyperframesConfig } from "../telemetry/config.js";
import {
readConfigFresh,
recordRecentRender,
writeConfig,
type HyperframesConfig,
} from "../telemetry/config.js";
import { shouldTrack } from "../telemetry/client.js";
import { renderJobObservabilityTelemetryPayload } from "../telemetry/renderObservability.js";
import { bytesToMb } from "../telemetry/system.js";
@@ -1339,6 +1344,9 @@ function handleRenderError(
...renderJobObservabilityTelemetryPayload(job),
...getMemorySnapshot(),
});
// Failed renders join the recent-renders ring too — a bug report filed via
// `hyperframes feedback` is MOST likely to be about a failed render.
if (job?.id) recordRecentRender(job.id, false);
if (options.throwOnError) {
throw new Error(message);
}
@@ -1376,6 +1384,9 @@ function trackRenderMetrics(
options: RenderOptions,
docker: boolean,
): void {
// Successful render → recent-renders ring, so a later `hyperframes
// feedback` can attach this render's telemetry id to the report.
recordRecentRender(job.id, true);
const perf = job.perfSummary;
const compositionDurationMs = perf
? Math.round(perf.compositionDurationSeconds * 1000)
@@ -1393,6 +1404,13 @@ function trackRenderMetrics(
fps: fpsToNumber(options.fps),
quality: options.quality,
workers: options.workers ?? perf?.workers,
workersBoundBy: perf?.workerSizing?.boundBy,
workersCpuBased: perf?.workerSizing?.cpuBasedWorkers,
workersMemoryBased: perf?.workerSizing?.memoryBasedWorkers,
workersHeapBased: perf?.workerSizing?.heapBasedWorkers,
workersFrameBased: perf?.workerSizing?.frameBasedWorkers,
workersHeapLimitMb: perf?.workerSizing?.heapLimitMb,
workersExceedHeapAdvisory: perf?.workerSizing?.exceedsHeapAdvisory,
docker,
gpu: options.gpu,
authoringSkill: options.authoringSkill,
+45
View File
@@ -85,6 +85,39 @@ export interface HyperframesConfig {
* `DE_PARALLEL_ROUTER_TRIAL_MAX_RENDERS` in `render.ts`.
*/
deParallelRouterTrialRenderCount?: number;
/**
* Ring of the last few local renders (newest last). `hyperframes feedback`
* attaches these ids — which are the `render_job_id` /
* `observability_render_job_id` on this install's PostHog events — to the
* feedback it submits, so a wild bug report can be joined to the exact
* telemetry rows of the renders it describes.
*/
recentRenders?: RecentRenderRecord[];
}
/** One entry in {@link HyperframesConfig.recentRenders}. */
export interface RecentRenderRecord {
/** The render job id (`RenderJob.id` — the telemetry `render_job_id`). */
id: string;
/** ISO timestamp of when the render finished. */
at: string;
/** Whether the render completed successfully. */
ok: boolean;
}
/** Ring size for {@link HyperframesConfig.recentRenders}. */
const MAX_RECENT_RENDERS = 5;
/**
* Append a finished render to the recent-renders ring (newest last, capped).
* Fresh read-modify-write like the trial counters — narrows, but does not
* eliminate, lost updates against a concurrent CLI process.
*/
export function recordRecentRender(id: string, ok: boolean): void {
const config = readConfigFresh();
const ring = [...(config.recentRenders ?? []), { id, at: new Date().toISOString(), ok }];
config.recentRenders = ring.slice(-MAX_RECENT_RENDERS);
writeConfig(config);
}
const DEFAULT_CONFIG: HyperframesConfig = {
@@ -142,6 +175,18 @@ export function readConfig(): HyperframesConfig {
typeof parsed.deParallelRouterTrialRenderCount === "number"
? parsed.deParallelRouterTrialRenderCount
: undefined,
recentRenders: Array.isArray(parsed.recentRenders)
? parsed.recentRenders
.filter(
(r): r is RecentRenderRecord =>
typeof r === "object" &&
r !== null &&
typeof (r as RecentRenderRecord).id === "string" &&
typeof (r as RecentRenderRecord).at === "string" &&
typeof (r as RecentRenderRecord).ok === "boolean",
)
.slice(-MAX_RECENT_RENDERS)
: undefined,
};
cachedConfig = config;
+53
View File
@@ -186,6 +186,59 @@ describe("render telemetry events", () => {
expect(flush).toHaveBeenCalledTimes(2);
});
// The enforcement decision for the advisory heap budget reads these fleet
// props (see computeWorkerSizing) — a silent drop in the summary→event hop
// would invalidate that decision without anyone noticing.
it("carries every worker-sizing provenance prop on render_complete", () => {
trackRenderComplete({
durationMs: 1000,
fps: 30,
quality: "high",
docker: false,
gpu: false,
workers: 6,
workersBoundBy: "max_workers",
workersCpuBased: 16,
workersMemoryBased: 8,
workersHeapBased: 4,
workersFrameBased: 24,
workersHeapLimitMb: 4096,
workersExceedHeapAdvisory: true,
});
expect(trackEvent).toHaveBeenCalledWith(
"render_complete",
expect.objectContaining({
workers: 6,
workers_bound_by: "max_workers",
workers_cpu_based: 16,
workers_memory_based: 8,
workers_heap_based: 4,
workers_frame_based: 24,
workers_heap_limit_mb: 4096,
workers_exceed_heap_advisory: true,
}),
undefined,
);
});
it("ties feedback to its report and recent renders via feedback_id + recent_render_ids", () => {
trackRenderFeedback({
rating: 3,
comment: "hook scene blank",
feedbackId: "feedback-uuid",
recentRenderIds: ["render-a", "render-b"],
});
expect(trackEvent).toHaveBeenCalledWith(
"survey sent",
expect.objectContaining({
feedback_id: "feedback-uuid",
recent_render_ids: "render-a,render-b",
}),
);
});
it("redacts paths and URL query strings from render error messages", () => {
trackRenderError({
fps: 30,
+31
View File
@@ -149,6 +149,17 @@ export function trackRenderComplete(
/** Authoring workflow skill that drove this render (e.g. "product-launch-video"). */
authoringSkill?: string;
workers?: number;
// Worker auto-sizing provenance (RenderPerfSummary.workerSizing). Answers
// "why N workers?" fleet-wide, and validates the advisory per-worker heap
// budget before it's enforced (field OOM: 6 auto workers on a 24GB/4GB-heap
// machine — see computeWorkerSizing in @hyperframes/engine).
workersBoundBy?: string;
workersCpuBased?: number;
workersMemoryBased?: number;
workersHeapBased?: number;
workersFrameBased?: number;
workersHeapLimitMb?: number;
workersExceedHeapAdvisory?: boolean;
docker: boolean;
gpu: boolean;
// Static-frame dedup outcome (opt-out HF_STATIC_DEDUP=false). Undefined on
@@ -245,6 +256,13 @@ export function trackRenderComplete(
quality: props.quality,
authoring_skill: props.authoringSkill,
workers: props.workers,
workers_bound_by: props.workersBoundBy,
workers_cpu_based: props.workersCpuBased,
workers_memory_based: props.workersMemoryBased,
workers_heap_based: props.workersHeapBased,
workers_frame_based: props.workersFrameBased,
workers_heap_limit_mb: props.workersHeapLimitMb,
workers_exceed_heap_advisory: props.workersExceedHeapAdvisory,
docker: props.docker,
gpu: props.gpu,
static_dedup_enabled: props.staticDedupEnabled,
@@ -609,6 +627,14 @@ export function trackRenderFeedback(props: {
renderDurationMs?: number;
comment?: string;
doctorSummary?: string;
/**
* Join key shared with the forwarded feedback report (Slack/backend): the
* same uuid rides in the report's env string as `fid=…`, so a wild report
* resolves to exactly one PostHog "survey sent" event and vice versa.
*/
feedbackId?: string;
/** render_job_id values of this install's recent renders (newest last). */
recentRenderIds?: string[];
}): void {
trackEvent("survey sent", {
$survey_id: "render_satisfaction",
@@ -617,6 +643,11 @@ export function trackRenderFeedback(props: {
...(props.comment ? { $survey_response_2: props.comment } : {}),
...(props.renderDurationMs !== undefined ? { render_duration_ms: props.renderDurationMs } : {}),
...(props.doctorSummary ? { doctor_summary: props.doctorSummary } : {}),
...(props.feedbackId ? { feedback_id: props.feedbackId } : {}),
// Comma-joined: EventProperties values are scalars only.
...(props.recentRenderIds?.length
? { recent_render_ids: props.recentRenderIds.join(",") }
: {}),
});
}
+3
View File
@@ -208,6 +208,7 @@ export type {
// ── Parallel rendering ─────────────────────────────────────────────────────────
export {
calculateOptimalWorkers,
computeWorkerSizing,
distributeFrames,
distributeFramesInterleaved,
executeParallelCapture,
@@ -215,6 +216,8 @@ export {
getSystemResources,
type WorkerTask,
type WorkerResult,
type WorkerSizing,
type WorkerSizingBound,
type ParallelProgress,
} from "./services/parallelCoordinator.js";
@@ -1,6 +1,7 @@
import { describe, it, expect } from "vitest";
import {
calculateOptimalWorkers,
computeWorkerSizing,
distributeFrames,
expectedFramesForTask,
flagSilentWorkerExits,
@@ -93,6 +94,50 @@ describe("calculateOptimalWorkers", () => {
});
});
describe("computeWorkerSizing", () => {
it("matches calculateOptimalWorkers and reports every constraint", () => {
const config = { concurrency: "auto" as const };
const sizing = computeWorkerSizing(900, undefined, config);
expect(sizing.workers).toBe(calculateOptimalWorkers(900, undefined, config));
expect(sizing.cpuBasedWorkers).toBeGreaterThanOrEqual(1);
expect(sizing.memoryBasedWorkers).toBeGreaterThanOrEqual(1);
expect(sizing.heapBasedWorkers).toBeGreaterThanOrEqual(1);
expect(sizing.frameBasedWorkers).toBe(30); // 900 / MIN_FRAMES_PER_WORKER(30)
expect(sizing.heapLimitMb).toBeGreaterThan(0);
expect(sizing.totalMemoryMb).toBeGreaterThan(0);
expect(sizing.cpuCount).toBeGreaterThan(0);
});
it("labels explicit requests and clamps them like the legacy wrapper", () => {
const sizing = computeWorkerSizing(900, 25, { concurrency: "auto" });
expect(sizing.workers).toBe(24);
expect(sizing.boundBy).toBe("explicit");
});
it("labels tiny renders as too_few_frames", () => {
const sizing = computeWorkerSizing(30, undefined, { concurrency: "auto" });
expect(sizing.workers).toBe(1);
expect(sizing.boundBy).toBe("too_few_frames");
});
it("labels the contention cap when high capture cost binds", () => {
const sizing = computeWorkerSizing(180, undefined, {
concurrency: 6,
coresPerWorker: 100,
minParallelFrames: 120,
largeRenderThreshold: 1000,
captureCostMultiplier: 4,
});
expect(sizing.workers).toBe(1);
expect(sizing.boundBy).toBe("contention");
});
it("flags exceedsHeapAdvisory exactly when workers exceed the heap budget", () => {
const sizing = computeWorkerSizing(900, 24, { concurrency: "auto" });
expect(sizing.exceedsHeapAdvisory).toBe(sizing.workers > sizing.heapBasedWorkers);
});
});
describe("shouldDisableBrowserPoolForParallelWorker", () => {
const linuxHeadlessWorker = {
parallel: true,
@@ -9,6 +9,7 @@ import { cpus, freemem } from "os";
import { existsSync, mkdirSync, readdirSync } from "fs";
import { copyFile, rename } from "fs/promises";
import { join } from "path";
import { getHeapStatistics } from "v8";
import {
createCaptureSession,
@@ -108,7 +109,22 @@ type WorkerBrowserPoolDecision = {
headlessShellPath?: string;
};
const MEMORY_PER_WORKER_MB = 256;
// System-memory budget per parallel worker. Each worker is a full Chrome
// process (SwiftShader compositor + raster threads) plus its share of parent-
// process frame buffering — the old 256MB figure was ~6× under a measured
// Chrome-under-capture RSS and let memory-constrained hosts overcommit (wild
// 16GB black-slab report; 0.7.66 heap-OOM report on 24GB with 6 auto workers).
const MEMORY_PER_WORKER_MB = 1536;
// Parent-process V8 heap the coordinator itself needs regardless of worker
// count (compile artifacts, puppeteer sessions, encoder state).
const HEAP_RESERVED_MB = 1024;
// Parent-process V8 heap consumed per worker (protocol buffers + in-flight
// frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB
// default heap ⇒ >~500MB/worker + base. ponytail: advisory-only until the
// workers_heap_* telemetry added alongside this constant validates the figure
// — enforcing a guessed budget could silently cut worker counts fleet-wide.
// TODO(PRINFRA-341): decide enforcement after ~2 weeks of fleet soak.
const HEAP_PER_WORKER_MB = 640;
const MIN_WORKERS = 1;
const MAX_WORKER_DIAGNOSTIC_LINES = 8;
// Hard ceiling on explicit `--workers N` requests. Above this, the cost of
@@ -229,13 +245,95 @@ export function formatWorkerFailure(result: WorkerResult): string {
return `${base}; diagnostics: ${diagnostics}`;
}
export function calculateOptimalWorkers(
/** Which constraint produced the final auto-sized worker count. */
export type WorkerSizingBound =
| "explicit"
| "too_few_frames"
| "cpu"
| "memory"
| "frames"
| "max_workers"
| "min_parallel_floor"
| "contention";
/**
* Full provenance of a worker-sizing decision. Threaded into render
* observability/telemetry so fleet data can answer "why N workers?" and
* "would the heap budget have prevented this OOM?" without a repro.
*/
export interface WorkerSizing {
workers: number;
boundBy: WorkerSizingBound;
cpuBasedWorkers: number;
memoryBasedWorkers: number;
frameBasedWorkers: number;
effectiveMaxWorkers: number;
/**
* ADVISORY, not enforced (see HEAP_PER_WORKER_MB): how many workers the
* parent process's V8 heap could feed. Compare against `workers` in
* telemetry to validate the budget before enforcement.
*/
heapBasedWorkers: number;
/** V8 `heap_size_limit` for the parent process, MB. */
heapLimitMb: number;
totalMemoryMb: number;
cpuCount: number;
captureCostMultiplier: number;
/** true when the chosen count exceeds the advisory heap budget. */
exceedsHeapAdvisory: boolean;
}
/**
* Compute the auto worker count together with the full sizing provenance.
* `calculateOptimalWorkers` is the thin legacy wrapper returning `.workers`.
*/
// The branch count is the decision provenance itself — each if records WHICH
// constraint bound, which is the entire point of the function.
// fallow-ignore-next-line complexity
export function computeWorkerSizing(
totalFrames: number,
requested?: number,
config?: WorkerSizingConfig,
): number {
): WorkerSizing {
const cpuCount = cpus().length;
// Use total memory instead of free memory — macOS reports misleadingly low
// freemem() because it aggressively caches files in "inactive" memory that
// is immediately reclaimable.
const totalMemoryMb = getSystemTotalMb();
const heapLimitMb = Math.round(getHeapStatistics().heap_size_limit / (1024 * 1024));
const heapBasedWorkers = Math.max(
1,
Math.floor((heapLimitMb - HEAP_RESERVED_MB) / HEAP_PER_WORKER_MB),
);
const cpuBasedWorkers = Math.max(1, cpuCount - 2);
const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMb * 0.5) / MEMORY_PER_WORKER_MB));
const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER);
const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1);
const base = {
cpuBasedWorkers,
memoryBasedWorkers,
frameBasedWorkers,
heapBasedWorkers,
heapLimitMb,
totalMemoryMb,
cpuCount,
captureCostMultiplier,
};
const finish = (workers: number, boundBy: WorkerSizingBound, effectiveMaxWorkers: number) => ({
workers,
boundBy,
effectiveMaxWorkers,
...base,
exceedsHeapAdvisory: workers > heapBasedWorkers,
});
if (requested !== undefined) {
return Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, requested));
return finish(
Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, requested)),
"explicit",
ABSOLUTE_MAX_WORKERS,
);
}
// Resolve effective values: config overrides → DEFAULT_CONFIG fallback.
@@ -250,24 +348,22 @@ export function calculateOptimalWorkers(
const effectiveMinParallelFrames = config?.minParallelFrames ?? DEFAULT_CONFIG.minParallelFrames;
const effectiveLargeRenderThreshold =
config?.largeRenderThreshold ?? DEFAULT_CONFIG.largeRenderThreshold;
const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1);
if (totalFrames < MIN_FRAMES_PER_WORKER * 2) return 1;
const cpuCount = cpus().length;
const cpuBasedWorkers = Math.max(1, cpuCount - 2);
// Use total memory instead of free memory — macOS reports misleadingly low
// freemem() because it aggressively caches files in "inactive" memory that
// is immediately reclaimable.
const totalMemoryMB = getSystemTotalMb();
const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMB * 0.5) / MEMORY_PER_WORKER_MB));
const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER);
if (totalFrames < MIN_FRAMES_PER_WORKER * 2) {
return finish(1, "too_few_frames", effectiveMaxWorkers);
}
const optimal = Math.min(cpuBasedWorkers, memoryBasedWorkers, frameBasedWorkers);
const optimalBound: WorkerSizingBound =
optimal === cpuBasedWorkers ? "cpu" : optimal === memoryBasedWorkers ? "memory" : "frames";
const minWorkersForJob = totalFrames >= effectiveMinParallelFrames ? 2 : MIN_WORKERS;
let finalWorkers = Math.max(minWorkersForJob, Math.min(effectiveMaxWorkers, optimal));
let boundBy: WorkerSizingBound =
finalWorkers === optimal
? optimalBound
: finalWorkers === effectiveMaxWorkers && effectiveMaxWorkers < optimal
? "max_workers"
: "min_parallel_floor";
// Adaptive scaling: cap workers for large or expensive renders to prevent
// CPU contention. Each Chrome process (with SwiftShader) is CPU-heavy; too
@@ -284,10 +380,19 @@ export function calculateOptimalWorkers(
const cpuScaledMax = Math.max(MIN_WORKERS, Math.floor(cpuCount / weightedCoresPerWorker));
if (finalWorkers > cpuScaledMax) {
finalWorkers = cpuScaledMax;
boundBy = "contention";
}
}
return finalWorkers;
return finish(finalWorkers, boundBy, effectiveMaxWorkers);
}
export function calculateOptimalWorkers(
totalFrames: number,
requested?: number,
config?: WorkerSizingConfig,
): number {
return computeWorkerSizing(totalFrames, requested, config).workers;
}
export function distributeFrames(
@@ -0,0 +1,42 @@
import { describe, expect, it } from "vitest";
import type { WorkerSizing } from "@hyperframes/engine";
import { buildHeapAdvisoryWarning } from "./captureCost.js";
function sizing(overrides: Partial<WorkerSizing>): WorkerSizing {
return {
workers: 6,
boundBy: "max_workers",
cpuBasedWorkers: 16,
memoryBasedWorkers: 8,
frameBasedWorkers: 24,
effectiveMaxWorkers: 6,
heapBasedWorkers: 4,
heapLimitMb: 4096,
totalMemoryMb: 24576,
cpuCount: 18,
captureCostMultiplier: 1,
exceedsHeapAdvisory: true,
...overrides,
};
}
describe("buildHeapAdvisoryWarning", () => {
it("names the chosen count, heap limit, safe count, and both remediation knobs", () => {
const message = buildHeapAdvisoryWarning(sizing({}), undefined);
expect(message).toContain("6 capture workers");
expect(message).toContain("limit 4096MB");
expect(message).toContain("~4");
expect(message).toContain("NODE_OPTIONS=--max-old-space-size=8192");
expect(message).toContain("--workers 4");
});
it("stays silent for explicit --workers requests (operator's own call)", () => {
expect(buildHeapAdvisoryWarning(sizing({}), 6)).toBeUndefined();
});
it("stays silent when the chosen count fits the heap budget", () => {
expect(
buildHeapAdvisoryWarning(sizing({ exceedsHeapAdvisory: false }), undefined),
).toBeUndefined();
});
});
@@ -16,9 +16,11 @@ import {
type CaptureOptions,
type CaptureSession,
type EngineConfig,
type WorkerSizing,
calculateOptimalWorkers,
captureFrameToBuffer,
closeCaptureSession,
computeWorkerSizing,
createCaptureSession,
initializeSession,
} from "@hyperframes/engine";
@@ -106,6 +108,36 @@ function combineCaptureCostEstimates(
};
}
/**
* Advisory heap warning (field OOM, 0.7.66 on 24GB: auto-picked 6 workers
* blew Node's default ~4GB heap). Returns the warning message, or undefined
* when it should not fire:
*
* - Auto-sized renders only (`requestedWorkers === undefined`) the field
* failure was auto sizing, and an explicit `--workers N` is the operator's
* own call.
* - Not enforced as a cap yet the per-worker budget constant is derived
* from one field report; the `workers_heap_*` telemetry emitted with the
* sizing decides whether to enforce (see the TODO on HEAP_PER_WORKER_MB in
* @hyperframes/engine's parallelCoordinator). The message gives the
* operator the actionable knobs today.
*
* Pure so the message shape + firing condition are unit-testable with a
* synthetic `WorkerSizing` (the real one depends on the host's heap).
*/
export function buildHeapAdvisoryWarning(
sizing: WorkerSizing,
requestedWorkers: number | undefined,
): string | undefined {
if (requestedWorkers !== undefined || !sizing.exceedsHeapAdvisory) return undefined;
return (
`[Render] ${sizing.workers} capture workers may exceed this process's V8 heap ` +
`(limit ${sizing.heapLimitMb}MB supports ~${sizing.heapBasedWorkers}). If the render ` +
`dies with "JavaScript heap out of memory", raise the heap ` +
`(NODE_OPTIONS=--max-old-space-size=8192) or pass --workers ${sizing.heapBasedWorkers}.`
);
}
export function resolveRenderWorkerCount(
totalFrames: number,
requestedWorkers: number | undefined,
@@ -113,6 +145,8 @@ export function resolveRenderWorkerCount(
compiled: Pick<CompiledComposition, "hasShaderTransitions" | "renderModeHints">,
log: ProducerLogger = defaultLogger,
measuredCaptureCost?: CaptureCostEstimate,
/** Sink for the sizing provenance so the orchestrator can thread it into telemetry. */
onSizing?: (sizing: WorkerSizing) => void,
): number {
// TODO(htmlInCanvas): workaround — Chrome's experimental drawElementImage
// API (CanvasDrawElement) is non-deterministic across concurrent browser
@@ -144,10 +178,16 @@ export function resolveRenderWorkerCount(
estimateCaptureCostMultiplier(compiled),
measuredCaptureCost,
);
const workerCount = calculateOptimalWorkers(totalFrames, requestedWorkers, {
const sizing = computeWorkerSizing(totalFrames, requestedWorkers, {
...cfg,
captureCostMultiplier: captureCost.multiplier,
});
// The heap advisory is deliberately NOT logged here: this function's
// "no unexpected warns" unit-test contract would become dependent on the
// test machine's actual V8 heap limit. The orchestrator's onSizing consumer
// emits it via buildHeapAdvisoryWarning.
onSizing?.(sizing);
const workerCount = sizing.workers;
if (requestedWorkers !== undefined || captureCost.multiplier <= 1) {
return workerCount;
@@ -4,7 +4,7 @@
*/
import { fpsToNumber } from "@hyperframes/core";
import type { CapturePerfSummary, SubTimelineWaitOutcome } from "@hyperframes/engine";
import type { CapturePerfSummary, SubTimelineWaitOutcome, WorkerSizing } from "@hyperframes/engine";
import type { CaptureCalibrationSample, CaptureCostEstimate } from "./captureCost.js";
import type {
CaptureAttemptSummary,
@@ -181,6 +181,8 @@ function aggregateBeginFrameReuse(
export function buildRenderPerfSummary(input: {
job: RenderJob;
workerCount: number;
/** Auto-sizing provenance; undefined when a pin (htmlInCanvas / low-memory) short-circuited sizing. */
workerSizing?: WorkerSizing;
enableChunkedEncode: boolean;
chunkedEncodeSize: number;
compositionDurationSeconds: number;
@@ -217,6 +219,7 @@ export function buildRenderPerfSummary(input: {
fps: fpsToNumber(input.job.config.fps),
quality: input.job.config.quality,
workers: input.workerCount,
workerSizing: input.workerSizing,
chunkedEncode: input.enableChunkedEncode,
chunkSizeFrames: input.enableChunkedEncode ? input.chunkedEncodeSize : null,
compositionDurationSeconds: input.compositionDurationSeconds,
@@ -76,6 +76,7 @@ import {
type CapturePerfSummary,
type CaptureWarning,
type SubTimelineWaitOutcome,
type WorkerSizing,
resolveBrowserGpuMode,
resolveHeadlessShellPath,
applyConcreteGpuScreenshotClamp,
@@ -124,6 +125,7 @@ import { resolveVideoCaptureBeyondViewport } from "./render/captureBeyondViewpor
import {
type CaptureCalibrationSample,
type CaptureCostEstimate,
buildHeapAdvisoryWarning,
resolveRenderWorkerCount,
runCaptureCalibration,
} from "./render/captureCost.js";
@@ -372,6 +374,14 @@ export interface RenderPerfSummary {
fps: number;
quality: string;
workers: number;
/**
* Provenance of the auto worker-sizing decision (undefined when the
* htmlInCanvas / low-memory pins short-circuited sizing). `boundBy` names
* the binding constraint; the heap fields are the advisory budget being
* validated by fleet telemetry before enforcement see
* `computeWorkerSizing` in @hyperframes/engine.
*/
workerSizing?: WorkerSizing;
chunkedEncode: boolean;
chunkSizeFrames: number | null;
compositionDurationSeconds: number;
@@ -1702,6 +1712,7 @@ async function executeRenderPipeline(input: {
// "routed" = the parallel router fired and held; "reverted" = fired but
// the self-verify retry rolled back; undefined = never fired.
let deParallelRouter: "routed" | "reverted" | undefined;
let workerSizing: WorkerSizing | undefined;
execution.defer("rollback staged artifact", () => artifactTransaction.rollback());
execution.defer("close file server", () => {
@@ -2473,6 +2484,16 @@ async function executeRenderPipeline(input: {
compiled,
log,
captureCalibration?.estimate,
(sizing) => {
workerSizing = sizing;
const heapAdvisory = buildHeapAdvisoryWarning(sizing, job.config.workers);
if (heapAdvisory) {
log.warn(heapAdvisory, {
heapLimitMb: sizing.heapLimitMb,
heapBasedWorkers: sizing.heapBasedWorkers,
});
}
},
);
// DE priority inversion — see shouldPreferSingleWorkerDrawElement for the
// policy and benchmark rationale (eligibility resolved above, before
@@ -3279,6 +3300,7 @@ async function executeRenderPipeline(input: {
const perfSummary = buildRenderPerfSummary({
job,
workerCount,
workerSizing,
enableChunkedEncode,
chunkedEncodeSize,
compositionDurationSeconds: composition.duration,