fix(producer): emit heap advisory at orchestrator, lock message + telemetry props with tests

This commit is contained in:
Vance Ingalls
2026-07-21 20:39:06 -07:00
parent 12e599a6ba
commit c6462a0a22
5 changed files with 138 additions and 15 deletions
@@ -0,0 +1,42 @@
import { describe, expect, it } from "vitest";
import type { WorkerSizing } from "@hyperframes/engine";
import { buildHeapAdvisoryWarning } from "./captureCost.js";
function sizing(overrides: Partial<WorkerSizing>): WorkerSizing {
return {
workers: 6,
boundBy: "max_workers",
cpuBasedWorkers: 16,
memoryBasedWorkers: 8,
frameBasedWorkers: 24,
effectiveMaxWorkers: 6,
heapBasedWorkers: 4,
heapLimitMb: 4096,
totalMemoryMb: 24576,
cpuCount: 18,
captureCostMultiplier: 1,
exceedsHeapAdvisory: true,
...overrides,
};
}
describe("buildHeapAdvisoryWarning", () => {
it("names the chosen count, heap limit, safe count, and both remediation knobs", () => {
const message = buildHeapAdvisoryWarning(sizing({}), undefined);
expect(message).toContain("6 capture workers");
expect(message).toContain("limit 4096MB");
expect(message).toContain("~4");
expect(message).toContain("NODE_OPTIONS=--max-old-space-size=8192");
expect(message).toContain("--workers 4");
});
it("stays silent for explicit --workers requests (operator's own call)", () => {
expect(buildHeapAdvisoryWarning(sizing({}), 6)).toBeUndefined();
});
it("stays silent when the chosen count fits the heap budget", () => {
expect(
buildHeapAdvisoryWarning(sizing({ exceedsHeapAdvisory: false }), undefined),
).toBeUndefined();
});
});
@@ -108,6 +108,36 @@ function combineCaptureCostEstimates(
};
}
/**
* Advisory heap warning (field OOM, 0.7.66 on 24GB: auto-picked 6 workers
* blew Node's default ~4GB heap). Returns the warning message, or undefined
* when it should not fire:
*
* - Auto-sized renders only (`requestedWorkers === undefined`) — the field
* failure was auto sizing, and an explicit `--workers N` is the operator's
* own call.
* - Not enforced as a cap yet — the per-worker budget constant is derived
* from one field report; the `workers_heap_*` telemetry emitted with the
* sizing decides whether to enforce (see the TODO on HEAP_PER_WORKER_MB in
* @hyperframes/engine's parallelCoordinator). The message gives the
* operator the actionable knobs today.
*
* Pure so the message shape + firing condition are unit-testable with a
* synthetic `WorkerSizing` (the real one depends on the host's heap).
*/
export function buildHeapAdvisoryWarning(
sizing: WorkerSizing,
requestedWorkers: number | undefined,
): string | undefined {
if (requestedWorkers !== undefined || !sizing.exceedsHeapAdvisory) return undefined;
return (
`[Render] ${sizing.workers} capture workers may exceed this process's V8 heap ` +
`(limit ${sizing.heapLimitMb}MB supports ~${sizing.heapBasedWorkers}). If the render ` +
`dies with "JavaScript heap out of memory", raise the heap ` +
`(NODE_OPTIONS=--max-old-space-size=8192) or pass --workers ${sizing.heapBasedWorkers}.`
);
}
export function resolveRenderWorkerCount(
totalFrames: number,
requestedWorkers: number | undefined,
@@ -152,24 +182,13 @@ export function resolveRenderWorkerCount(
...cfg,
captureCostMultiplier: captureCost.multiplier,
});
// The heap advisory is deliberately NOT logged here: this function's
// "no unexpected warns" unit-test contract would become dependent on the
// test machine's actual V8 heap limit. The orchestrator's onSizing consumer
// emits it via buildHeapAdvisoryWarning.
onSizing?.(sizing);
const workerCount = sizing.workers;
// Advisory heap check (field OOM, 0.7.66 on 24GB: auto-picked 6 workers
// blew Node's default ~4GB heap). Not enforced yet — the budget constant is
// derived from one field report; the workers_heap_* telemetry emitted with
// this sizing decides whether to enforce. The warning gives the operator
// the actionable knobs today.
if (sizing.exceedsHeapAdvisory) {
log.warn(
`[Render] ${workerCount} capture workers may exceed this process's V8 heap ` +
`(limit ${sizing.heapLimitMb}MB supports ~${sizing.heapBasedWorkers}). If the render ` +
`dies with "JavaScript heap out of memory", raise the heap ` +
`(NODE_OPTIONS=--max-old-space-size=8192) or pass --workers ${sizing.heapBasedWorkers}.`,
{ heapLimitMb: sizing.heapLimitMb, heapBasedWorkers: sizing.heapBasedWorkers },
);
}
if (requestedWorkers !== undefined || captureCost.multiplier <= 1) {
return workerCount;
}
@@ -125,6 +125,7 @@ import { resolveVideoCaptureBeyondViewport } from "./render/captureBeyondViewpor
import {
type CaptureCalibrationSample,
type CaptureCostEstimate,
buildHeapAdvisoryWarning,
resolveRenderWorkerCount,
runCaptureCalibration,
} from "./render/captureCost.js";
@@ -2487,6 +2488,13 @@ async function executeRenderPipeline(input: {
captureCalibration?.estimate,
(sizing) => {
workerSizing = sizing;
const heapAdvisory = buildHeapAdvisoryWarning(sizing, job.config.workers);
if (heapAdvisory) {
log.warn(heapAdvisory, {
heapLimitMb: sizing.heapLimitMb,
heapBasedWorkers: sizing.heapBasedWorkers,
});
}
},
);
// DE priority inversion — see shouldPreferSingleWorkerDrawElement for the