mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 14:50:02 +00:00
fix(producer): emit heap advisory at orchestrator, lock message + telemetry props with tests
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { WorkerSizing } from "@hyperframes/engine";
|
||||
import { buildHeapAdvisoryWarning } from "./captureCost.js";
|
||||
|
||||
function sizing(overrides: Partial<WorkerSizing>): WorkerSizing {
|
||||
return {
|
||||
workers: 6,
|
||||
boundBy: "max_workers",
|
||||
cpuBasedWorkers: 16,
|
||||
memoryBasedWorkers: 8,
|
||||
frameBasedWorkers: 24,
|
||||
effectiveMaxWorkers: 6,
|
||||
heapBasedWorkers: 4,
|
||||
heapLimitMb: 4096,
|
||||
totalMemoryMb: 24576,
|
||||
cpuCount: 18,
|
||||
captureCostMultiplier: 1,
|
||||
exceedsHeapAdvisory: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildHeapAdvisoryWarning", () => {
|
||||
it("names the chosen count, heap limit, safe count, and both remediation knobs", () => {
|
||||
const message = buildHeapAdvisoryWarning(sizing({}), undefined);
|
||||
expect(message).toContain("6 capture workers");
|
||||
expect(message).toContain("limit 4096MB");
|
||||
expect(message).toContain("~4");
|
||||
expect(message).toContain("NODE_OPTIONS=--max-old-space-size=8192");
|
||||
expect(message).toContain("--workers 4");
|
||||
});
|
||||
|
||||
it("stays silent for explicit --workers requests (operator's own call)", () => {
|
||||
expect(buildHeapAdvisoryWarning(sizing({}), 6)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("stays silent when the chosen count fits the heap budget", () => {
|
||||
expect(
|
||||
buildHeapAdvisoryWarning(sizing({ exceedsHeapAdvisory: false }), undefined),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -108,6 +108,36 @@ function combineCaptureCostEstimates(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Advisory heap warning (field OOM, 0.7.66 on 24GB: auto-picked 6 workers
|
||||
* blew Node's default ~4GB heap). Returns the warning message, or undefined
|
||||
* when it should not fire:
|
||||
*
|
||||
* - Auto-sized renders only (`requestedWorkers === undefined`) — the field
|
||||
* failure was auto sizing, and an explicit `--workers N` is the operator's
|
||||
* own call.
|
||||
* - Not enforced as a cap yet — the per-worker budget constant is derived
|
||||
* from one field report; the `workers_heap_*` telemetry emitted with the
|
||||
* sizing decides whether to enforce (see the TODO on HEAP_PER_WORKER_MB in
|
||||
* @hyperframes/engine's parallelCoordinator). The message gives the
|
||||
* operator the actionable knobs today.
|
||||
*
|
||||
* Pure so the message shape + firing condition are unit-testable with a
|
||||
* synthetic `WorkerSizing` (the real one depends on the host's heap).
|
||||
*/
|
||||
export function buildHeapAdvisoryWarning(
|
||||
sizing: WorkerSizing,
|
||||
requestedWorkers: number | undefined,
|
||||
): string | undefined {
|
||||
if (requestedWorkers !== undefined || !sizing.exceedsHeapAdvisory) return undefined;
|
||||
return (
|
||||
`[Render] ${sizing.workers} capture workers may exceed this process's V8 heap ` +
|
||||
`(limit ${sizing.heapLimitMb}MB supports ~${sizing.heapBasedWorkers}). If the render ` +
|
||||
`dies with "JavaScript heap out of memory", raise the heap ` +
|
||||
`(NODE_OPTIONS=--max-old-space-size=8192) or pass --workers ${sizing.heapBasedWorkers}.`
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveRenderWorkerCount(
|
||||
totalFrames: number,
|
||||
requestedWorkers: number | undefined,
|
||||
@@ -152,24 +182,13 @@ export function resolveRenderWorkerCount(
|
||||
...cfg,
|
||||
captureCostMultiplier: captureCost.multiplier,
|
||||
});
|
||||
// The heap advisory is deliberately NOT logged here: this function's
|
||||
// "no unexpected warns" unit-test contract would become dependent on the
|
||||
// test machine's actual V8 heap limit. The orchestrator's onSizing consumer
|
||||
// emits it via buildHeapAdvisoryWarning.
|
||||
onSizing?.(sizing);
|
||||
const workerCount = sizing.workers;
|
||||
|
||||
// Advisory heap check (field OOM, 0.7.66 on 24GB: auto-picked 6 workers
|
||||
// blew Node's default ~4GB heap). Not enforced yet — the budget constant is
|
||||
// derived from one field report; the workers_heap_* telemetry emitted with
|
||||
// this sizing decides whether to enforce. The warning gives the operator
|
||||
// the actionable knobs today.
|
||||
if (sizing.exceedsHeapAdvisory) {
|
||||
log.warn(
|
||||
`[Render] ${workerCount} capture workers may exceed this process's V8 heap ` +
|
||||
`(limit ${sizing.heapLimitMb}MB supports ~${sizing.heapBasedWorkers}). If the render ` +
|
||||
`dies with "JavaScript heap out of memory", raise the heap ` +
|
||||
`(NODE_OPTIONS=--max-old-space-size=8192) or pass --workers ${sizing.heapBasedWorkers}.`,
|
||||
{ heapLimitMb: sizing.heapLimitMb, heapBasedWorkers: sizing.heapBasedWorkers },
|
||||
);
|
||||
}
|
||||
|
||||
if (requestedWorkers !== undefined || captureCost.multiplier <= 1) {
|
||||
return workerCount;
|
||||
}
|
||||
|
||||
@@ -125,6 +125,7 @@ import { resolveVideoCaptureBeyondViewport } from "./render/captureBeyondViewpor
|
||||
import {
|
||||
type CaptureCalibrationSample,
|
||||
type CaptureCostEstimate,
|
||||
buildHeapAdvisoryWarning,
|
||||
resolveRenderWorkerCount,
|
||||
runCaptureCalibration,
|
||||
} from "./render/captureCost.js";
|
||||
@@ -2487,6 +2488,13 @@ async function executeRenderPipeline(input: {
|
||||
captureCalibration?.estimate,
|
||||
(sizing) => {
|
||||
workerSizing = sizing;
|
||||
const heapAdvisory = buildHeapAdvisoryWarning(sizing, job.config.workers);
|
||||
if (heapAdvisory) {
|
||||
log.warn(heapAdvisory, {
|
||||
heapLimitMb: sizing.heapLimitMb,
|
||||
heapBasedWorkers: sizing.heapBasedWorkers,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
// DE priority inversion — see shouldPreferSingleWorkerDrawElement for the
|
||||
|
||||
Reference in New Issue
Block a user