Merge pull request #2723 from heygen-com/07-21-fix_engine_worker_autoscaler_memory_budget

fix(engine): realistic worker memory budget + sizing/feedback telemetry

packages/producer/src/services/render/captureCost.heapAdvisory.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import type { WorkerSizing } from "@hyperframes/engine";
+import { buildHeapAdvisoryWarning } from "./captureCost.js";
+
+function sizing(overrides: Partial<WorkerSizing>): WorkerSizing {
+  return {
+    workers: 6,
+    boundBy: "max_workers",
+    cpuBasedWorkers: 16,
+    memoryBasedWorkers: 8,
+    frameBasedWorkers: 24,
+    effectiveMaxWorkers: 6,
+    heapBasedWorkers: 4,
+    heapLimitMb: 4096,
+    totalMemoryMb: 24576,
+    cpuCount: 18,
+    captureCostMultiplier: 1,
+    exceedsHeapAdvisory: true,
+    ...overrides,
+  };
+}
+
+describe("buildHeapAdvisoryWarning", () => {
+  it("names the chosen count, heap limit, safe count, and both remediation knobs", () => {
+    const message = buildHeapAdvisoryWarning(sizing({}), undefined);
+    expect(message).toContain("6 capture workers");
+    expect(message).toContain("limit 4096MB");
+    expect(message).toContain("~4");
+    expect(message).toContain("NODE_OPTIONS=--max-old-space-size=8192");
+    expect(message).toContain("--workers 4");
+  });
+
+  it("stays silent for explicit --workers requests (operator's own call)", () => {
+    expect(buildHeapAdvisoryWarning(sizing({}), 6)).toBeUndefined();
+  });
+
+  it("stays silent when the chosen count fits the heap budget", () => {
+    expect(
+      buildHeapAdvisoryWarning(sizing({ exceedsHeapAdvisory: false }), undefined),
+    ).toBeUndefined();
+  });
+});

packages/producer/src/services/render/captureCost.ts

@@ -16,9 +16,11 @@ import {
   type CaptureOptions,
   type CaptureSession,
   type EngineConfig,
+  type WorkerSizing,
   calculateOptimalWorkers,
   captureFrameToBuffer,
   closeCaptureSession,
+  computeWorkerSizing,
   createCaptureSession,
   initializeSession,
 } from "@hyperframes/engine";
@@ -106,6 +108,36 @@ function combineCaptureCostEstimates(
   };
 }
 
+/**
+ * Advisory heap warning (field OOM, 0.7.66 on 24GB: auto-picked 6 workers
+ * blew Node's default ~4GB heap). Returns the warning message, or undefined
+ * when it should not fire:
+ *
+ * - Auto-sized renders only (`requestedWorkers === undefined`) — the field
+ *   failure was auto sizing, and an explicit `--workers N` is the operator's
+ *   own call.
+ * - Not enforced as a cap yet — the per-worker budget constant is derived
+ *   from one field report; the `workers_heap_*` telemetry emitted with the
+ *   sizing decides whether to enforce (see the TODO on HEAP_PER_WORKER_MB in
+ *   @hyperframes/engine's parallelCoordinator). The message gives the
+ *   operator the actionable knobs today.
+ *
+ * Pure so the message shape + firing condition are unit-testable with a
+ * synthetic `WorkerSizing` (the real one depends on the host's heap).
+ */
+export function buildHeapAdvisoryWarning(
+  sizing: WorkerSizing,
+  requestedWorkers: number | undefined,
+): string | undefined {
+  if (requestedWorkers !== undefined || !sizing.exceedsHeapAdvisory) return undefined;
+  return (
+    `[Render] ${sizing.workers} capture workers may exceed this process's V8 heap ` +
+    `(limit ${sizing.heapLimitMb}MB supports ~${sizing.heapBasedWorkers}). If the render ` +
+    `dies with "JavaScript heap out of memory", raise the heap ` +
+    `(NODE_OPTIONS=--max-old-space-size=8192) or pass --workers ${sizing.heapBasedWorkers}.`
+  );
+}
+
 export function resolveRenderWorkerCount(
   totalFrames: number,
   requestedWorkers: number | undefined,
@@ -113,6 +145,8 @@ export function resolveRenderWorkerCount(
   compiled: Pick<CompiledComposition, "hasShaderTransitions" | "renderModeHints">,
   log: ProducerLogger = defaultLogger,
   measuredCaptureCost?: CaptureCostEstimate,
+  /** Sink for the sizing provenance so the orchestrator can thread it into telemetry. */
+  onSizing?: (sizing: WorkerSizing) => void,
 ): number {
   // TODO(htmlInCanvas): workaround — Chrome's experimental drawElementImage
   // API (CanvasDrawElement) is non-deterministic across concurrent browser
@@ -144,10 +178,16 @@ export function resolveRenderWorkerCount(
     estimateCaptureCostMultiplier(compiled),
     measuredCaptureCost,
   );
-  const workerCount = calculateOptimalWorkers(totalFrames, requestedWorkers, {
+  const sizing = computeWorkerSizing(totalFrames, requestedWorkers, {
     ...cfg,
     captureCostMultiplier: captureCost.multiplier,
   });
+  // The heap advisory is deliberately NOT logged here: this function's
+  // "no unexpected warns" unit-test contract would become dependent on the
+  // test machine's actual V8 heap limit. The orchestrator's onSizing consumer
+  // emits it via buildHeapAdvisoryWarning.
+  onSizing?.(sizing);
+  const workerCount = sizing.workers;
 
   if (requestedWorkers !== undefined || captureCost.multiplier <= 1) {
     return workerCount;

packages/producer/src/services/render/perfSummary.ts

@@ -4,7 +4,7 @@
  */
 
 import { fpsToNumber } from "@hyperframes/core";
-import type { CapturePerfSummary, SubTimelineWaitOutcome } from "@hyperframes/engine";
+import type { CapturePerfSummary, SubTimelineWaitOutcome, WorkerSizing } from "@hyperframes/engine";
 import type { CaptureCalibrationSample, CaptureCostEstimate } from "./captureCost.js";
 import type {
   CaptureAttemptSummary,
@@ -181,6 +181,8 @@ function aggregateBeginFrameReuse(
 export function buildRenderPerfSummary(input: {
   job: RenderJob;
   workerCount: number;
+  /** Auto-sizing provenance; undefined when a pin (htmlInCanvas / low-memory) short-circuited sizing. */
+  workerSizing?: WorkerSizing;
   enableChunkedEncode: boolean;
   chunkedEncodeSize: number;
   compositionDurationSeconds: number;
@@ -217,6 +219,7 @@ export function buildRenderPerfSummary(input: {
     fps: fpsToNumber(input.job.config.fps),
     quality: input.job.config.quality,
     workers: input.workerCount,
+    workerSizing: input.workerSizing,
     chunkedEncode: input.enableChunkedEncode,
     chunkSizeFrames: input.enableChunkedEncode ? input.chunkedEncodeSize : null,
     compositionDurationSeconds: input.compositionDurationSeconds,

packages/producer/src/services/renderOrchestrator.ts

@@ -76,6 +76,7 @@ import {
   type CapturePerfSummary,
   type CaptureWarning,
   type SubTimelineWaitOutcome,
+  type WorkerSizing,
   resolveBrowserGpuMode,
   resolveHeadlessShellPath,
   applyConcreteGpuScreenshotClamp,
@@ -124,6 +125,7 @@ import { resolveVideoCaptureBeyondViewport } from "./render/captureBeyondViewpor
 import {
   type CaptureCalibrationSample,
   type CaptureCostEstimate,
+  buildHeapAdvisoryWarning,
   resolveRenderWorkerCount,
   runCaptureCalibration,
 } from "./render/captureCost.js";
@@ -372,6 +374,14 @@ export interface RenderPerfSummary {
   fps: number;
   quality: string;
   workers: number;
+  /**
+   * Provenance of the auto worker-sizing decision (undefined when the
+   * htmlInCanvas / low-memory pins short-circuited sizing). `boundBy` names
+   * the binding constraint; the heap fields are the advisory budget being
+   * validated by fleet telemetry before enforcement — see
+   * `computeWorkerSizing` in @hyperframes/engine.
+   */
+  workerSizing?: WorkerSizing;
   chunkedEncode: boolean;
   chunkSizeFrames: number | null;
   compositionDurationSeconds: number;
@@ -1702,6 +1712,7 @@ async function executeRenderPipeline(input: {
   // "routed" = the parallel router fired and held; "reverted" = fired but
   // the self-verify retry rolled back; undefined = never fired.
   let deParallelRouter: "routed" | "reverted" | undefined;
+  let workerSizing: WorkerSizing | undefined;
 
   execution.defer("rollback staged artifact", () => artifactTransaction.rollback());
   execution.defer("close file server", () => {
@@ -2473,6 +2484,16 @@ async function executeRenderPipeline(input: {
       compiled,
       log,
       captureCalibration?.estimate,
+      (sizing) => {
+        workerSizing = sizing;
+        const heapAdvisory = buildHeapAdvisoryWarning(sizing, job.config.workers);
+        if (heapAdvisory) {
+          log.warn(heapAdvisory, {
+            heapLimitMb: sizing.heapLimitMb,
+            heapBasedWorkers: sizing.heapBasedWorkers,
+          });
+        }
+      },
     );
     // DE priority inversion — see shouldPreferSingleWorkerDrawElement for the
     // policy and benchmark rationale (eligibility resolved above, before
@@ -3279,6 +3300,7 @@ async function executeRenderPipeline(input: {
     const perfSummary = buildRenderPerfSummary({
       job,
       workerCount,
+      workerSizing,
       enableChunkedEncode,
       chunkedEncodeSize,
       compositionDurationSeconds: composition.duration,