mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
Merge pull request #2723 from heygen-com/07-21-fix_engine_worker_autoscaler_memory_budget
fix(engine): realistic worker memory budget + sizing/feedback telemetry
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildTelemetryJoinKeys } from "./feedback.js";
|
||||
|
||||
describe("buildTelemetryJoinKeys", () => {
|
||||
it("emits fid + tid and omits renders when the ring is empty", () => {
|
||||
const keys = buildTelemetryJoinKeys({
|
||||
feedbackId: "feedback-uuid",
|
||||
anonymousId: "install-uuid",
|
||||
});
|
||||
expect(keys).toBe("fid=feedback-uuid tid=install-uuid");
|
||||
});
|
||||
|
||||
it("appends recent render ids newest-last with a ! marking failed renders", () => {
|
||||
const keys = buildTelemetryJoinKeys({
|
||||
feedbackId: "f",
|
||||
anonymousId: "t",
|
||||
recentRenders: [
|
||||
{ id: "render-a", at: "2026-07-21T00:00:00Z", ok: true },
|
||||
{ id: "render-b", at: "2026-07-21T01:00:00Z", ok: false },
|
||||
],
|
||||
});
|
||||
expect(keys).toBe("fid=f tid=t renders=render-a,render-b!");
|
||||
});
|
||||
|
||||
it("stays within the backend env cap for a full ring of uuid render ids", () => {
|
||||
const uuid = "01234567-89ab-cdef-0123-456789abcdef";
|
||||
const keys = buildTelemetryJoinKeys({
|
||||
feedbackId: uuid,
|
||||
anonymousId: uuid,
|
||||
recentRenders: Array.from({ length: 5 }, (_, i) => ({
|
||||
id: uuid,
|
||||
at: "2026-07-21T00:00:00Z",
|
||||
ok: i % 2 === 0,
|
||||
})),
|
||||
});
|
||||
// submitFeedback caps env at 500 chars; the doctor summary consumes ~100.
|
||||
expect(keys.length).toBeLessThan(400);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,5 @@
|
||||
import { failCommand } from "../utils/commandResult.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { resolve } from "node:path";
|
||||
import { defineCommand } from "citty";
|
||||
import * as clack from "@clack/prompts";
|
||||
@@ -7,6 +8,7 @@ import type { Example } from "./_examples.js";
|
||||
import { trackRenderFeedback } from "../telemetry/events.js";
|
||||
import { shouldTrack, flush } from "../telemetry/client.js";
|
||||
import { getDoctorSummary } from "../telemetry/feedback.js";
|
||||
import { readConfig, type RecentRenderRecord } from "../telemetry/config.js";
|
||||
import { publishProjectArchive } from "../utils/publishProject.js";
|
||||
import { submitFeedback } from "../utils/submitFeedback.js";
|
||||
import { buildIssueUrl, HYPERFRAMES_REPO_URL } from "../utils/feedbackIssue.js";
|
||||
@@ -28,6 +30,27 @@ function normalizeComment(raw?: string): string | undefined {
|
||||
return raw || undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact PostHog join keys appended to the environment string that rides
|
||||
* along with the forwarded report (and therefore lands verbatim in the wild
|
||||
* feedback channel): `fid` = this submission's PostHog `survey sent`
|
||||
* `feedback_id`; `tid` = the install's telemetry distinct_id; `renders` =
|
||||
* recent `render_job_id`s (newest last, `!` suffix = the render failed).
|
||||
* Together they turn a wild report into an exact telemetry lookup instead of
|
||||
* a hardware-fingerprint hunt.
|
||||
*/
|
||||
export function buildTelemetryJoinKeys(input: {
|
||||
feedbackId: string;
|
||||
anonymousId: string;
|
||||
recentRenders?: RecentRenderRecord[];
|
||||
}): string {
|
||||
const parts = [`fid=${input.feedbackId}`, `tid=${input.anonymousId}`];
|
||||
if (input.recentRenders?.length) {
|
||||
parts.push(`renders=${input.recentRenders.map((r) => `${r.id}${r.ok ? "" : "!"}`).join(",")}`);
|
||||
}
|
||||
return parts.join(" ");
|
||||
}
|
||||
|
||||
function printIssueConsent(dir: string): void {
|
||||
console.log();
|
||||
console.log(
|
||||
@@ -170,6 +193,18 @@ export default defineCommand({
|
||||
const comment = normalizeComment(args.comment);
|
||||
const doctorSummary = await getDoctorSummary();
|
||||
|
||||
// Join keys tying this report to the install's PostHog rows — see
|
||||
// buildTelemetryJoinKeys. Appended to the env string so they surface in
|
||||
// the forwarded report; mirrored as structured props on the PostHog event.
|
||||
const feedbackId = randomUUID();
|
||||
const config = readConfig();
|
||||
const joinKeys = buildTelemetryJoinKeys({
|
||||
feedbackId,
|
||||
anonymousId: config.anonymousId,
|
||||
recentRenders: config.recentRenders,
|
||||
});
|
||||
const envWithJoinKeys = doctorSummary ? `${doctorSummary} ${joinKeys}` : joinKeys;
|
||||
|
||||
// Soft-warn (never blocks) when the comment for a non-clean report is
|
||||
// missing the mandated reproduction-packet markers. Prints before the
|
||||
// submission ack so the reporter sees the nudge while their run is fresh.
|
||||
@@ -177,13 +212,19 @@ export default defineCommand({
|
||||
|
||||
// The standalone command runs separately from `render`, so it has no real
|
||||
// elapsed time to report. Omit it rather than recording a fake duration.
|
||||
trackRenderFeedback({ rating, comment, doctorSummary });
|
||||
trackRenderFeedback({
|
||||
rating,
|
||||
comment,
|
||||
doctorSummary,
|
||||
feedbackId,
|
||||
recentRenderIds: config.recentRenders?.map((r) => r.id),
|
||||
});
|
||||
|
||||
await flush();
|
||||
// Ack first so the user isn't kept waiting on the best-effort forward (which
|
||||
// is bounded to a few seconds and never surfaces an error either way).
|
||||
console.log(c.dim("Thanks for the feedback!"));
|
||||
await submitFeedback({ rating, comment, cliVersion: VERSION, env: doctorSummary });
|
||||
await submitFeedback({ rating, comment, cliVersion: VERSION, env: envWithJoinKeys });
|
||||
|
||||
if (args["file-issue"] === true) {
|
||||
await fileGithubIssue({
|
||||
|
||||
@@ -61,7 +61,12 @@ import {
|
||||
trackRenderObservation,
|
||||
} from "../telemetry/events.js";
|
||||
import { maybePromptRenderFeedback } from "../telemetry/feedback.js";
|
||||
import { readConfigFresh, writeConfig, type HyperframesConfig } from "../telemetry/config.js";
|
||||
import {
|
||||
readConfigFresh,
|
||||
recordRecentRender,
|
||||
writeConfig,
|
||||
type HyperframesConfig,
|
||||
} from "../telemetry/config.js";
|
||||
import { shouldTrack } from "../telemetry/client.js";
|
||||
import { renderJobObservabilityTelemetryPayload } from "../telemetry/renderObservability.js";
|
||||
import { bytesToMb } from "../telemetry/system.js";
|
||||
@@ -1339,6 +1344,9 @@ function handleRenderError(
|
||||
...renderJobObservabilityTelemetryPayload(job),
|
||||
...getMemorySnapshot(),
|
||||
});
|
||||
// Failed renders join the recent-renders ring too — a bug report filed via
|
||||
// `hyperframes feedback` is MOST likely to be about a failed render.
|
||||
if (job?.id) recordRecentRender(job.id, false);
|
||||
if (options.throwOnError) {
|
||||
throw new Error(message);
|
||||
}
|
||||
@@ -1376,6 +1384,9 @@ function trackRenderMetrics(
|
||||
options: RenderOptions,
|
||||
docker: boolean,
|
||||
): void {
|
||||
// Successful render → recent-renders ring, so a later `hyperframes
|
||||
// feedback` can attach this render's telemetry id to the report.
|
||||
recordRecentRender(job.id, true);
|
||||
const perf = job.perfSummary;
|
||||
const compositionDurationMs = perf
|
||||
? Math.round(perf.compositionDurationSeconds * 1000)
|
||||
@@ -1393,6 +1404,13 @@ function trackRenderMetrics(
|
||||
fps: fpsToNumber(options.fps),
|
||||
quality: options.quality,
|
||||
workers: options.workers ?? perf?.workers,
|
||||
workersBoundBy: perf?.workerSizing?.boundBy,
|
||||
workersCpuBased: perf?.workerSizing?.cpuBasedWorkers,
|
||||
workersMemoryBased: perf?.workerSizing?.memoryBasedWorkers,
|
||||
workersHeapBased: perf?.workerSizing?.heapBasedWorkers,
|
||||
workersFrameBased: perf?.workerSizing?.frameBasedWorkers,
|
||||
workersHeapLimitMb: perf?.workerSizing?.heapLimitMb,
|
||||
workersExceedHeapAdvisory: perf?.workerSizing?.exceedsHeapAdvisory,
|
||||
docker,
|
||||
gpu: options.gpu,
|
||||
authoringSkill: options.authoringSkill,
|
||||
|
||||
@@ -85,6 +85,39 @@ export interface HyperframesConfig {
|
||||
* `DE_PARALLEL_ROUTER_TRIAL_MAX_RENDERS` in `render.ts`.
|
||||
*/
|
||||
deParallelRouterTrialRenderCount?: number;
|
||||
/**
|
||||
* Ring of the last few local renders (newest last). `hyperframes feedback`
|
||||
* attaches these ids — which are the `render_job_id` /
|
||||
* `observability_render_job_id` on this install's PostHog events — to the
|
||||
* feedback it submits, so a wild bug report can be joined to the exact
|
||||
* telemetry rows of the renders it describes.
|
||||
*/
|
||||
recentRenders?: RecentRenderRecord[];
|
||||
}
|
||||
|
||||
/** One entry in {@link HyperframesConfig.recentRenders}. */
|
||||
export interface RecentRenderRecord {
|
||||
/** The render job id (`RenderJob.id` — the telemetry `render_job_id`). */
|
||||
id: string;
|
||||
/** ISO timestamp of when the render finished. */
|
||||
at: string;
|
||||
/** Whether the render completed successfully. */
|
||||
ok: boolean;
|
||||
}
|
||||
|
||||
/** Ring size for {@link HyperframesConfig.recentRenders}. */
|
||||
const MAX_RECENT_RENDERS = 5;
|
||||
|
||||
/**
|
||||
* Append a finished render to the recent-renders ring (newest last, capped).
|
||||
* Fresh read-modify-write like the trial counters — narrows, but does not
|
||||
* eliminate, lost updates against a concurrent CLI process.
|
||||
*/
|
||||
export function recordRecentRender(id: string, ok: boolean): void {
|
||||
const config = readConfigFresh();
|
||||
const ring = [...(config.recentRenders ?? []), { id, at: new Date().toISOString(), ok }];
|
||||
config.recentRenders = ring.slice(-MAX_RECENT_RENDERS);
|
||||
writeConfig(config);
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG: HyperframesConfig = {
|
||||
@@ -142,6 +175,18 @@ export function readConfig(): HyperframesConfig {
|
||||
typeof parsed.deParallelRouterTrialRenderCount === "number"
|
||||
? parsed.deParallelRouterTrialRenderCount
|
||||
: undefined,
|
||||
recentRenders: Array.isArray(parsed.recentRenders)
|
||||
? parsed.recentRenders
|
||||
.filter(
|
||||
(r): r is RecentRenderRecord =>
|
||||
typeof r === "object" &&
|
||||
r !== null &&
|
||||
typeof (r as RecentRenderRecord).id === "string" &&
|
||||
typeof (r as RecentRenderRecord).at === "string" &&
|
||||
typeof (r as RecentRenderRecord).ok === "boolean",
|
||||
)
|
||||
.slice(-MAX_RECENT_RENDERS)
|
||||
: undefined,
|
||||
};
|
||||
|
||||
cachedConfig = config;
|
||||
|
||||
@@ -186,6 +186,59 @@ describe("render telemetry events", () => {
|
||||
expect(flush).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
// The enforcement decision for the advisory heap budget reads these fleet
|
||||
// props (see computeWorkerSizing) — a silent drop in the summary→event hop
|
||||
// would invalidate that decision without anyone noticing.
|
||||
it("carries every worker-sizing provenance prop on render_complete", () => {
|
||||
trackRenderComplete({
|
||||
durationMs: 1000,
|
||||
fps: 30,
|
||||
quality: "high",
|
||||
docker: false,
|
||||
gpu: false,
|
||||
workers: 6,
|
||||
workersBoundBy: "max_workers",
|
||||
workersCpuBased: 16,
|
||||
workersMemoryBased: 8,
|
||||
workersHeapBased: 4,
|
||||
workersFrameBased: 24,
|
||||
workersHeapLimitMb: 4096,
|
||||
workersExceedHeapAdvisory: true,
|
||||
});
|
||||
|
||||
expect(trackEvent).toHaveBeenCalledWith(
|
||||
"render_complete",
|
||||
expect.objectContaining({
|
||||
workers: 6,
|
||||
workers_bound_by: "max_workers",
|
||||
workers_cpu_based: 16,
|
||||
workers_memory_based: 8,
|
||||
workers_heap_based: 4,
|
||||
workers_frame_based: 24,
|
||||
workers_heap_limit_mb: 4096,
|
||||
workers_exceed_heap_advisory: true,
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("ties feedback to its report and recent renders via feedback_id + recent_render_ids", () => {
|
||||
trackRenderFeedback({
|
||||
rating: 3,
|
||||
comment: "hook scene blank",
|
||||
feedbackId: "feedback-uuid",
|
||||
recentRenderIds: ["render-a", "render-b"],
|
||||
});
|
||||
|
||||
expect(trackEvent).toHaveBeenCalledWith(
|
||||
"survey sent",
|
||||
expect.objectContaining({
|
||||
feedback_id: "feedback-uuid",
|
||||
recent_render_ids: "render-a,render-b",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("redacts paths and URL query strings from render error messages", () => {
|
||||
trackRenderError({
|
||||
fps: 30,
|
||||
|
||||
@@ -149,6 +149,17 @@ export function trackRenderComplete(
|
||||
/** Authoring workflow skill that drove this render (e.g. "product-launch-video"). */
|
||||
authoringSkill?: string;
|
||||
workers?: number;
|
||||
// Worker auto-sizing provenance (RenderPerfSummary.workerSizing). Answers
|
||||
// "why N workers?" fleet-wide, and validates the advisory per-worker heap
|
||||
// budget before it's enforced (field OOM: 6 auto workers on a 24GB/4GB-heap
|
||||
// machine — see computeWorkerSizing in @hyperframes/engine).
|
||||
workersBoundBy?: string;
|
||||
workersCpuBased?: number;
|
||||
workersMemoryBased?: number;
|
||||
workersHeapBased?: number;
|
||||
workersFrameBased?: number;
|
||||
workersHeapLimitMb?: number;
|
||||
workersExceedHeapAdvisory?: boolean;
|
||||
docker: boolean;
|
||||
gpu: boolean;
|
||||
// Static-frame dedup outcome (opt-out HF_STATIC_DEDUP=false). Undefined on
|
||||
@@ -245,6 +256,13 @@ export function trackRenderComplete(
|
||||
quality: props.quality,
|
||||
authoring_skill: props.authoringSkill,
|
||||
workers: props.workers,
|
||||
workers_bound_by: props.workersBoundBy,
|
||||
workers_cpu_based: props.workersCpuBased,
|
||||
workers_memory_based: props.workersMemoryBased,
|
||||
workers_heap_based: props.workersHeapBased,
|
||||
workers_frame_based: props.workersFrameBased,
|
||||
workers_heap_limit_mb: props.workersHeapLimitMb,
|
||||
workers_exceed_heap_advisory: props.workersExceedHeapAdvisory,
|
||||
docker: props.docker,
|
||||
gpu: props.gpu,
|
||||
static_dedup_enabled: props.staticDedupEnabled,
|
||||
@@ -609,6 +627,14 @@ export function trackRenderFeedback(props: {
|
||||
renderDurationMs?: number;
|
||||
comment?: string;
|
||||
doctorSummary?: string;
|
||||
/**
|
||||
* Join key shared with the forwarded feedback report (Slack/backend): the
|
||||
* same uuid rides in the report's env string as `fid=…`, so a wild report
|
||||
* resolves to exactly one PostHog "survey sent" event and vice versa.
|
||||
*/
|
||||
feedbackId?: string;
|
||||
/** render_job_id values of this install's recent renders (newest last). */
|
||||
recentRenderIds?: string[];
|
||||
}): void {
|
||||
trackEvent("survey sent", {
|
||||
$survey_id: "render_satisfaction",
|
||||
@@ -617,6 +643,11 @@ export function trackRenderFeedback(props: {
|
||||
...(props.comment ? { $survey_response_2: props.comment } : {}),
|
||||
...(props.renderDurationMs !== undefined ? { render_duration_ms: props.renderDurationMs } : {}),
|
||||
...(props.doctorSummary ? { doctor_summary: props.doctorSummary } : {}),
|
||||
...(props.feedbackId ? { feedback_id: props.feedbackId } : {}),
|
||||
// Comma-joined: EventProperties values are scalars only.
|
||||
...(props.recentRenderIds?.length
|
||||
? { recent_render_ids: props.recentRenderIds.join(",") }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -208,6 +208,7 @@ export type {
|
||||
// ── Parallel rendering ─────────────────────────────────────────────────────────
|
||||
export {
|
||||
calculateOptimalWorkers,
|
||||
computeWorkerSizing,
|
||||
distributeFrames,
|
||||
distributeFramesInterleaved,
|
||||
executeParallelCapture,
|
||||
@@ -215,6 +216,8 @@ export {
|
||||
getSystemResources,
|
||||
type WorkerTask,
|
||||
type WorkerResult,
|
||||
type WorkerSizing,
|
||||
type WorkerSizingBound,
|
||||
type ParallelProgress,
|
||||
} from "./services/parallelCoordinator.js";
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
calculateOptimalWorkers,
|
||||
computeWorkerSizing,
|
||||
distributeFrames,
|
||||
expectedFramesForTask,
|
||||
flagSilentWorkerExits,
|
||||
@@ -93,6 +94,50 @@ describe("calculateOptimalWorkers", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeWorkerSizing", () => {
|
||||
it("matches calculateOptimalWorkers and reports every constraint", () => {
|
||||
const config = { concurrency: "auto" as const };
|
||||
const sizing = computeWorkerSizing(900, undefined, config);
|
||||
expect(sizing.workers).toBe(calculateOptimalWorkers(900, undefined, config));
|
||||
expect(sizing.cpuBasedWorkers).toBeGreaterThanOrEqual(1);
|
||||
expect(sizing.memoryBasedWorkers).toBeGreaterThanOrEqual(1);
|
||||
expect(sizing.heapBasedWorkers).toBeGreaterThanOrEqual(1);
|
||||
expect(sizing.frameBasedWorkers).toBe(30); // 900 / MIN_FRAMES_PER_WORKER(30)
|
||||
expect(sizing.heapLimitMb).toBeGreaterThan(0);
|
||||
expect(sizing.totalMemoryMb).toBeGreaterThan(0);
|
||||
expect(sizing.cpuCount).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("labels explicit requests and clamps them like the legacy wrapper", () => {
|
||||
const sizing = computeWorkerSizing(900, 25, { concurrency: "auto" });
|
||||
expect(sizing.workers).toBe(24);
|
||||
expect(sizing.boundBy).toBe("explicit");
|
||||
});
|
||||
|
||||
it("labels tiny renders as too_few_frames", () => {
|
||||
const sizing = computeWorkerSizing(30, undefined, { concurrency: "auto" });
|
||||
expect(sizing.workers).toBe(1);
|
||||
expect(sizing.boundBy).toBe("too_few_frames");
|
||||
});
|
||||
|
||||
it("labels the contention cap when high capture cost binds", () => {
|
||||
const sizing = computeWorkerSizing(180, undefined, {
|
||||
concurrency: 6,
|
||||
coresPerWorker: 100,
|
||||
minParallelFrames: 120,
|
||||
largeRenderThreshold: 1000,
|
||||
captureCostMultiplier: 4,
|
||||
});
|
||||
expect(sizing.workers).toBe(1);
|
||||
expect(sizing.boundBy).toBe("contention");
|
||||
});
|
||||
|
||||
it("flags exceedsHeapAdvisory exactly when workers exceed the heap budget", () => {
|
||||
const sizing = computeWorkerSizing(900, 24, { concurrency: "auto" });
|
||||
expect(sizing.exceedsHeapAdvisory).toBe(sizing.workers > sizing.heapBasedWorkers);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldDisableBrowserPoolForParallelWorker", () => {
|
||||
const linuxHeadlessWorker = {
|
||||
parallel: true,
|
||||
|
||||
@@ -9,6 +9,7 @@ import { cpus, freemem } from "os";
|
||||
import { existsSync, mkdirSync, readdirSync } from "fs";
|
||||
import { copyFile, rename } from "fs/promises";
|
||||
import { join } from "path";
|
||||
import { getHeapStatistics } from "v8";
|
||||
|
||||
import {
|
||||
createCaptureSession,
|
||||
@@ -108,7 +109,22 @@ type WorkerBrowserPoolDecision = {
|
||||
headlessShellPath?: string;
|
||||
};
|
||||
|
||||
const MEMORY_PER_WORKER_MB = 256;
|
||||
// System-memory budget per parallel worker. Each worker is a full Chrome
|
||||
// process (SwiftShader compositor + raster threads) plus its share of parent-
|
||||
// process frame buffering — the old 256MB figure was ~6× under a measured
|
||||
// Chrome-under-capture RSS and let memory-constrained hosts overcommit (wild
|
||||
// 16GB black-slab report; 0.7.66 heap-OOM report on 24GB with 6 auto workers).
|
||||
const MEMORY_PER_WORKER_MB = 1536;
|
||||
// Parent-process V8 heap the coordinator itself needs regardless of worker
|
||||
// count (compile artifacts, puppeteer sessions, encoder state).
|
||||
const HEAP_RESERVED_MB = 1024;
|
||||
// Parent-process V8 heap consumed per worker (protocol buffers + in-flight
|
||||
// frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB
|
||||
// default heap ⇒ >~500MB/worker + base. ponytail: advisory-only until the
|
||||
// workers_heap_* telemetry added alongside this constant validates the figure
|
||||
// — enforcing a guessed budget could silently cut worker counts fleet-wide.
|
||||
// TODO(PRINFRA-341): decide enforcement after ~2 weeks of fleet soak.
|
||||
const HEAP_PER_WORKER_MB = 640;
|
||||
const MIN_WORKERS = 1;
|
||||
const MAX_WORKER_DIAGNOSTIC_LINES = 8;
|
||||
// Hard ceiling on explicit `--workers N` requests. Above this, the cost of
|
||||
@@ -229,13 +245,95 @@ export function formatWorkerFailure(result: WorkerResult): string {
|
||||
return `${base}; diagnostics: ${diagnostics}`;
|
||||
}
|
||||
|
||||
export function calculateOptimalWorkers(
|
||||
/** Which constraint produced the final auto-sized worker count. */
|
||||
export type WorkerSizingBound =
|
||||
| "explicit"
|
||||
| "too_few_frames"
|
||||
| "cpu"
|
||||
| "memory"
|
||||
| "frames"
|
||||
| "max_workers"
|
||||
| "min_parallel_floor"
|
||||
| "contention";
|
||||
|
||||
/**
|
||||
* Full provenance of a worker-sizing decision. Threaded into render
|
||||
* observability/telemetry so fleet data can answer "why N workers?" and
|
||||
* "would the heap budget have prevented this OOM?" without a repro.
|
||||
*/
|
||||
export interface WorkerSizing {
|
||||
workers: number;
|
||||
boundBy: WorkerSizingBound;
|
||||
cpuBasedWorkers: number;
|
||||
memoryBasedWorkers: number;
|
||||
frameBasedWorkers: number;
|
||||
effectiveMaxWorkers: number;
|
||||
/**
|
||||
* ADVISORY, not enforced (see HEAP_PER_WORKER_MB): how many workers the
|
||||
* parent process's V8 heap could feed. Compare against `workers` in
|
||||
* telemetry to validate the budget before enforcement.
|
||||
*/
|
||||
heapBasedWorkers: number;
|
||||
/** V8 `heap_size_limit` for the parent process, MB. */
|
||||
heapLimitMb: number;
|
||||
totalMemoryMb: number;
|
||||
cpuCount: number;
|
||||
captureCostMultiplier: number;
|
||||
/** true when the chosen count exceeds the advisory heap budget. */
|
||||
exceedsHeapAdvisory: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the auto worker count together with the full sizing provenance.
|
||||
* `calculateOptimalWorkers` is the thin legacy wrapper returning `.workers`.
|
||||
*/
|
||||
// The branch count is the decision provenance itself — each if records WHICH
|
||||
// constraint bound, which is the entire point of the function.
|
||||
// fallow-ignore-next-line complexity
|
||||
export function computeWorkerSizing(
|
||||
totalFrames: number,
|
||||
requested?: number,
|
||||
config?: WorkerSizingConfig,
|
||||
): number {
|
||||
): WorkerSizing {
|
||||
const cpuCount = cpus().length;
|
||||
// Use total memory instead of free memory — macOS reports misleadingly low
|
||||
// freemem() because it aggressively caches files in "inactive" memory that
|
||||
// is immediately reclaimable.
|
||||
const totalMemoryMb = getSystemTotalMb();
|
||||
const heapLimitMb = Math.round(getHeapStatistics().heap_size_limit / (1024 * 1024));
|
||||
const heapBasedWorkers = Math.max(
|
||||
1,
|
||||
Math.floor((heapLimitMb - HEAP_RESERVED_MB) / HEAP_PER_WORKER_MB),
|
||||
);
|
||||
const cpuBasedWorkers = Math.max(1, cpuCount - 2);
|
||||
const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMb * 0.5) / MEMORY_PER_WORKER_MB));
|
||||
const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER);
|
||||
const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1);
|
||||
|
||||
const base = {
|
||||
cpuBasedWorkers,
|
||||
memoryBasedWorkers,
|
||||
frameBasedWorkers,
|
||||
heapBasedWorkers,
|
||||
heapLimitMb,
|
||||
totalMemoryMb,
|
||||
cpuCount,
|
||||
captureCostMultiplier,
|
||||
};
|
||||
const finish = (workers: number, boundBy: WorkerSizingBound, effectiveMaxWorkers: number) => ({
|
||||
workers,
|
||||
boundBy,
|
||||
effectiveMaxWorkers,
|
||||
...base,
|
||||
exceedsHeapAdvisory: workers > heapBasedWorkers,
|
||||
});
|
||||
|
||||
if (requested !== undefined) {
|
||||
return Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, requested));
|
||||
return finish(
|
||||
Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, requested)),
|
||||
"explicit",
|
||||
ABSOLUTE_MAX_WORKERS,
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve effective values: config overrides → DEFAULT_CONFIG fallback.
|
||||
@@ -250,24 +348,22 @@ export function calculateOptimalWorkers(
|
||||
const effectiveMinParallelFrames = config?.minParallelFrames ?? DEFAULT_CONFIG.minParallelFrames;
|
||||
const effectiveLargeRenderThreshold =
|
||||
config?.largeRenderThreshold ?? DEFAULT_CONFIG.largeRenderThreshold;
|
||||
const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1);
|
||||
|
||||
if (totalFrames < MIN_FRAMES_PER_WORKER * 2) return 1;
|
||||
|
||||
const cpuCount = cpus().length;
|
||||
const cpuBasedWorkers = Math.max(1, cpuCount - 2);
|
||||
|
||||
// Use total memory instead of free memory — macOS reports misleadingly low
|
||||
// freemem() because it aggressively caches files in "inactive" memory that
|
||||
// is immediately reclaimable.
|
||||
const totalMemoryMB = getSystemTotalMb();
|
||||
const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMB * 0.5) / MEMORY_PER_WORKER_MB));
|
||||
|
||||
const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER);
|
||||
if (totalFrames < MIN_FRAMES_PER_WORKER * 2) {
|
||||
return finish(1, "too_few_frames", effectiveMaxWorkers);
|
||||
}
|
||||
|
||||
const optimal = Math.min(cpuBasedWorkers, memoryBasedWorkers, frameBasedWorkers);
|
||||
const optimalBound: WorkerSizingBound =
|
||||
optimal === cpuBasedWorkers ? "cpu" : optimal === memoryBasedWorkers ? "memory" : "frames";
|
||||
const minWorkersForJob = totalFrames >= effectiveMinParallelFrames ? 2 : MIN_WORKERS;
|
||||
let finalWorkers = Math.max(minWorkersForJob, Math.min(effectiveMaxWorkers, optimal));
|
||||
let boundBy: WorkerSizingBound =
|
||||
finalWorkers === optimal
|
||||
? optimalBound
|
||||
: finalWorkers === effectiveMaxWorkers && effectiveMaxWorkers < optimal
|
||||
? "max_workers"
|
||||
: "min_parallel_floor";
|
||||
|
||||
// Adaptive scaling: cap workers for large or expensive renders to prevent
|
||||
// CPU contention. Each Chrome process (with SwiftShader) is CPU-heavy; too
|
||||
@@ -284,10 +380,19 @@ export function calculateOptimalWorkers(
|
||||
const cpuScaledMax = Math.max(MIN_WORKERS, Math.floor(cpuCount / weightedCoresPerWorker));
|
||||
if (finalWorkers > cpuScaledMax) {
|
||||
finalWorkers = cpuScaledMax;
|
||||
boundBy = "contention";
|
||||
}
|
||||
}
|
||||
|
||||
return finalWorkers;
|
||||
return finish(finalWorkers, boundBy, effectiveMaxWorkers);
|
||||
}
|
||||
|
||||
export function calculateOptimalWorkers(
|
||||
totalFrames: number,
|
||||
requested?: number,
|
||||
config?: WorkerSizingConfig,
|
||||
): number {
|
||||
return computeWorkerSizing(totalFrames, requested, config).workers;
|
||||
}
|
||||
|
||||
export function distributeFrames(
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { WorkerSizing } from "@hyperframes/engine";
|
||||
import { buildHeapAdvisoryWarning } from "./captureCost.js";
|
||||
|
||||
function sizing(overrides: Partial<WorkerSizing>): WorkerSizing {
|
||||
return {
|
||||
workers: 6,
|
||||
boundBy: "max_workers",
|
||||
cpuBasedWorkers: 16,
|
||||
memoryBasedWorkers: 8,
|
||||
frameBasedWorkers: 24,
|
||||
effectiveMaxWorkers: 6,
|
||||
heapBasedWorkers: 4,
|
||||
heapLimitMb: 4096,
|
||||
totalMemoryMb: 24576,
|
||||
cpuCount: 18,
|
||||
captureCostMultiplier: 1,
|
||||
exceedsHeapAdvisory: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildHeapAdvisoryWarning", () => {
|
||||
it("names the chosen count, heap limit, safe count, and both remediation knobs", () => {
|
||||
const message = buildHeapAdvisoryWarning(sizing({}), undefined);
|
||||
expect(message).toContain("6 capture workers");
|
||||
expect(message).toContain("limit 4096MB");
|
||||
expect(message).toContain("~4");
|
||||
expect(message).toContain("NODE_OPTIONS=--max-old-space-size=8192");
|
||||
expect(message).toContain("--workers 4");
|
||||
});
|
||||
|
||||
it("stays silent for explicit --workers requests (operator's own call)", () => {
|
||||
expect(buildHeapAdvisoryWarning(sizing({}), 6)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("stays silent when the chosen count fits the heap budget", () => {
|
||||
expect(
|
||||
buildHeapAdvisoryWarning(sizing({ exceedsHeapAdvisory: false }), undefined),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -16,9 +16,11 @@ import {
|
||||
type CaptureOptions,
|
||||
type CaptureSession,
|
||||
type EngineConfig,
|
||||
type WorkerSizing,
|
||||
calculateOptimalWorkers,
|
||||
captureFrameToBuffer,
|
||||
closeCaptureSession,
|
||||
computeWorkerSizing,
|
||||
createCaptureSession,
|
||||
initializeSession,
|
||||
} from "@hyperframes/engine";
|
||||
@@ -106,6 +108,36 @@ function combineCaptureCostEstimates(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Advisory heap warning (field OOM, 0.7.66 on 24GB: auto-picked 6 workers
|
||||
* blew Node's default ~4GB heap). Returns the warning message, or undefined
|
||||
* when it should not fire:
|
||||
*
|
||||
* - Auto-sized renders only (`requestedWorkers === undefined`) — the field
|
||||
* failure was auto sizing, and an explicit `--workers N` is the operator's
|
||||
* own call.
|
||||
* - Not enforced as a cap yet — the per-worker budget constant is derived
|
||||
* from one field report; the `workers_heap_*` telemetry emitted with the
|
||||
* sizing decides whether to enforce (see the TODO on HEAP_PER_WORKER_MB in
|
||||
* @hyperframes/engine's parallelCoordinator). The message gives the
|
||||
* operator the actionable knobs today.
|
||||
*
|
||||
* Pure so the message shape + firing condition are unit-testable with a
|
||||
* synthetic `WorkerSizing` (the real one depends on the host's heap).
|
||||
*/
|
||||
export function buildHeapAdvisoryWarning(
|
||||
sizing: WorkerSizing,
|
||||
requestedWorkers: number | undefined,
|
||||
): string | undefined {
|
||||
if (requestedWorkers !== undefined || !sizing.exceedsHeapAdvisory) return undefined;
|
||||
return (
|
||||
`[Render] ${sizing.workers} capture workers may exceed this process's V8 heap ` +
|
||||
`(limit ${sizing.heapLimitMb}MB supports ~${sizing.heapBasedWorkers}). If the render ` +
|
||||
`dies with "JavaScript heap out of memory", raise the heap ` +
|
||||
`(NODE_OPTIONS=--max-old-space-size=8192) or pass --workers ${sizing.heapBasedWorkers}.`
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveRenderWorkerCount(
|
||||
totalFrames: number,
|
||||
requestedWorkers: number | undefined,
|
||||
@@ -113,6 +145,8 @@ export function resolveRenderWorkerCount(
|
||||
compiled: Pick<CompiledComposition, "hasShaderTransitions" | "renderModeHints">,
|
||||
log: ProducerLogger = defaultLogger,
|
||||
measuredCaptureCost?: CaptureCostEstimate,
|
||||
/** Sink for the sizing provenance so the orchestrator can thread it into telemetry. */
|
||||
onSizing?: (sizing: WorkerSizing) => void,
|
||||
): number {
|
||||
// TODO(htmlInCanvas): workaround — Chrome's experimental drawElementImage
|
||||
// API (CanvasDrawElement) is non-deterministic across concurrent browser
|
||||
@@ -144,10 +178,16 @@ export function resolveRenderWorkerCount(
|
||||
estimateCaptureCostMultiplier(compiled),
|
||||
measuredCaptureCost,
|
||||
);
|
||||
const workerCount = calculateOptimalWorkers(totalFrames, requestedWorkers, {
|
||||
const sizing = computeWorkerSizing(totalFrames, requestedWorkers, {
|
||||
...cfg,
|
||||
captureCostMultiplier: captureCost.multiplier,
|
||||
});
|
||||
// The heap advisory is deliberately NOT logged here: this function's
|
||||
// "no unexpected warns" unit-test contract would become dependent on the
|
||||
// test machine's actual V8 heap limit. The orchestrator's onSizing consumer
|
||||
// emits it via buildHeapAdvisoryWarning.
|
||||
onSizing?.(sizing);
|
||||
const workerCount = sizing.workers;
|
||||
|
||||
if (requestedWorkers !== undefined || captureCost.multiplier <= 1) {
|
||||
return workerCount;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
*/
|
||||
|
||||
import { fpsToNumber } from "@hyperframes/core";
|
||||
import type { CapturePerfSummary, SubTimelineWaitOutcome } from "@hyperframes/engine";
|
||||
import type { CapturePerfSummary, SubTimelineWaitOutcome, WorkerSizing } from "@hyperframes/engine";
|
||||
import type { CaptureCalibrationSample, CaptureCostEstimate } from "./captureCost.js";
|
||||
import type {
|
||||
CaptureAttemptSummary,
|
||||
@@ -181,6 +181,8 @@ function aggregateBeginFrameReuse(
|
||||
export function buildRenderPerfSummary(input: {
|
||||
job: RenderJob;
|
||||
workerCount: number;
|
||||
/** Auto-sizing provenance; undefined when a pin (htmlInCanvas / low-memory) short-circuited sizing. */
|
||||
workerSizing?: WorkerSizing;
|
||||
enableChunkedEncode: boolean;
|
||||
chunkedEncodeSize: number;
|
||||
compositionDurationSeconds: number;
|
||||
@@ -217,6 +219,7 @@ export function buildRenderPerfSummary(input: {
|
||||
fps: fpsToNumber(input.job.config.fps),
|
||||
quality: input.job.config.quality,
|
||||
workers: input.workerCount,
|
||||
workerSizing: input.workerSizing,
|
||||
chunkedEncode: input.enableChunkedEncode,
|
||||
chunkSizeFrames: input.enableChunkedEncode ? input.chunkedEncodeSize : null,
|
||||
compositionDurationSeconds: input.compositionDurationSeconds,
|
||||
|
||||
@@ -76,6 +76,7 @@ import {
|
||||
type CapturePerfSummary,
|
||||
type CaptureWarning,
|
||||
type SubTimelineWaitOutcome,
|
||||
type WorkerSizing,
|
||||
resolveBrowserGpuMode,
|
||||
resolveHeadlessShellPath,
|
||||
applyConcreteGpuScreenshotClamp,
|
||||
@@ -124,6 +125,7 @@ import { resolveVideoCaptureBeyondViewport } from "./render/captureBeyondViewpor
|
||||
import {
|
||||
type CaptureCalibrationSample,
|
||||
type CaptureCostEstimate,
|
||||
buildHeapAdvisoryWarning,
|
||||
resolveRenderWorkerCount,
|
||||
runCaptureCalibration,
|
||||
} from "./render/captureCost.js";
|
||||
@@ -372,6 +374,14 @@ export interface RenderPerfSummary {
|
||||
fps: number;
|
||||
quality: string;
|
||||
workers: number;
|
||||
/**
|
||||
* Provenance of the auto worker-sizing decision (undefined when the
|
||||
* htmlInCanvas / low-memory pins short-circuited sizing). `boundBy` names
|
||||
* the binding constraint; the heap fields are the advisory budget being
|
||||
* validated by fleet telemetry before enforcement — see
|
||||
* `computeWorkerSizing` in @hyperframes/engine.
|
||||
*/
|
||||
workerSizing?: WorkerSizing;
|
||||
chunkedEncode: boolean;
|
||||
chunkSizeFrames: number | null;
|
||||
compositionDurationSeconds: number;
|
||||
@@ -1702,6 +1712,7 @@ async function executeRenderPipeline(input: {
|
||||
// "routed" = the parallel router fired and held; "reverted" = fired but
|
||||
// the self-verify retry rolled back; undefined = never fired.
|
||||
let deParallelRouter: "routed" | "reverted" | undefined;
|
||||
let workerSizing: WorkerSizing | undefined;
|
||||
|
||||
execution.defer("rollback staged artifact", () => artifactTransaction.rollback());
|
||||
execution.defer("close file server", () => {
|
||||
@@ -2473,6 +2484,16 @@ async function executeRenderPipeline(input: {
|
||||
compiled,
|
||||
log,
|
||||
captureCalibration?.estimate,
|
||||
(sizing) => {
|
||||
workerSizing = sizing;
|
||||
const heapAdvisory = buildHeapAdvisoryWarning(sizing, job.config.workers);
|
||||
if (heapAdvisory) {
|
||||
log.warn(heapAdvisory, {
|
||||
heapLimitMb: sizing.heapLimitMb,
|
||||
heapBasedWorkers: sizing.heapBasedWorkers,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
// DE priority inversion — see shouldPreferSingleWorkerDrawElement for the
|
||||
// policy and benchmark rationale (eligibility resolved above, before
|
||||
@@ -3279,6 +3300,7 @@ async function executeRenderPipeline(input: {
|
||||
const perfSummary = buildRenderPerfSummary({
|
||||
job,
|
||||
workerCount,
|
||||
workerSizing,
|
||||
enableChunkedEncode,
|
||||
chunkedEncodeSize,
|
||||
compositionDurationSeconds: composition.duration,
|
||||
|
||||
Reference in New Issue
Block a user