From 36b3fc8cd95639424f211cd4a1a8f232d3529124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Mon, 27 Apr 2026 21:27:16 -0400 Subject: [PATCH] fix: budget workers for expensive captures --- packages/cli/src/commands/render.ts | 18 +- packages/cli/src/telemetry/events.ts | 2 +- packages/cli/src/utils/dockerRunArgs.test.ts | 18 +- packages/cli/src/utils/dockerRunArgs.ts | 5 +- .../src/services/parallelCoordinator.test.ts | 28 +- .../src/services/parallelCoordinator.ts | 40 +- .../src/services/htmlCompiler.test.ts | 31 + .../producer/src/services/htmlCompiler.ts | 20 + .../src/services/renderOrchestrator.test.ts | 359 +++++++++-- .../src/services/renderOrchestrator.ts | 558 +++++++++++++++++- 10 files changed, 978 insertions(+), 101 deletions(-) diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index b123c3972..f91954823 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -35,11 +35,6 @@ const FORMAT_EXT: Record = { mp4: ".mp4", webm: ".webm", mov: ". const CPU_CORE_COUNT = cpus().length; -/** 3/4 of CPU cores, capped at 8. Each worker spawns a Chrome process (~256 MB). */ -function defaultWorkerCount(): number { - return Math.max(1, Math.min(Math.floor((CPU_CORE_COUNT * 3) / 4), 8)); -} - export default defineCommand({ meta: { name: "render", @@ -216,12 +211,9 @@ export default defineCommand({ } // ── Print render plan ───────────────────────────────────────────────── - const workerCount = workers ?? defaultWorkerCount(); if (!quiet) { const workerLabel = - args.workers != null - ? `${workerCount} workers` - : `${workerCount} workers (auto — ${CPU_CORE_COUNT} cores detected)`; + workers != null ? `${workers} workers` : `auto workers (${CPU_CORE_COUNT} cores detected)`; console.log(""); console.log( c.accent("\u25C6") + @@ -307,7 +299,7 @@ export default defineCommand({ fps, quality, format, - workers: workerCount, + workers, gpu: useGpu, hdr: args.hdr ?? false, crf, @@ -319,7 +311,7 @@ export default defineCommand({ fps, quality, format, - workers: workerCount, + workers, gpu: useGpu, hdr: args.hdr ?? false, crf, @@ -335,7 +327,7 @@ interface RenderOptions { fps: 24 | 30 | 60; quality: "draft" | "standard" | "high"; format: "mp4" | "webm" | "mov"; - workers: number; + workers?: number; gpu: boolean; hdr: boolean; crf?: number; @@ -604,7 +596,7 @@ function trackRenderMetrics( durationMs: elapsedMs, fps: options.fps, quality: options.quality, - workers: options.workers, + workers: options.workers ?? perf?.workers, docker, gpu: options.gpu, compositionDurationMs, diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index c89016a0b..57269361f 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -8,7 +8,7 @@ export function trackRenderComplete(props: { durationMs: number; fps: number; quality: string; - workers: number; + workers?: number; docker: boolean; gpu: boolean; // Composition metadata diff --git a/packages/cli/src/utils/dockerRunArgs.test.ts b/packages/cli/src/utils/dockerRunArgs.test.ts index 287afba65..42a4c416a 100644 --- a/packages/cli/src/utils/dockerRunArgs.test.ts +++ b/packages/cli/src/utils/dockerRunArgs.test.ts @@ -5,7 +5,6 @@ const BASE: DockerRenderOptions = { fps: 30, quality: "standard", format: "mp4", - workers: 4, gpu: false, hdr: false, crf: undefined, @@ -43,17 +42,28 @@ describe("buildDockerRunArgs", () => { "standard", "--format", "mp4", - "--workers", - "4", ] `); }); + it("omits --workers when auto sizing should happen inside the container", () => { + const args = buildDockerRunArgs({ ...FIXED_INPUT, options: BASE }); + expect(args).not.toContain("--workers"); + }); + it("matches snapshot when every renderer flag is enabled", () => { expect( buildDockerRunArgs({ ...FIXED_INPUT, - options: { ...BASE, gpu: true, hdr: true, crf: 18, videoBitrate: undefined, quiet: true }, + options: { + ...BASE, + workers: 4, + gpu: true, + hdr: true, + crf: 18, + videoBitrate: undefined, + quiet: true, + }, }), ).toMatchInlineSnapshot(` [ diff --git a/packages/cli/src/utils/dockerRunArgs.ts b/packages/cli/src/utils/dockerRunArgs.ts index b56b5aef5..6ec926103 100644 --- a/packages/cli/src/utils/dockerRunArgs.ts +++ b/packages/cli/src/utils/dockerRunArgs.ts @@ -22,7 +22,7 @@ export interface DockerRenderOptions { fps: 24 | 30 | 60; quality: "draft" | "standard" | "high"; format: "mp4" | "webm" | "mov"; - workers: number; + workers?: number; gpu: boolean; hdr: boolean; crf?: number; @@ -54,8 +54,7 @@ export function buildDockerRunArgs(input: DockerRunArgsInput): string[] { options.quality, "--format", options.format, - "--workers", - String(options.workers), + ...(options.workers != null ? ["--workers", String(options.workers)] : []), ...(options.crf != null ? ["--crf", String(options.crf)] : []), ...(options.videoBitrate ? ["--video-bitrate", options.videoBitrate] : []), ...(options.quiet ? ["--quiet"] : []), diff --git a/packages/engine/src/services/parallelCoordinator.test.ts b/packages/engine/src/services/parallelCoordinator.test.ts index cddcc5721..696343618 100644 --- a/packages/engine/src/services/parallelCoordinator.test.ts +++ b/packages/engine/src/services/parallelCoordinator.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { distributeFrames } from "./parallelCoordinator.js"; +import { calculateOptimalWorkers, distributeFrames } from "./parallelCoordinator.js"; describe("distributeFrames", () => { it("distributes frames evenly across workers", () => { @@ -42,3 +42,29 @@ describe("distributeFrames", () => { expect(tasks.map((t) => t.workerId)).toEqual([0, 1, 2]); }); }); + +describe("calculateOptimalWorkers", () => { + it("lets high-cost auto renders fall back to one worker when CPU budget requires it", () => { + const workers = calculateOptimalWorkers(180, undefined, { + concurrency: 6, + coresPerWorker: 100, + minParallelFrames: 120, + largeRenderThreshold: 1000, + captureCostMultiplier: 4, + }); + + expect(workers).toBe(1); + }); + + it("does not apply capture cost to explicit worker requests", () => { + const workers = calculateOptimalWorkers(180, 4, { + concurrency: 6, + coresPerWorker: 100, + minParallelFrames: 120, + largeRenderThreshold: 1000, + captureCostMultiplier: 4, + }); + + expect(workers).toBe(4); + }); +}); diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts index 96c35485e..38c3472b6 100644 --- a/packages/engine/src/services/parallelCoordinator.ts +++ b/packages/engine/src/services/parallelCoordinator.ts @@ -48,6 +48,20 @@ export interface ParallelProgress { workerProgress: Map; } +export interface WorkerSizingConfig extends Partial< + Pick< + EngineConfig, + "concurrency" | "coresPerWorker" | "minParallelFrames" | "largeRenderThreshold" + > +> { + /** + * Relative per-frame capture cost for auto worker sizing. Values above 1 + * represent compositions that put more CPU pressure on each Chrome worker + * than a plain DOM screenshot. Explicit --workers requests ignore this hint. + */ + captureCostMultiplier?: number; +} + const MEMORY_PER_WORKER_MB = 256; const MIN_WORKERS = 1; const ABSOLUTE_MAX_WORKERS = 10; @@ -57,12 +71,7 @@ const MIN_FRAMES_PER_WORKER = 30; export function calculateOptimalWorkers( totalFrames: number, requested?: number, - config?: Partial< - Pick< - EngineConfig, - "concurrency" | "coresPerWorker" | "minParallelFrames" | "largeRenderThreshold" - > - >, + config?: WorkerSizingConfig, ): number { // Resolve effective values: config overrides → DEFAULT_CONFIG fallback. const effectiveMaxWorkers = (() => { @@ -76,6 +85,7 @@ export function calculateOptimalWorkers( const effectiveMinParallelFrames = config?.minParallelFrames ?? DEFAULT_CONFIG.minParallelFrames; const effectiveLargeRenderThreshold = config?.largeRenderThreshold ?? DEFAULT_CONFIG.largeRenderThreshold; + const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1); if (requested !== undefined) { return Math.max(MIN_WORKERS, Math.min(effectiveMaxWorkers, requested)); @@ -98,13 +108,19 @@ export function calculateOptimalWorkers( const minWorkersForJob = totalFrames >= effectiveMinParallelFrames ? 2 : MIN_WORKERS; let finalWorkers = Math.max(minWorkersForJob, Math.min(effectiveMaxWorkers, optimal)); - // Adaptive scaling: cap workers for large renders to prevent CPU contention. - // Each Chrome process (with SwiftShader) is CPU-heavy; too many on a long - // render causes protocol timeouts from compositor starvation. - // Scale proportionally to CPU count: ~3 cores per worker (benchmarked). + // Adaptive scaling: cap workers for large or expensive renders to prevent + // CPU contention. Each Chrome process (with SwiftShader) is CPU-heavy; too + // many concurrent captures can starve the compositor and surface as CDP + // protocol timeouts. Scale proportionally to CPU count and composition cost: // 8 cores → 2 workers, 16 cores → 5 workers, 32 cores → 10 workers. - if (totalFrames >= effectiveLargeRenderThreshold) { - const cpuScaledMax = Math.max(2, Math.floor(cpuCount / effectiveCoresPerWorker)); + const weightedFrames = totalFrames * captureCostMultiplier; + const contentionThreshold = Math.max( + effectiveMinParallelFrames, + Math.floor(effectiveLargeRenderThreshold / 3), + ); + if (totalFrames >= effectiveLargeRenderThreshold || weightedFrames >= contentionThreshold) { + const weightedCoresPerWorker = effectiveCoresPerWorker * captureCostMultiplier; + const cpuScaledMax = Math.max(MIN_WORKERS, Math.floor(cpuCount / weightedCoresPerWorker)); if (finalWorkers > cpuScaledMax) { finalWorkers = cpuScaledMax; } diff --git a/packages/producer/src/services/htmlCompiler.test.ts b/packages/producer/src/services/htmlCompiler.test.ts index 9a0851318..b566f6942 100644 --- a/packages/producer/src/services/htmlCompiler.test.ts +++ b/packages/producer/src/services/htmlCompiler.test.ts @@ -7,6 +7,7 @@ import { collectExternalAssets, compileForRender, detectRenderModeHints, + detectShaderTransitionUsage, inlineExternalScripts, recompileWithResolutions, } from "./htmlCompiler.js"; @@ -459,6 +460,36 @@ describe("detectRenderModeHints", () => { }); }); +describe("detectShaderTransitionUsage", () => { + it("detects authored HyperShader initialization", () => { + const html = ` + + + +`; + + expect(detectShaderTransitionUsage(html)).toBe(true); + }); + + it("ignores comments and external scripts by themselves", () => { + const html = ` + + + +`; + + expect(detectShaderTransitionUsage(html)).toBe(false); + }); +}); + describe("template-wrapped sub-composition media offsets", () => { function writeTemplateWrappedProject( hostAttrs: string, diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index 80d08acba..557ec2dee 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -51,6 +51,7 @@ export interface CompiledComposition { height: number; staticDuration: number; renderModeHints: RenderModeHints; + hasShaderTransitions: boolean; } export type RenderModeHintCode = "iframe" | "requestAnimationFrame"; @@ -124,6 +125,22 @@ export function detectRenderModeHints(html: string): RenderModeHints { }; } +const SHADER_TRANSITION_USAGE_PATTERN = + /\b(?:(?:window|globalThis)\s*\.\s*)?HyperShader\s*\.\s*init\s*\(|\b__hf\s*\.\s*transitions\s*=/; + +export function detectShaderTransitionUsage(html: string): boolean { + let scriptMatch: RegExpExecArray | null; + const scriptPattern = new RegExp(INLINE_SCRIPT_PATTERN.source, INLINE_SCRIPT_PATTERN.flags); + while ((scriptMatch = scriptPattern.exec(html)) !== null) { + const attrs = scriptMatch[1] || ""; + if (/\bsrc\s*=/i.test(attrs)) continue; + const content = stripJsComments(stripCompilerMountBootstrap(scriptMatch[2] || "")); + if (SHADER_TRANSITION_USAGE_PATTERN.test(content)) return true; + } + + return false; +} + async function resolveMediaDuration( src: string, mediaStart: number, @@ -932,6 +949,7 @@ export async function compileForRender( "$1", ); const renderModeHints = detectRenderModeHints(sanitizedHtml); + const hasShaderTransitions = detectShaderTransitionUsage(sanitizedHtml); const coalescedHtml = await injectDeterministicFontFaces( coalesceHeadStylesAndBodyScripts(promoteCssImportsToLinkTags(sanitizedHtml)), @@ -1014,6 +1032,7 @@ export async function compileForRender( height, staticDuration, renderModeHints, + hasShaderTransitions, }; } @@ -1189,5 +1208,6 @@ export async function recompileWithResolutions( images, unresolvedCompositions: remaining, renderModeHints: compiled.renderModeHints, + hasShaderTransitions: compiled.hasShaderTransitions, }; } diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index 7902bde8a..5c524204a 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -7,8 +7,18 @@ import type { CompiledComposition } from "./htmlCompiler.js"; import { applyRenderModeHints, + buildMissingFrameRetryBatches, + createCaptureCalibrationConfig, + estimateMeasuredCaptureCostMultiplier, + estimateCaptureCostMultiplier, extractStandaloneEntryFromIndex, + findMissingFrameRanges, + getNextRetryWorkerCount, + isRecoverableParallelCaptureError, projectBrowserEndToCompositionTimeline, + resolveRenderWorkerCount, + selectCaptureCalibrationFrames, + shouldFallbackToScreenshotAfterCalibrationError, writeCompiledArtifacts, } from "./renderOrchestrator.js"; import { toExternalAssetKey } from "../utils/paths.js"; @@ -119,6 +129,7 @@ describe("writeCompiledArtifacts — external assets on Windows drive-letter pat recommendScreenshot: false, reasons: [], }, + hasShaderTransitions: false, }; writeCompiledArtifacts(compiled, workDir, false); @@ -150,6 +161,7 @@ describe("writeCompiledArtifacts — external assets on Windows drive-letter pat recommendScreenshot: false, reasons: [], }, + hasShaderTransitions: false, }; writeCompiledArtifacts(compiled, workDir, false); @@ -159,60 +171,61 @@ describe("writeCompiledArtifacts — external assets on Windows drive-letter pat }); }); +function createCompiledComposition( + reasonCodes: Array<"iframe" | "requestAnimationFrame">, +): CompiledComposition { + return { + html: "", + subCompositions: new Map(), + videos: [], + audios: [], + unresolvedCompositions: [], + externalAssets: new Map(), + width: 1920, + height: 1080, + staticDuration: 5, + renderModeHints: { + recommendScreenshot: reasonCodes.length > 0, + reasons: reasonCodes.map((code) => ({ + code, + message: `reason: ${code}`, + })), + }, + hasShaderTransitions: false, + }; +} + +function createConfig(): EngineConfig { + return { + fps: 30, + quality: "standard", + format: "jpeg", + jpegQuality: 80, + concurrency: "auto", + coresPerWorker: 2.5, + minParallelFrames: 120, + largeRenderThreshold: 1000, + disableGpu: false, + enableBrowserPool: false, + browserTimeout: 120000, + protocolTimeout: 300000, + forceScreenshot: false, + enableChunkedEncode: false, + chunkSizeFrames: 360, + enableStreamingEncode: false, + ffmpegEncodeTimeout: 600000, + ffmpegProcessTimeout: 300000, + ffmpegStreamingTimeout: 600000, + audioGain: 1, + frameDataUriCacheLimit: 256, + playerReadyTimeout: 45000, + renderReadyTimeout: 15000, + verifyRuntime: true, + debug: false, + }; +} + describe("applyRenderModeHints", () => { - function createCompiledComposition( - reasonCodes: Array<"iframe" | "requestAnimationFrame">, - ): CompiledComposition { - return { - html: "", - subCompositions: new Map(), - videos: [], - audios: [], - unresolvedCompositions: [], - externalAssets: new Map(), - width: 1920, - height: 1080, - staticDuration: 5, - renderModeHints: { - recommendScreenshot: reasonCodes.length > 0, - reasons: reasonCodes.map((code) => ({ - code, - message: `reason: ${code}`, - })), - }, - }; - } - - function createConfig(): EngineConfig { - return { - fps: 30, - quality: "standard", - format: "jpeg", - jpegQuality: 80, - concurrency: "auto", - coresPerWorker: 2.5, - minParallelFrames: 120, - largeRenderThreshold: 1000, - disableGpu: false, - enableBrowserPool: false, - browserTimeout: 120000, - protocolTimeout: 300000, - forceScreenshot: false, - enableChunkedEncode: false, - chunkSizeFrames: 360, - enableStreamingEncode: false, - ffmpegEncodeTimeout: 600000, - ffmpegProcessTimeout: 300000, - ffmpegStreamingTimeout: 600000, - audioGain: 1, - frameDataUriCacheLimit: 256, - playerReadyTimeout: 45000, - renderReadyTimeout: 15000, - verifyRuntime: true, - debug: false, - }; - } - it("forces screenshot mode when compatibility hints recommend it", () => { const cfg = createConfig(); const compiled = createCompiledComposition(["iframe", "requestAnimationFrame"]); @@ -246,6 +259,246 @@ describe("applyRenderModeHints", () => { }); }); +describe("resolveRenderWorkerCount", () => { + const cfg = { ...createConfig(), coresPerWorker: 100 }; + const audio = { + id: "narration", + src: "narration.wav", + start: 0, + end: 3, + mediaStart: 0, + layer: 9, + type: "audio" as const, + }; + + it("reduces auto workers for expensive capture workloads", () => { + const log = { + error: vi.fn(), + warn: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + }; + + const workers = resolveRenderWorkerCount( + 180, + undefined, + cfg, + { + hasShaderTransitions: true, + renderModeHints: { recommendScreenshot: false, reasons: [] }, + }, + { videos: [], audios: [audio] }, + log, + ); + + expect(workers).toBe(1); + expect(log.warn).toHaveBeenCalledOnce(); + }); + + it("respects explicit worker requests", () => { + const log = { + error: vi.fn(), + warn: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + }; + + const workers = resolveRenderWorkerCount( + 180, + 6, + cfg, + { + hasShaderTransitions: true, + renderModeHints: { recommendScreenshot: false, reasons: [] }, + }, + { videos: [], audios: [audio] }, + log, + ); + + expect(workers).toBe(6); + expect(log.warn).not.toHaveBeenCalled(); + }); + + it("uses measured capture cost when static hints miss an expensive composition", () => { + const workers = resolveRenderWorkerCount( + 180, + undefined, + cfg, + { + hasShaderTransitions: false, + renderModeHints: { recommendScreenshot: false, reasons: [] }, + }, + { videos: [], audios: [] }, + undefined, + { multiplier: 4, reasons: ["calibration-p95=2400ms"] }, + ); + + expect(workers).toBe(1); + }); +}); + +describe("estimateCaptureCostMultiplier", () => { + it("weights shader transitions, media, and render mode hints", () => { + const cost = estimateCaptureCostMultiplier( + { + hasShaderTransitions: true, + renderModeHints: { + recommendScreenshot: true, + reasons: [{ code: "requestAnimationFrame", message: "raw rAF" }], + }, + }, + { + videos: [], + audios: [ + { + id: "narration", + src: "narration.wav", + start: 0, + end: 3, + mediaStart: 0, + layer: 9, + type: "audio" as const, + }, + ], + }, + ); + + expect(cost.multiplier).toBe(4.75); + expect(cost.reasons).toEqual(["shader-transitions", "requestAnimationFrame", "1 audio"]); + }); +}); + +describe("estimateMeasuredCaptureCostMultiplier", () => { + it("turns slow calibration samples into a capture cost multiplier", () => { + const estimate = estimateMeasuredCaptureCostMultiplier([ + { frameIndex: 0, captureTimeMs: 180 }, + { frameIndex: 45, captureTimeMs: 700 }, + { frameIndex: 90, captureTimeMs: 2400 }, + { frameIndex: 135, captureTimeMs: 900 }, + ]); + + expect(estimate.multiplier).toBe(4); + expect(estimate.reasons).toEqual(["calibration-p95=2400ms"]); + }); + + it("keeps fast calibration samples at baseline cost", () => { + const estimate = estimateMeasuredCaptureCostMultiplier([ + { frameIndex: 0, captureTimeMs: 120 }, + { frameIndex: 60, captureTimeMs: 180 }, + { frameIndex: 119, captureTimeMs: 220 }, + ]); + + expect(estimate.multiplier).toBe(1); + expect(estimate.reasons).toEqual([]); + }); +}); + +describe("selectCaptureCalibrationFrames", () => { + it("samples the start, middle, end, and quartiles without duplicates", () => { + expect(selectCaptureCalibrationFrames(180)).toEqual([0, 45, 90, 135, 179]); + expect(selectCaptureCalibrationFrames(3)).toEqual([0, 1, 2]); + }); +}); + +describe("capture calibration safeguards", () => { + it("uses a bounded protocol timeout for calibration probes", () => { + const cfg = createConfig(); + const calibrationCfg = createCaptureCalibrationConfig(cfg); + + expect(calibrationCfg.protocolTimeout).toBe(30000); + expect(cfg.protocolTimeout).toBe(300000); + }); + + it("preserves smaller explicit protocol timeouts for calibration probes", () => { + const cfg = createConfig(); + cfg.protocolTimeout = 5000; + + expect(createCaptureCalibrationConfig(cfg).protocolTimeout).toBe(5000); + }); + + it("falls back to screenshot mode after beginFrame calibration failures", () => { + expect( + shouldFallbackToScreenshotAfterCalibrationError( + new Error("HeadlessExperimental.beginFrame timed out"), + ), + ).toBe(true); + expect(shouldFallbackToScreenshotAfterCalibrationError(new Error("ffmpeg exited"))).toBe(false); + }); +}); + +describe("adaptive missing-frame retry helpers", () => { + const tempDirs: string[] = []; + + afterEach(() => { + while (tempDirs.length > 0) { + const d = tempDirs.pop(); + if (d) rmSync(d, { recursive: true, force: true }); + } + }); + + function makeFramesDir(): string { + const d = mkdtempSync(join(tmpdir(), "hf-missing-frames-")); + tempDirs.push(d); + return d; + } + + it("finds contiguous missing frame ranges from captured disk frames", () => { + const framesDir = makeFramesDir(); + for (const frameIndex of [0, 1, 4]) { + writeFileSync(join(framesDir, `frame_${String(frameIndex).padStart(6, "0")}.jpg`), "x"); + } + + expect(findMissingFrameRanges(6, framesDir, "jpg")).toEqual([ + { startFrame: 2, endFrame: 4 }, + { startFrame: 5, endFrame: 6 }, + ]); + }); + + it("builds retry batches that cap active workers per attempt", () => { + const batches = buildMissingFrameRetryBatches( + [ + { startFrame: 2, endFrame: 4 }, + { startFrame: 5, endFrame: 6 }, + { startFrame: 9, endFrame: 12 }, + ], + 2, + "/tmp/work", + 1, + ); + + expect(batches).toHaveLength(2); + expect(batches[0]).toMatchObject([ + { workerId: 0, startFrame: 2, endFrame: 4 }, + { workerId: 1, startFrame: 5, endFrame: 6 }, + ]); + expect(batches[1]).toMatchObject([{ workerId: 0, startFrame: 9, endFrame: 12 }]); + expect(batches[0][0].outputDir).toContain("retry-1-batch-0-worker-0"); + }); + + it("halves retry workers until sequential fallback", () => { + expect(getNextRetryWorkerCount(8)).toBe(4); + expect(getNextRetryWorkerCount(3)).toBe(1); + expect(getNextRetryWorkerCount(2)).toBe(1); + expect(getNextRetryWorkerCount(1)).toBe(1); + }); + + it("only retries parallel capture timeout failures", () => { + expect( + isRecoverableParallelCaptureError( + new Error("[Parallel] Capture failed: Worker 0: Runtime.callFunctionOn timed out"), + ), + ).toBe(true); + expect( + isRecoverableParallelCaptureError( + new Error("[Parallel] Capture failed: Worker 1: HeadlessExperimental.beginFrame timed out"), + ), + ).toBe(true); + expect(isRecoverableParallelCaptureError(new Error("Encoding failed: ffmpeg exited"))).toBe( + false, + ); + }); +}); + describe("projectBrowserEndToCompositionTimeline", () => { it("keeps end unchanged when browser and compiled starts share the same origin", () => { expect(projectBrowserEndToCompositionTimeline(2, 2, 6)).toBe(6); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index b66279b82..d6cc3300a 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -58,6 +58,8 @@ import { distributeFrames, executeParallelCapture, mergeWorkerFrames, + type ParallelProgress, + type WorkerTask, spawnStreamingEncoder, createFrameReorderBuffer, type StreamingEncoder, @@ -278,6 +280,13 @@ export interface RenderPerfSummary { tmpPeakBytes?: number; captureAvgMs?: number; capturePeakMs?: number; + captureCalibration?: { + sampledFrames: number[]; + p95Ms?: number; + multiplier: number; + reasons: string[]; + }; + captureAttempts?: CaptureAttemptSummary[]; /** * Peak resident set size (RSS) observed during the render, in MiB. * @@ -303,6 +312,29 @@ export interface HdrDiagnostics { imageDecodeFailures: number; } +export interface CaptureCostEstimate { + multiplier: number; + reasons: string[]; + p95Ms?: number; +} + +export interface CaptureCalibrationSample { + frameIndex: number; + captureTimeMs: number; +} + +export interface FrameRange { + startFrame: number; + endFrame: number; +} + +export interface CaptureAttemptSummary { + attempt: number; + workers: number; + frameCount: number; + reason: "initial" | "retry"; +} + export interface RenderJob { id: string; config: RenderConfig; @@ -480,6 +512,7 @@ export function writeCompiledArtifacts( })), subCompositions: Array.from(compiled.subCompositions.keys()), renderModeHints: compiled.renderModeHints, + hasShaderTransitions: compiled.hasShaderTransitions, }; writeFileSync(join(compileDir, "summary.json"), JSON.stringify(summary, null, 2), "utf-8"); } @@ -499,6 +532,383 @@ export function applyRenderModeHints( }); } +export function resolveRenderWorkerCount( + totalFrames: number, + requestedWorkers: number | undefined, + cfg: EngineConfig, + compiled: Pick, + composition: Pick, + log: ProducerLogger = defaultLogger, + measuredCaptureCost?: CaptureCostEstimate, +): number { + const captureCost = combineCaptureCostEstimates( + estimateCaptureCostMultiplier(compiled, composition), + measuredCaptureCost, + ); + const workerCount = calculateOptimalWorkers(totalFrames, requestedWorkers, { + ...cfg, + captureCostMultiplier: captureCost.multiplier, + }); + + if (requestedWorkers !== undefined || captureCost.multiplier <= 1) { + return workerCount; + } + + const baselineWorkers = calculateOptimalWorkers(totalFrames, undefined, cfg); + if (workerCount < baselineWorkers) { + log.warn( + "[Render] Reduced auto worker count for high-cost capture workload to avoid Chrome compositor starvation.", + { + from: baselineWorkers, + to: workerCount, + costMultiplier: captureCost.multiplier, + reasons: captureCost.reasons, + }, + ); + } + + return workerCount; +} + +export function estimateCaptureCostMultiplier( + compiled: Pick, + composition: Pick, +): CaptureCostEstimate { + let multiplier = 1; + const reasons: string[] = []; + + if (compiled.hasShaderTransitions) { + multiplier += 2; + reasons.push("shader-transitions"); + } + + const reasonCodes = new Set(compiled.renderModeHints.reasons.map((reason) => reason.code)); + if (reasonCodes.has("requestAnimationFrame")) { + multiplier += 1; + reasons.push("requestAnimationFrame"); + } + if (reasonCodes.has("iframe")) { + multiplier += 0.5; + reasons.push("iframe"); + } + + if (composition.videos.length > 0) { + multiplier += Math.min(2, composition.videos.length * 0.75); + reasons.push(`${composition.videos.length} video${composition.videos.length === 1 ? "" : "s"}`); + } + + if (composition.audios.length > 0) { + multiplier += Math.min(1, composition.audios.length * 0.75); + reasons.push(`${composition.audios.length} audio${composition.audios.length === 1 ? "" : "s"}`); + } + + return { + multiplier: Math.round(multiplier * 100) / 100, + reasons, + }; +} + +function combineCaptureCostEstimates( + staticCost: CaptureCostEstimate, + measuredCost?: CaptureCostEstimate, +): CaptureCostEstimate { + if (!measuredCost || measuredCost.multiplier <= 1) return staticCost; + if (staticCost.multiplier >= measuredCost.multiplier) { + return { + multiplier: staticCost.multiplier, + reasons: [...staticCost.reasons, ...measuredCost.reasons], + p95Ms: measuredCost.p95Ms, + }; + } + return { + multiplier: measuredCost.multiplier, + reasons: [...measuredCost.reasons, ...staticCost.reasons], + p95Ms: measuredCost.p95Ms, + }; +} + +const CAPTURE_CALIBRATION_TARGET_MS = 600; +const MAX_MEASURED_CAPTURE_COST_MULTIPLIER = 8; +const CAPTURE_CALIBRATION_PROTOCOL_TIMEOUT_MS = 30_000; + +export function createCaptureCalibrationConfig(cfg: EngineConfig): EngineConfig { + return { + ...cfg, + protocolTimeout: Math.min(cfg.protocolTimeout, CAPTURE_CALIBRATION_PROTOCOL_TIMEOUT_MS), + }; +} + +export function estimateMeasuredCaptureCostMultiplier( + samples: CaptureCalibrationSample[], +): CaptureCostEstimate { + if (samples.length === 0) { + return { multiplier: 1, reasons: [] }; + } + + const sorted = [...samples].sort((a, b) => a.captureTimeMs - b.captureTimeMs); + const p95Index = Math.max(0, Math.ceil(sorted.length * 0.95) - 1); + const p95Sample = sorted[p95Index] ?? sorted[sorted.length - 1]; + if (!p95Sample) { + return { multiplier: 1, reasons: [] }; + } + const p95Ms = Math.round(p95Sample.captureTimeMs); + const multiplier = Math.min( + MAX_MEASURED_CAPTURE_COST_MULTIPLIER, + Math.max(1, Math.round((p95Ms / CAPTURE_CALIBRATION_TARGET_MS) * 100) / 100), + ); + + return { + multiplier, + reasons: multiplier > 1 ? [`calibration-p95=${p95Ms}ms`] : [], + p95Ms, + }; +} + +export function selectCaptureCalibrationFrames(totalFrames: number): number[] { + if (totalFrames <= 0) return []; + const lastFrame = totalFrames - 1; + const candidates = [ + 0, + Math.floor(totalFrames * 0.25), + Math.floor(totalFrames * 0.5), + Math.floor(totalFrames * 0.75), + lastFrame, + ]; + return Array.from( + new Set(candidates.map((frame) => Math.max(0, Math.min(lastFrame, frame)))), + ).sort((a, b) => a - b); +} + +export function findMissingFrameRanges( + totalFrames: number, + framesDir: string, + frameExt: "jpg" | "png", +): FrameRange[] { + const ranges: FrameRange[] = []; + let rangeStart: number | null = null; + + for (let frameIndex = 0; frameIndex < totalFrames; frameIndex++) { + const framePath = join(framesDir, `frame_${String(frameIndex).padStart(6, "0")}.${frameExt}`); + const missing = !existsSync(framePath); + if (missing && rangeStart === null) { + rangeStart = frameIndex; + } else if (!missing && rangeStart !== null) { + ranges.push({ startFrame: rangeStart, endFrame: frameIndex }); + rangeStart = null; + } + } + + if (rangeStart !== null) { + ranges.push({ startFrame: rangeStart, endFrame: totalFrames }); + } + + return ranges; +} + +export function buildMissingFrameRetryBatches( + ranges: FrameRange[], + maxWorkers: number, + workDir: string, + attempt: number, +): WorkerTask[][] { + const workersPerBatch = Math.max(1, Math.floor(maxWorkers)); + const batches: WorkerTask[][] = []; + + for (let i = 0; i < ranges.length; i += workersPerBatch) { + const batchIndex = batches.length; + const batch = ranges.slice(i, i + workersPerBatch).map((range, workerId) => ({ + workerId, + startFrame: range.startFrame, + endFrame: range.endFrame, + outputDir: join(workDir, `retry-${attempt}-batch-${batchIndex}-worker-${workerId}`), + })); + batches.push(batch); + } + + return batches; +} + +export function getNextRetryWorkerCount(currentWorkers: number): number { + return Math.max(1, Math.floor(currentWorkers / 2)); +} + +export function isRecoverableParallelCaptureError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes("[Parallel] Capture failed") && + /Runtime\.callFunctionOn timed out|HeadlessExperimental\.beginFrame timed out|Waiting failed|timeout exceeded|timed out|Navigation timeout|Protocol error|Target closed/i.test( + message, + ) + ); +} + +export function shouldFallbackToScreenshotAfterCalibrationError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /HeadlessExperimental\.beginFrame timed out|beginFrame probe timeout|Another frame is pending|Frame still pending|Protocol error.*HeadlessExperimental\.beginFrame/i.test( + message, + ); +} + +function countCapturedFrames( + totalFrames: number, + framesDir: string, + frameExt: "jpg" | "png", +): number { + let captured = 0; + for (let frameIndex = 0; frameIndex < totalFrames; frameIndex++) { + const framePath = join(framesDir, `frame_${String(frameIndex).padStart(6, "0")}.${frameExt}`); + if (existsSync(framePath)) captured++; + } + return captured; +} + +function countFrameRanges(ranges: FrameRange[]): number { + return ranges.reduce((sum, range) => sum + (range.endFrame - range.startFrame), 0); +} + +async function measureCaptureCostFromSession( + session: CaptureSession, + totalFrames: number, + fps: number, +): Promise<{ estimate: CaptureCostEstimate; samples: CaptureCalibrationSample[] }> { + const sampledFrames = selectCaptureCalibrationFrames(totalFrames); + const samples: CaptureCalibrationSample[] = []; + + for (const frameIndex of sampledFrames) { + const time = frameIndex / fps; + const startedAt = Date.now(); + const result = await captureFrameToBuffer(session, frameIndex, time); + samples.push({ + frameIndex, + captureTimeMs: result.captureTimeMs || Date.now() - startedAt, + }); + } + + return { + estimate: estimateMeasuredCaptureCostMultiplier(samples), + samples, + }; +} + +async function executeDiskCaptureWithAdaptiveRetry(options: { + serverUrl: string; + workDir: string; + framesDir: string; + totalFrames: number; + initialWorkerCount: number; + allowRetry: boolean; + frameExt: "jpg" | "png"; + captureOptions: CaptureOptions; + createBeforeCaptureHook: () => BeforeCaptureHook | null; + abortSignal?: AbortSignal; + onProgress?: (progress: ParallelProgress) => void; + cfg: EngineConfig; + log: ProducerLogger; +}): Promise { + const attempts: CaptureAttemptSummary[] = []; + let currentWorkers = options.initialWorkerCount; + let missingRanges: FrameRange[] | null = null; + let attempt = 0; + + while (true) { + const frameCount = missingRanges ? countFrameRanges(missingRanges) : options.totalFrames; + attempts.push({ + attempt, + workers: currentWorkers, + frameCount, + reason: attempt === 0 ? "initial" : "retry", + }); + + const attemptWorkDir = join(options.workDir, `capture-attempt-${attempt}`); + const batches = missingRanges + ? buildMissingFrameRetryBatches(missingRanges, currentWorkers, attemptWorkDir, attempt) + : [distributeFrames(options.totalFrames, currentWorkers, attemptWorkDir)]; + + try { + for (const tasks of batches) { + const capturedBeforeBatch = countCapturedFrames( + options.totalFrames, + options.framesDir, + options.frameExt, + ); + try { + await executeParallelCapture( + options.serverUrl, + attemptWorkDir, + tasks, + options.captureOptions, + options.createBeforeCaptureHook, + options.abortSignal, + options.onProgress + ? (progress) => { + options.onProgress?.({ + ...progress, + totalFrames: options.totalFrames, + capturedFrames: Math.min( + options.totalFrames, + capturedBeforeBatch + progress.capturedFrames, + ), + }); + } + : undefined, + undefined, + options.cfg, + ); + } finally { + await mergeWorkerFrames(attemptWorkDir, tasks, options.framesDir); + } + } + + const remaining = findMissingFrameRanges( + options.totalFrames, + options.framesDir, + options.frameExt, + ); + if (remaining.length === 0) { + return attempts; + } + if (!options.allowRetry || currentWorkers <= 1) { + throw new Error( + `[Render] Capture completed but ${countFrameRanges(remaining)} frame(s) are missing`, + ); + } + + const nextWorkers = getNextRetryWorkerCount(currentWorkers); + options.log.warn("[Render] Retrying missing captured frames with fewer workers.", { + fromWorkers: currentWorkers, + toWorkers: nextWorkers, + missingFrames: countFrameRanges(remaining), + }); + currentWorkers = nextWorkers; + missingRanges = remaining; + attempt++; + } catch (error) { + const remaining = findMissingFrameRanges( + options.totalFrames, + options.framesDir, + options.frameExt, + ); + if (remaining.length === 0) { + return attempts; + } + if (!options.allowRetry || currentWorkers <= 1 || !isRecoverableParallelCaptureError(error)) { + throw error; + } + + const nextWorkers = getNextRetryWorkerCount(currentWorkers); + options.log.warn("[Render] Parallel capture timed out; retrying missing frames.", { + fromWorkers: currentWorkers, + toWorkers: nextWorkers, + missingFrames: countFrameRanges(remaining), + error: error instanceof Error ? error.message : String(error), + }); + currentWorkers = nextWorkers; + missingRanges = remaining; + attempt++; + } + } +} + /** * Blit a single HDR video layer onto an rgb48le canvas. * @@ -1607,7 +2017,113 @@ export async function executeRenderJob( skipReadinessVideoIds: Array.from(nativeHdrVideoIds), }); - const workerCount = calculateOptimalWorkers(totalFrames, job.config.workers, cfg); + let captureCalibration: + | { + estimate: CaptureCostEstimate; + samples: CaptureCalibrationSample[]; + } + | undefined; + let switchedToScreenshotAfterCalibration = false; + + if (job.config.workers === undefined && totalFrames >= 60) { + const calibrationDir = join(workDir, "capture-calibration"); + const calibrationCfg = createCaptureCalibrationConfig(cfg); + const videoInjector = createVideoFrameInjector(frameLookup); + let calibrationSession: CaptureSession | null = null; + try { + calibrationSession = await createCaptureSession( + fileServer.url, + calibrationDir, + buildHdrCaptureOptions(), + videoInjector, + calibrationCfg, + ); + if (!calibrationSession.isInitialized) { + await initializeSession(calibrationSession); + } + assertNotAborted(); + + captureCalibration = await measureCaptureCostFromSession( + calibrationSession, + totalFrames, + job.config.fps, + ); + if (captureCalibration.estimate.multiplier > 1) { + log.warn("[Render] Measured slow frame capture during auto-worker calibration.", { + multiplier: captureCalibration.estimate.multiplier, + p95Ms: captureCalibration.estimate.p95Ms, + sampledFrames: captureCalibration.samples.map((sample) => sample.frameIndex), + }); + } else { + log.debug("[Render] Auto-worker calibration kept baseline capture cost.", { + p95Ms: captureCalibration.estimate.p95Ms, + sampledFrames: captureCalibration.samples.map((sample) => sample.frameIndex), + }); + } + } catch (error) { + const shouldFallbackToScreenshot = + !cfg.forceScreenshot && shouldFallbackToScreenshotAfterCalibrationError(error); + if (shouldFallbackToScreenshot) { + cfg.forceScreenshot = true; + switchedToScreenshotAfterCalibration = true; + if (probeSession) { + lastBrowserConsole = probeSession.browserConsoleBuffer; + await closeCaptureSession(probeSession).catch(() => {}); + probeSession = null; + } + } + captureCalibration = { + estimate: { + multiplier: MAX_MEASURED_CAPTURE_COST_MULTIPLIER, + reasons: shouldFallbackToScreenshot + ? ["calibration-beginframe-timeout", "screenshot-fallback"] + : ["calibration-failed"], + }, + samples: [], + }; + if (shouldFallbackToScreenshot) { + log.warn( + "[Render] BeginFrame auto-worker calibration timed out; falling back to screenshot capture mode.", + { + protocolTimeout: calibrationCfg.protocolTimeout, + error: error instanceof Error ? error.message : String(error), + }, + ); + } else { + log.warn("[Render] Auto-worker calibration failed; using conservative worker budget.", { + protocolTimeout: calibrationCfg.protocolTimeout, + error: error instanceof Error ? error.message : String(error), + }); + } + } finally { + if (calibrationSession) { + lastBrowserConsole = calibrationSession.browserConsoleBuffer; + await closeCaptureSession(calibrationSession).catch(() => {}); + } + } + } + + let workerCount = resolveRenderWorkerCount( + totalFrames, + job.config.workers, + cfg, + compiled, + composition, + log, + captureCalibration?.estimate, + ); + + if (switchedToScreenshotAfterCalibration && workerCount > 1) { + workerCount = 1; + } + + if (workerCount > 1 && probeSession) { + lastBrowserConsole = probeSession.browserConsoleBuffer; + await closeCaptureSession(probeSession); + probeSession = null; + } + + const captureAttempts: CaptureAttemptSummary[] = []; // png-sequence is "no container" — outputPath is treated as a directory and // the encode/mux/faststart stages are skipped entirely. The empty extension @@ -2500,16 +3016,18 @@ export async function executeRenderJob( // ── Disk-based capture (original flow) ──────────────────────────── if (workerCount > 1) { // Parallel capture - const tasks = distributeFrames(job.totalFrames, workerCount, workDir); - - await executeParallelCapture( - fileServer.url, + const attempts = await executeDiskCaptureWithAdaptiveRetry({ + serverUrl: fileServer.url, workDir, - tasks, - buildHdrCaptureOptions(), - () => createVideoFrameInjector(frameLookup), + framesDir, + totalFrames: job.totalFrames, + initialWorkerCount: workerCount, + allowRetry: job.config.workers === undefined, + frameExt: needsAlpha ? "png" : "jpg", + captureOptions: buildHdrCaptureOptions(), + createBeforeCaptureHook: () => createVideoFrameInjector(frameLookup), abortSignal, - (progress) => { + onProgress: (progress) => { job.framesRendered = progress.capturedFrames; const frameProgress = progress.capturedFrames / progress.totalFrames; const progressPct = 25 + frameProgress * 45; @@ -2521,17 +3039,20 @@ export async function executeRenderJob( updateJobStatus( job, "rendering", - `Capturing frame ${progress.capturedFrames}/${progress.totalFrames} (${workerCount} workers)`, + `Capturing frame ${progress.capturedFrames}/${progress.totalFrames} (${progress.activeWorkers} workers)`, Math.round(progressPct), onProgress, ); } }, - undefined, cfg, - ); - - await mergeWorkerFrames(workDir, tasks, framesDir); + log, + }); + captureAttempts.push(...attempts); + const lastAttempt = attempts[attempts.length - 1]; + if (lastAttempt) { + workerCount = lastAttempt.workers; + } if (probeSession) { lastBrowserConsole = probeSession.browserConsoleBuffer; await closeCaptureSession(probeSession); @@ -2747,6 +3268,15 @@ export async function executeRenderJob( stages: perfStages, videoExtractBreakdown: extractionResult?.phaseBreakdown, tmpPeakBytes, + captureCalibration: captureCalibration + ? { + sampledFrames: captureCalibration.samples.map((sample) => sample.frameIndex), + p95Ms: captureCalibration.estimate.p95Ms, + multiplier: captureCalibration.estimate.multiplier, + reasons: captureCalibration.estimate.reasons, + } + : undefined, + captureAttempts: captureAttempts.length > 0 ? captureAttempts : undefined, hdrDiagnostics: hdrDiagnostics.videoExtractionFailures > 0 || hdrDiagnostics.imageDecodeFailures > 0 ? { ...hdrDiagnostics }