From 1d97ed24c647553cefb251c59b1bccc48ff4f09c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 28 Oct 2026 07:36:32 +0000 Subject: [PATCH] perf(producer): the audio mix runs while capture starts its browsers (#6632) * perf(producer): the audio mix runs while capture starts its browsers * test(producer): master-bus test stops the render at capture setup --- .../renderOrchestrator.audioOverlap.test.ts | 132 ++++++++++++++++++ .../renderOrchestrator.masterBus.test.ts | 9 +- .../src/services/renderOrchestrator.ts | 76 +++++----- 3 files changed, 181 insertions(+), 36 deletions(-) create mode 100644 packages/producer/src/services/renderOrchestrator.audioOverlap.test.ts diff --git a/packages/producer/src/services/renderOrchestrator.audioOverlap.test.ts b/packages/producer/src/services/renderOrchestrator.audioOverlap.test.ts new file mode 100644 index 0000000000..c0b93d5d3b --- /dev/null +++ b/packages/producer/src/services/renderOrchestrator.audioOverlap.test.ts @@ -0,0 +1,132 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveConfig } from "@hyperframes/engine"; +import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js"; + +// Entry: executeRenderJob. The audio mix is held open; worker resolution (capture setup) records +// itself and then stops the render, so the test sees what ran while audio was still mixing. +const trace = vi.hoisted(() => ({ events: [] as string[], releaseAudio: () => {} })); + +vi.mock("./render/shared.js", async (importOriginal) => ({ + ...(await importOriginal()), + createMemorySampler: () => ({ + peakRssBytes: () => 0, + peakHeapUsedBytes: () => 0, + stop: () => {}, + }), +})); +vi.mock("./fileServer.js", async (importOriginal) => ({ + ...(await importOriginal()), + createFileServer: async () => ({ + url: "http://127.0.0.1:1", + port: 1, + close: () => {}, + addPreHeadScript: () => {}, + }), +})); +vi.mock("./render/stages/compileStage.js", () => ({ + runCompileStage: async () => ({ + compiled: { + html: "
", + hasShaderTransitions: false, + renderModeHints: { reasons: [] }, + }, + composition: { width: 16, height: 16, duration: 40 / 30, videos: [], images: [], audios: [] }, + deviceScaleFactor: 1, + outputWidth: 16, + outputHeight: 16, + compileOnlyMs: 0, + forceScreenshot: true, + }), +})); +vi.mock("./render/stages/probeStage.js", () => ({ + runProbeStage: async ({ compiled }: { compiled: object }) => ({ + compiled, + fileServer: null, + probeSession: null, + lastBrowserConsole: [], + duration: 40 / 30, + totalFrames: 40, + browserProbeMs: 0, + beginFrameStalled: false, + }), +})); +vi.mock("./render/stages/extractVideosStage.js", () => ({ + runExtractVideosStage: async () => ({ + extractionResult: null, + frameLookup: null, + videoReadinessSkipIds: [], + videoMetadataHints: [], + nativeHdrVideoIds: new Set(), + videoTransfers: new Map(), + nativeHdrImageIds: new Set(), + imageTransfers: new Map(), + hdrImageSrcPaths: new Map(), + imageColorSpaces: [], + videoExtractMs: 0, + failureToEnforce: null, + }), +})); +vi.mock("./render/stages/audioStage.js", () => ({ + runAudioStage: ({ workDir }: { workDir: string }) => + new Promise((resolve) => { + trace.events.push("audio started"); + trace.releaseAudio = () => { + trace.events.push("audio done"); + resolve({ + audioOutputPath: join(workDir, "audio.wav"), + hasAudio: false, + audioProcessMs: 0, + }); + }; + }), +})); +vi.mock("./render/captureCost.js", async (importOriginal) => ({ + ...(await importOriginal()), + resolveRenderWorkerCount: () => { + trace.events.push("workers resolved"); + throw new Error("stop after capture setup"); + }, +})); + +const dirs: string[] = []; +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +async function until(condition: () => boolean): Promise { + for (let turn = 0; turn < 5_000 && !condition(); turn++) { + await new Promise((resolve) => setImmediate(resolve)); + } + return condition(); +} + +it("sets up capture while the audio mixes, and a failure waits for the mix", async () => { + const root = mkdtempSync(join(tmpdir(), "hf-audio-overlap-")); + dirs.push(root); + writeFileSync(join(root, "index.html"), "
"); + const job = createRenderJob({ + fps: 30, + quality: "draft", + format: "mp4", + hdrMode: "force-sdr", + producerConfig: resolveConfig({ enablePageSideCompositing: false }), + logger: { info() {}, warn() {}, error() {}, debug() {} }, + }); + let failure: unknown; + const render = executeRenderJob(job, root, join(root, "frames")).catch((error: unknown) => { + failure = error; + }); + + expect(await until(() => trace.events.includes("workers resolved"))).toBe(true); + expect(trace.events).toEqual(["audio started", "workers resolved"]); + await until(() => failure !== undefined); + expect(failure).toBeUndefined(); + + trace.releaseAudio(); + await render; + expect(trace.events).toEqual(["audio started", "workers resolved", "audio done"]); + expect(String(failure)).toContain("stop after capture setup"); +}); diff --git a/packages/producer/src/services/renderOrchestrator.masterBus.test.ts b/packages/producer/src/services/renderOrchestrator.masterBus.test.ts index ceec4d1485..e06060da9c 100644 --- a/packages/producer/src/services/renderOrchestrator.masterBus.test.ts +++ b/packages/producer/src/services/renderOrchestrator.masterBus.test.ts @@ -72,6 +72,13 @@ vi.mock("./render/stages/audioStage.js", () => ({ throw new Error("audio stage reached"); }, })); +// The mix runs while capture sets up, so the render stops at setup's first step instead. +vi.mock("./fileServer.js", async (importOriginal) => ({ + ...(await importOriginal()), + createFileServer: async () => { + throw new Error("capture setup reached"); + }, +})); const dirs: string[] = []; afterEach(() => { @@ -89,7 +96,7 @@ it("the local renderer gives the shared audio stage its master chain and video c logger: { info() {}, warn() {}, error() {}, debug() {} }, }); await expect(executeRenderJob(job, root, join(root, "output.mp4"))).rejects.toThrow( - "audio stage reached", + "capture setup reached", ); expect(seen.audioInput?.masterFxChain).toBe(MASTER_CHAIN); expect(seen.audioInput?.totalFrames).toBe(30); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 3d0d0804d5..c3c7a380f7 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -3016,6 +3016,8 @@ async function executeRenderPipeline(input: { let captureCompositionHeight: number | undefined; let captureTotalFrames: number | undefined; const perfStages: Record = {}; + // The audio mix runs while capture sets up; a failure in between still waits for it. + let pendingAudio: Promise | undefined; const hdrDiagnostics: HdrDiagnostics = { videoExtractionFailures: 0, imageDecodeFailures: 0, @@ -3516,7 +3518,7 @@ async function executeRenderPipeline(input: { code: "process_audio", }); - const audioResult = await observeRenderStage( + const audioStage = observeRenderStage( observability, "audio_process", { audioCount: composition.audios.length }, @@ -3536,40 +3538,7 @@ async function executeRenderPipeline(input: { log, }), ); - const { audioOutputPath, hasAudio } = audioResult; - perfStages.audioProcessMs = audioResult.audioProcessMs; - recordLimiterAttenuation(job, audioResult.audioLoweredDb); - if (audioResult.audioError) { - const audioFailures = audioResult.audioFailures ?? []; - const failureOwner = - audioFailures.length === 0 - ? undefined - : audioFailures.some((failure) => failure.owner === "system") - ? "system" - : "user"; - const retryable = - audioFailures.length === 0 - ? undefined - : audioFailures.every((failure) => failure.retryable); - applyRenderWarningPolicy( - job, - [ - { - code: "audio_processing_failed", - message: `Audio mix failed; output would be video-only: ${audioResult.audioError}`, - details: { - mediaType: "audio", - audioFailures: audioFailureDiagnostics(audioFailures), - failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))], - failureStages: [...new Set(audioFailures.map((failure) => failure.stage))], - failureOwner, - retryable, - }, - }, - ], - log, - ); - } + pendingAudio = audioStage.catch(() => undefined); // ── Stage 4: Frame capture ────────────────────────────────────────── const stage4Start = Date.now(); @@ -4584,6 +4553,42 @@ async function executeRenderPipeline(input: { const effectiveQuality = job.config.crf ?? preset.quality; const effectiveBitrate = job.config.crf != null ? undefined : job.config.videoBitrate; + const audioResult = await audioStage; + const { audioOutputPath, hasAudio } = audioResult; + perfStages.audioProcessMs = audioResult.audioProcessMs; + recordLimiterAttenuation(job, audioResult.audioLoweredDb); + if (audioResult.audioError) { + const audioFailures = audioResult.audioFailures ?? []; + const failureOwner = + audioFailures.length === 0 + ? undefined + : audioFailures.some((failure) => failure.owner === "system") + ? "system" + : "user"; + const retryable = + audioFailures.length === 0 + ? undefined + : audioFailures.every((failure) => failure.retryable); + applyRenderWarningPolicy( + job, + [ + { + code: "audio_processing_failed", + message: `Audio mix failed; output would be video-only: ${audioResult.audioError}`, + details: { + mediaType: "audio", + audioFailures: audioFailureDiagnostics(audioFailures), + failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))], + failureStages: [...new Set(audioFailures.map((failure) => failure.stage))], + failureOwner, + retryable, + }, + }, + ], + log, + ); + } + resetCaptureAttemptProgress(job); // ── Z-ordered multi-layer compositing ───────────────────────────────── @@ -5488,6 +5493,7 @@ async function executeRenderPipeline(input: { updateJobStatus(job, "complete", "Render complete", 100, onProgress); await eventPublisher.flush(); } catch (caught) { + await pendingAudio; const error = tooSlowError ?? caught; if (!tooSlowError && (error instanceof RenderCancelledError || executionSignal?.aborted)) { job.error = error instanceof Error ? error.message : "render_cancelled";