From 0be1a0719ca199119a2d719ef017f55cd274753f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 17 Nov 2026 14:35:07 +0000 Subject: [PATCH] fix(producer): frames start drawing while the audio mix finishes (#7985) * fix(producer): frames start drawing while the audio mix finishes * fix(producer): keep the encode stage's comment share * fix(producer): on machines with under 4 CPUs frames wait for the audio mix * test(producer): a mix that fails during capture is reported as the audio stage's failure * fix(producer): a mix that fails during capture names the audio stage as where the render failed --- .../src/services/render/stages/encodeStage.ts | 12 +- .../renderOrchestrator.audioWait.test.ts | 33 ++- ...derOrchestrator.captureDuringAudio.test.ts | 222 ++++++++++++++++++ .../src/services/renderOrchestrator.ts | 124 ++++++---- 4 files changed, 327 insertions(+), 64 deletions(-) create mode 100644 packages/producer/src/services/renderOrchestrator.captureDuringAudio.test.ts diff --git a/packages/producer/src/services/render/stages/encodeStage.ts b/packages/producer/src/services/render/stages/encodeStage.ts index ea0de99e77..8fd3192869 100644 --- a/packages/producer/src/services/render/stages/encodeStage.ts +++ b/packages/producer/src/services/render/stages/encodeStage.ts @@ -141,6 +141,8 @@ export interface EncodeStageInput { gopSize?: number; } +export type VideoEncodeInput = Omit; + export interface EncodeStageResult { /** Wall-clock ms for the encode (or png-copy) phase. */ encodeMs: number; @@ -264,7 +266,7 @@ function startEncodeProgress( return (frames = total) => reportEncodeProgress(job, frames, total, onProgress, 75); } -function resolveEngineConfig(input: EncodeStageInput) { +function resolveEngineConfig(input: VideoEncodeInput) { return input.engineConfig ?? input.job.config.producerConfig ?? resolveConfig(); } @@ -273,7 +275,7 @@ export function overlapEncoderThreads(workerCount: number, cpuCount: number): nu } export function videoEncoderOptions( - input: EncodeStageInput, + input: VideoEncodeInput, engineCfg: Pick, softwareEncoderThreads?: number, ) { @@ -306,7 +308,7 @@ export function videoEncoderOptions( * arguments. Needs a core the capture workers leave free. */ export function canEncodeWhileCapturing( - input: EncodeStageInput, + input: VideoEncodeInput, workerCount: number, cpuCount: number, ): boolean { @@ -350,7 +352,7 @@ async function readLandedFrame(workerPath: string | undefined, mergedPath: strin * On failure `finish` re-runs `runEncodeStage` on the same frames. */ export function encodeWhileCapturing( - input: EncodeStageInput, + input: VideoEncodeInput, totalFrames: number, captureBudget?: { workerCount: number; cpuCount: number }, ): EncodeWhileCapturing { @@ -432,7 +434,7 @@ export function encodeWhileCapturing( error: String(failure ?? result?.error), }, ); - return runEncodeStage(input); + return runEncodeStage({ ...input, hasAudio: false }); }, abort: async () => { stop.abort(); diff --git a/packages/producer/src/services/renderOrchestrator.audioWait.test.ts b/packages/producer/src/services/renderOrchestrator.audioWait.test.ts index 6ded56265c..c4c5a8071b 100644 --- a/packages/producer/src/services/renderOrchestrator.audioWait.test.ts +++ b/packages/producer/src/services/renderOrchestrator.audioWait.test.ts @@ -5,14 +5,18 @@ import { join } from "node:path"; import { resolveConfig } from "@hyperframes/engine"; import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js"; -// Entry: executeRenderJob. The audio mix is held open past capture setup, so capture waits on it; the test drives the -// mix's FFmpeg progress and reads the lines the render reports meanwhile. +// Entry: executeRenderJob. The audio mix is held open past capture, so the steps after it wait on it; the test drives +// the mix's FFmpeg progress and reads the lines the render reports meanwhile. const trace = vi.hoisted(() => ({ ffmpegProgressed: (): void => {}, failAudio: (_error: Error): void => {}, setUp: false, })); +vi.mock("@hyperframes/engine", async (importOriginal) => ({ + ...(await importOriginal()), + usableCpuCount: () => 8, +})); vi.mock("./render/shared.js", async (importOriginal) => ({ ...(await importOriginal()), createMemorySampler: () => ({ @@ -82,12 +86,21 @@ vi.mock("./render/stages/audioStage.js", () => ({ })); vi.mock("./render/captureCost.js", async (importOriginal) => ({ ...(await importOriginal()), - resolveRenderWorkerCount: () => { - // The mix writes more while capture is still setting up: not a line of its own yet. - trace.ffmpegProgressed(); - trace.setUp = true; - return 1; - }, + resolveRenderWorkerCount: () => 1, +})); +/** Capture finishes while the mix still runs; the mix writes more meanwhile: not a line of its own yet. */ +async function capturedWhileMixing() { + trace.ffmpegProgressed(); + trace.setUp = true; + return { success: true, workerCount: 1, probeSession: null, lastBrowserConsole: [], encodeMs: 0 }; +} +vi.mock("./render/stages/captureStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCaptureStage: capturedWhileMixing, +})); +vi.mock("./render/stages/captureStreamingStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCaptureStreamingStage: capturedWhileMixing, })); const dirs: string[] = []; @@ -98,7 +111,7 @@ afterEach(() => { const turn = () => new Promise((resolve) => setImmediate(resolve)); -it("says it is processing audio while capture waits on a mix whose FFmpeg writes more, at most every 5 s", async () => { +it("says it is processing audio while the render waits on a mix whose FFmpeg writes more, at most every 5 s", async () => { vi.useFakeTimers({ toFake: ["Date"] }); const root = mkdtempSync(join(tmpdir(), "hf-audio-wait-")); dirs.push(root); @@ -123,7 +136,7 @@ it("says it is processing audio while capture waits on a mix whose FFmpeg writes // Only the stage's own first line. const before = audioLines().length; expect(before).toBe(1); - // The mix writes more every 6 s until capture, now waiting on it, says so. + // The mix writes more every 6 s until the render, now waiting on it, says so. for (let at = 0; at < 200 && audioLines().length === before; at++) { vi.setSystemTime(Date.now() + 6_000); trace.ffmpegProgressed(); diff --git a/packages/producer/src/services/renderOrchestrator.captureDuringAudio.test.ts b/packages/producer/src/services/renderOrchestrator.captureDuringAudio.test.ts new file mode 100644 index 0000000000..7c098fedb1 --- /dev/null +++ b/packages/producer/src/services/renderOrchestrator.captureDuringAudio.test.ts @@ -0,0 +1,222 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveConfig } from "@hyperframes/engine"; +import { createRenderJob, executeRenderJob, RenderQualityError } from "./renderOrchestrator.js"; +import { AUDIO_STAGE } from "./render/shared.js"; + +// Entry: executeRenderJob. The audio mix is held open; every capture stage records that it started, then waits for the +// test or for the render's signal, so the test sees whether frames are drawn while the mix runs and what stops them. +const trace = vi.hoisted(() => ({ + events: [] as string[], + finishAudio: (_result: object): void => {}, + failAudio: (_error: Error): void => {}, + finishCapture: (): void => {}, + cpus: 8, +})); + +vi.mock("@hyperframes/engine", async (importOriginal) => ({ + ...(await importOriginal()), + usableCpuCount: () => trace.cpus, +})); + +vi.mock("./render/shared.js", async (importOriginal) => ({ + ...(await importOriginal()), + createMemorySampler: () => ({ + peakRssBytes: () => 0, + peakHeapUsedBytes: () => 0, + stop: () => {}, + }), +})); +vi.mock("./fileServer.js", async (importOriginal) => ({ + ...(await importOriginal()), + createFileServer: async () => ({ + url: "http://127.0.0.1:1", + port: 1, + close: () => {}, + addPreHeadScript: () => {}, + }), +})); +vi.mock("./render/stages/compileStage.js", () => ({ + runCompileStage: async () => ({ + compiled: { + html: "
", + hasShaderTransitions: false, + renderModeHints: { reasons: [] }, + }, + composition: { width: 16, height: 16, duration: 40 / 30, videos: [], images: [], audios: [] }, + deviceScaleFactor: 1, + outputWidth: 16, + outputHeight: 16, + compileOnlyMs: 0, + forceScreenshot: true, + }), +})); +vi.mock("./render/stages/probeStage.js", () => ({ + runProbeStage: async ({ compiled }: { compiled: object }) => ({ + compiled, + fileServer: null, + probeSession: null, + lastBrowserConsole: [], + duration: 40 / 30, + totalFrames: 40, + browserProbeMs: 0, + beginFrameStalled: false, + }), +})); +vi.mock("./render/stages/extractVideosStage.js", () => ({ + runExtractVideosStage: async () => ({ + extractionResult: null, + frameLookup: null, + videoReadinessSkipIds: [], + videoMetadataHints: [], + nativeHdrVideoIds: new Set(), + videoTransfers: new Map(), + nativeHdrImageIds: new Set(), + imageTransfers: new Map(), + hdrImageSrcPaths: new Map(), + imageColorSpaces: [], + videoExtractMs: 0, + failureToEnforce: null, + }), +})); +vi.mock("./render/stages/audioStage.js", () => ({ + runAudioStage: () => + new Promise((resolve, reject) => { + trace.events.push("audio started"); + trace.finishAudio = (result) => { + trace.events.push("audio done"); + resolve(result); + }; + trace.failAudio = reject; + }), +})); +vi.mock("./render/captureCost.js", async (importOriginal) => ({ + ...(await importOriginal()), + resolveRenderWorkerCount: () => 1, +})); + +/** Started, then held until the test finishes it (it then stops the render) or the render's signal aborts it. */ +function heldCapture({ abortSignal }: { abortSignal?: AbortSignal }): Promise { + trace.events.push("capture started"); + return new Promise((_resolve, reject) => { + trace.finishCapture = () => reject(new Error("stop after capture")); + abortSignal?.addEventListener("abort", () => { + trace.events.push("capture aborted"); + reject(abortSignal.reason); + }); + }); +} +vi.mock("./render/stages/captureStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCaptureStage: heldCapture, +})); +vi.mock("./render/stages/captureStreamingStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCaptureStreamingStage: heldCapture, +})); +vi.mock("./render/stages/captureSegmentedStage.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCaptureSegmentedStage: heldCapture, +})); + +const dirs: string[] = []; +afterEach(() => { + trace.events.length = 0; + trace.cpus = 8; + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +async function until(condition: () => boolean): Promise { + for (let turn = 0; turn < 5_000 && !condition(); turn++) { + await new Promise((resolve) => setImmediate(resolve)); + } + return condition(); +} + +function startRender() { + const root = mkdtempSync(join(tmpdir(), "hf-capture-during-audio-")); + dirs.push(root); + writeFileSync(join(root, "index.html"), "
"); + const job = createRenderJob({ + fps: 30, + quality: "draft", + format: "mp4", + hdrMode: "force-sdr", + producerConfig: resolveConfig({ enablePageSideCompositing: false }), + logger: { info() {}, warn() {}, error() {}, debug() {} }, + }); + const failure = executeRenderJob(job, root, join(root, "out.mp4")).then( + () => undefined, + (error: unknown) => error, + ); + return { job, root, failure }; +} + +it("draws frames while the audio still mixes", async () => { + const { root, failure } = startRender(); + + expect(await until(() => trace.events.includes("capture started"))).toBe(true); + expect(trace.events).toEqual(["audio started", "capture started"]); + + trace.finishAudio({ + audioOutputPath: join(root, "audio.wav"), + hasAudio: false, + audioProcessMs: 0, + }); + trace.finishCapture(); + expect(String(await failure)).toContain("stop after capture"); +}); + +it("an audio mix that fails while frames are drawn stops them and fails the render with the mix's message", async () => { + const { job, root, failure } = startRender(); + expect(await until(() => trace.events.includes("capture started"))).toBe(true); + + trace.finishAudio({ + audioOutputPath: join(root, "audio.wav"), + hasAudio: false, + audioProcessMs: 0, + audioError: "the music track could not be read", + audioFailures: [], + }); + const error = await failure; + + expect(trace.events).toContain("capture aborted"); + expect(error).toBeInstanceOf(RenderQualityError); + expect(String(error)).toContain( + "Audio mix failed; output would be video-only: the music track could not be read", + ); + expect(job.status).not.toBe("cancelled"); + expect(job.failedStage).toBe(AUDIO_STAGE); +}); + +it("an audio stage that throws while frames are drawn stops them and fails the render with its error", async () => { + const { job, failure } = startRender(); + expect(await until(() => trace.events.includes("capture started"))).toBe(true); + + trace.failAudio(new Error("the mix's FFmpeg exited with code 1")); + const error = await failure; + + expect(trace.events).toContain("capture aborted"); + expect(String(error)).toContain("the mix's FFmpeg exited with code 1"); + expect(job.status).not.toBe("cancelled"); + expect(job.failedStage).toBe(AUDIO_STAGE); +}); + +it("on fewer than 4 CPUs draws frames only after the mix", async () => { + trace.cpus = 2; + const { root, failure } = startRender(); + expect(await until(() => trace.events.includes("audio started"))).toBe(true); + expect(await until(() => trace.events.includes("capture started"))).toBe(false); + + trace.finishAudio({ + audioOutputPath: join(root, "audio.wav"), + hasAudio: false, + audioProcessMs: 0, + }); + expect(await until(() => trace.events.includes("capture started"))).toBe(true); + expect(trace.events).toEqual(["audio started", "audio done", "capture started"]); + trace.finishCapture(); + expect(String(await failure)).toContain("stop after capture"); +}); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 718dd13c57..b1b19a3743 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1,5 +1,5 @@ import type { ExpectedOutputDimensions } from "./render/outputDimensions.js"; -import { audioFailureDiagnostics } from "@hyperframes/engine"; +import { audioFailureDiagnostics, usableCpuCount } from "@hyperframes/engine"; // fallow-ignore-file unused-type circular-dependency code-duplication complexity /** * Render Orchestrator Service @@ -275,7 +275,7 @@ import { canEncodeWhileCapturing, encodeWhileCapturing, runEncodeStage, - type EncodeStageInput, + type VideoEncodeInput, } from "./render/stages/encodeStage.js"; import { recordLimiterAttenuation, runAssembleStage } from "./render/stages/assembleStage.js"; import { shouldUseLayeredComposite } from "./hdrCompositor.js"; @@ -857,6 +857,8 @@ export class RenderQualityError extends Error { } } +const MIN_CPUS_TO_CAPTURE_DURING_MIX = 4; + export function applyRenderWarningPolicy( job: RenderJob, captureWarnings: readonly CaptureWarning[], @@ -3153,10 +3155,15 @@ async function executeRenderPipeline(input: { const onProgress = execution.onProgress; const tooSlow = new AbortController(); let tooSlowError: RenderTooSlowError | undefined; + // Frames are drawn while the audio mixes; a failed mix stops them and is the render's error. + const audioFailed = new AbortController(); + let audioFailure: unknown; let calibrationEvaluateTimedOut = false; - const executionSignal = job.config.maxRenderMs - ? AbortSignal.any([tooSlow.signal, ...(execution.signal ? [execution.signal] : [])]) - : execution.signal; + const executionSignal = AbortSignal.any([ + audioFailed.signal, + ...(job.config.maxRenderMs ? [tooSlow.signal] : []), + ...(execution.signal ? [execution.signal] : []), + ]); let fileServer: FileServerHandle | null = null; let probeSession: CaptureSession | null = null; let lastBrowserConsole: string[] = []; @@ -4940,41 +4947,50 @@ async function executeRenderPipeline(input: { const effectiveQuality = job.config.crf ?? preset.quality; const effectiveBitrate = job.config.crf != null ? undefined : job.config.videoBitrate; - const audioResult = await audioWait.wait(audioStage); - const { audioOutputPath, deliverySourcePath, hasAudio } = audioResult; - perfStages.audioProcessMs = audioResult.audioProcessMs; - recordLimiterAttenuation(job, audioResult.audioLoweredDb); - if (audioResult.audioError) { - const audioFailures = audioResult.audioFailures ?? []; - const failureOwner = - audioFailures.length === 0 - ? undefined - : audioFailures.some((failure) => failure.owner === "system") - ? "system" - : "user"; - const retryable = - audioFailures.length === 0 - ? undefined - : audioFailures.every((failure) => failure.retryable); - applyRenderWarningPolicy( - job, - [ - { - code: "audio_processing_failed", - message: `Audio mix failed; output would be video-only: ${audioResult.audioError}`, - details: { - mediaType: "audio", - audioFailures: audioFailureDiagnostics(audioFailures), - failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))], - failureStages: [...new Set(audioFailures.map((failure) => failure.stage))], - failureOwner, - retryable, + // Capture never reads the mix: it starts now, and the first step that needs the mix waits for it. + const audioReady = audioStage.then((audioResult) => { + perfStages.audioProcessMs = audioResult.audioProcessMs; + recordLimiterAttenuation(job, audioResult.audioLoweredDb); + if (audioResult.audioError) { + const audioFailures = audioResult.audioFailures ?? []; + const failureOwner = + audioFailures.length === 0 + ? undefined + : audioFailures.some((failure) => failure.owner === "system") + ? "system" + : "user"; + const retryable = + audioFailures.length === 0 + ? undefined + : audioFailures.every((failure) => failure.retryable); + applyRenderWarningPolicy( + job, + [ + { + code: "audio_processing_failed", + message: `Audio mix failed; output would be video-only: ${audioResult.audioError}`, + details: { + mediaType: "audio", + audioFailures: audioFailureDiagnostics(audioFailures), + failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))], + failureStages: [...new Set(audioFailures.map((failure) => failure.stage))], + failureOwner, + retryable, + }, }, - }, - ], - log, - ); - } + ], + log, + ); + } + return audioResult; + }); + audioReady.catch((error: unknown) => { + if (executionSignal.aborted) return; + audioFailure = error; + audioFailed.abort(error); + }); + const mixedAudio = () => audioWait.wait(audioReady); + if (usableCpuCount() < MIN_CPUS_TO_CAPTURE_DURING_MIX) await mixedAudio(); resetCaptureAttemptProgress(job); @@ -5180,7 +5196,7 @@ async function executeRenderPipeline(input: { if ( (!isVerifyError && !isDrawElementCaptureError(err) && !isDeStall) || err instanceof RenderCancelledError || - executionSignal?.aborted === true + executionSignal.aborted === true ) { throw err; } @@ -5360,7 +5376,7 @@ async function executeRenderPipeline(input: { const isParallelStall = isParallelCaptureStallError(err); const isEncoderDeath = isRetryableEncoderDeath(err); const isCancellation = - err instanceof RenderCancelledError || executionSignal?.aborted === true; + err instanceof RenderCancelledError || executionSignal.aborted === true; if ( !shouldRetryViaPinnedFallback({ isVerifyError, @@ -5549,7 +5565,7 @@ async function executeRenderPipeline(input: { } // ── Disk-based capture (original flow) ──────────────────────────── resetCaptureAttemptProgress(job); - const encodeInput: EncodeStageInput = { + const encodeInput: VideoEncodeInput = { job, log, outputPath: stagedOutputPath, @@ -5559,8 +5575,6 @@ async function executeRenderPipeline(input: { height, needsAlpha, captureImageFormat: captureOptions.format ?? "jpeg", - hasAudio, - audioOutputPath, isPngSequence, isGif, preset, @@ -5628,7 +5642,7 @@ async function executeRenderPipeline(input: { if ( (!isDrawElementVerificationError(err) && !isDrawElementCaptureError(err)) || err instanceof RenderCancelledError || - executionSignal?.aborted === true + executionSignal.aborted === true ) { throw err; } @@ -5701,6 +5715,7 @@ async function executeRenderPipeline(input: { perfStages.captureFrameMs = captureFrameMs; perfStages.captureSetupMs = Math.max(0, perfStages.captureMs - captureFrameMs); + const { hasAudio, audioOutputPath } = await mixedAudio(); const encodeRes = await observeRenderStage( observability, "encode", @@ -5710,7 +5725,10 @@ async function executeRenderPipeline(input: { isGif, chunkedEncode: enableChunkedEncode, }), - () => (overlap ? overlap.finish() : runEncodeStage(encodeInput)), + () => + overlap + ? overlap.finish() + : runEncodeStage({ ...encodeInput, hasAudio, audioOutputPath }), ); perfStages.encodeMs = encodeRes.encodeMs; } @@ -5751,6 +5769,7 @@ async function executeRenderPipeline(input: { // directory deliverable, and gif is written directly to outputPath by the // two-pass palette encoder. hls is a directory deliverable but still runs // this stage — the HLS packaging IS its assemble step. + const { audioOutputPath, deliverySourcePath, hasAudio } = await mixedAudio(); if (!isPngSequence && !isGif) { const assembleRes = await observeRenderStage( observability, @@ -5900,8 +5919,15 @@ async function executeRenderPipeline(input: { framesRendered: job.framesRendered ?? 0, error: caught, }); - const error = surfaceScriptFailure(tooSlowError ?? pageNeverReady ?? caught, job); - if (!tooSlowError && (error instanceof RenderCancelledError || executionSignal?.aborted)) { + const error = surfaceScriptFailure( + tooSlowError ?? audioFailure ?? pageNeverReady ?? caught, + job, + ); + if ( + !tooSlowError && + !audioFailure && + (error instanceof RenderCancelledError || executionSignal.aborted) + ) { job.error = error instanceof Error ? error.message : "render_cancelled"; updateJobStatus(job, "cancelled", "Render cancelled", job.progress, onProgress); await eventPublisher.flush(); @@ -5991,7 +6017,7 @@ async function executeRenderPipeline(input: { ); } - const failedStage = job.currentStage || "pipeline"; + const failedStage = audioFailure === undefined ? job.currentStage || "pipeline" : AUDIO_STAGE; const observabilitySummary = observability.summary({ lastBrowserConsole, capture: withCapturedBeyondViewport(captureObservability, dedupPerfs),