From df698a0290c71f3a2936f3591a8bdf14a56a7163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Mon, 16 Nov 2026 16:11:05 +0000 Subject: [PATCH] fix(producer): a render waiting on a long audio mix says it is processing audio (#7952) * fix(producer): a render waiting on a long audio mix says it is processing audio * fix(producer): drop the comment the field name already says --- packages/engine/src/index.ts | 1 + packages/engine/src/utils/runFfmpeg.test.ts | 42 +++++ packages/engine/src/utils/runFfmpeg.ts | 10 ++ .../producer/src/services/render/shared.ts | 28 ++++ .../src/services/render/stages/audioStage.ts | 5 +- .../renderOrchestrator.audioWait.test.ts | 148 ++++++++++++++++++ .../src/services/renderOrchestrator.ts | 8 +- 7 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 packages/producer/src/services/renderOrchestrator.audioWait.test.ts diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 0aaca8b949..dfe673d774 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -453,6 +453,7 @@ export { onRenderRefusal } from "./services/publicNetwork/captureProxy.js"; export type { RefusedHostClass } from "./services/publicNetwork/publicNetwork.js"; export { runFfmpeg, + withFfmpegProgress, formatFfmpegError, describeFfmpegFailure, isExternalFfmpegInterruption, diff --git a/packages/engine/src/utils/runFfmpeg.test.ts b/packages/engine/src/utils/runFfmpeg.test.ts index 9ecc5dd1ce..6ec45d0772 100644 --- a/packages/engine/src/utils/runFfmpeg.test.ts +++ b/packages/engine/src/utils/runFfmpeg.test.ts @@ -311,6 +311,48 @@ describe("a run's timeout", () => { }); }); +describe("withFfmpegProgress", () => { + afterEach(() => { + vi.doUnmock("child_process"); + vi.resetModules(); + }); + + it("tells its listener each time a run inside it writes more, and never a run outside it", async () => { + const procs: EventEmitter[] = []; + vi.resetModules(); + vi.doMock("child_process", () => ({ + spawn: () => { + const proc = Object.assign(new EventEmitter(), { + stderr: new EventEmitter(), + kill: vi.fn(), + }); + procs.push(proc); + return proc; + }, + })); + const { runFfmpeg, withFfmpegProgress } = await import("./runFfmpeg.js"); + const heard = vi.fn(); + const write = (proc: EventEmitter, time: string) => + (proc as unknown as { stderr: EventEmitter }).stderr.emit( + "data", + Buffer.from(`size= 100KiB time=${time}\r`), + ); + + const inside = withFfmpegProgress(heard, () => runFfmpeg(["-i", "in.wav", "out.m4a"])); + const outside = runFfmpeg(["-i", "in.wav", "other.m4a"]); + const [mine, other] = procs; + write(mine, "00:00:01.00"); + write(mine, "00:00:01.00"); + write(mine, "00:00:02.00"); + write(other, "00:00:05.00"); + expect(heard).toHaveBeenCalledTimes(2); + + mine.emit("close", 0, null); + other.emit("close", 0, null); + await Promise.all([inside, outside]); + }); +}); + describe("a run whose signal already aborted", () => { afterEach(() => { vi.doUnmock("child_process"); diff --git a/packages/engine/src/utils/runFfmpeg.ts b/packages/engine/src/utils/runFfmpeg.ts index 092c0c4d06..f3ae250d75 100644 --- a/packages/engine/src/utils/runFfmpeg.ts +++ b/packages/engine/src/utils/runFfmpeg.ts @@ -6,6 +6,7 @@ * that appears across audioMixer and chunkEncoder into a single helper. */ +import { AsyncLocalStorage } from "node:async_hooks"; import { spawn, type ChildProcess } from "child_process"; import { getFfmpegBinary } from "./ffmpegBinaries.js"; import { trackChildProcess } from "./processTracker.js"; @@ -199,8 +200,16 @@ const ABORTED_BEFORE_START: RunFfmpegResult = { terminationReason: "abort", }; +const progressListener = new AsyncLocalStorage<() => void>(); + +/** Runs `work` with `onProgress` called each time an FFmpeg it starts writes a higher frame count or output time. */ +export function withFfmpegProgress(onProgress: () => void, work: () => Promise): Promise { + return progressListener.run(onProgress, work); +} + export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promise { if (opts?.signal?.aborted) return { ...ABORTED_BEFORE_START }; + const onProgress = progressListener.getStore(); let framesWritten = 0; let secondsWritten = 0; const readStats = ffmpegStatsReader(({ frames = 0, seconds = 0 }) => { @@ -208,6 +217,7 @@ export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promis framesWritten = Math.max(frames, framesWritten); secondsWritten = Math.max(seconds, secondsWritten); managed.markActivity(); + onProgress?.(); }); const managed = new ManagedChildProcess(spawnFfmpeg(args), { signal: opts?.signal, diff --git a/packages/producer/src/services/render/shared.ts b/packages/producer/src/services/render/shared.ts index ef3482e3ca..b11e3c1198 100644 --- a/packages/producer/src/services/render/shared.ts +++ b/packages/producer/src/services/render/shared.ts @@ -379,6 +379,34 @@ export function reportEncodeProgress( ); } +export const AUDIO_STAGE = "Processing audio tracks"; +const AUDIO_WAIT_REPORT_MS = 5_000; + +/** Once capture waits on the audio mix: a line at most every 5 s in which an FFmpeg of the mix wrote more, so whoever + * watches the render for silence hears it working, and a mix that writes nothing stays silent. */ +export function audioWaitReporter(job: RenderJob, onProgress: ProgressCallback | undefined) { + let waiting = false; + let reportedAt = -Infinity; + return { + progressed(): void { + const now = Date.now(); + if (!waiting || now - reportedAt < AUDIO_WAIT_REPORT_MS) return; + reportedAt = now; + updateJobStatus(job, job.status, AUDIO_STAGE, job.progress, onProgress, { + code: "process_audio", + }); + }, + async wait(audio: Promise): Promise { + waiting = true; + try { + return await audio; + } finally { + waiting = false; + } + }, + }; +} + const lastAssembleReportAt = new WeakMap(); /** Seconds of the final file written, 90% to 99%: only a complete file reaches 100. */ diff --git a/packages/producer/src/services/render/stages/audioStage.ts b/packages/producer/src/services/render/stages/audioStage.ts index 374975a247..64f973effb 100644 --- a/packages/producer/src/services/render/stages/audioStage.ts +++ b/packages/producer/src/services/render/stages/audioStage.ts @@ -26,6 +26,7 @@ import { processCompositionAudio, removeScratch, collectScratchLeftovers, + withFfmpegProgress, type AudioProcessingFailure, type MixResult, } from "@hyperframes/engine"; @@ -60,6 +61,7 @@ export interface AudioStageInput { log?: ProducerLogger; /** Leave the delivered PCM in `workDir` for the assemble stage; its owner removes the dir. */ keepDeliverySource?: boolean; + onFfmpegProgress?: () => void; } export interface AudioStageResult { @@ -100,7 +102,8 @@ function assertAudioNotInterrupted(failures: readonly AudioProcessingFailure[] | export function runAudioStage(input: AudioStageInput): Promise { return collectScratchLeftovers( - () => runAudioStageScoped(input), + () => + withFfmpegProgress(input.onFfmpegProgress ?? (() => {}), () => runAudioStageScoped(input)), (left) => { if (left.length > 0) input.log?.warn("[audio] scratch left behind", { count: left.length }); }, diff --git a/packages/producer/src/services/renderOrchestrator.audioWait.test.ts b/packages/producer/src/services/renderOrchestrator.audioWait.test.ts new file mode 100644 index 0000000000..6ded56265c --- /dev/null +++ b/packages/producer/src/services/renderOrchestrator.audioWait.test.ts @@ -0,0 +1,148 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveConfig } from "@hyperframes/engine"; +import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js"; + +// Entry: executeRenderJob. The audio mix is held open past capture setup, so capture waits on it; the test drives the +// mix's FFmpeg progress and reads the lines the render reports meanwhile. +const trace = vi.hoisted(() => ({ + ffmpegProgressed: (): void => {}, + failAudio: (_error: Error): void => {}, + setUp: false, +})); + +vi.mock("./render/shared.js", async (importOriginal) => ({ + ...(await importOriginal()), + createMemorySampler: () => ({ + peakRssBytes: () => 0, + peakHeapUsedBytes: () => 0, + stop: () => {}, + }), +})); +vi.mock("./fileServer.js", async (importOriginal) => ({ + ...(await importOriginal()), + createFileServer: async () => ({ + url: "http://127.0.0.1:1", + port: 1, + close: () => {}, + addPreHeadScript: () => {}, + }), +})); +vi.mock("./render/stages/compileStage.js", () => ({ + runCompileStage: async () => ({ + compiled: { + html: "
", + hasShaderTransitions: false, + renderModeHints: { reasons: [] }, + }, + composition: { width: 16, height: 16, duration: 40 / 30, videos: [], images: [], audios: [] }, + deviceScaleFactor: 1, + outputWidth: 16, + outputHeight: 16, + compileOnlyMs: 0, + forceScreenshot: true, + }), +})); +vi.mock("./render/stages/probeStage.js", () => ({ + runProbeStage: async ({ compiled }: { compiled: object }) => ({ + compiled, + fileServer: null, + probeSession: null, + lastBrowserConsole: [], + duration: 40 / 30, + totalFrames: 40, + browserProbeMs: 0, + beginFrameStalled: false, + }), +})); +vi.mock("./render/stages/extractVideosStage.js", () => ({ + runExtractVideosStage: async () => ({ + extractionResult: null, + frameLookup: null, + videoReadinessSkipIds: [], + videoMetadataHints: [], + nativeHdrVideoIds: new Set(), + videoTransfers: new Map(), + nativeHdrImageIds: new Set(), + imageTransfers: new Map(), + hdrImageSrcPaths: new Map(), + imageColorSpaces: [], + videoExtractMs: 0, + failureToEnforce: null, + }), +})); +vi.mock("./render/stages/audioStage.js", () => ({ + runAudioStage: ({ onFfmpegProgress }: { onFfmpegProgress?: () => void }) => + new Promise((_resolve, reject) => { + trace.ffmpegProgressed = onFfmpegProgress ?? (() => {}); + trace.failAudio = reject; + }), +})); +vi.mock("./render/captureCost.js", async (importOriginal) => ({ + ...(await importOriginal()), + resolveRenderWorkerCount: () => { + // The mix writes more while capture is still setting up: not a line of its own yet. + trace.ffmpegProgressed(); + trace.setUp = true; + return 1; + }, +})); + +const dirs: string[] = []; +afterEach(() => { + vi.useRealTimers(); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +const turn = () => new Promise((resolve) => setImmediate(resolve)); + +it("says it is processing audio while capture waits on a mix whose FFmpeg writes more, at most every 5 s", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const root = mkdtempSync(join(tmpdir(), "hf-audio-wait-")); + dirs.push(root); + writeFileSync(join(root, "index.html"), "
"); + const job = createRenderJob({ + fps: 30, + quality: "draft", + format: "mp4", + hdrMode: "force-sdr", + producerConfig: resolveConfig({ enablePageSideCompositing: false }), + logger: { info() {}, warn() {}, error() {}, debug() {} }, + }); + const lines: string[] = []; + const render = executeRenderJob(job, root, join(root, "out.mp4"), (reported, stage) => { + lines.push(`${reported.stageProgress?.code}:${stage}`); + }).catch((error: unknown) => error); + + for (let at = 0; at < 5_000 && !trace.setUp; at++) await turn(); + expect(trace.setUp).toBe(true); + const audioLines = () => lines.filter((line) => line === "process_audio:Processing audio tracks"); + await turn(); + // Only the stage's own first line. + const before = audioLines().length; + expect(before).toBe(1); + // The mix writes more every 6 s until capture, now waiting on it, says so. + for (let at = 0; at < 200 && audioLines().length === before; at++) { + vi.setSystemTime(Date.now() + 6_000); + trace.ffmpegProgressed(); + await turn(); + } + expect(audioLines().length).toBe(before + 1); + expect(lines.at(-1)).toBe("process_audio:Processing audio tracks"); + + // Writing more within 5 s of that line says nothing new; after 5 s, one more line. + vi.setSystemTime(Date.now() + 4_000); + trace.ffmpegProgressed(); + await turn(); + expect(audioLines().length).toBe(before + 1); + vi.setSystemTime(Date.now() + 1_000); + trace.ffmpegProgressed(); + await turn(); + expect(audioLines().length).toBe(before + 2); + + vi.useRealTimers(); + trace.failAudio(new Error("stop after the wait")); + expect(String(await render)).toContain("stop after the wait"); +}); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index a1c6e538ec..9399ff4f74 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -162,6 +162,8 @@ import { type MemorySampler, reportExtractProgress, updateJobStatus, + audioWaitReporter, + AUDIO_STAGE, audioWorkDir, } from "./render/shared.js"; import { frameCacheOptions } from "./render/frameCacheBudget.js"; @@ -3739,10 +3741,11 @@ async function executeRenderPipeline(input: { }); // ── Stage 3: Audio processing ─────────────────────────────────────── - updateJobStatus(job, "preprocessing", "Processing audio tracks", 20, onProgress, { + updateJobStatus(job, "preprocessing", AUDIO_STAGE, 20, onProgress, { code: "process_audio", }); + const audioWait = audioWaitReporter(job, onProgress); const audioStage = observeRenderStage( observability, "audio_process", @@ -3763,6 +3766,7 @@ async function executeRenderPipeline(input: { assertNotAborted, log, keepDeliverySource: true, + onFfmpegProgress: audioWait.progressed, }), ); pendingAudio = audioStage.catch(() => undefined); @@ -4930,7 +4934,7 @@ async function executeRenderPipeline(input: { const effectiveQuality = job.config.crf ?? preset.quality; const effectiveBitrate = job.config.crf != null ? undefined : job.config.videoBitrate; - const audioResult = await audioStage; + const audioResult = await audioWait.wait(audioStage); const { audioOutputPath, deliverySourcePath, hasAudio } = audioResult; perfStages.audioProcessMs = audioResult.audioProcessMs; recordLimiterAttenuation(job, audioResult.audioLoweredDb);