fix(producer): a render waiting on a long audio mix says it is processing audio (#7952)

* fix(producer): a render waiting on a long audio mix says it is processing audio

* fix(producer): drop the comment the field name already says
This commit is contained in:
Miguel Ángel
2026-11-16 16:11:05 +00:00
committed by GitHub
parent 9e50487720
commit df698a0290
7 changed files with 239 additions and 3 deletions
+1
View File
@@ -453,6 +453,7 @@ export { onRenderRefusal } from "./services/publicNetwork/captureProxy.js";
export type { RefusedHostClass } from "./services/publicNetwork/publicNetwork.js";
export {
runFfmpeg,
withFfmpegProgress,
formatFfmpegError,
describeFfmpegFailure,
isExternalFfmpegInterruption,
@@ -311,6 +311,48 @@ describe("a run's timeout", () => {
});
});
describe("withFfmpegProgress", () => {
afterEach(() => {
vi.doUnmock("child_process");
vi.resetModules();
});
it("tells its listener each time a run inside it writes more, and never a run outside it", async () => {
const procs: EventEmitter[] = [];
vi.resetModules();
vi.doMock("child_process", () => ({
spawn: () => {
const proc = Object.assign(new EventEmitter(), {
stderr: new EventEmitter(),
kill: vi.fn(),
});
procs.push(proc);
return proc;
},
}));
const { runFfmpeg, withFfmpegProgress } = await import("./runFfmpeg.js");
const heard = vi.fn();
const write = (proc: EventEmitter, time: string) =>
(proc as unknown as { stderr: EventEmitter }).stderr.emit(
"data",
Buffer.from(`size= 100KiB time=${time}\r`),
);
const inside = withFfmpegProgress(heard, () => runFfmpeg(["-i", "in.wav", "out.m4a"]));
const outside = runFfmpeg(["-i", "in.wav", "other.m4a"]);
const [mine, other] = procs;
write(mine, "00:00:01.00");
write(mine, "00:00:01.00");
write(mine, "00:00:02.00");
write(other, "00:00:05.00");
expect(heard).toHaveBeenCalledTimes(2);
mine.emit("close", 0, null);
other.emit("close", 0, null);
await Promise.all([inside, outside]);
});
});
describe("a run whose signal already aborted", () => {
afterEach(() => {
vi.doUnmock("child_process");
+10
View File
@@ -6,6 +6,7 @@
* that appears across audioMixer and chunkEncoder into a single helper.
*/
import { AsyncLocalStorage } from "node:async_hooks";
import { spawn, type ChildProcess } from "child_process";
import { getFfmpegBinary } from "./ffmpegBinaries.js";
import { trackChildProcess } from "./processTracker.js";
@@ -199,8 +200,16 @@ const ABORTED_BEFORE_START: RunFfmpegResult = {
terminationReason: "abort",
};
const progressListener = new AsyncLocalStorage<() => void>();
/** Runs `work` with `onProgress` called each time an FFmpeg it starts writes a higher frame count or output time. */
export function withFfmpegProgress<T>(onProgress: () => void, work: () => Promise<T>): Promise<T> {
return progressListener.run(onProgress, work);
}
export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promise<RunFfmpegResult> {
if (opts?.signal?.aborted) return { ...ABORTED_BEFORE_START };
const onProgress = progressListener.getStore();
let framesWritten = 0;
let secondsWritten = 0;
const readStats = ffmpegStatsReader(({ frames = 0, seconds = 0 }) => {
@@ -208,6 +217,7 @@ export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promis
framesWritten = Math.max(frames, framesWritten);
secondsWritten = Math.max(seconds, secondsWritten);
managed.markActivity();
onProgress?.();
});
const managed = new ManagedChildProcess(spawnFfmpeg(args), {
signal: opts?.signal,
@@ -379,6 +379,34 @@ export function reportEncodeProgress(
);
}
export const AUDIO_STAGE = "Processing audio tracks";
const AUDIO_WAIT_REPORT_MS = 5_000;
/** Once capture waits on the audio mix: a line at most every 5 s in which an FFmpeg of the mix wrote more, so whoever
* watches the render for silence hears it working, and a mix that writes nothing stays silent. */
export function audioWaitReporter(job: RenderJob, onProgress: ProgressCallback | undefined) {
let waiting = false;
let reportedAt = -Infinity;
return {
progressed(): void {
const now = Date.now();
if (!waiting || now - reportedAt < AUDIO_WAIT_REPORT_MS) return;
reportedAt = now;
updateJobStatus(job, job.status, AUDIO_STAGE, job.progress, onProgress, {
code: "process_audio",
});
},
async wait<T>(audio: Promise<T>): Promise<T> {
waiting = true;
try {
return await audio;
} finally {
waiting = false;
}
},
};
}
const lastAssembleReportAt = new WeakMap<RenderJob, number>();
/** Seconds of the final file written, 90% to 99%: only a complete file reaches 100. */
@@ -26,6 +26,7 @@ import {
processCompositionAudio,
removeScratch,
collectScratchLeftovers,
withFfmpegProgress,
type AudioProcessingFailure,
type MixResult,
} from "@hyperframes/engine";
@@ -60,6 +61,7 @@ export interface AudioStageInput {
log?: ProducerLogger;
/** Leave the delivered PCM in `workDir` for the assemble stage; its owner removes the dir. */
keepDeliverySource?: boolean;
onFfmpegProgress?: () => void;
}
export interface AudioStageResult {
@@ -100,7 +102,8 @@ function assertAudioNotInterrupted(failures: readonly AudioProcessingFailure[] |
export function runAudioStage(input: AudioStageInput): Promise<AudioStageResult> {
return collectScratchLeftovers(
() => runAudioStageScoped(input),
() =>
withFfmpegProgress(input.onFfmpegProgress ?? (() => {}), () => runAudioStageScoped(input)),
(left) => {
if (left.length > 0) input.log?.warn("[audio] scratch left behind", { count: left.length });
},
@@ -0,0 +1,148 @@
import { afterEach, expect, it, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { resolveConfig } from "@hyperframes/engine";
import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js";
// Entry: executeRenderJob. The audio mix is held open past capture setup, so capture waits on it; the test drives the
// mix's FFmpeg progress and reads the lines the render reports meanwhile.
const trace = vi.hoisted(() => ({
ffmpegProgressed: (): void => {},
failAudio: (_error: Error): void => {},
setUp: false,
}));
vi.mock("./render/shared.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/shared.js")>()),
createMemorySampler: () => ({
peakRssBytes: () => 0,
peakHeapUsedBytes: () => 0,
stop: () => {},
}),
}));
vi.mock("./fileServer.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./fileServer.js")>()),
createFileServer: async () => ({
url: "http://127.0.0.1:1",
port: 1,
close: () => {},
addPreHeadScript: () => {},
}),
}));
vi.mock("./render/stages/compileStage.js", () => ({
runCompileStage: async () => ({
compiled: {
html: "<div></div>",
hasShaderTransitions: false,
renderModeHints: { reasons: [] },
},
composition: { width: 16, height: 16, duration: 40 / 30, videos: [], images: [], audios: [] },
deviceScaleFactor: 1,
outputWidth: 16,
outputHeight: 16,
compileOnlyMs: 0,
forceScreenshot: true,
}),
}));
vi.mock("./render/stages/probeStage.js", () => ({
runProbeStage: async ({ compiled }: { compiled: object }) => ({
compiled,
fileServer: null,
probeSession: null,
lastBrowserConsole: [],
duration: 40 / 30,
totalFrames: 40,
browserProbeMs: 0,
beginFrameStalled: false,
}),
}));
vi.mock("./render/stages/extractVideosStage.js", () => ({
runExtractVideosStage: async () => ({
extractionResult: null,
frameLookup: null,
videoReadinessSkipIds: [],
videoMetadataHints: [],
nativeHdrVideoIds: new Set(),
videoTransfers: new Map(),
nativeHdrImageIds: new Set(),
imageTransfers: new Map(),
hdrImageSrcPaths: new Map(),
imageColorSpaces: [],
videoExtractMs: 0,
failureToEnforce: null,
}),
}));
vi.mock("./render/stages/audioStage.js", () => ({
runAudioStage: ({ onFfmpegProgress }: { onFfmpegProgress?: () => void }) =>
new Promise((_resolve, reject) => {
trace.ffmpegProgressed = onFfmpegProgress ?? (() => {});
trace.failAudio = reject;
}),
}));
vi.mock("./render/captureCost.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./render/captureCost.js")>()),
resolveRenderWorkerCount: () => {
// The mix writes more while capture is still setting up: not a line of its own yet.
trace.ffmpegProgressed();
trace.setUp = true;
return 1;
},
}));
const dirs: string[] = [];
afterEach(() => {
vi.useRealTimers();
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
const turn = () => new Promise((resolve) => setImmediate(resolve));
it("says it is processing audio while capture waits on a mix whose FFmpeg writes more, at most every 5 s", async () => {
vi.useFakeTimers({ toFake: ["Date"] });
const root = mkdtempSync(join(tmpdir(), "hf-audio-wait-"));
dirs.push(root);
writeFileSync(join(root, "index.html"), "<div></div>");
const job = createRenderJob({
fps: 30,
quality: "draft",
format: "mp4",
hdrMode: "force-sdr",
producerConfig: resolveConfig({ enablePageSideCompositing: false }),
logger: { info() {}, warn() {}, error() {}, debug() {} },
});
const lines: string[] = [];
const render = executeRenderJob(job, root, join(root, "out.mp4"), (reported, stage) => {
lines.push(`${reported.stageProgress?.code}:${stage}`);
}).catch((error: unknown) => error);
for (let at = 0; at < 5_000 && !trace.setUp; at++) await turn();
expect(trace.setUp).toBe(true);
const audioLines = () => lines.filter((line) => line === "process_audio:Processing audio tracks");
await turn();
// Only the stage's own first line.
const before = audioLines().length;
expect(before).toBe(1);
// The mix writes more every 6 s until capture, now waiting on it, says so.
for (let at = 0; at < 200 && audioLines().length === before; at++) {
vi.setSystemTime(Date.now() + 6_000);
trace.ffmpegProgressed();
await turn();
}
expect(audioLines().length).toBe(before + 1);
expect(lines.at(-1)).toBe("process_audio:Processing audio tracks");
// Writing more within 5 s of that line says nothing new; after 5 s, one more line.
vi.setSystemTime(Date.now() + 4_000);
trace.ffmpegProgressed();
await turn();
expect(audioLines().length).toBe(before + 1);
vi.setSystemTime(Date.now() + 1_000);
trace.ffmpegProgressed();
await turn();
expect(audioLines().length).toBe(before + 2);
vi.useRealTimers();
trace.failAudio(new Error("stop after the wait"));
expect(String(await render)).toContain("stop after the wait");
});
@@ -162,6 +162,8 @@ import {
type MemorySampler,
reportExtractProgress,
updateJobStatus,
audioWaitReporter,
AUDIO_STAGE,
audioWorkDir,
} from "./render/shared.js";
import { frameCacheOptions } from "./render/frameCacheBudget.js";
@@ -3739,10 +3741,11 @@ async function executeRenderPipeline(input: {
});
// ── Stage 3: Audio processing ───────────────────────────────────────
updateJobStatus(job, "preprocessing", "Processing audio tracks", 20, onProgress, {
updateJobStatus(job, "preprocessing", AUDIO_STAGE, 20, onProgress, {
code: "process_audio",
});
const audioWait = audioWaitReporter(job, onProgress);
const audioStage = observeRenderStage(
observability,
"audio_process",
@@ -3763,6 +3766,7 @@ async function executeRenderPipeline(input: {
assertNotAborted,
log,
keepDeliverySource: true,
onFfmpegProgress: audioWait.progressed,
}),
);
pendingAudio = audioStage.catch(() => undefined);
@@ -4930,7 +4934,7 @@ async function executeRenderPipeline(input: {
const effectiveQuality = job.config.crf ?? preset.quality;
const effectiveBitrate = job.config.crf != null ? undefined : job.config.videoBitrate;
const audioResult = await audioStage;
const audioResult = await audioWait.wait(audioStage);
const { audioOutputPath, deliverySourcePath, hasAudio } = audioResult;
perfStages.audioProcessMs = audioResult.audioProcessMs;
recordLimiterAttenuation(job, audioResult.audioLoweredDb);