mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(producer): a render waiting on a long audio mix says it is processing audio (#7952)
* fix(producer): a render waiting on a long audio mix says it is processing audio * fix(producer): drop the comment the field name already says
This commit is contained in:
@@ -453,6 +453,7 @@ export { onRenderRefusal } from "./services/publicNetwork/captureProxy.js";
|
||||
export type { RefusedHostClass } from "./services/publicNetwork/publicNetwork.js";
|
||||
export {
|
||||
runFfmpeg,
|
||||
withFfmpegProgress,
|
||||
formatFfmpegError,
|
||||
describeFfmpegFailure,
|
||||
isExternalFfmpegInterruption,
|
||||
|
||||
@@ -311,6 +311,48 @@ describe("a run's timeout", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("withFfmpegProgress", () => {
|
||||
afterEach(() => {
|
||||
vi.doUnmock("child_process");
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("tells its listener each time a run inside it writes more, and never a run outside it", async () => {
|
||||
const procs: EventEmitter[] = [];
|
||||
vi.resetModules();
|
||||
vi.doMock("child_process", () => ({
|
||||
spawn: () => {
|
||||
const proc = Object.assign(new EventEmitter(), {
|
||||
stderr: new EventEmitter(),
|
||||
kill: vi.fn(),
|
||||
});
|
||||
procs.push(proc);
|
||||
return proc;
|
||||
},
|
||||
}));
|
||||
const { runFfmpeg, withFfmpegProgress } = await import("./runFfmpeg.js");
|
||||
const heard = vi.fn();
|
||||
const write = (proc: EventEmitter, time: string) =>
|
||||
(proc as unknown as { stderr: EventEmitter }).stderr.emit(
|
||||
"data",
|
||||
Buffer.from(`size= 100KiB time=${time}\r`),
|
||||
);
|
||||
|
||||
const inside = withFfmpegProgress(heard, () => runFfmpeg(["-i", "in.wav", "out.m4a"]));
|
||||
const outside = runFfmpeg(["-i", "in.wav", "other.m4a"]);
|
||||
const [mine, other] = procs;
|
||||
write(mine, "00:00:01.00");
|
||||
write(mine, "00:00:01.00");
|
||||
write(mine, "00:00:02.00");
|
||||
write(other, "00:00:05.00");
|
||||
expect(heard).toHaveBeenCalledTimes(2);
|
||||
|
||||
mine.emit("close", 0, null);
|
||||
other.emit("close", 0, null);
|
||||
await Promise.all([inside, outside]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a run whose signal already aborted", () => {
|
||||
afterEach(() => {
|
||||
vi.doUnmock("child_process");
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
* that appears across audioMixer and chunkEncoder into a single helper.
|
||||
*/
|
||||
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { spawn, type ChildProcess } from "child_process";
|
||||
import { getFfmpegBinary } from "./ffmpegBinaries.js";
|
||||
import { trackChildProcess } from "./processTracker.js";
|
||||
@@ -199,8 +200,16 @@ const ABORTED_BEFORE_START: RunFfmpegResult = {
|
||||
terminationReason: "abort",
|
||||
};
|
||||
|
||||
const progressListener = new AsyncLocalStorage<() => void>();
|
||||
|
||||
/** Runs `work` with `onProgress` called each time an FFmpeg it starts writes a higher frame count or output time. */
|
||||
export function withFfmpegProgress<T>(onProgress: () => void, work: () => Promise<T>): Promise<T> {
|
||||
return progressListener.run(onProgress, work);
|
||||
}
|
||||
|
||||
export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promise<RunFfmpegResult> {
|
||||
if (opts?.signal?.aborted) return { ...ABORTED_BEFORE_START };
|
||||
const onProgress = progressListener.getStore();
|
||||
let framesWritten = 0;
|
||||
let secondsWritten = 0;
|
||||
const readStats = ffmpegStatsReader(({ frames = 0, seconds = 0 }) => {
|
||||
@@ -208,6 +217,7 @@ export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promis
|
||||
framesWritten = Math.max(frames, framesWritten);
|
||||
secondsWritten = Math.max(seconds, secondsWritten);
|
||||
managed.markActivity();
|
||||
onProgress?.();
|
||||
});
|
||||
const managed = new ManagedChildProcess(spawnFfmpeg(args), {
|
||||
signal: opts?.signal,
|
||||
|
||||
@@ -379,6 +379,34 @@ export function reportEncodeProgress(
|
||||
);
|
||||
}
|
||||
|
||||
export const AUDIO_STAGE = "Processing audio tracks";
|
||||
const AUDIO_WAIT_REPORT_MS = 5_000;
|
||||
|
||||
/** Once capture waits on the audio mix: a line at most every 5 s in which an FFmpeg of the mix wrote more, so whoever
|
||||
* watches the render for silence hears it working, and a mix that writes nothing stays silent. */
|
||||
export function audioWaitReporter(job: RenderJob, onProgress: ProgressCallback | undefined) {
|
||||
let waiting = false;
|
||||
let reportedAt = -Infinity;
|
||||
return {
|
||||
progressed(): void {
|
||||
const now = Date.now();
|
||||
if (!waiting || now - reportedAt < AUDIO_WAIT_REPORT_MS) return;
|
||||
reportedAt = now;
|
||||
updateJobStatus(job, job.status, AUDIO_STAGE, job.progress, onProgress, {
|
||||
code: "process_audio",
|
||||
});
|
||||
},
|
||||
async wait<T>(audio: Promise<T>): Promise<T> {
|
||||
waiting = true;
|
||||
try {
|
||||
return await audio;
|
||||
} finally {
|
||||
waiting = false;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const lastAssembleReportAt = new WeakMap<RenderJob, number>();
|
||||
|
||||
/** Seconds of the final file written, 90% to 99%: only a complete file reaches 100. */
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
processCompositionAudio,
|
||||
removeScratch,
|
||||
collectScratchLeftovers,
|
||||
withFfmpegProgress,
|
||||
type AudioProcessingFailure,
|
||||
type MixResult,
|
||||
} from "@hyperframes/engine";
|
||||
@@ -60,6 +61,7 @@ export interface AudioStageInput {
|
||||
log?: ProducerLogger;
|
||||
/** Leave the delivered PCM in `workDir` for the assemble stage; its owner removes the dir. */
|
||||
keepDeliverySource?: boolean;
|
||||
onFfmpegProgress?: () => void;
|
||||
}
|
||||
|
||||
export interface AudioStageResult {
|
||||
@@ -100,7 +102,8 @@ function assertAudioNotInterrupted(failures: readonly AudioProcessingFailure[] |
|
||||
|
||||
export function runAudioStage(input: AudioStageInput): Promise<AudioStageResult> {
|
||||
return collectScratchLeftovers(
|
||||
() => runAudioStageScoped(input),
|
||||
() =>
|
||||
withFfmpegProgress(input.onFfmpegProgress ?? (() => {}), () => runAudioStageScoped(input)),
|
||||
(left) => {
|
||||
if (left.length > 0) input.log?.warn("[audio] scratch left behind", { count: left.length });
|
||||
},
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { resolveConfig } from "@hyperframes/engine";
|
||||
import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js";
|
||||
|
||||
// Entry: executeRenderJob. The audio mix is held open past capture setup, so capture waits on it; the test drives the
|
||||
// mix's FFmpeg progress and reads the lines the render reports meanwhile.
|
||||
const trace = vi.hoisted(() => ({
|
||||
ffmpegProgressed: (): void => {},
|
||||
failAudio: (_error: Error): void => {},
|
||||
setUp: false,
|
||||
}));
|
||||
|
||||
vi.mock("./render/shared.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./render/shared.js")>()),
|
||||
createMemorySampler: () => ({
|
||||
peakRssBytes: () => 0,
|
||||
peakHeapUsedBytes: () => 0,
|
||||
stop: () => {},
|
||||
}),
|
||||
}));
|
||||
vi.mock("./fileServer.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./fileServer.js")>()),
|
||||
createFileServer: async () => ({
|
||||
url: "http://127.0.0.1:1",
|
||||
port: 1,
|
||||
close: () => {},
|
||||
addPreHeadScript: () => {},
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/compileStage.js", () => ({
|
||||
runCompileStage: async () => ({
|
||||
compiled: {
|
||||
html: "<div></div>",
|
||||
hasShaderTransitions: false,
|
||||
renderModeHints: { reasons: [] },
|
||||
},
|
||||
composition: { width: 16, height: 16, duration: 40 / 30, videos: [], images: [], audios: [] },
|
||||
deviceScaleFactor: 1,
|
||||
outputWidth: 16,
|
||||
outputHeight: 16,
|
||||
compileOnlyMs: 0,
|
||||
forceScreenshot: true,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/probeStage.js", () => ({
|
||||
runProbeStage: async ({ compiled }: { compiled: object }) => ({
|
||||
compiled,
|
||||
fileServer: null,
|
||||
probeSession: null,
|
||||
lastBrowserConsole: [],
|
||||
duration: 40 / 30,
|
||||
totalFrames: 40,
|
||||
browserProbeMs: 0,
|
||||
beginFrameStalled: false,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/extractVideosStage.js", () => ({
|
||||
runExtractVideosStage: async () => ({
|
||||
extractionResult: null,
|
||||
frameLookup: null,
|
||||
videoReadinessSkipIds: [],
|
||||
videoMetadataHints: [],
|
||||
nativeHdrVideoIds: new Set(),
|
||||
videoTransfers: new Map(),
|
||||
nativeHdrImageIds: new Set(),
|
||||
imageTransfers: new Map(),
|
||||
hdrImageSrcPaths: new Map(),
|
||||
imageColorSpaces: [],
|
||||
videoExtractMs: 0,
|
||||
failureToEnforce: null,
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/stages/audioStage.js", () => ({
|
||||
runAudioStage: ({ onFfmpegProgress }: { onFfmpegProgress?: () => void }) =>
|
||||
new Promise((_resolve, reject) => {
|
||||
trace.ffmpegProgressed = onFfmpegProgress ?? (() => {});
|
||||
trace.failAudio = reject;
|
||||
}),
|
||||
}));
|
||||
vi.mock("./render/captureCost.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./render/captureCost.js")>()),
|
||||
resolveRenderWorkerCount: () => {
|
||||
// The mix writes more while capture is still setting up: not a line of its own yet.
|
||||
trace.ffmpegProgressed();
|
||||
trace.setUp = true;
|
||||
return 1;
|
||||
},
|
||||
}));
|
||||
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const turn = () => new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
it("says it is processing audio while capture waits on a mix whose FFmpeg writes more, at most every 5 s", async () => {
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-audio-wait-"));
|
||||
dirs.push(root);
|
||||
writeFileSync(join(root, "index.html"), "<div></div>");
|
||||
const job = createRenderJob({
|
||||
fps: 30,
|
||||
quality: "draft",
|
||||
format: "mp4",
|
||||
hdrMode: "force-sdr",
|
||||
producerConfig: resolveConfig({ enablePageSideCompositing: false }),
|
||||
logger: { info() {}, warn() {}, error() {}, debug() {} },
|
||||
});
|
||||
const lines: string[] = [];
|
||||
const render = executeRenderJob(job, root, join(root, "out.mp4"), (reported, stage) => {
|
||||
lines.push(`${reported.stageProgress?.code}:${stage}`);
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
for (let at = 0; at < 5_000 && !trace.setUp; at++) await turn();
|
||||
expect(trace.setUp).toBe(true);
|
||||
const audioLines = () => lines.filter((line) => line === "process_audio:Processing audio tracks");
|
||||
await turn();
|
||||
// Only the stage's own first line.
|
||||
const before = audioLines().length;
|
||||
expect(before).toBe(1);
|
||||
// The mix writes more every 6 s until capture, now waiting on it, says so.
|
||||
for (let at = 0; at < 200 && audioLines().length === before; at++) {
|
||||
vi.setSystemTime(Date.now() + 6_000);
|
||||
trace.ffmpegProgressed();
|
||||
await turn();
|
||||
}
|
||||
expect(audioLines().length).toBe(before + 1);
|
||||
expect(lines.at(-1)).toBe("process_audio:Processing audio tracks");
|
||||
|
||||
// Writing more within 5 s of that line says nothing new; after 5 s, one more line.
|
||||
vi.setSystemTime(Date.now() + 4_000);
|
||||
trace.ffmpegProgressed();
|
||||
await turn();
|
||||
expect(audioLines().length).toBe(before + 1);
|
||||
vi.setSystemTime(Date.now() + 1_000);
|
||||
trace.ffmpegProgressed();
|
||||
await turn();
|
||||
expect(audioLines().length).toBe(before + 2);
|
||||
|
||||
vi.useRealTimers();
|
||||
trace.failAudio(new Error("stop after the wait"));
|
||||
expect(String(await render)).toContain("stop after the wait");
|
||||
});
|
||||
@@ -162,6 +162,8 @@ import {
|
||||
type MemorySampler,
|
||||
reportExtractProgress,
|
||||
updateJobStatus,
|
||||
audioWaitReporter,
|
||||
AUDIO_STAGE,
|
||||
audioWorkDir,
|
||||
} from "./render/shared.js";
|
||||
import { frameCacheOptions } from "./render/frameCacheBudget.js";
|
||||
@@ -3739,10 +3741,11 @@ async function executeRenderPipeline(input: {
|
||||
});
|
||||
|
||||
// ── Stage 3: Audio processing ───────────────────────────────────────
|
||||
updateJobStatus(job, "preprocessing", "Processing audio tracks", 20, onProgress, {
|
||||
updateJobStatus(job, "preprocessing", AUDIO_STAGE, 20, onProgress, {
|
||||
code: "process_audio",
|
||||
});
|
||||
|
||||
const audioWait = audioWaitReporter(job, onProgress);
|
||||
const audioStage = observeRenderStage(
|
||||
observability,
|
||||
"audio_process",
|
||||
@@ -3763,6 +3766,7 @@ async function executeRenderPipeline(input: {
|
||||
assertNotAborted,
|
||||
log,
|
||||
keepDeliverySource: true,
|
||||
onFfmpegProgress: audioWait.progressed,
|
||||
}),
|
||||
);
|
||||
pendingAudio = audioStage.catch(() => undefined);
|
||||
@@ -4930,7 +4934,7 @@ async function executeRenderPipeline(input: {
|
||||
const effectiveQuality = job.config.crf ?? preset.quality;
|
||||
const effectiveBitrate = job.config.crf != null ? undefined : job.config.videoBitrate;
|
||||
|
||||
const audioResult = await audioStage;
|
||||
const audioResult = await audioWait.wait(audioStage);
|
||||
const { audioOutputPath, deliverySourcePath, hasAudio } = audioResult;
|
||||
perfStages.audioProcessMs = audioResult.audioProcessMs;
|
||||
recordLimiterAttenuation(job, audioResult.audioLoweredDb);
|
||||
|
||||
Reference in New Issue
Block a user