ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-03 04:38:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(engine): write the audio mix filter graph to a file, not the command line (#1890)

Browse Source 
* fix(engine): write the audio mix filter graph to a file, not the command line

mixAudioTracks built the ffmpeg -filter_complex argument as one inline
string scaling linearly with track count. Reported in the wild at 146
timed audio clips: the resulting command line exceeded the OS length
limit and spawn failed with ENAMETOOLONG, dropping audio entirely until
the user manually consolidated clips to reduce the count.

FFmpeg supports -filter_complex_script specifically for this - the same
filter graph read from a file instead of inlined as an argument. The -i
pairs for each track still scale with count but stay short and fixed-size
each, so the one component that actually grew unbounded (the filter
string) no longer sits on the command line at all. The temp file is
cleaned up immediately after ffmpeg exits, matching the existing sibling
temp-file convention in audioVolumeEnvelope.ts.

Verified end-to-end against a real ffmpeg binary (not just mocked): a
two-track mix produced correct output audio with no leftover temp files.

* fix(engine): create audio filter scripts safely
This commit is contained in:
Miguel Ángel
2026-07-04 14:08:15 -07:00
committed by GitHub
parent 78069da140
commit 8a3227f548
2 changed files with 126 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/engine/src/services/audioMixer.test.ts
+96 -34
View File
@@ -3,14 +3,29 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
const { runFfmpegMock } = vi.hoisted(() => ({
runFfmpegMock: vi.fn(async () => ({
success: true,
durationMs: 1,
stderr: "",
exitCode: 0,
})),
}));
// The mix filter graph is written to a temp file and passed via
// -filter_complex_script (not inlined via -filter_complex) so the command
// line doesn't scale with track count — production code deletes that file
// the moment the (real) ffmpeg process exits. The mock captures each call's
// filter content synchronously, while the file still exists, into an
// index-aligned side array (rather than re-reading it from disk after
// processCompositionAudio resolves, by which point it's already gone).
const { runFfmpegMock, capturedFilterScripts } = vi.hoisted(() => {
const capturedFilterScripts: string[] = [];
return {
capturedFilterScripts,
runFfmpegMock: vi.fn(async (args: string[]) => {
const idx = args.indexOf("-filter_complex_script");
if (idx >= 0) {
const { readFileSync } = await import("node:fs");
capturedFilterScripts.push(readFileSync(args[idx + 1], "utf8"));
} else {
capturedFilterScripts.push("");
}
return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
}),
};
});
vi.mock("../utils/runFfmpeg.js", () => ({
runFfmpeg: runFfmpegMock,
@@ -23,6 +38,7 @@ describe("processCompositionAudio", () => {
afterEach(() => {
runFfmpegMock.mockClear();
capturedFilterScripts.length = 0;
for (const dir of tempDirs.splice(0)) {
rmSync(dir, { recursive: true, force: true });
}
@@ -57,9 +73,7 @@ describe("processCompositionAudio", () => {
expect(result.success).toBe(true);
expect(runFfmpegMock).toHaveBeenCalledTimes(2);
const mixArgs = runFfmpegMock.mock.calls[1]?.[0];
const filterIndex = mixArgs.indexOf("-filter_complex");
const filter = mixArgs[filterIndex + 1];
const filter = capturedFilterScripts[1];
expect(filter).toContain("volume=0");
expect(filter).toContain("[mixed]volume=1[out]");
@@ -119,8 +133,7 @@ describe("processCompositionAudio", () => {
// 3 prepare calls (one per track via Promise.all) precede the mix call,
// so the mix is at index 3, not index 1.
expect(runFfmpegMock).toHaveBeenCalledTimes(4);
const mixArgs = runFfmpegMock.mock.calls[3]?.[0];
const filter = mixArgs[mixArgs.indexOf("-filter_complex") + 1];
const filter = capturedFilterScripts[3];
expect(filter).toContain("amix=inputs=3");
expect(filter).not.toContain("normalize=");
@@ -161,9 +174,7 @@ describe("processCompositionAudio", () => {
expect(result.success).toBe(true);
const mixArgs = runFfmpegMock.mock.calls[1]?.[0];
const filterIndex = mixArgs.indexOf("-filter_complex");
const filter = mixArgs[filterIndex + 1];
const filter = capturedFilterScripts[1];
expect(filter).toContain("volume=");
expect(filter).toContain(":eval=frame");
@@ -211,9 +222,7 @@ describe("processCompositionAudio", () => {
expect(result.success).toBe(true);
const mixArgs = runFfmpegMock.mock.calls[1]?.[0];
const filterIndex = mixArgs.indexOf("-filter_complex");
const filter = mixArgs[filterIndex + 1];
const filter = capturedFilterScripts[1];
// One nested `if(lt(...))` is emitted per segment; cap it well under the
// FFmpeg evaluator's nesting limit (MAX_VOLUME_SEGMENTS = 32).
@@ -235,20 +244,24 @@ describe("processCompositionAudio", () => {
// Simulate an ffmpeg build that rejects the automation expression: the
// first mix attempt fails, the static-volume retry succeeds. (prepare =
// call 0, automated mix = call 1, fallback mix = call 2.)
// call 0, automated mix = call 1, fallback mix = call 2.) These two
// one-time overrides bypass the default mock's capturedFilterScripts
// push, so they push an empty placeholder themselves to keep the array
// index-aligned with call order for the fallback mix's assertion below.
runFfmpegMock
.mockImplementationOnce(async () => ({
success: true,
durationMs: 1,
stderr: "",
exitCode: 0,
}))
.mockImplementationOnce(async () => ({
success: false,
durationMs: 1,
stderr: "Error initializing filters",
exitCode: 234,
}));
.mockImplementationOnce(async () => {
capturedFilterScripts.push("");
return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
})
.mockImplementationOnce(async () => {
capturedFilterScripts.push("");
return {
success: false,
durationMs: 1,
stderr: "Error initializing filters",
exitCode: 234,
};
});
const result = await processCompositionAudio(
[
@@ -280,12 +293,61 @@ describe("processCompositionAudio", () => {
expect(result.error).toMatch(/base volume/i);
// The fallback mix omits the automation expression (base volume only).
const fallbackArgs = runFfmpegMock.mock.calls[2]?.[0];
const fallbackFilter = fallbackArgs[fallbackArgs.indexOf("-filter_complex") + 1];
const fallbackFilter = capturedFilterScripts[2];
expect(fallbackFilter).not.toContain(":eval=frame");
expect(fallbackFilter).toContain("volume=0.8");
});
it("keeps the ffmpeg command line short with a large track count (regression for spawn ENAMETOOLONG)", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
// Reported in the wild at 146 timed audio clips: the old inline
// -filter_complex string scaled with track count and blew past the OS
// command-line length limit. 150 tracks reproduces the same shape.
const trackCount = 150;
const elements = Array.from({ length: trackCount }, (_, i) => {
const filename = `clip-${i}.wav`;
writeFileSync(join(baseDir, filename), "stub");
return {
id: `clip-${i}`,
src: filename,
start: i * 0.1,
end: i * 0.1 + 0.5,
mediaStart: 0,
layer: i,
volume: 1,
type: "audio" as const,
};
});
const result = await processCompositionAudio(
elements,
baseDir,
workDir,
join(baseDir, "out.m4a"),
trackCount * 0.1 + 0.5,
);
expect(result.success).toBe(true);
expect(result.tracksProcessed).toBe(trackCount);
const mixArgs = runFfmpegMock.mock.calls.at(-1)?.[0] as string[];
expect(mixArgs).toContain("-filter_complex_script");
expect(mixArgs).not.toContain("-filter_complex");
// The only things that scale with track count are the -i pairs (short,
// fixed-size each) and the filter SCRIPT FILE's content (off the command
// line entirely) — not the args array's own total character length.
const argsLength = mixArgs.join(" ").length;
expect(argsLength).toBeLessThan(20_000);
const filter = capturedFilterScripts.at(-1);
expect(filter).toContain(`amix=inputs=${trackCount}`);
expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount);
});
it("prepares percent-encoded non-Latin audio srcs from decoded filesystem paths", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
packages/engine/src/services/audioMixer.ts
+30 -10
View File
@@ -4,7 +4,7 @@
* Processes and mixes audio tracks using FFmpeg.
*/
import { existsSync, mkdirSync, rmSync } from "fs";
import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs";
import { isAbsolute, join, dirname } from "path";
import { parseHTML } from "linkedom";
import { extractAudioMetadata } from "../utils/ffprobe.js";
@@ -384,11 +384,9 @@ async function mixAudioTracks(
const outputDir = dirname(outputPath);
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
const buildArgs = (ignoreAutomation: boolean): string[] => {
const inputs: string[] = [];
const buildFilterComplex = (ignoreAutomation: boolean): string => {
const filterParts: string[] = [];
tracks.forEach((track, i) => {
inputs.push("-i", track.srcPath);
const delayMs = Math.round(track.start * 1000);
const trimDuration = track.end - track.start;
const volumeFilter = buildVolumeExpression(track, ignoreAutomation);
@@ -403,12 +401,31 @@ async function mixAudioTracks(
// gain by track count so per-track volumes authored in data-volume are preserved.
const compensatedGain = masterOutputGain * tracks.length;
const postMixGainFilter = `[mixed]volume=${formatFilterNumber(compensatedGain)}[out]`;
const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
return [...filterParts, mixFilter, postMixGainFilter].join(";");
};
return [
// A large track count (100+) makes the inline `-filter_complex <string>`
// argument scale linearly with track count until it exceeds the OS
// command-line length limit — spawn ENAMETOOLONG, seen in practice at 146
// tracks — even though every individual filter segment is short. FFmpeg's
// own `-filter_complex_script <file>` reads the same graph from disk
// instead, sidestepping the argv limit for the one component of this
// command line that actually grows with the composition.
const runMix = (ignoreAutomation: boolean) => {
const inputs: string[] = [];
tracks.forEach((track) => inputs.push("-i", track.srcPath));
const scriptDir = mkdtempSync(join(outputDir, ".filter-complex-"));
const scriptPath = join(scriptDir, "graph.txt");
const fd = openSync(scriptPath, "wx", 0o600);
try {
writeFileSync(fd, buildFilterComplex(ignoreAutomation));
} finally {
closeSync(fd);
}
const args = [
...inputs,
"-filter_complex",
fullFilter,
"-filter_complex_script",
scriptPath,
"-map",
"[out]",
"-acodec",
@@ -420,9 +437,12 @@ async function mixAudioTracks(
"-y",
outputPath,
];
return runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout }).finally(() =>
rmSync(scriptDir, { recursive: true, force: true }),
);
};
let result = await runFfmpeg(buildArgs(false), { signal, timeout: ffmpegProcessTimeout });
let result = await runMix(false);
// Defense in depth: volume automation is folded into an FFmpeg `volume`
// expression whose evaluator limits are build-dependent (see
@@ -432,7 +452,7 @@ async function mixAudioTracks(
let degradedAutomation = false;
const hasAutomation = tracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0);
if (!result.success && !signal?.aborted && hasAutomation) {
const retry = await runFfmpeg(buildArgs(true), { signal, timeout: ffmpegProcessTimeout });
const retry = await runMix(true);
if (retry.success) {
result = retry;
degradedAutomation = true;