fix(engine): mix any number of audio clips by summing them in stages (#6120)

One ffmpeg graph held every clip of a group (or of the whole film) as its own
input, so a long project ran into the open-file and input limits and the
export failed. Past 96 clips the mixer now folds slices into one running
float sum, with no cap on the total; mixes of 96 clips or fewer take the same
single-graph path as before.

A mix process now also gets a deadline that grows with the film length, and a
failed ffmpeg run keeps how long it ran and the last lines of its stderr.

packages/engine/src/services/audioMixer.manyTracks.test.ts

@@ -0,0 +1,135 @@
+import { spawnSync } from "node:child_process";
+import { copyFileSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
+import { MAX_MIX_INPUTS, MIXED_AUDIO_FILENAME, processCompositionAudio } from "./audioMixer.js";
+
+const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
+const tempDirs: string[] = [];
+
+function meanVolumeDb(path: string): number {
+  const result = spawnSync(
+    getFfmpegBinary(),
+    ["-nostdin", "-hide_banner", "-i", path, "-af", "volumedetect", "-f", "null", "-"],
+    { encoding: "utf-8" },
+  );
+  const match = result.stderr.match(/mean_volume:\s*(-?[\d.]+) dB/);
+  if (result.status !== 0 || !match?.[1]) throw new Error(`Could not measure: ${result.stderr}`);
+  return Number(match[1]);
+}
+
+function peakDb(path: string): number {
+  const result = spawnSync(
+    getFfmpegBinary(),
+    ["-nostdin", "-hide_banner", "-i", path, "-af", "astats", "-f", "null", "-"],
+    { encoding: "utf-8" },
+  );
+  const match = result.stderr.match(/Peak level dB:\s*(-?[\d.]+)/);
+  if (result.status !== 0 || !match?.[1]) throw new Error(`Could not measure: ${result.stderr}`);
+  return Number(match[1]);
+}
+
+function writeTone(path: string, seconds: number, gain: number): void {
+  const result = spawnSync(
+    getFfmpegBinary(),
+    [
+      "-nostdin",
+      "-v",
+      "error",
+      "-f",
+      "lavfi",
+      "-i",
+      `sine=frequency=440:duration=${seconds}:sample_rate=48000`,
+      "-af",
+      `volume=${gain}`,
+      "-c:a",
+      "pcm_s16le",
+      path,
+    ],
+    { encoding: "utf-8" },
+  );
+  if (result.status !== 0) throw new Error(`Could not write tone: ${result.stderr}`);
+}
+
+const clip = (i: number, start: number, end: number, groupId?: string) => ({
+  id: `c${i}`,
+  src: `a${i}.wav`,
+  start,
+  end,
+  mediaStart: 0,
+  layer: i,
+  volume: 1,
+  type: "audio" as const,
+  ...(groupId ? { groupId } : {}),
+});
+
+describe.skipIf(!HAS_FFMPEG)("a mix with more clips than one ffmpeg graph can hold", () => {
+  afterEach(() => {
+    for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+  });
+
+  async function mix(
+    count: number,
+    groupId: string | undefined,
+    shape: { stackGain: number } | "sequence",
+  ) {
+    const projectDir = mkdtempSync(join(tmpdir(), "hf-many-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-many-work-"));
+    tempDirs.push(projectDir, workDir);
+    const stacked = shape !== "sequence";
+    writeTone(join(projectDir, "a.wav"), stacked ? 2 : 0.2, stacked ? shape.stackGain : 1);
+    for (let i = 0; i < count; i++)
+      copyFileSync(join(projectDir, "a.wav"), join(projectDir, `a${i}.wav`));
+    const clips = Array.from({ length: count }, (_, i) =>
+      stacked ? clip(i, 0, 2, groupId) : clip(i, i * 0.05, (i + 1) * 0.05, groupId),
+    );
+    const out = join(projectDir, MIXED_AUDIO_FILENAME);
+    const sum = join(projectDir, "sum.wav");
+    const result = await processCompositionAudio(
+      clips,
+      projectDir,
+      workDir,
+      out,
+      stacked ? 2 : count * 0.05,
+      undefined,
+      { keepEncoderInputAt: sum },
+    );
+    return { result, out, sum };
+  }
+
+  it("mixes 200 clips in one group at the level of a short group", async () => {
+    const small = await mix(8, "sfx", "sequence");
+    const many = await mix(200, "sfx", "sequence");
+    expect(many.result.failures ?? []).toEqual([]);
+    expect(many.result.success).toBe(true);
+    expect(Math.abs(meanVolumeDb(many.out) - meanVolumeDb(small.out))).toBeLessThan(1);
+  }, 120_000);
+
+  it("mixes 200 ungrouped clips at the level of a short mix", async () => {
+    const small = await mix(8, undefined, "sequence");
+    const many = await mix(200, undefined, "sequence");
+    expect(many.result.failures ?? []).toEqual([]);
+    expect(many.result.success).toBe(true);
+    expect(Math.abs(meanVolumeDb(many.out) - meanVolumeDb(small.out))).toBeLessThan(1);
+    expect(readdirSync(dirname(many.out)).filter((f) => f.startsWith(".mix-batches-"))).toEqual([]);
+  }, 120_000);
+
+  it.each([
+    ["ungrouped", undefined],
+    ["grouped", "sfx"],
+  ])(
+    "sums %s clips past the slice size at the gain the flat mix gives",
+    async (_name, groupId) => {
+      // Stacked coherent clips add, so 120 at gain g and 60 at gain 2g have the same sum.
+      // 60 is one flat graph; 120 is staged. The float sums must peak alike.
+      const flat = await mix(60, groupId, { stackGain: 0.1 });
+      const staged = await mix(MAX_MIX_INPUTS + 24, groupId, { stackGain: 0.05 });
+      expect(flat.result.success).toBe(true);
+      expect(staged.result.success).toBe(true);
+      expect(Math.abs(peakDb(staged.sum) - peakDb(flat.sum))).toBeLessThan(0.1);
+    },
+    120_000,
+  );
+});

packages/engine/src/services/audioMixer.test.ts

@@ -83,7 +83,7 @@ vi.mock("../utils/ffprobe.js", async (importOriginal) => {
 
 import { RATE_RANGE } from "@hyperframes/core/audio-automation";
 import { AudioFxRenderError } from "./audioFxRender.js";
-import { parseAudioElements, processCompositionAudio } from "./audioMixer.js";
+import { MAX_MIX_INPUTS, parseAudioElements, processCompositionAudio } from "./audioMixer.js";
 import { encodeAac } from "./aacEncode.js";
 
 describe("parseAudioElements strict literal timing", () => {
@@ -1093,6 +1093,140 @@ describe("processCompositionAudio", () => {
     expect(runFfmpegMock).toHaveBeenCalledTimes(2);
   });
 
+  it("keeps how long a killed group mix ran and ffmpeg's last words", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+    writeFileSync(join(baseDir, "voice.wav"), "stub");
+
+    const streamLines = Array.from(
+      { length: 400 },
+      (_, i) =>
+        `  Stream #${i}:0 -> #0:${i} (pcm_s16le (native) -> atrim (graph 0 input from stream ${i}:0))`,
+    );
+    runFfmpegMock
+      .mockResolvedValueOnce({
+        success: true,
+        durationMs: 1,
+        stderr: "",
+        exitCode: 0,
+        terminationReason: "exit",
+      })
+      .mockResolvedValueOnce({
+        success: false,
+        durationMs: 530_000,
+        stderr: [...streamLines, "Exiting normally, received signal 15."].join("\n"),
+        exitCode: 255,
+        signal: null,
+        terminationReason: "exit",
+        failureReason: "external_interruption",
+      });
+
+    const result = await processCompositionAudio(
+      [
+        {
+          id: "voice",
+          src: "voice.wav",
+          start: 0,
+          end: 2,
+          mediaStart: 0,
+          layer: 0,
+          volume: 1,
+          type: "audio",
+          groupId: "sfx",
+        },
+      ],
+      baseDir,
+      workDir,
+      join(baseDir, "out.m4a"),
+      2,
+    );
+
+    const failure = result.failures?.find((f) => f.elementId === "sfx");
+    expect(failure?.reason).toBe("external_interruption");
+    expect(failure?.subprocess).toMatchObject({ exitCode: 255 });
+    expect(failure?.detail).toContain("ran 530000 ms");
+    expect(failure?.detail).toContain("Exiting normally, received signal 15.");
+  });
+
+  it.each([
+    ["ungrouped", undefined],
+    ["grouped", "sfx"],
+  ])(
+    "never hands one ffmpeg more inputs than a graph holds, %s",
+    async (_name, groupId) => {
+      const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+      const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+      tempDirs.push(baseDir, workDir);
+      const trackCount = 2_000;
+      const elements = Array.from({ length: trackCount }, (_, i) => {
+        writeFileSync(join(baseDir, `clip-${i}.wav`), "stub");
+        return {
+          id: `clip-${i}`,
+          src: `clip-${i}.wav`,
+          start: i * 0.1,
+          end: i * 0.1 + 0.5,
+          mediaStart: 0,
+          layer: i,
+          volume: 1,
+          type: "audio" as const,
+          ...(groupId ? { groupId } : {}),
+        };
+      });
+      runFfmpegMock.mockClear();
+
+      const result = await processCompositionAudio(
+        elements,
+        baseDir,
+        workDir,
+        join(baseDir, "out.m4a"),
+        trackCount * 0.1 + 0.5,
+      );
+
+      // The mocked ffmpeg writes no files, so only the ungrouped mix runs to the end.
+      if (!groupId) expect(result.success).toBe(true);
+      const inputsPerCall = runFfmpegMock.mock.calls.map(
+        (call) => (call[0] as string[]).filter((arg) => arg === "-i").length,
+      );
+      expect(Math.max(...inputsPerCall)).toBeLessThanOrEqual(MAX_MIX_INPUTS);
+    },
+    60_000,
+  );
+
+  it("lets a mix's ffmpeg deadline grow with the film length", async () => {
+    const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+    tempDirs.push(baseDir, workDir);
+    writeFileSync(join(baseDir, "voice.wav"), "stub");
+    const voice = {
+      id: "voice",
+      src: "voice.wav",
+      start: 0,
+      end: 2,
+      mediaStart: 0,
+      layer: 0,
+      volume: 1,
+      type: "audio" as const,
+    };
+    const mixTimeout = async (totalDuration: number) => {
+      runFfmpegMock.mockClear();
+      await processCompositionAudio(
+        [voice],
+        baseDir,
+        workDir,
+        join(baseDir, "out.m4a"),
+        totalDuration,
+      );
+      const mixCall = runFfmpegMock.mock.calls.find((call) =>
+        (call[0] as string[]).includes("-filter_complex_script"),
+      );
+      return (mixCall?.[1] as { timeout?: number } | undefined)?.timeout;
+    };
+
+    expect(await mixTimeout(60)).toBe(300_000);
+    expect(await mixTimeout(36_000)).toBe(3_600_000);
+  });
+
   it("preserves and classifies unsupported FFmpeg filter failures", async () => {
     const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
     const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
@@ -1466,21 +1600,22 @@ describe("processCompositionAudio", () => {
     expect(result.success).toBe(true);
     expect(result.tracksProcessed).toBe(trackCount);
 
-    const mixArgs = runFfmpegMock.mock.calls.at(-1)?.[0] as string[];
-    expect(mixArgs).toContain("-filter_complex_script");
-    expect(mixArgs).not.toContain("-filter_complex");
+    // 150 tracks exceed one graph: two slice sums, the fold of those two, then the final mix.
+    const mixCalls = runFfmpegMock.mock.calls.slice(-4).map((call) => call[0] as string[]);
+    for (const mixArgs of mixCalls) {
+      expect(mixArgs).toContain("-filter_complex_script");
+      expect(mixArgs).not.toContain("-filter_complex");
+      // Only the -i pairs and the filter script's content scale with track count.
+      expect(mixArgs.join(" ").length).toBeLessThan(20_000);
+    }
 
-    // The only things that scale with track count are the -i pairs (short,
-    // fixed-size each) and the filter SCRIPT FILE's content (off the command
-    // line entirely) — not the args array's own total character length.
-    const argsLength = mixArgs.join(" ").length;
-    expect(argsLength).toBeLessThan(20_000);
-
-    const filter = capturedFilterScripts.at(-1);
-    expect(filter).toContain(`amix=inputs=${trackCount}`);
+    const [first, second, folded, last] = capturedFilterScripts.slice(-4);
+    expect(first).toContain(`amix=inputs=${MAX_MIX_INPUTS}`);
+    expect(second).toContain(`amix=inputs=${trackCount - MAX_MIX_INPUTS}`);
+    expect(folded).toContain("amix=inputs=2");
+    expect(last).toContain("amix=inputs=1");
     // Each track is trimmed to its authored clip; only the mixed output is padded.
-    expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount + 1);
-    expect((filter?.match(/apad,/g) ?? []).length).toBe(1);
+    expect((last?.match(/apad,/g) ?? []).length).toBe(1);
     // 150 real files + 150 existence checks: ~60ms on Linux, but well past the 5s
     // default on the Windows lane, where each file create is orders slower.
   }, 30_000);

packages/engine/src/services/audioMixer.ts

@@ -392,7 +392,11 @@ function boundedDetail(message: string, maxLength = 2_000): string {
       /(^|[\s"'(])\/.+?(?=:\s[A-Z]|\s(?:ENOENT|EACCES|EPERM)\b|\r?$)/gm,
       "$1<redacted-path>",
     );
-  return redacted.length <= maxLength ? redacted : `${redacted.slice(0, maxLength - 1)}…`;
+  // Keep both ends: the head names the failure, the tail holds ffmpeg's last words.
+  const head = Math.floor(maxLength / 4);
+  return redacted.length <= maxLength
+    ? redacted
+    : `${redacted.slice(0, head)}…${redacted.slice(-(maxLength - head - 1))}`;
 }
 
 function probeFailure(message: string, elementId: string): AudioProcessingFailure {
@@ -486,9 +490,10 @@ function ffmpegFailure(
     elementId,
     subprocess: audioSubprocessEvidence(result),
     detail: boundedDetail(
-      result.error?.message
-        ? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`
-        : describeFfmpegFailure(result, timeoutMs),
+      `(ran ${result.durationMs} ms, limit ${timeoutMs} ms) ` +
+        (result.error?.message
+          ? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`
+          : describeFfmpegFailure(result, timeoutMs)),
     ),
   };
 }
@@ -793,14 +798,25 @@ async function generateSilence(
   };
 }
 
-async function mixAudioTracks(
+/** A mix graph's work grows with the film length (about 11 ms per film second at 96 inputs); so does its deadline. */
+const MIX_TIMEOUT_MS_PER_FILM_SECOND = 100;
+
+function mixProcessTimeout(
+  config: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">> | undefined,
+  totalDuration: number,
+): number {
+  const configured = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+  return Math.max(configured, Math.ceil(totalDuration * MIX_TIMEOUT_MS_PER_FILM_SECOND));
+}
+
+async function mixAudioTracksFlat(
   tracks: AudioTrack[],
   outputPath: string,
   totalDuration: number,
   signal?: AbortSignal,
   config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,
 ): Promise<MixResult> {
-  const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+  const ffmpegProcessTimeout = mixProcessTimeout(config, totalDuration);
   const masterOutputGain = config?.audioGain ?? DEFAULT_CONFIG.audioGain;
 
   if (tracks.length === 0) {
@@ -989,7 +1005,7 @@ function groupNormalizeOptionUnsupported(stderr: string): boolean {
  * compensation at all and is preferred; the manual per-own-count fallback
  * only runs when this ffmpeg build's `amix` rejects the option.
  */
-async function mixGroupMembers(
+async function mixGroupMembersFlat(
   memberTracks: AudioTrack[],
   outputPath: string,
   totalDuration: number,
@@ -999,7 +1015,7 @@ async function mixGroupMembers(
   | { success: true; degradedAutomation?: boolean }
   | { success: false; error: string; failure: AudioProcessingFailure }
 > {
-  const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+  const ffmpegProcessTimeout = mixProcessTimeout(config, totalDuration);
   const outputDir = dirname(outputPath);
   if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
 
@@ -1099,6 +1115,135 @@ async function mixGroupMembers(
   return { success: true, degradedAutomation };
 }
 
+/** Tracks per ffmpeg graph; a larger mix is summed in stages, so there is no cap on the total. */
+export const MAX_MIX_INPUTS = 96;
+
+type GroupMixResult =
+  | { success: true; degradedAutomation?: boolean }
+  | { success: false; error: string; failure: AudioProcessingFailure };
+
+/** Unity-sum `tracks` (the group gain law) slice by slice into one float WAV, with few full-length files at once. */
+async function sumInSlices(
+  tracks: AudioTrack[],
+  scratchDir: string,
+  totalDuration: number,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<
+  | { success: true; track: AudioTrack; degradedAutomation: boolean }
+  | Extract<GroupMixResult, { success: false }>
+> {
+  const wavTrack = (id: string, srcPath: string): AudioTrack => ({
+    id,
+    srcPath,
+    start: 0,
+    end: totalDuration,
+    mediaStart: 0,
+    duration: totalDuration,
+    volume: 1,
+  });
+  let running: AudioTrack | null = null;
+  let degradedAutomation = false;
+  for (let at = 0; at < tracks.length; at += MAX_MIX_INPUTS) {
+    const id = `sum-${at / MAX_MIX_INPUTS}`;
+    const slicePath = join(scratchDir, `${id}-slice.wav`);
+    const slice = await mixGroupMembersFlat(
+      tracks.slice(at, at + MAX_MIX_INPUTS),
+      slicePath,
+      totalDuration,
+      signal,
+      config,
+    );
+    if (!slice.success) return slice;
+    degradedAutomation ||= slice.degradedAutomation === true;
+    if (!running) {
+      running = wavTrack(id, slicePath);
+      continue;
+    }
+    const foldedPath = join(scratchDir, `${id}.wav`);
+    const folded = await mixGroupMembersFlat(
+      [running, wavTrack(`${id}-slice`, slicePath)],
+      foldedPath,
+      totalDuration,
+      signal,
+      config,
+    );
+    rmSync(running.srcPath, { force: true });
+    rmSync(slicePath, { force: true });
+    if (!folded.success) return folded;
+    running = wavTrack(id, foldedPath);
+  }
+  if (!running) throw new Error("[audioMixer] sumInSlices needs at least one track");
+  return { success: true, track: running, degradedAutomation };
+}
+
+async function mixGroupMembers(
+  memberTracks: AudioTrack[],
+  outputPath: string,
+  totalDuration: number,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<GroupMixResult> {
+  if (memberTracks.length <= MAX_MIX_INPUTS) {
+    return mixGroupMembersFlat(memberTracks, outputPath, totalDuration, signal, config);
+  }
+  mkdirSync(dirname(outputPath), { recursive: true });
+  const scratchDir = mkdtempSync(join(dirname(outputPath), ".mix-batches-"));
+  try {
+    const summed = await sumInSlices(memberTracks, scratchDir, totalDuration, signal, config);
+    if (!summed.success) return summed;
+    renameSync(summed.track.srcPath, outputPath);
+    return { success: true, degradedAutomation: summed.degradedAutomation };
+  } finally {
+    rmSync(scratchDir, { recursive: true, force: true });
+  }
+}
+
+async function mixAudioTracks(
+  tracks: AudioTrack[],
+  outputPath: string,
+  totalDuration: number,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,
+): Promise<MixResult> {
+  if (tracks.length <= MAX_MIX_INPUTS) {
+    return mixAudioTracksFlat(tracks, outputPath, totalDuration, signal, config);
+  }
+  mkdirSync(dirname(outputPath), { recursive: true });
+  const scratchDir = mkdtempSync(join(dirname(outputPath), ".mix-batches-"));
+  try {
+    const summed = await sumInSlices(tracks, scratchDir, totalDuration, signal, config);
+    if (!summed.success) {
+      return {
+        success: false,
+        outputPath,
+        durationMs: 0,
+        tracksProcessed: 0,
+        error: summed.error,
+        failures: [summed.failure],
+      };
+    }
+    const mixed = await mixAudioTracksFlat(
+      [summed.track],
+      outputPath,
+      totalDuration,
+      signal,
+      config,
+    );
+    return mixed.success
+      ? {
+          ...mixed,
+          tracksProcessed: tracks.length,
+          error: summed.degradedAutomation
+            ? "Volume automation exceeded this ffmpeg build's expression limits; rendered at base volume"
+            : mixed.error,
+        }
+      : mixed;
+  } finally {
+    rmSync(scratchDir, { recursive: true, force: true });
+  }
+}
+
 /**
  * Sum the tracks to a float WAV, run the master chain (if any) over the sum, then AAC-encode.
  * Groups are already folded into `tracks`, and the sum stays float until the encoder.