fix(engine): sample-accurate volume automation so dense fades keep their audio (#1117)

Animated media volume (GSAP/JS fades) dropped the audio track entirely for dense fades. The 60 Hz timeline probe emits 100-300 keyframes for a multi-second fade, which were folded into an FFmpeg `volume` expression nesting one `if(lt(t,...))` per keyframe. Past ~95 nested levels (build-dependent, lower on some Linux ffmpeg builds) the expression overflows FFmpeg's evaluator, fails filter-graph init, fails the whole mix, and the muxer omits audio — so a `data-volume="0"` fade-in rendered with no audio at all (follow-up to #1066; this is why #1064's own scenario regressed once the fade was dense enough). Apply volume automation as sample-accurate gain, layered so audio is never lost: 1. Primary: bake the envelope into the prepared PCM samples in-process (audioVolumeEnvelope.ts). The track WAV is always pcm_s16le/48k/stereo; multiply its samples by the interpolated envelope and atomically rename the result into place, then mix at unity. No expression, no keyframe ceiling, exact at every sample, and the downstream ffmpeg amix/AAC encode is untouched so golden baselines only change where a fade is applied. The RIFF parser scans chunks order-independently and accepts only 16-bit PCM, falling back otherwise. The output is written to a random-named sibling and renamed, so a crash can't leave a truncated WAV and there's no predictable-path write. 2. Fallback: RDP-bounded ffmpeg `volume` expression (0.5% tolerance, capped at 32 segments) for the rare case a WAV is not 16-bit PCM. 0.5% keeps the rendered envelope within ~0.2 dB of the source curve. 3. Backstop: if an automated mix still fails, retry once at base volume and surface the degradation rather than dropping the track. This mirrors how OSS NLEs render automation (sample-level gain): MoviePy, Kdenlive/Shotcut (MLT), Remotion. Verified end-to-end: a 297-keyframe fade that rendered with no audio now bakes all 297 keyframes sample-accurately. Adds unit tests for sample-accurate gain, track-start offset, base/tail holds, thousands of keyframes, order-independent chunk parsing, and format rejection, plus mixer regression tests for bounded nesting and the base-volume backstop.
2026-09-07 10:06:21 +00:00 · 2026-05-28 23:49:47 -04:00
parent b1f9587aa1
commit 95d2a949b7
4 changed files with 616 additions and 42 deletions
@@ -0,0 +1,165 @@
+/**
+ * Sample-accurate volume automation.
+ *
+ * The audio mixer's primary path for time-varying volume bakes the envelope
+ * directly into the prepared PCM rather than encoding it as an FFmpeg `volume`
+ * expression. The expression approach nests one `if(lt(t,...))` per keyframe and
+ * overflows FFmpeg's expression evaluator past ~95 levels (a dense GSAP fade
+ * emits hundreds of keyframes), which fails the whole mix and drops the audio
+ * track. Multiplying the samples in-house has no such ceiling, is exact at every
+ * sample, and keeps the downstream ffmpeg `amix`/AAC encode untouched — so the
+ * output (and the golden baselines) only change where a fade is actually applied.
+ *
+ * The prepared tracks are always `pcm_s16le`, 48 kHz, stereo (see
+ * `prepareAudioTrack` / `extractAudioFromVideo`). Anything else is rejected so
+ * the caller can fall back to the expression path rather than corrupting audio.
+ */
+
+import { readFileSync, renameSync, writeFileSync } from "fs";
+import { randomBytes } from "crypto";
+import type { AudioVolumeKeyframe } from "./audioMixer.types.js";
+
+const PCM_FORMAT = 1; // WAVE_FORMAT_PCM
+const SUPPORTED_BITS = 16;
+
+interface WavLayout {
+  numChannels: number;
+  sampleRate: number;
+  dataOffset: number;
+  dataSize: number;
+}
+
+/**
+ * Locate the `fmt ` and `data` chunks and validate the format we know how to edit.
+ *
+ * Scans every chunk rather than assuming an ordering: the loop always advances
+ * past a chunk's body (using its declared size), so `data` may precede `fmt `
+ * and trailing chunks (LIST/fact/etc.) are skipped harmlessly. Returns null on
+ * anything unexpected so the caller falls back to the expression path.
+ */
+function parseWavLayout(buffer: Buffer): WavLayout | null {
+  if (buffer.length < 12 || buffer.toString("ascii", 0, 4) !== "RIFF") return null;
+  if (buffer.toString("ascii", 8, 12) !== "WAVE") return null;
+
+  let offset = 12;
+  let fmt: { numChannels: number; sampleRate: number; bitsPerSample: number } | null = null;
+  let data: { offset: number; size: number } | null = null;
+
+  while (offset + 8 <= buffer.length) {
+    const chunkId = buffer.toString("ascii", offset, offset + 4);
+    const chunkSize = buffer.readUInt32LE(offset + 4);
+    const body = offset + 8;
+    if (chunkId === "fmt " && body + 16 <= buffer.length) {
+      if (buffer.readUInt16LE(body) !== PCM_FORMAT) return null;
+      fmt = {
+        numChannels: buffer.readUInt16LE(body + 2),
+        sampleRate: buffer.readUInt32LE(body + 4),
+        bitsPerSample: buffer.readUInt16LE(body + 14),
+      };
+    } else if (chunkId === "data") {
+      data = { offset: body, size: Math.min(chunkSize, buffer.length - body) };
+    }
+    // Chunks are word-aligned: an odd size carries a trailing pad byte.
+    offset = body + chunkSize + (chunkSize % 2);
+  }
+
+  if (!fmt || !data) return null;
+  if (fmt.bitsPerSample !== SUPPORTED_BITS || fmt.numChannels < 1) return null;
+  return {
+    numChannels: fmt.numChannels,
+    sampleRate: fmt.sampleRate,
+    dataOffset: data.offset,
+    dataSize: data.size,
+  };
+}
+
+/**
+ * Normalise keyframes to track-relative seconds, sorted and de-duplicated, with
+ * `baseVolume` filling any gap before the first keyframe. Returns the breakpoints
+ * the gain envelope is linearly interpolated between.
+ */
+function toRelativeEnvelope(
+  keyframes: AudioVolumeKeyframe[],
+  trackStart: number,
+  baseVolume: number,
+): { time: number; volume: number }[] {
+  const points = keyframes
+    .filter((k) => Number.isFinite(k.time) && Number.isFinite(k.volume))
+    .map((k) => ({
+      time: Math.max(0, k.time - trackStart),
+      volume: Math.max(0, Math.min(1, k.volume)),
+    }))
+    .sort((a, b) => a.time - b.time);
+
+  const deduped: { time: number; volume: number }[] = [];
+  for (const point of points) {
+    const previous = deduped.at(-1);
+    if (previous && Math.abs(previous.time - point.time) < 1e-9) previous.volume = point.volume;
+    else deduped.push(point);
+  }
+
+  if (deduped.length === 0) return deduped;
+  if (deduped[0]!.time > 0) {
+    deduped.unshift({ time: 0, volume: Math.max(0, Math.min(1, baseVolume)) });
+  }
+  return deduped;
+}
+
+/**
+ * Multiply a prepared WAV's samples by a time-varying gain envelope in place.
+ *
+ * @returns `true` if the envelope was applied; `false` if the file isn't the
+ *   expected 16-bit PCM (caller should fall back to the expression path).
+ */
+export function applyVolumeEnvelopeToWav(
+  wavPath: string,
+  keyframes: AudioVolumeKeyframe[],
+  trackStart: number,
+  baseVolume: number,
+): boolean {
+  const envelope = toRelativeEnvelope(keyframes, trackStart, baseVolume);
+  if (envelope.length === 0) return false;
+
+  try {
+    const buffer = readFileSync(wavPath);
+    const layout = parseWavLayout(buffer);
+    if (!layout) return false;
+
+    const { numChannels, sampleRate, dataOffset, dataSize } = layout;
+    const bytesPerSample = SUPPORTED_BITS / 8;
+    const frameBytes = numChannels * bytesPerSample;
+    const frameCount = Math.floor(dataSize / frameBytes);
+
+    let segment = 0;
+    for (let frame = 0; frame < frameCount; frame += 1) {
+      const time = frame / sampleRate;
+      while (segment < envelope.length - 2 && time >= envelope[segment + 1]!.time) segment += 1;
+
+      const a = envelope[segment]!;
+      const b = envelope[segment + 1] ?? a;
+      const span = b.time - a.time;
+      const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (time - a.time) / span));
+      const gain = a.volume + (b.volume - a.volume) * progress;
+
+      const base = dataOffset + frame * frameBytes;
+      for (let channel = 0; channel < numChannels; channel += 1) {
+        const at = base + channel * bytesPerSample;
+        const scaled = Math.round(buffer.readInt16LE(at) * gain);
+        buffer.writeInt16LE(scaled < -32768 ? -32768 : scaled > 32767 ? 32767 : scaled, at);
+      }
+    }
+
+    // Write to a uniquely-named sibling then atomically rename over the
+    // original. The random name avoids following a pre-planted symlink at a
+    // predictable path, and the rename means a crash mid-write can't leave a
+    // truncated WAV for the downstream mix.
+    const tempPath = `${wavPath}.${randomBytes(6).toString("hex")}.tmp`;
+    writeFileSync(tempPath, buffer);
+    renameSync(tempPath, wavPath);
+    return true;
+  } catch {
+    // Any read/parse/write failure → leave the file untouched and let the
+    // caller fall back to the ffmpeg expression path rather than losing audio.
+    return false;
+  }
+}