/** * Audio Mixer Service * * Processes and mixes audio tracks using FFmpeg. */ import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs"; import { isAbsolute, join, dirname } from "path"; import { parseHTML } from "linkedom"; import { extractAudioMetadata } from "../utils/ffprobe.js"; import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { runFfmpeg } from "../utils/runFfmpeg.js"; import { unwrapTemplate } from "../utils/htmlTemplate.js"; import { resolveProjectRelativeSrc } from "./videoFrameExtractor.js"; import type { AudioElement, AudioTrack, MixResult } from "./audioMixer.types.js"; import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js"; export type { AudioElement, MixResult } from "./audioMixer.types.js"; function clampVolume(volume: number): number { if (!Number.isFinite(volume)) return 1; return Math.max(0, Math.min(1, volume)); } function formatFilterNumber(value: number): string { return Number(value.toFixed(6)).toString(); } function escapeExpressionCommas(expression: string): string { return expression.replace(/\\/g, "\\\\").replace(/,/g, "\\,"); } /** * Upper bound on volume-automation keyframes folded into the FFmpeg `volume` * expression. The expression nests one `if(lt(...))` per keyframe, and * FFmpeg's expression evaluator has a finite nesting depth: past ~95 levels * (build-dependent — lower on some Linux ffmpeg builds) `volume=...:eval=frame` * fails filter-graph init, which fails the whole mix and drops the audio track * entirely. The 60 Hz timeline probe routinely emits 100–300 keyframes for a * multi-second fade (GH #1066 follow-up: a 171-keyframe GSAP fade rendered with * no audio). 32 segments keeps a wide safety margin and is far more resolution * than a piecewise-linear volume envelope needs. */ const MAX_VOLUME_SEGMENTS = 32; /** * Volume delta below which a keyframe is collinear enough to drop. Kept tight * (0.5% linear) so the rendered piecewise-linear envelope tracks the GSAP curve * the browser plays in preview to within ~0.2 dB across the audible range — well * under the ~1 dB loudness JND, so render stays WYSIWYG with preview. A full * ease-in/ease-out fade still reduces to ~25 segments, inside MAX_VOLUME_SEGMENTS. */ const VOLUME_SIMPLIFY_EPSILON = 0.005; /** * Reduce a sorted keyframe list to a perceptually-equivalent piecewise-linear * envelope with a bounded segment count. * * Ramer–Douglas–Peucker drops control points lying within * `VOLUME_SIMPLIFY_EPSILON` of the line through their neighbours (a linear fade * collapses to its two endpoints; an eased fade to a handful). A uniform * downsample backstop then bounds pathological inputs (e.g. audio-rate volume * oscillation) to `MAX_VOLUME_SEGMENTS`. Endpoints are always preserved so the * envelope still spans the full clip. */ function simplifyVolumeKeyframes( keyframes: { time: number; volume: number }[], ): { time: number; volume: number }[] { if (keyframes.length < 3) return keyframes; const keep = new Array(keyframes.length).fill(false); keep[0] = true; keep[keyframes.length - 1] = true; const stack: [number, number][] = [[0, keyframes.length - 1]]; while (stack.length > 0) { const [startIndex, endIndex] = stack.pop()!; const start = keyframes[startIndex]!; const end = keyframes[endIndex]!; const span = end.time - start.time; let maxDistance = VOLUME_SIMPLIFY_EPSILON; let splitIndex = -1; for (let i = startIndex + 1; i < endIndex; i += 1) { const point = keyframes[i]!; const interpolated = span === 0 ? start.volume : start.volume + ((end.volume - start.volume) * (point.time - start.time)) / span; const distance = Math.abs(point.volume - interpolated); if (distance > maxDistance) { maxDistance = distance; splitIndex = i; } } if (splitIndex !== -1) { keep[splitIndex] = true; stack.push([startIndex, splitIndex], [splitIndex, endIndex]); } } const simplified = keyframes.filter((_, i) => keep[i]); if (simplified.length <= MAX_VOLUME_SEGMENTS) return simplified; const step = (simplified.length - 1) / (MAX_VOLUME_SEGMENTS - 1); const sampled: { time: number; volume: number }[] = []; for (let i = 0; i < MAX_VOLUME_SEGMENTS; i += 1) { const point = simplified[Math.round(i * step)]!; if (sampled.length === 0 || point.time > sampled.at(-1)!.time) sampled.push(point); } return sampled; } function buildVolumeExpression(track: AudioTrack, ignoreKeyframes = false): string { const trimDuration = track.end - track.start; const staticVolume = clampVolume(track.volume); const keyframes = (ignoreKeyframes ? [] : (track.volumeKeyframes ?? [])) .filter((keyframe) => Number.isFinite(keyframe.time) && Number.isFinite(keyframe.volume)) .map((keyframe) => ({ time: Math.max(0, Math.min(trimDuration, keyframe.time - track.start)), volume: clampVolume(keyframe.volume), })) .sort((a, b) => a.time - b.time); if (keyframes.length === 0) return `volume=${formatFilterNumber(staticVolume)}`; if (keyframes[0]!.time > 0) { keyframes.unshift({ time: 0, volume: staticVolume }); } const deduped: typeof keyframes = []; for (const keyframe of keyframes) { const previous = deduped.at(-1); if (previous && Math.abs(previous.time - keyframe.time) < 0.000001) { previous.volume = keyframe.volume; } else { deduped.push(keyframe); } } // Collapse the densely-sampled probe output to a bounded piecewise-linear // envelope. Without this, the nested-if expression below grows one level per // keyframe and overflows FFmpeg's expression evaluator (see MAX_VOLUME_SEGMENTS). const simplified = simplifyVolumeKeyframes(deduped); if (simplified.length === 1) { return `volume=${formatFilterNumber(simplified[0]!.volume)}`; } let expression = formatFilterNumber(simplified.at(-1)!.volume); for (let i = simplified.length - 2; i >= 0; i -= 1) { const current = simplified[i]!; const next = simplified[i + 1]!; const currentTime = formatFilterNumber(current.time); const nextTime = formatFilterNumber(next.time); const currentVolume = formatFilterNumber(current.volume); const span = Math.max(0.000001, next.time - current.time); const slope = formatFilterNumber((next.volume - current.volume) / span); const segment = `${currentVolume}+(${slope})*(t-${currentTime})`; expression = `if(lt(t,${nextTime}),${segment},${expression})`; } return `volume=${escapeExpressionCommas(expression)}:eval=frame`; } interface ExtractResult { success: boolean; outputPath: string; durationMs: number; error?: string; } export function parseAudioElements(html: string): AudioElement[] { const elements: AudioElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); // Parse