// fallow-ignore-file complexity code-duplication /** * Audio Mixer Service * * Processes and mixes audio tracks using FFmpeg. */ import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs"; import { join, dirname } from "path"; import { parseHTML } from "linkedom"; import { extractAudioMetadata } from "../utils/ffprobe.js"; import { isNotMediaPayload } from "../utils/notMediaPayload.js"; import { clampAudioGain } from "@hyperframes/core/audio-gain"; import { downloadToTemp, isHttpUrl, UrlDownloadError, writeUrlDownloadTelemetry, } from "../utils/urlDownloader.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js"; import { unwrapTemplate } from "../utils/htmlTemplate.js"; import { resolveMediaElementSrc, resolveProjectRelativeSrc } from "./videoFrameExtractor.js"; import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js"; import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js"; import type { AudioElement, AudioFailureStage, AudioProcessingFailure, AudioTrack, MixResult, } from "./audioMixer.types.js"; import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js"; import { HF_AUDIO_FX_ATTR, parseAudioFxChain } from "@hyperframes/core/audio-fx"; import { HF_AUDIO_AUTOMATION_ATTR, parseAutomation, resolveAutomation, sampleAutomationLane, VOLUME_TARGET, type HfAutomationLane, } from "@hyperframes/core/audio-automation"; import { chainTailSeconds } from "@hyperframes/core/audio-fx-tail"; import { MEDIA_RENDER_ID_ATTR, normalizePlaybackRate, parseStrictFiniteTimingNumber, readMediaStart, } from "@hyperframes/core"; import { HF_AUDIO_GROUP_ATTR, resolveAudioGroups } from "@hyperframes/core/audio-groups"; import { AUDIO_GROUP_RENDER_ID_ATTR } from "@hyperframes/core"; import { applyAudioFxChain, AudioFxRenderError } from "./audioFxRender.js"; import type { AudioVolumeKeyframe } from "./audioMixer.types.js"; export type { AudioElement, MixResult } from "./audioMixer.types.js"; /** * Filename every caller must use for the mixed-audio artifact. * * The extension is load-bearing, not cosmetic: FFmpeg picks the muxer from it, * and the mix is AAC-encoded. A raw ADTS `.aac` stream has nowhere to record * the encoder's priming delay, so those leading samples decode as real silence * and shift the whole track ~1024 samples (21.33 ms at 48 kHz) late against a * frame-accurate video track. An MP4-family container carries the delay as an * edit list, which every decoder then strips, so the mix lands on its authored * start. Keep the choice here rather than at each call site: the same file is * muxed into the video, shipped in a distributed plan, and handed to users as * the PNG-sequence sidecar, and all three have to agree. */ export const MIXED_AUDIO_FILENAME = "audio.m4a"; /** * The bus key a member belongs to, as `resolveAudioGroups` keys them. * * The compiler's `data-hf-group-render-id` names one INSTANCE of a bus; the * author's `data-audio-group` names it only within its own composition file. A * sub-composition declaring a bus and its members, used twice, therefore had * both instances' members under one key: one sub-mix for two independent buses, * one instance's fader and chain over the other's audio, and — with only the * second muted — BOTH instances dropped from the export. Uncompiled documents * (the live preview) carry no stamp and read exactly as before. */ function memberGroupKey(el: RefResolverEl): string | null { return el.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR) ?? el.getAttribute(HF_AUDIO_GROUP_ATTR); } function clampVolume(volume: number): number { return clampAudioGain(volume); } /** * An author-controlled id, made safe to put in a filename. * * `data-audio-group` reaches this file straight from the document — the * studio's `GROUP_ID_PATTERN` guards only ids the studio itself mints, and a * hand-authored or agent-written one is unvalidated. Interpolated raw it could * carry `/` or `..`, and `mkdirSync(recursive)` inside ffmpeg's own path * handling would then write outside `workDir`, where `bail()`'s `rmSync` never * cleans it up. Everything outside [A-Za-z0-9_-] collapses to `_`, and every * result gets a stable positional suffix so distinct ids that sanitize alike * cannot share one intermediate file. */ function safePathSegment(id: string, fallbackIndex: number): string { const cleaned = id.replace(/[^A-Za-z0-9_-]/g, "_"); // Sanitisation is many-to-one (`bed/a` and `bed?a` both become `bed_a`). // The stable position keeps every authored group on a distinct temp path // even when their readable portions collide. return `${cleaned || "group"}-${fallbackIndex}`; } function formatFilterNumber(value: number): string { return Number(value.toFixed(6)).toString(); } /** Build an FFmpeg-compatible, pitch-preserving tempo chain. */ function buildAtempoFilter(playbackRate: number): string | null { let remaining = normalizePlaybackRate(playbackRate); if (Math.abs(remaining - 1) < 1e-9) return null; const stages: number[] = []; while (remaining < 0.5 - 1e-9) { stages.push(0.5); remaining /= 0.5; } while (remaining > 2 + 1e-9) { stages.push(2); remaining /= 2; } if (Math.abs(remaining - 1) >= 1e-9) stages.push(remaining); return stages.map((stage) => `atempo=${formatFilterNumber(stage)}`).join(","); } function preparedAudioOutputArgs(srcPath: string, playbackRate: number): Promise { return stereoOutputArgs(srcPath).then((channelArgs) => { const filters: string[] = []; const outputArgs: string[] = []; if (channelArgs[0] === "-af" && channelArgs[1]) { filters.push(channelArgs[1]); } else { outputArgs.push(...channelArgs); } const atempo = buildAtempoFilter(playbackRate); if (atempo) filters.push(atempo); if (filters.length > 0) outputArgs.push("-af", filters.join(",")); return outputArgs; }); } function escapeExpressionCommas(expression: string): string { return expression.replace(/\\/g, "\\\\").replace(/,/g, "\\,"); } function legacyFilterScriptOptionIsUnsupported(stderr: string): boolean { return ( /filter_complex_script/i.test(stderr) && /(?:unrecognized option|option (?:was )?not found)/i.test(stderr) ); } /** * Upper bound on volume-automation keyframes folded into the FFmpeg `volume` * expression. The expression nests one `if(lt(...))` per keyframe, and * FFmpeg's expression evaluator has a finite nesting depth: past ~95 levels * (build-dependent — lower on some Linux ffmpeg builds) `volume=...:eval=frame` * fails filter-graph init, which fails the whole mix and drops the audio track * entirely. The 60 Hz timeline probe routinely emits 100–300 keyframes for a * multi-second fade (GH #1066 follow-up: a 171-keyframe GSAP fade rendered with * no audio). 32 segments keeps a wide safety margin and is far more resolution * than a piecewise-linear volume envelope needs. */ const MAX_VOLUME_SEGMENTS = 32; /** * Volume delta below which a keyframe is collinear enough to drop. Kept tight * (0.5% linear) so the rendered piecewise-linear envelope tracks the GSAP curve * the browser plays in preview to within ~0.2 dB across the audible range — well * under the ~1 dB loudness JND, so render stays WYSIWYG with preview. A full * ease-in/ease-out fade still reduces to ~25 segments, inside MAX_VOLUME_SEGMENTS. */ const VOLUME_SIMPLIFY_EPSILON = 0.005; // `-ac 2` uses FFmpeg's default mono-to-stereo rematrix, which attenuates a // mono source by 3 dB. Explicitly map front-center into both stereo channels; // native stereo sources have FL/FR and pass through unchanged. const STEREO_CHANNEL_FILTER = "pan=stereo|FL=FL+FC|FR=FR+FC"; async function stereoOutputArgs(srcPath: string): Promise { try { const { channels } = await extractAudioMetadata(srcPath); if (channels === 1) return ["-af", STEREO_CHANNEL_FILTER]; } catch { // Preserve the previous FFmpeg conversion path when metadata probing fails. } return ["-ac", "2"]; } /** * Reduce a sorted keyframe list to a perceptually-equivalent piecewise-linear * envelope with a bounded segment count. * * Ramer–Douglas–Peucker drops control points lying within * `VOLUME_SIMPLIFY_EPSILON` of the line through their neighbours (a linear fade * collapses to its two endpoints; an eased fade to a handful). A uniform * downsample backstop then bounds pathological inputs (e.g. audio-rate volume * oscillation) to `MAX_VOLUME_SEGMENTS`. Endpoints are always preserved so the * envelope still spans the full clip. */ function simplifyVolumeKeyframes( keyframes: { time: number; volume: number }[], ): { time: number; volume: number }[] { if (keyframes.length < 3) return keyframes; const keep = new Array(keyframes.length).fill(false); keep[0] = true; keep[keyframes.length - 1] = true; const stack: [number, number][] = [[0, keyframes.length - 1]]; while (stack.length > 0) { const [startIndex, endIndex] = stack.pop()!; const start = keyframes[startIndex]!; const end = keyframes[endIndex]!; const span = end.time - start.time; let maxDistance = VOLUME_SIMPLIFY_EPSILON; let splitIndex = -1; for (let i = startIndex + 1; i < endIndex; i += 1) { const point = keyframes[i]!; const interpolated = span === 0 ? start.volume : start.volume + ((end.volume - start.volume) * (point.time - start.time)) / span; const distance = Math.abs(point.volume - interpolated); if (distance > maxDistance) { maxDistance = distance; splitIndex = i; } } if (splitIndex !== -1) { keep[splitIndex] = true; stack.push([startIndex, splitIndex], [splitIndex, endIndex]); } } const simplified = keyframes.filter((_, i) => keep[i]); if (simplified.length <= MAX_VOLUME_SEGMENTS) return simplified; const step = (simplified.length - 1) / (MAX_VOLUME_SEGMENTS - 1); const sampled: { time: number; volume: number }[] = []; for (let i = 0; i < MAX_VOLUME_SEGMENTS; i += 1) { const point = simplified[Math.round(i * step)]!; if (sampled.length === 0 || point.time > sampled.at(-1)!.time) sampled.push(point); } return sampled; } function buildVolumeExpression(track: AudioTrack, ignoreKeyframes = false): string { const trimDuration = track.end - track.start; const staticVolume = clampVolume(track.volume); const keyframes = (ignoreKeyframes ? [] : (track.volumeKeyframes ?? [])) .filter((keyframe) => Number.isFinite(keyframe.time) && Number.isFinite(keyframe.volume)) .map((keyframe) => ({ time: Math.max(0, Math.min(trimDuration, keyframe.time - track.start)), volume: clampVolume(keyframe.volume), })) .sort((a, b) => a.time - b.time); if (keyframes.length === 0) return `volume=${formatFilterNumber(staticVolume)}`; if (keyframes[0]!.time > 0) { keyframes.unshift({ time: 0, volume: staticVolume }); } const deduped: typeof keyframes = []; for (const keyframe of keyframes) { const previous = deduped.at(-1); if (previous && Math.abs(previous.time - keyframe.time) < 0.000001) { previous.volume = keyframe.volume; } else { deduped.push(keyframe); } } // Collapse the densely-sampled probe output to a bounded piecewise-linear // envelope. Without this, the nested-if expression below grows one level per // keyframe and overflows FFmpeg's expression evaluator (see MAX_VOLUME_SEGMENTS). const simplified = simplifyVolumeKeyframes(deduped); if (simplified.length === 1) { return `volume=${formatFilterNumber(simplified[0]!.volume)}`; } let expression = formatFilterNumber(simplified.at(-1)!.volume); for (let i = simplified.length - 2; i >= 0; i -= 1) { const current = simplified[i]!; const next = simplified[i + 1]!; const currentTime = formatFilterNumber(current.time); const nextTime = formatFilterNumber(next.time); const currentVolume = formatFilterNumber(current.volume); const span = Math.max(0.000001, next.time - current.time); const slope = formatFilterNumber((next.volume - current.volume) / span); const segment = `${currentVolume}+(${slope})*(t-${currentTime})`; expression = `if(lt(t,${nextTime}),${segment},${expression})`; } return `volume=${escapeExpressionCommas(expression)}:eval=frame`; } interface ExtractResult { success: boolean; outputPath: string; durationMs: number; error?: string; failure?: AudioProcessingFailure; } function boundedDetail(message: string, maxLength = 2_000): string { const redacted = message .replace(/\bhttps?:\/\/[^\s"'<>]+/gi, "") .replace(/\bfile:\/\/[^\s"'<>]+/gi, "") .replace( /\b[A-Za-z]:[\\/].+?(?=:\s[A-Z]|\s(?:ENOENT|EACCES|EPERM)\b|\r?$)/gm, "", ) .replace( /(^|[\s"'(])\/.+?(?=:\s[A-Z]|\s(?:ENOENT|EACCES|EPERM)\b|\r?$)/gm, "$1", ); return redacted.length <= maxLength ? redacted : `${redacted.slice(0, maxLength - 1)}…`; } function probeFailure(message: string, elementId: string): AudioProcessingFailure { const unavailable = /(?:not found|ENOENT|spawn)/i.test(message); const cancelled = /(?:aborted|AbortError|cancelled|canceled)/i.test(message); const timedOut = /(?:timed?\s*out|timeout|deadline|inactivity)/i.test(message); const invalidMedia = /(?:invalid data found|could not find codec parameters|moov atom not found|no audio stream)/i.test( message, ); return { stage: "probe", reason: cancelled ? "cancelled" : invalidMedia ? "invalid_media" : unavailable ? "ffmpeg_unavailable" : timedOut ? "ffmpeg_timeout" : "probe_failed", owner: cancelled || invalidMedia ? "user" : "system", retryable: !cancelled && !invalidMedia && (unavailable || timedOut), elementId, detail: boundedDetail(`Audio probe failed for element ${elementId}: ${message}`), }; } function downloadFailure(error: unknown, elementId: string): AudioProcessingFailure { const message = error instanceof Error ? error.message : String(error); const invalidSource = error instanceof UrlDownloadError ? error.kind === "http_not_found" || error.kind === "http_rejected" || error.kind === "invalid_payload" || error.kind === "cancelled" : /(?:invalid URL|only HTTPS|private\/reserved|HTTP (?:400|401|403|404|405|410|422)\b)/i.test( message, ); const retryable = error instanceof UrlDownloadError ? error.retryable : !invalidSource; return { stage: "download", reason: "download_failed", owner: invalidSource ? "user" : "system", retryable, elementId, detail: boundedDetail(`Download failed for audio element ${elementId}: ${message}`), }; } function ffmpegFailure( stage: Extract, result: RunFfmpegResult, elementId?: string, ): AudioProcessingFailure { const stderr = result.stderr ?? ""; let reason: AudioProcessingFailure["reason"] = "ffmpeg_failed"; let owner: AudioProcessingFailure["owner"] = "system"; let retryable = false; if (result.terminationReason === "abort") { reason = "cancelled"; owner = "user"; } else if (result.terminationReason === "deadline" || result.terminationReason === "inactivity") { reason = "ffmpeg_timeout"; retryable = true; } else if (result.terminationReason === "spawn_error") { reason = "ffmpeg_unavailable"; retryable = true; } else if ( /(?:unrecognized option|option (?:was )?not found|no option name near)/i.test(stderr) ) { reason = "ffmpeg_unsupported"; } else if ( (stage === "extract" || stage === "prepare") && /(?:invalid data found|could not find codec parameters|moov atom not found)/i.test(stderr) ) { reason = "invalid_media"; owner = "user"; } return { stage, reason, owner, retryable, elementId, detail: boundedDetail( result.error?.message ? `${formatFfmpegError(result.exitCode, stderr)}: ${result.error.message}` : formatFfmpegError(result.exitCode, stderr), ), }; } /** Extra samples per second inside a segment the baker cannot draw straight. */ const CURVED_SEGMENT_SAMPLES_PER_SEC = 30; /** * A volume lane as keyframes for the PCM baker. * * The baker interpolates linearly between keyframes, so a straight segment * needs only its two ends — a simple fade stays two keyframes. A bent one is * sampled, or the bake would quietly straighten it. * * Times come out in composition seconds, which is what the baker subtracts the * track start from. Returns null when the track has no volume lane. */ export function volumeLaneKeyframes( automation: { lanes: HfAutomationLane[] }, trackStart: number, duration: number, ): AudioVolumeKeyframe[] | null { const lane = automation.lanes.find((l) => l.target === VOLUME_TARGET); if (!lane || lane.points.length === 0) return null; const out: AudioVolumeKeyframe[] = []; const push = (t: number, v: number): void => { out.push({ time: trackStart + t, volume: v }); }; // The envelope holds its first value before the first point, rather than // falling back to `data-volume`. const first = lane.points[0]!; if (first.t > 0) push(0, first.v); for (let i = 0; i < lane.points.length; i += 1) { const a = lane.points[i]!; push(a.t, a.v); const b = lane.points[i + 1]; if (!b || !a.curve) continue; const steps = Math.max(2, Math.ceil((b.t - a.t) * CURVED_SEGMENT_SAMPLES_PER_SEC)); for (let k = 1; k < steps; k += 1) { const t = a.t + ((b.t - a.t) * k) / steps; push(t, sampleAutomationLane(lane, t)); } } // Hold the last value to the clip's end, so the baker does not ramp away // from it toward whatever it would otherwise assume. const last = lane.points[lane.points.length - 1]!; if (duration > last.t) push(duration, last.v); return out; } export function parseAudioElements(html: string): AudioElement[] { const elements: AudioElement[] = []; const { document } = parseHTML(unwrapTemplate(html)); interface AudioMediaElement extends RefResolverEl { hasAttribute(name: string): boolean; parentElement: AudioMediaElement | null; } // Shared resolver state so a relative `data-start` ("start when clip X ends") // resolves against every clip in the composition — exactly as // parseVideoElements does. Without this, `parseFloat("clipId")` yields NaN and // the mixer silently drops the track (the segment renders as pure digital // silence), even though the same reference places the *video* correctly. const startCache = new Map(); const visiting = new Set(); const resolveStart = (el: RefResolverEl): number => el.getAttribute("data-start") ? resolveReferencedStart(document, el, startCache, visiting) : 0; // `end` stays a plain numeric read (the mixer derives the real segment length // from data-duration / natural media downstream); guard NaN so a malformed // value never poisons the mix instead of falling back to 0. const parseEnd = (raw: string | null): number => { return parseStrictFiniteTimingNumber(raw) ?? 0; }; const isHidden = (el: AudioMediaElement): boolean => { for (let current: AudioMediaElement | null = el; current; current = current.parentElement) { if (current.hasAttribute("data-hidden")) return true; } return false; }; // Resolved once per parse. A group element carrying `data-hidden` drops // every member from the render (RULES: mute-by-drop, never // mute-by-volume-0) — members never enter the sub-mix. const groupsById = new Map( resolveAudioGroups(document).map((group) => [group.id, group] as const), ); const memberGroupHidden = (el: AudioMediaElement): boolean => { const groupId = memberGroupKey(el); return groupId ? (groupsById.get(groupId)?.hidden ?? false) : false; }; //