diff --git a/packages/core/src/audioGroups.ts b/packages/core/src/audioGroups.ts index 5fc6fb49a..31f3cdb90 100644 --- a/packages/core/src/audioGroups.ts +++ b/packages/core/src/audioGroups.ts @@ -35,7 +35,13 @@ export interface HfAudioGroup { hidden: boolean; } -function parseGroupVolume(el: Element | undefined): number { +/** + * A group element's `data-volume`, defaulting to 1 for a missing, unparseable + * or absent element. Shared so the preview bus and `resolveAudioGroups` (which + * the render reads) cannot drift: the export was ~8 dB quieter than what was + * auditioned for exactly as long as the preview ignored this. + */ +export function readAudioGroupVolume(el: Element | null | undefined): number { const raw = el?.getAttribute("data-volume"); const parsed = raw ? parseFloat(raw) : 1; return Number.isFinite(parsed) ? parsed : 1; @@ -50,7 +56,7 @@ function buildGroup(id: string, memberIds: string[], el: Element | undefined): H memberIds, ...(fxChain ? { fxChain } : {}), ...(automation ? { automation } : {}), - volume: parseGroupVolume(el), + volume: readAudioGroupVolume(el), hidden: el?.hasAttribute("data-hidden") ?? false, }; } diff --git a/packages/core/src/runtime/audioFx.ts b/packages/core/src/runtime/audioFx.ts index 0068a129e..bd459af01 100644 --- a/packages/core/src/runtime/audioFx.ts +++ b/packages/core/src/runtime/audioFx.ts @@ -98,6 +98,16 @@ export function readElementAutomation(el: { */ export interface ElementFxHandle { dispose(): void; + /** + * Re-book every envelope against a fresh reference frame. + * + * For a graph that OUTLIVES the source feeding it — a group bus, which is + * built once and kept for the session — a replay or a seek starts a new pass + * over the same chain. Without re-anchoring, the envelopes stay committed to + * the first pass's absolute context times: past their last point that is a + * stuck value, which for a fade-out is silence for the rest of the session. + */ + reanchor(timing: AutomationTiming): void; /** * Re-aim every booked envelope at a new playback rate. * @@ -277,6 +287,13 @@ export function attachElementFxChain( } return { + reanchor: (next: AutomationTiming) => { + if (disposed) return; + const at = timingNow(); + if (at) cancelParamLane(automated, at.scheduledAt); + frame = { ...next }; + scheduleFor(readChain(el).chain, frame); + }, setRate: (rate: number) => { const at = timingNow(); if (disposed || !at || !Number.isFinite(rate) || rate <= 0 || rate === at.rate) return; diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index dd3851420..14d5a25e9 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -726,16 +726,17 @@ describe("WebAudioTransport", () => { await scheduleGrouped(transport, gen, "b", "vo"); // Creation order for a: a-gain(0), a-solo(1), groupInput(2), groupOutput(3), - // muteGain(4) — the group bus is built lazily inside a's schedule call. - // Then b: b-gain(5), b-solo(6). - expect(mock.gainNodes.length).toBeGreaterThanOrEqual(7); + // muteGain(4), fader(5) — the group bus is built lazily inside a's + // schedule call. Then b: b-gain(6), b-solo(7). + expect(mock.gainNodes.length).toBeGreaterThanOrEqual(8); const aGain = mock.gainNodes[0]!; const aSolo = mock.gainNodes[1]!; const groupInput = firstGroupInput(mock); const groupOutput = mock.gainNodes[3]!; const muteGain = mock.gainNodes[4]!; - const bGain = mock.gainNodes[5]!; - const bSolo = mock.gainNodes[6]!; + const fader = mock.gainNodes[5]!; + const bGain = mock.gainNodes[6]!; + const bSolo = mock.gainNodes[7]!; // Each member feeds its own solo gain, and both solo gains feed the // shared bus — neither connects straight to master. @@ -746,11 +747,14 @@ describe("WebAudioTransport", () => { expect(aSolo.connect).not.toHaveBeenCalledWith(mock.masterGain); expect(bSolo.connect).not.toHaveBeenCalledWith(mock.masterGain); - // The bus's input never reaches master directly — it lands on the mute - // gain (B5) first (the dry passthrough, since neither member's group has - // a chain-bearing ``), then the output gain, then master. + // The bus's input never reaches master directly. It runs through the + // chain (dry here — neither member's group has a chain-bearing + // ``) onto the FADER, then the mute gain (B5), then the + // output gain, then master. The fader sits POST-FX because that is where + // the render bakes group volume in. expect(groupInput.connect).not.toHaveBeenCalledWith(mock.masterGain); - expect(groupInput.connect).toHaveBeenCalledWith(muteGain); + expect(groupInput.connect).toHaveBeenCalledWith(fader); + expect(fader.connect).toHaveBeenCalledWith(muteGain); expect(muteGain.connect).toHaveBeenCalledWith(groupOutput); expect(groupOutput.connect).toHaveBeenCalledWith(mock.masterGain); }); @@ -773,18 +777,36 @@ describe("WebAudioTransport", () => { const muteGain = mock.gainNodes[4]!; const groupOutput = mock.gainNodes[3]!; - expect(firstGroupInput(mock).connect).toHaveBeenCalledWith(muteGain); + const fader = mock.gainNodes[5]!; + expect(firstGroupInput(mock).connect).toHaveBeenCalledWith(fader); + expect(fader.connect).toHaveBeenCalledWith(muteGain); expect(muteGain.connect).toHaveBeenCalledWith(groupOutput); expect(groupOutput.connect).toHaveBeenCalledWith(mock.masterGain); + // No element to read, so the fader sits at unity. + expect(fader.gain.value).toBe(1); }); - it("group volume rides the group's own data-volume via its automation lane, not the member's", async () => { + // The old assertion here was `resolves.not.toBeNull()` — it was named for + // group volume and checked only that scheduling did not throw, so the bus + // sitting at unity while the render applied data-volume went unseen. The + // export was ~8 dB quieter than what had been auditioned. + it("puts the group's own data-volume on the bus fader", async () => { + document.body.innerHTML = ``; + const { transport, mock, gen } = setupGroupTransport(); + + await expect(scheduleGrouped(transport, gen, "a", "vo")).resolves.not.toBeNull(); + + expect(mock.gainNodes[5]!.gain.value).toBeCloseTo(0.4, 6); + }); + + it("leaves the fader at unity when the group carries no data-volume", async () => { document.body.innerHTML = ``; - const { transport, gen } = setupGroupTransport(); + const { transport, mock, gen } = setupGroupTransport(); // No throw wiring the group's automation reader against a real // element that carries no fx/automation attrs. await expect(scheduleGrouped(transport, gen, "a", "vo")).resolves.not.toBeNull(); + expect(mock.gainNodes[5]!.gain.value).toBe(1); }); it("destroy() disposes every group bus", async () => { @@ -811,13 +833,37 @@ describe("WebAudioTransport", () => { expect(mock.gainNodes.filter((n) => n === groupInput)).toHaveLength(1); }); + // Surviving stopAll() is the point of the bus — and the trap. Its envelopes + // were booked against the FIRST pass's absolute context times, so a replay + // or a seek left the fader holding that pass's last value: 0 after a + // fade-out, i.e. silent for the rest of the session. + it("re-anchors the reused bus once per play generation, and only once", async () => { + document.body.innerHTML = ``; + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + const fader = mock.gainNodes[5]!; + + // Something moved the fader mid-pass (a ramp reaching its last point). + fader.gain.value = 0; + transport.stopAll(); + + const gen2 = transport.startGeneration(); + await scheduleGrouped(transport, gen2, "a", "vo"); + expect(fader.gain.value).toBeCloseTo(0.5, 6); + + // A second member in the SAME pass must not re-book on top of the first. + fader.gain.value = 0; + await scheduleGrouped(transport, gen2, "b", "vo"); + expect(fader.gain.value).toBe(0); + }); + describe('solo — "Hear only this" (B5)', () => { it("silences a non-soloed member via its own solo gain, without touching the group bus", async () => { const { transport, mock, gen } = setupGroupTransport(); await scheduleGrouped(transport, gen, "a", "vo"); await scheduleGrouped(transport, gen, "b", "vo"); const aSolo = mock.gainNodes[1]!; - const bSolo = mock.gainNodes[6]!; + const bSolo = mock.gainNodes[7]!; const groupInput = firstGroupInput(mock); transport.setSolo(new Set(["other-clip"])); @@ -834,7 +880,7 @@ describe("WebAudioTransport", () => { await scheduleGrouped(transport, gen, "a", "vo"); await scheduleGrouped(transport, gen, "b", "vo"); const aSolo = mock.gainNodes[1]!; - const bSolo = mock.gainNodes[6]!; + const bSolo = mock.gainNodes[7]!; const groupInput = firstGroupInput(mock); transport.setSolo(new Set(["a"])); @@ -849,7 +895,7 @@ describe("WebAudioTransport", () => { await scheduleGrouped(transport, gen, "a", "vo"); await scheduleGrouped(transport, gen, "b", "vo"); const aSolo = mock.gainNodes[1]!; - const bSolo = mock.gainNodes[6]!; + const bSolo = mock.gainNodes[7]!; transport.setSolo(new Set(["vo"])); diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index 56e917355..3d32e4d2f 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -5,7 +5,7 @@ import { type AutomationTiming, } from "../audio/audioFxAutomation.js"; import { VOLUME_RANGE } from "../audioAutomation.js"; -import { audioGroupOf, isAudibleUnderSolo } from "../audioGroups.js"; +import { audioGroupOf, isAudibleUnderSolo, readAudioGroupVolume } from "../audioGroups.js"; import { swallow } from "./diagnostics"; import { clampAudioGain } from "../audioGain.js"; import { getDebugSurface } from "./globals.js"; @@ -136,9 +136,17 @@ export class WebAudioTransport { string, { input: GainNode; + /** Post-FX fader: `data-volume` plus the volume lane. */ + fader: GainNode; muteGain: GainNode; analyser: AnalyserNode; levelBuf: Float32Array; + /** Kept so `setRate` can re-aim this bus's FX automation, the way it does + * every source's — its docblock claims it already did. */ + fx: ElementFxHandle | null; + /** Play generation the current envelopes were booked against. */ + generation: number; + reanchor(timing: AutomationTiming): void; dispose(): void; } >(); @@ -308,7 +316,18 @@ export class WebAudioTransport { */ private groupInput(groupId: string, doc: Document, timing: AutomationTiming): GainNode | null { const existing = this._groups.get(groupId); - if (existing) return existing.input; + if (existing) { + // The bus outlives `stopAll()` on purpose, so a replay or a seek reuses + // this graph — but its envelopes were committed to the FIRST pass's + // absolute context times. Left alone they hold their last value forever, + // which for a fade-out is silence for the rest of the session. Re-anchor + // once per play generation, not once per member scheduled. + if (existing.generation !== this._playGeneration) { + existing.generation = this._playGeneration; + existing.reanchor(timing); + } + return existing.input; + } if (!this._ctx || !this._masterGain) return null; const input = this._ctx.createGain(); @@ -329,24 +348,42 @@ export class WebAudioTransport { const muteGain = this._ctx.createGain(); muteGain.gain.value = groupEl?.hasAttribute("data-hidden") ? 0 : 1; muteGain.connect(output); + // The group's fader, POST-FX: `data-volume` is the static position and the + // volume lane rides it, which is where a DAW puts it and the order the + // render bakes it in (`scheduleVolumeLane`'s own contract). Scheduling it + // on `input` instead put the fader ahead of the effects, so any nonlinear + // group effect — a compressor, the Giant preset — previewed differently + // than it rendered. + const fader = this._ctx.createGain(); + fader.gain.value = readAudioGroupVolume(groupEl); + fader.connect(muteGain); const fx = attachElementFxChain( this._ctx, groupEl ?? { getAttribute: () => null }, input, - muteGain, + fader, timing, ); - if (groupEl) scheduleVolumeLane(groupEl, input, timing); + if (groupEl) scheduleVolumeLane(groupEl, fader, timing); this._groups.set(groupId, { input, + fader, muteGain, analyser, levelBuf: new Float32Array(analyser.fftSize), + fx, + generation: this._playGeneration, + reanchor: (at: AutomationTiming) => { + fader.gain.value = readAudioGroupVolume(groupEl); + fx?.reanchor(at); + if (groupEl) scheduleVolumeLane(groupEl, fader, at); + }, dispose: () => { try { fx?.dispose(); input.disconnect(); + fader.disconnect(); muteGain.disconnect(); output.disconnect(); analyser.disconnect(); @@ -588,6 +625,16 @@ export class WebAudioTransport { swallow("webAudioTransport.setRate", err); } } + // Group buses are not in `_activeSources` — they outlive it — so their FX + // automation needs re-aiming here too, or a rate change leaves a group's + // envelopes running the old plan over audio at the new speed. + for (const group of this._groups.values()) { + try { + group.fx?.setRate(safeRate); + } catch (err) { + swallow("webAudioTransport.setRate.group", err); + } + } return true; } diff --git a/packages/engine/src/services/audioMixer.grouping.test.ts b/packages/engine/src/services/audioMixer.grouping.test.ts index d148b72cb..51658815e 100644 --- a/packages/engine/src/services/audioMixer.grouping.test.ts +++ b/packages/engine/src/services/audioMixer.grouping.test.ts @@ -63,6 +63,35 @@ function writeTone(path: string, freq: number, seconds: number, gain: number): v if (result.status !== 0) throw new Error(`Could not write tone: ${result.stderr}`); } +/** + * A sine of an EXACT peak amplitude. + * + * `writeTone` builds on ffmpeg's `sine` source, whose output is ~0.125 full + * scale, so its `gain` argument is a relative knob rather than a level: a tone + * asked for at 0.7 lands at about -21 dBFS. Fine for the relative comparisons + * above, useless for anything about headroom — `aevalsrc` states the amplitude + * outright. + */ +function writePeakTone(path: string, freq: number, seconds: number, peak: number): void { + const result = spawnSync( + getFfmpegBinary(), + [ + "-nostdin", + "-v", + "error", + "-f", + "lavfi", + "-i", + `aevalsrc=${peak}*sin(2*PI*${freq}*t):d=${seconds}:s=48000`, + "-c:a", + "pcm_s16le", + path, + ], + { encoding: "utf-8" }, + ); + if (result.status !== 0) throw new Error(`Could not write tone: ${result.stderr}`); +} + const track = (id: string, end: number, volume = 1) => ({ id, src: `${id}.wav`, @@ -268,4 +297,54 @@ describe.skipIf(!HAS_FFMPEG)("mix level arithmetic", () => { const flatTail = meanVolumeDb(flatOut, 3, 4); expect(Math.abs(groupedTail - flatTail)).toBeLessThan(0.5); }); + + // Members sum at unity (normalize=0), so an over-unity sum used to hard-clip + // at ±1 in the 16-bit intermediate BEFORE the group's fader and FX chain ran + // — pulling the group down then operated on distortion. Every existing probe + // here sums to ≤ 0.8, which is exactly why nothing caught it; preview cannot + // reproduce it either, because its bus is float. + it("does not clip an over-unity member sum before the group fader", async () => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-grp-clip-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-grp-clip-work-")); + tempDirs.push(projectDir, workDir); + + // Two coherent copies of the same tone: 0.7 + 0.7 = 1.4, comfortably over. + // `writePeakTone`, not `writeTone` — ffmpeg's `sine` source is nowhere near + // full scale (its output peaks at ~0.125), so every tone in this file sits + // around -21 dBFS and NOTHING here can reach a clip no matter what gain is + // asked for. That is a large part of why this class of bug survived. + writePeakTone(join(projectDir, "a.wav"), 440, 2, 0.7); + writePeakTone(join(projectDir, "b.wav"), 440, 2, 0.7); + // The reference: the level that sum SHOULD reach once the group's 0.5 + // fader has been applied — 1.4 × 0.5 = 0.7, one tone's worth. + writePeakTone(join(projectDir, "ref.wav"), 440, 2, 0.7); + + const groupedOut = join(projectDir, `clip-grouped-${MIXED_AUDIO_FILENAME}`); + const refOut = join(projectDir, `clip-ref-${MIXED_AUDIO_FILENAME}`); + + const grouped = await processCompositionAudio( + [ + { ...track("a", 2), groupId: "vo", groupVolume: 0.5 }, + { ...track("b", 2), groupId: "vo", groupVolume: 0.5 }, + ], + projectDir, + workDir, + groupedOut, + 2, + ); + const reference = await processCompositionAudio( + [track("ref", 2)], + projectDir, + workDir, + refOut, + 2, + ); + expect(grouped.success).toBe(true); + expect(reference.success).toBe(true); + + // Both are ONE track into the outer mix, so the outer graph is identical + // and the levels are directly comparable. Clipped, the flat-topped sum + // reads well over a dB hot even after the fader halves it. + expect(Math.abs(meanVolumeDb(groupedOut) - meanVolumeDb(refOut))).toBeLessThan(0.5); + }); }); diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 28f151104..7c3d3d891 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -885,20 +885,25 @@ async function mixGroupMembers( totalDuration: number, signal?: AbortSignal, config?: Partial>, -): Promise<{ success: boolean; error?: string }> { +): Promise<{ success: boolean; error?: string; degradedAutomation?: boolean }> { const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; const outputDir = dirname(outputPath); if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); - const inputFilters = memberTracks.map((track, i) => { - const delayMs = Math.round(track.start * 1000); - const trimDuration = track.end - track.start + (track.tailSeconds ?? 0); - const volumeFilter = buildVolumeExpression(track); - return `[${i}:a]atrim=0:${formatFilterNumber(trimDuration)},${volumeFilter},adelay=${delayMs}|${delayMs},apad,atrim=0:${formatFilterNumber(totalDuration)}[a${i}]`; - }); + const buildInputFilters = (ignoreKeyframes: boolean) => + memberTracks.map((track, i) => { + const delayMs = Math.round(track.start * 1000); + const trimDuration = track.end - track.start + (track.tailSeconds ?? 0); + const volumeFilter = buildVolumeExpression(track, ignoreKeyframes); + return `[${i}:a]atrim=0:${formatFilterNumber(trimDuration)},${volumeFilter},adelay=${delayMs}|${delayMs},apad,atrim=0:${formatFilterNumber(totalDuration)}[a${i}]`; + }); const mixInputs = memberTracks.map((_, i) => `[a${i}]`).join(""); - const runOnce = async (useNormalize: boolean): Promise => { + const runOnce = async ( + useNormalize: boolean, + ignoreKeyframes = false, + ): Promise => { + const inputFilters = buildInputFilters(ignoreKeyframes); const mixFilter = useNormalize ? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0[out]` : // amix's default normalize divides by input count; compensate by THIS @@ -922,8 +927,15 @@ async function mixGroupMembers( scriptPath, "-map", "[out]", + // Float, not pcm_s16le: `normalize=0` sums the members at unity, so any + // over-unity sum hard-clipped at ±1 in the intermediate — BEFORE the + // group's FX chain and its fader ran. Pulling the group down, or the + // Giant preset's compressor, then operated on distortion. Preview + // cannot reproduce it (its bus is float), and it only shows up in the + // export. Both readers downstream take float: `readWav` (format 3) and + // `applyVolumeEnvelopeToWav`. "-acodec", - "pcm_s16le", + "pcm_f32le", "-ar", "48000", "-t", @@ -943,14 +955,32 @@ async function mixGroupMembers( } }; - let result = await runOnce(true); + let useNormalize = true; + let result = await runOnce(useNormalize); if (!result.success && groupNormalizeOptionUnsupported(result.stderr)) { - result = await runOnce(false); + useNormalize = false; + result = await runOnce(useNormalize); } + + // The same defence `mixAudioTracks` has, which this forked without: a + // member's volume automation becomes an ffmpeg `volume` expression whose + // evaluator limits are build-dependent, so a dense envelope can fail the + // whole run. Ungrouped, that track degrades to base volume with a warning; + // grouped, it took the entire composition's audio down with it. + let degradedAutomation = false; + const hasAutomation = memberTracks.some((track) => (track.volumeKeyframes?.length ?? 0) > 0); + if (!result.success && !signal?.aborted && hasAutomation) { + const retry = await runOnce(useNormalize, true); + if (retry.success) { + result = retry; + degradedAutomation = true; + } + } + if (signal?.aborted) return { success: false, error: "Group sub-mix cancelled" }; if (!result.success) return { success: false, error: formatFfmpegError(result.exitCode, result.stderr) }; - return { success: true }; + return { success: true, degradedAutomation }; } export async function processCompositionAudio( @@ -1305,6 +1335,7 @@ export async function processCompositionAudio( // clock is composition time (offset 0): a group has no `data-start` of its // own, and members are already delayed to their composition positions // inside the sub-mix, so the group WAV's t=0 IS composition time. + const groupsDegradedAutomation: string[] = []; for (const [groupId, memberTracks] of groupTracks) { const meta = groupMeta.get(groupId); if (!meta) continue; @@ -1329,6 +1360,7 @@ export async function processCompositionAudio( }); continue; } + if (subMix.degradedAutomation) groupsDegradedAutomation.push(groupId); // Composition-time automation (offset 0, duration totalDuration) — same // resolve/lane/bake path a member uses, just anchored at the group clock @@ -1385,6 +1417,10 @@ export async function processCompositionAudio( mediaStart: 0, duration: totalDuration, volume: bakedEnvelope ? 1.0 : meta.volume, + // Same fallback an ungrouped track gets: when the envelope could not be + // baked into the samples, hand the keyframes to the outer mix's volume + // expression instead of dropping the group's automation on the floor. + ...(bakedEnvelope || !laneKeyframes?.length ? {} : { volumeKeyframes: laneKeyframes }), }); } if (failures.length > 0) return bail(); @@ -1397,9 +1433,18 @@ export async function processCompositionAudio( /* ignore */ } + // A group whose sub-mix had to drop member automation reports it the same + // way mixAudioTracks reports its own degradation: on a SUCCESSFUL result, so + // the render ships and the caller can still say what was lost. + const degradedGroups = [...groupsDegradedAutomation]; + const degradedNote = + degradedGroups.length > 0 + ? `Volume automation exceeded this ffmpeg build's expression limits in group(s) ${degradedGroups.join(", ")}; rendered at base volume` + : undefined; + return { ...mixResult, durationMs: Date.now() - startMs, - error: mixResult.error, + error: mixResult.error ?? degradedNote, }; } diff --git a/packages/engine/src/services/audioVolumeEnvelope.test.ts b/packages/engine/src/services/audioVolumeEnvelope.test.ts index d0cb183de..d1a2dd1bd 100644 --- a/packages/engine/src/services/audioVolumeEnvelope.test.ts +++ b/packages/engine/src/services/audioVolumeEnvelope.test.ts @@ -173,4 +173,76 @@ describe("applyVolumeEnvelopeToWav", () => { ), ).toBe(false); }); + + // The group sub-mix writes float precisely so an over-unity member sum keeps + // its headroom until the group's own fader and FX act on it. If the baker + // could not read that file it returned false, and the group's automation was + // dropped on the floor by the caller. + describe("32-bit float input", () => { + /** Stereo float32 WAV, every sample `value` — over 1.0 on purpose. */ + function writeConstantFloatWav(path: string, frames: number, value: number): void { + const dataSize = frames * CHANNELS * 4; + const buffer = Buffer.alloc(44 + dataSize); + buffer.write("RIFF", 0, "ascii"); + buffer.writeUInt32LE(36 + dataSize, 4); + buffer.write("WAVE", 8, "ascii"); + buffer.write("fmt ", 12, "ascii"); + buffer.writeUInt32LE(16, 16); + buffer.writeUInt16LE(3, 20); // WAVE_FORMAT_IEEE_FLOAT + buffer.writeUInt16LE(CHANNELS, 22); + buffer.writeUInt32LE(SAMPLE_RATE, 24); + buffer.writeUInt32LE(SAMPLE_RATE * CHANNELS * 4, 28); + buffer.writeUInt16LE(CHANNELS * 4, 32); + buffer.writeUInt16LE(32, 34); + buffer.write("data", 36, "ascii"); + buffer.writeUInt32LE(dataSize, 40); + for (let i = 0; i < frames * CHANNELS; i += 1) buffer.writeFloatLE(value, 44 + i * 4); + writeFileSync(path, buffer); + } + + const floatSampleAt = (path: string, frame: number, channel = 0): number => + readFileSync(path).readFloatLE(44 + (frame * CHANNELS + channel) * 4); + + it("scales float samples and keeps them above 1.0 unclamped", () => { + const path = join(tmp(), "float.wav"); + writeConstantFloatWav(path, SAMPLE_RATE, 1.4); + + expect( + applyVolumeEnvelopeToWav( + path, + [ + { time: 0, volume: 1 }, + { time: 1, volume: 1 }, + ], + 0, + 1, + ), + ).toBe(true); + + // Unity envelope: unchanged, and NOT clamped down to 1.0. + expect(floatSampleAt(path, 0)).toBeCloseTo(1.4, 5); + expect(floatSampleAt(path, SAMPLE_RATE - 1)).toBeCloseTo(1.4, 5); + }); + + it("applies the envelope across the file", () => { + const path = join(tmp(), "float-fade.wav"); + writeConstantFloatWav(path, SAMPLE_RATE, 1.4); + + expect( + applyVolumeEnvelopeToWav( + path, + [ + { time: 0, volume: 1 }, + { time: 1, volume: 0 }, + ], + 0, + 1, + ), + ).toBe(true); + + expect(floatSampleAt(path, 0)).toBeCloseTo(1.4, 5); + expect(floatSampleAt(path, Math.floor(SAMPLE_RATE / 2))).toBeCloseTo(0.7, 2); + expect(floatSampleAt(path, SAMPLE_RATE - 1)).toBeCloseTo(0, 3); + }); + }); }); diff --git a/packages/engine/src/services/audioVolumeEnvelope.ts b/packages/engine/src/services/audioVolumeEnvelope.ts index d64cc33a4..4213e2939 100644 --- a/packages/engine/src/services/audioVolumeEnvelope.ts +++ b/packages/engine/src/services/audioVolumeEnvelope.ts @@ -22,13 +22,17 @@ import { normaliseEnvelope } from "@hyperframes/core/media-volume-envelope"; import { riffChunks } from "./wavChunks.js"; const PCM_FORMAT = 1; // WAVE_FORMAT_PCM -const SUPPORTED_BITS = 16; +const FLOAT_FORMAT = 3; // WAVE_FORMAT_IEEE_FLOAT interface WavLayout { numChannels: number; sampleRate: number; dataOffset: number; dataSize: number; + /** 16-bit integer, or 32-bit float — the group sub-mix writes float so an + * over-unity member sum is not hard-clipped before the group's own FX and + * fader get to act on it. */ + float: boolean; } /** @@ -39,34 +43,49 @@ interface WavLayout { * and trailing chunks (LIST/fact/etc.) are skipped harmlessly. Returns null on * anything unexpected so the caller falls back to the expression path. */ -function parseWavLayout(buffer: Buffer): WavLayout | null { - if (buffer.length < 12 || buffer.toString("ascii", 0, 4) !== "RIFF") return null; - if (buffer.toString("ascii", 8, 12) !== "WAVE") return null; +interface WavFmt { + numChannels: number; + sampleRate: number; + /** 16-bit integer PCM, or 32-bit IEEE float. Anything else is unreadable. */ + float: boolean; +} - let fmt: { numChannels: number; sampleRate: number; bitsPerSample: number } | null = null; +/** The `fmt ` chunk, or null for a format this cannot safely edit in place. */ +function readFmtChunk(buffer: Buffer, body: number): WavFmt | null { + const format = buffer.readUInt16LE(body); + const bits = buffer.readUInt16LE(body + 14); + const float = format === FLOAT_FORMAT; + if (!float && format !== PCM_FORMAT) return null; + if (bits !== (float ? 32 : 16)) return null; + const numChannels = buffer.readUInt16LE(body + 2); + if (numChannels < 1) return null; + return { numChannels, sampleRate: buffer.readUInt32LE(body + 4), float }; +} + +function isRiffWave(buffer: Buffer): boolean { + return ( + buffer.length >= 12 && + buffer.toString("ascii", 0, 4) === "RIFF" && + buffer.toString("ascii", 8, 12) === "WAVE" + ); +} + +function parseWavLayout(buffer: Buffer): WavLayout | null { + if (!isRiffWave(buffer)) return null; + + let fmt: WavFmt | null = null; let data: { offset: number; size: number } | null = null; for (const { id, body, size } of riffChunks(buffer)) { if (id === "fmt " && body + 16 <= buffer.length) { - if (buffer.readUInt16LE(body) !== PCM_FORMAT) return null; - fmt = { - numChannels: buffer.readUInt16LE(body + 2), - sampleRate: buffer.readUInt32LE(body + 4), - bitsPerSample: buffer.readUInt16LE(body + 14), - }; + fmt = readFmtChunk(buffer, body); } else if (id === "data") { data = { offset: body, size: Math.min(size, buffer.length - body) }; } } if (!fmt || !data) return null; - if (fmt.bitsPerSample !== SUPPORTED_BITS || fmt.numChannels < 1) return null; - return { - numChannels: fmt.numChannels, - sampleRate: fmt.sampleRate, - dataOffset: data.offset, - dataSize: data.size, - }; + return { ...fmt, dataOffset: data.offset, dataSize: data.size }; } /** @@ -103,11 +122,41 @@ export function createEnvelopeWalker( }; } +/** +/** Every sample scaled by the envelope, in place, in whichever of the two + * formats the layout reports. Float is NOT clamped: it is the format the group + * sub-mix writes precisely so an over-unity sum keeps its headroom until + * something downstream chooses to reduce it. */ +function scaleSamples( + buffer: Buffer, + layout: WavLayout, + gainAt: (seconds: number) => number, +): void { + const { numChannels, sampleRate, dataOffset, dataSize, float } = layout; + const bytesPerSample = float ? 4 : 2; + const frameBytes = numChannels * bytesPerSample; + const frameCount = Math.floor(dataSize / frameBytes); + const scaleOne = float + ? (at: number, gain: number) => buffer.writeFloatLE(buffer.readFloatLE(at) * gain, at) + : (at: number, gain: number) => { + const scaled = Math.round(buffer.readInt16LE(at) * gain); + buffer.writeInt16LE(scaled < -32768 ? -32768 : scaled > 32767 ? 32767 : scaled, at); + }; + + for (let frame = 0; frame < frameCount; frame += 1) { + const gain = gainAt(frame / sampleRate); + const base = dataOffset + frame * frameBytes; + for (let channel = 0; channel < numChannels; channel += 1) { + scaleOne(base + channel * bytesPerSample, gain); + } + } +} + /** * Multiply a prepared WAV's samples by a time-varying gain envelope in place. * - * @returns `true` if the envelope was applied; `false` if the file isn't the - * expected 16-bit PCM (caller should fall back to the expression path). + * @returns `true` if the envelope was applied; `false` if the file is neither + * 16-bit PCM nor 32-bit float (caller should fall back to the expression path). */ export function applyVolumeEnvelopeToWav( wavPath: string, @@ -123,20 +172,7 @@ export function applyVolumeEnvelopeToWav( const layout = parseWavLayout(buffer); if (!layout) return false; - const { numChannels, sampleRate, dataOffset, dataSize } = layout; - const bytesPerSample = SUPPORTED_BITS / 8; - const frameBytes = numChannels * bytesPerSample; - const frameCount = Math.floor(dataSize / frameBytes); - - for (let frame = 0; frame < frameCount; frame += 1) { - const gain = gainAt(frame / sampleRate); - const base = dataOffset + frame * frameBytes; - for (let channel = 0; channel < numChannels; channel += 1) { - const at = base + channel * bytesPerSample; - const scaled = Math.round(buffer.readInt16LE(at) * gain); - buffer.writeInt16LE(scaled < -32768 ? -32768 : scaled > 32767 ? 32767 : scaled, at); - } - } + scaleSamples(buffer, layout, gainAt); // Write to a uniquely-named sibling then atomically rename over the // original. The random name avoids following a pre-planted symlink at a