From 30359f80382686da98f88cc1907c44d07ef82c09 Mon Sep 17 00:00:00 2001 From: Jerrai Date: Sun, 11 Oct 2026 23:56:09 +0000 Subject: [PATCH] fix(audio): master bus follow-ups: a scene renders with its own chain; long mixes render in windows (#5474) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(core): a scene rendered on its own takes its master chain from its own root The standalone wrapper for a scene file reused the project index's root attributes, so the index's data-fx-chain became the master bus. The wrapper now carries the scene root's own chain, or none when the scene declares none, as the audio-effects reference describes. — Jerrai * fix(engine): run long audio chains window by window instead of reading the file whole applyAudioFxChain read the whole input WAV into memory, which Node refuses past 2 GiB (a stereo float mix beyond about 93 minutes), and the browser page then held the track twice. Inputs above 512 MiB of samples are now read, rendered and written in 5 minute windows, each with 10 s of preceding and 1 s of following audio rendered and discarded so stateful effects arrive warm. The output header is patched at the end. Inputs below the threshold keep the existing whole-file path. Parameter automation is clocked from the file start and is not windowed. An input past 2 GiB that carries automation now fails with an AudioFxRenderError naming that, instead of Node's raw file-size error. — Jerrai * chore(engine): trim the browser-lease comment to its live invariant — Jerrai * fix(core): a scene mounted as the index's own root takes its own master chain When the index's top-level composition is itself the scene's mount, the standalone extraction returned before the scene's chain was applied, so the index's chain still won. — Jerrai * fix(engine): windowed audio renders follow the file's clock and warm up for the whole chain - Pass each window's start time into the page and on to buildFxChain, and seed the pitch-shift worklet's tap phase from it, so chorus, phaser and pitch shift continue across a window seam instead of restarting from time zero. - Size the pre-roll from the chain (delay repeats, reverb room, dynamics release, to -60 dB), never below the configured floor and never above a cap. - Read a WAV data size of 0 or 0xFFFFFFFF as running to the end of the file, so audio past 4 GiB is no longer dropped. - Close the input descriptor when the output cannot be opened, and name the real cause when an input over 2 GiB cannot be streamed. - Cover reverb, a slow-release compressor, chorus and pitch shift in the windowed-vs-whole test, with an output-length check. — Jerrai * fix(engine): start each audio window on a render quantum and test the seams against a hard chain Windows that begin off a 128-frame block see worklet blocks shifted against a whole render, which left spikes in the tail after the last input sample. Align the window start, and make the windowed-vs-whole test cover modulation, delay, reverb and a slow-release compressor on windows whose start is not zero, failing above 0.001 where it previously tolerated 0.02. — Jerrai * fix(engine): keep a limiter's lookahead, seed bitcrush holds, close-check without /proc - Pad the render source with the limiter's lookahead of silence so the last samples of a clip leave the lookahead line instead of being dropped. - Seed the bitcrush hold counter from the window's start so holds that do not divide the render quantum stay in step across seams, and cover bitcrush in the windowed-vs-whole test. - Check that the input descriptor is closed through the fs mock instead of /proc/self/fd, which only exists on Linux. — Jerrai * test(engine): pick a bitcrush hold the window starts do not divide The earlier hold of 3 divided both window starts, so an unseeded counter passed. — Jerrai * fix(engine): warn at the pre-roll cap, share tail formulas, read a zero data size alike - Warn when a chain's memory exceeds the pre-roll cap, so cut repeats at window seams are not silent. - Export the delay and reverb tail formulas from audio-fx-tail and use them for the window warm-up too. - Both WAV readers now treat a declared data size of 0 or 0xFFFFFFFF as the rest of the file. — Jerrai --- packages/core/src/audio/audioFxGraph.ts | 6 +- packages/core/src/audio/audioFxTail.test.ts | 20 +- packages/core/src/audio/audioFxTail.ts | 10 +- packages/core/src/audio/audioFxWorklets.ts | 10 +- .../core/src/compiler/standaloneEntry.test.ts | 74 +++ packages/core/src/compiler/standaloneEntry.ts | 29 +- packages/core/stubs/audio-fx-runtime-entry.ts | 13 +- .../engine/src/services/audioFxRender.test.ts | 463 +++++++++++++++++- packages/engine/src/services/audioFxRender.ts | 438 ++++++++++++++--- .../engine/src/services/audioFxWarmup.test.ts | 59 +++ packages/engine/src/services/audioFxWarmup.ts | 58 +++ 11 files changed, 1085 insertions(+), 95 deletions(-) create mode 100644 packages/engine/src/services/audioFxWarmup.test.ts create mode 100644 packages/engine/src/services/audioFxWarmup.ts diff --git a/packages/core/src/audio/audioFxGraph.ts b/packages/core/src/audio/audioFxGraph.ts index 3d0a03b08f..fcee382345 100644 --- a/packages/core/src/audio/audioFxGraph.ts +++ b/packages/core/src/audio/audioFxGraph.ts @@ -253,8 +253,10 @@ function onePoleBuilder(kind: "highpass" | "lowpass"): Builder { } function workletBuilder(processor: string, resettable = false): Builder { - return (ctx, p) => { - const node = new AudioWorkletNode(ctx, processor, { processorOptions: { ...p } }); + return (ctx, p, elapsed) => { + const node = new AudioWorkletNode(ctx, processor, { + processorOptions: { ...p, __hfElapsed: elapsed }, + }); return { input: node, output: node, diff --git a/packages/core/src/audio/audioFxTail.test.ts b/packages/core/src/audio/audioFxTail.test.ts index 562c18bd35..2a363feed4 100644 --- a/packages/core/src/audio/audioFxTail.test.ts +++ b/packages/core/src/audio/audioFxTail.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from "vitest"; -import { chainLatencySamples, chainTailSeconds, MAX_FX_TAIL_SECONDS } from "./audioFxTail.js"; +import { + chainLatencySamples, + chainTailSeconds, + delayTailSeconds, + MAX_FX_TAIL_SECONDS, + reverbRoomSeconds, +} from "./audioFxTail.js"; import { truePeakLatencySamples } from "./audioFxTruePeak.js"; import { synthesizeReverbImpulse } from "./audioFxGraph.js"; import type { HfAudioFxChain } from "../audioFx.js"; @@ -130,3 +136,15 @@ describe("chainLatencySamples", () => { ); }); }); + +describe("tail helpers shared with the engine's window warm-up", () => { + it("counts a delay's repeats to -60 dB, uncapped", () => { + expect(delayTailSeconds(5000, 0.95)).toBeCloseTo(135 * 5, 5); + expect(delayTailSeconds(400, 0.5)).toBeCloseTo(10 * 0.4, 5); + }); + + it("is the reverb impulse's length", () => { + expect(reverbRoomSeconds(0.5)).toBeCloseTo(1.9, 5); + expect(reverbRoomSeconds(4)).toBeCloseTo(3.2, 5); + }); +}); diff --git a/packages/core/src/audio/audioFxTail.ts b/packages/core/src/audio/audioFxTail.ts index 963664e481..6fbe901093 100644 --- a/packages/core/src/audio/audioFxTail.ts +++ b/packages/core/src/audio/audioFxTail.ts @@ -56,7 +56,7 @@ function knobMax(node: HfAudioFxNode, key: string, automation?: HfAutomation): n * Repeats until a feedback loop falls under the floor, times the gap between * them. `feedback` is capped below 1 by the registry, so this terminates. */ -function delayTail(time: number, feedback: number): number { +export function delayTailSeconds(time: number, feedback: number): number { const gap = Math.min(5, time / 1000); if (gap <= 0) return 0; const fb = Math.max(0, Math.min(0.999, feedback)); @@ -66,15 +66,19 @@ function delayTail(time: number, feedback: number): number { // Exactly the generated impulse's length — see synthesizeReverbImpulse, which // is the same expression. A convolution is as long as its impulse. +export function reverbRoomSeconds(size: number): number { + return 0.6 + Math.max(0, Math.min(1, size)) * 2.6; +} + function reverbTail(node: HfAudioFxNode, automation?: HfAutomation): number { return knobMax(node, "wet", automation) > 0 - ? 0.6 + Math.max(0, Math.min(1, knobMax(node, "size", automation))) * 2.6 + ? reverbRoomSeconds(knobMax(node, "size", automation)) : 0; } function delayNodeTail(node: HfAudioFxNode, automation?: HfAutomation): number { return knobMax(node, "mix", automation) > 0 - ? delayTail(knobMax(node, "time", automation), knobMax(node, "feedback", automation)) + ? delayTailSeconds(knobMax(node, "time", automation), knobMax(node, "feedback", automation)) : 0; } diff --git a/packages/core/src/audio/audioFxWorklets.ts b/packages/core/src/audio/audioFxWorklets.ts index 9f9354401c..fba4d3de12 100644 --- a/packages/core/src/audio/audioFxWorklets.ts +++ b/packages/core/src/audio/audioFxWorklets.ts @@ -192,6 +192,7 @@ class HfBitcrush extends AudioWorkletProcessor { this.p = o.processorOptions || {}; this.holds = []; this.held = []; + this.start = Math.round((this.p.__hfElapsed ?? 0) * sampleRate); this.port.onmessage = (e) => { if (e.data && e.data.__hfDispose) { this.dead = true; return; } this.p = { ...this.p, ...e.data }; @@ -208,7 +209,8 @@ class HfBitcrush extends AudioWorkletProcessor { for (let ch = 0; ch < i.length; ch++) { const inp = i[ch], out = o[ch]; if (this.held[ch] === undefined) this.held[ch] = 0; - if (this.holds[ch] === undefined) this.holds[ch] = 0; + // The hold counter runs from the file's first sample, so a node built part-way in starts where it would be. + if (this.holds[ch] === undefined) this.holds[ch] = this.start % step; for (let n = 0; n < inp.length; n++) { // Per channel: advancing one shared counter only on the last channel // left every earlier channel either unheld or frozen for a whole @@ -255,7 +257,11 @@ class HfPitchshift extends AudioWorkletProcessor { this.grain = Math.round(sampleRate * 0.1); this.buf = []; this.write = 0; - this.phase = 0; + // The taps sweep at a constant rate from the start of the file, so a node built + // part-way in joins the sweep where it would already be. + const ratio0 = Math.pow(2, Math.max(-12, Math.min(12, this.p.semitones ?? 0)) / 12); + const turns = (this.p.__hfElapsed ?? 0) * sampleRate * ((1 - ratio0) / this.grain); + this.phase = turns - Math.floor(turns); // Samples written so far, capped at one grain. The taps read up to a grain // behind the write head, so until this fills they would read the ring's // zeros, so the head of every clip came out attenuated or silent. diff --git a/packages/core/src/compiler/standaloneEntry.test.ts b/packages/core/src/compiler/standaloneEntry.test.ts index 448f427c40..2939f5157c 100644 --- a/packages/core/src/compiler/standaloneEntry.test.ts +++ b/packages/core/src/compiler/standaloneEntry.test.ts @@ -1,3 +1,4 @@ +import { parseHTML } from "linkedom"; import { describe, expect, it } from "vitest"; import { extractStandaloneEntryFromIndex } from "./standaloneEntry"; @@ -112,4 +113,77 @@ describe("extractStandaloneEntryFromIndex", () => { expect(extracted).toContain('data-duration="2"'); expect(extracted).not.toContain('data-duration="12"'); }); + describe("master bus chain", () => { + const hostIndexHtml = ` + + +
+ +`; + const indexHtml = ` + + +
+
+
+ +`; + const chainOf = (html: string): string | null => { + const { document } = parseHTML(html); + const roots = Array.from(document.querySelectorAll("[data-composition-id]")) as Element[]; + const top = roots.find((el) => !el.parentElement?.closest("[data-composition-id]")); + return top?.getAttribute("data-fx-chain") ?? null; + }; + + it("carries the scene's own chain, not the index's", () => { + const chain = '{"version":1,"nodes":[{"type":"truepeak","id":"scene"}]}'; + const sceneHtml = ``; + + const extracted = extractStandaloneEntryFromIndex( + indexHtml, + "compositions/scene1.html", + sceneHtml, + ); + + expect(chainOf(extracted ?? "")).toBe(chain); + }); + + it("drops the index's chain when the scene declares none", () => { + const sceneHtml = ``; + + const extracted = extractStandaloneEntryFromIndex( + indexHtml, + "compositions/scene1.html", + sceneHtml, + ); + + expect(chainOf(extracted ?? "")).toBeNull(); + }); + + it("carries the scene's chain when the index's top-level root is the mount itself", () => { + const chain = '{"version":1,"nodes":[{"type":"truepeak","id":"scene"}]}'; + const sceneHtml = ``; + + const extracted = extractStandaloneEntryFromIndex( + hostIndexHtml, + "compositions/scene1.html", + sceneHtml, + ); + + expect(chainOf(extracted ?? "")).toBe(chain); + }); + + it("drops the index's chain from a top-level mount when the scene declares none", () => { + const sceneHtml = ``; + + const extracted = extractStandaloneEntryFromIndex( + hostIndexHtml, + "compositions/scene1.html", + sceneHtml, + ); + + expect(extracted).not.toBeNull(); + expect(chainOf(extracted ?? "")).toBeNull(); + }); + }); }); diff --git a/packages/core/src/compiler/standaloneEntry.ts b/packages/core/src/compiler/standaloneEntry.ts index b03435ac79..89b2b4c4dc 100644 --- a/packages/core/src/compiler/standaloneEntry.ts +++ b/packages/core/src/compiler/standaloneEntry.ts @@ -1,4 +1,5 @@ import { parseHTML } from "linkedom"; +import { HF_AUDIO_FX_ATTR } from "../audioFx"; export function isTemplateEntry(html: string): boolean { return html.trimStart().startsWith(" content, so the template's innerHTML is re-parsed. -function readSceneRootDuration(entryHtml: string | undefined): string | null { +function readSceneRoot(entryHtml: string | undefined): Element | null { if (!entryHtml) return null; const { document } = parseHTML(entryHtml); const template = document.querySelector("template"); const scope = template ? parseHTML(template.innerHTML).document : document; - const root = scope.querySelector("[data-composition-id]") as Element | null; - return root?.getAttribute("data-duration") ?? null; + return scope.querySelector("[data-composition-id]") as Element | null; +} + +// The scene's own chain replaces the index's, which belongs to the whole project. +function setSceneFxChain(el: Element, sceneFxChain: string | null): void { + if (sceneFxChain) { + el.setAttribute(HF_AUDIO_FX_ATTR, sceneFxChain); + } else { + el.removeAttribute(HF_AUDIO_FX_ATTR); + } } function createStandaloneEntryRenderClone( root: Element, host: Element, sceneDuration: string | null, + sceneFxChain: string | null, ): Element { const hostClone = host.cloneNode(true) as Element; hostClone.setAttribute("data-start", "0"); - if (root === host) return hostClone; + if (root === host) { + setSceneFxChain(hostClone, sceneFxChain); + return hostClone; + } const rootClone = root.cloneNode(false) as Element; // The wrapper takes the scene's length, not the whole project's; with none declared @@ -36,6 +49,7 @@ function createStandaloneEntryRenderClone( } else { rootClone.removeAttribute("data-duration"); } + setSceneFxChain(rootClone, sceneFxChain); rootClone.appendChild(hostClone); return rootClone; } @@ -78,9 +92,12 @@ export function extractStandaloneEntryFromIndex( if (!root) return null; // The scene file owns its duration; the mount's window is the fallback. - const sceneDuration = readSceneRootDuration(entryHtml) ?? host.getAttribute("data-duration"); + const sceneRoot = readSceneRoot(entryHtml); + const sceneDuration = + sceneRoot?.getAttribute("data-duration") ?? host.getAttribute("data-duration"); + const sceneFxChain = sceneRoot?.getAttribute(HF_AUDIO_FX_ATTR) || null; - const renderClone = createStandaloneEntryRenderClone(root, host, sceneDuration); + const renderClone = createStandaloneEntryRenderClone(root, host, sceneDuration, sceneFxChain); replaceBodyWithRenderClone(body, renderClone); return document.toString(); diff --git a/packages/core/stubs/audio-fx-runtime-entry.ts b/packages/core/stubs/audio-fx-runtime-entry.ts index 786122a325..4bd56f9d5a 100644 --- a/packages/core/stubs/audio-fx-runtime-entry.ts +++ b/packages/core/stubs/audio-fx-runtime-entry.ts @@ -7,7 +7,7 @@ * ends share a single implementation rather than two that have to be kept in * agreement. * - * Exposes `window.__HF_AUDIO_FX.render(pcm, sampleRate, chain, automation)`, + * Exposes `window.__HF_AUDIO_FX.render(pcm, sampleRate, chain, automation, elapsed)`, * which returns the processed samples. */ @@ -29,6 +29,7 @@ declare global { sampleRate: number, chainJson: string, automationJson?: string, + elapsed?: number, ): Promise; }; } @@ -45,6 +46,9 @@ declare global { * comes from the settings (`chainTailSeconds`), capped, and the returned planes * are correspondingly longer than what came in — the mixer decides how much of * that it lets through past the clip's end. + * + * `elapsed` is where in the file `planes` begins, for a window cut from a longer + * input: modulated effects follow the file's clock, not the window's. */ /** The clip's audio as an AudioBuffer, a plane per channel. */ @@ -77,6 +81,7 @@ async function render( sampleRate: number, chainJson: string, automationJson?: string, + elapsed = 0, ): Promise { const chain: HfAudioFxChain = parseAudioFxChain(chainJson); const channels = Math.max(1, planes.length); @@ -98,9 +103,11 @@ async function render( if (chainNeedsWorklets(chain)) await ensureAudioFxWorklets(ctx); const source = ctx.createBufferSource(); - source.buffer = toBuffer(ctx, planes, channels, frames, sampleRate); + // Padded with silence by the limiter's lookahead: once a source ends its worklets stop being + // fed, so without it the last `latency` samples of the clip never leave the lookahead line. + source.buffer = toBuffer(ctx, planes, channels, frames + latency, sampleRate); - const fx = buildFxChain(ctx, chain); + const fx = buildFxChain(ctx, chain, elapsed); // The input WAV is the clip's own audio from its first sample, so clip-local // time is offline time — the envelope needs no offset here. Same scheduler as diff --git a/packages/engine/src/services/audioFxRender.test.ts b/packages/engine/src/services/audioFxRender.test.ts index 6d2c944f97..eec49fb363 100644 --- a/packages/engine/src/services/audioFxRender.test.ts +++ b/packages/engine/src/services/audioFxRender.test.ts @@ -1,11 +1,28 @@ import { execFileSync, spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { + closeSync, + existsSync, + mkdtempSync, + openSync, + readFileSync, + rmSync, + truncateSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js"; import { defaultAudioFxParams, type HfAudioFxChain } from "@hyperframes/core/audio-fx"; -import { applyAudioFxChain, AudioFxRenderError, readWav, writeWav } from "./audioFxRender.js"; +import { + applyAudioFxChain, + AudioFxRenderError, + planAudioFxSegments, + readWav, + readWavHeader, + resolveDataBytes, + writeWav, +} from "./audioFxRender.js"; import { MIXED_AUDIO_FILENAME, parseAudioElements, @@ -15,6 +32,29 @@ import { import { resolveHeadlessShellPath } from "./browserManager.js"; import { getFfmpegBinary } from "../utils/ffmpegBinaries.js"; +const wholeFileReads = vi.hoisted(() => ({ paths: [] as string[] })); +const openedFds = vi.hoisted(() => ({ + log: [] as { path: string; fd: number; closed: boolean }[], +})); +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + const readFileSync = ((path: Parameters[0], ...rest: unknown[]) => { + wholeFileReads.paths.push(String(path)); + return (actual.readFileSync as (...a: unknown[]) => unknown)(path, ...rest); + }) as typeof actual.readFileSync; + const openSync = ((path: Parameters[0], ...rest: unknown[]) => { + const fd = (actual.openSync as (...a: unknown[]) => number)(path, ...rest); + openedFds.log.push({ path: String(path), fd, closed: false }); + return fd; + }) as typeof actual.openSync; + const closeSync = ((fd: number) => { + const open = openedFds.log.findLast((entry) => entry.fd === fd && !entry.closed); + if (open) open.closed = true; + return actual.closeSync(fd); + }) as typeof actual.closeSync; + return { ...actual, readFileSync, openSync, closeSync }; +}); + /** Same probe the ffmpeg-dependent suites use: ask the binary, don't assume it. */ const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0; @@ -54,7 +94,11 @@ const SR = 48000; const chainOf = (...types: string[]): HfAudioFxChain => ({ version: 1, - nodes: types.map((t) => ({ type: t, enabled: true, params: defaultAudioFxParams(t) })), + nodes: types.map((t) => ({ + type: t, + enabled: true, + params: defaultAudioFxParams(t), + })), }); let dir: string; @@ -232,7 +276,13 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => { input, { version: 1, - nodes: [{ type: "peaking", enabled: true, params: { frequency: 440, gain: -30, q: 1 } }], + nodes: [ + { + type: "peaking", + enabled: true, + params: { frequency: 440, gain: -30, q: 1 }, + }, + ], }, outPath, { trackId: "t" }, @@ -258,7 +308,11 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => { { type: "compressor", enabled: true, - params: { ...defaultAudioFxParams("compressor"), threshold: -40, ratio: 20 }, + params: { + ...defaultAudioFxParams("compressor"), + threshold: -40, + ratio: 20, + }, }, ], }, @@ -280,7 +334,11 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => { { type: "pitchshift", enabled: true, - params: { ...defaultAudioFxParams("pitchshift"), semitones: 12, mix: 1 }, + params: { + ...defaultAudioFxParams("pitchshift"), + semitones: 12, + mix: 1, + }, }, ], }, @@ -344,7 +402,14 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => { input, { version: 1, - nodes: [{ type: "lowpass", id: "n1", enabled: true, params: { frequency: 300, q: 0.707 } }], + nodes: [ + { + type: "lowpass", + id: "n1", + enabled: true, + params: { frequency: 300, q: 0.707 }, + }, + ], }, outPath, { @@ -398,7 +463,13 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => { input, { version: 1, - nodes: [{ type: "peaking", enabled: true, params: { frequency: 440, gain: 6, q: 1 } }], + nodes: [ + { + type: "peaking", + enabled: true, + params: { frequency: 440, gain: 6, q: 1 }, + }, + ], }, outPath, { @@ -470,7 +541,13 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => { // input and any difference is the transfer's doing. { version: 1, - nodes: [{ type: "peaking", enabled: true, params: { frequency: 1000, gain: 0, q: 1 } }], + nodes: [ + { + type: "peaking", + enabled: true, + params: { frequency: 1000, gain: 0, q: 1 }, + }, + ], }, outPath, { trackId: "t" }, @@ -552,7 +629,11 @@ function maxVolumeDb(path: string): number { const chainJson = (...nodes: Record[]): string => JSON.stringify({ version: 1, - nodes: nodes.map((node, i) => ({ id: `n${i + 1}`, enabled: true, ...node })), + nodes: nodes.map((node, i) => ({ + id: `n${i + 1}`, + enabled: true, + ...node, + })), }); describe("master bus: reading it", () => { @@ -730,3 +811,363 @@ describe.skipIf(!HAS_BROWSER || !HAS_FFMPEG)("master bus: rendering it", () => { expect(await mixedPeakDb("ordered", [element("g", group)], masterFxChain)).toBeCloseTo(-14, 0); }, 180_000); }); + +describe("windowed rendering of a long input", () => { + const limits = { + aboveBytes: 0, + segmentSeconds: 1, + prerollSeconds: 1, + maxPrerollSeconds: 1, + postrollSeconds: 0.1, + }; + + it("tiles the input exactly, each window carrying its pre- and post-roll", () => { + const segments = planAudioFxSegments(Math.round(3.5 * SR), SR, limits); + expect(segments.map((s) => [s.keepFrom, s.keepTo])).toEqual([ + [0, SR], + [SR, 2 * SR], + [2 * SR, 3 * SR], + [3 * SR, Math.round(3.5 * SR)], + ]); + expect(segments.map((s) => s.last)).toEqual([false, false, false, true]); + expect(segments[0]).toMatchObject({ readFrom: 0, readTo: SR + SR / 10 }); + expect(segments[2]).toMatchObject({ + readFrom: SR, + readTo: 3 * SR + SR / 10, + }); + expect(segments[3]?.readTo).toBe(Math.round(3.5 * SR)); + }); + + it("starts every window on a render quantum, so block-level state lines up with a whole render", () => { + const odd = { ...limits, prerollSeconds: 0.337 }; + const segments = planAudioFxSegments(Math.round(3.5 * SR), SR, odd); + for (const seg of segments) { + expect(seg.readFrom % 128).toBe(0); + expect(seg.keepFrom - seg.readFrom).toBeGreaterThanOrEqual( + Math.min(seg.keepFrom, Math.round(0.337 * SR)), + ); + } + }); + + it("reads the head of a file Node cannot read whole", () => { + // 2.5 GiB of samples: past the 2 GiB ceiling readFileSync enforces, sparse on disk. + const path = join(dir, "huge.wav"); + writeWav(path, new Float32Array(0), SR, 2, true); + const fd = openSync(path, "r+"); + truncateSync(path, 2.5 * 1024 ** 3); + closeSync(fd); + expect(() => readWav(path)).toThrow(/greater than 2 GiB/); + const header = readWavHeader(path); + expect(header).toMatchObject({ + channels: 2, + sampleRate: SR, + float: true, + dataOffset: 44, + }); + }); + + it("reads a data size of 0xFFFFFFFF, or 0, as running to the end of the file", () => { + const FIVE_GIB = 5 * 1024 ** 3; + expect(resolveDataBytes(0xffffffff, FIVE_GIB)).toBe(FIVE_GIB); + expect(resolveDataBytes(0, FIVE_GIB)).toBe(FIVE_GIB); + expect(resolveDataBytes(1000, FIVE_GIB)).toBe(1000); + expect(resolveDataBytes(2000, 1500)).toBe(1500); + + // The same header as ffmpeg leaves on a stream, on a file small enough to write. + const path = join(dir, "stream.wav"); + writeWav(path, new Float32Array(100), SR, 2, true); + const bytes = readFileSync(path); + bytes.writeUInt32LE(0xffffffff, 40); + writeFileSync(path, bytes); + expect(readWavHeader(path)).toMatchObject({ + dataOffset: 44, + dataBytes: 400, + }); + }); + + it("names the cause when a file past 2 GiB cannot be streamed", async () => { + const path = join(dir, "huge-bad-header.wav"); + writeFileSync(path, "not a wav"); + truncateSync(path, 2.5 * 1024 ** 3); + await expect( + applyAudioFxChain(path, chainOf("gain"), join(dir, "out.wav"), { + trackId: "t", + }), + ).rejects.toThrow(/WAV header of an input over 2 GiB could not be read/); + }); + + it("reads a data size of 0 or 0xFFFFFFFF as the rest of the file in both readers", () => { + for (const declared of [0, 0xffffffff]) { + const path = join(dir, `declared-${declared}.wav`); + writeWav(path, new Float32Array(200), SR, 2, true); + const bytes = readFileSync(path); + bytes.writeUInt32LE(declared, 40); + writeFileSync(path, bytes); + expect(readWav(path).samples.length).toBe(200); + expect(readWavHeader(path)?.dataBytes).toBe(800); + } + }); + + it.skipIf(!HAS_BROWSER)("warns when a chain remembers longer than the pre-roll cap", async () => { + const input = join(dir, "warn.wav"); + writeWav(input, new Float32Array(SR), SR, 1, true); + const delay = (feedback: number): HfAudioFxChain => ({ + version: 1, + nodes: [{ type: "delay", id: "d", enabled: true, params: { time: 400, feedback, mix: 0.5 } }], + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + const streaming = { ...limits, prerollSeconds: 0.5, maxPrerollSeconds: 2 }; + await applyAudioFxChain(input, delay(0.95), join(dir, "w1.wav"), { + trackId: "t1", + streaming, + }); + expect(warn).toHaveBeenCalledTimes(1); + expect(String(warn.mock.calls[0]?.[0])).toContain("track t1"); + await applyAudioFxChain(input, delay(0.1), join(dir, "w2.wav"), { trackId: "t2", streaming }); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + } + }); + + it.skipIf(!HAS_BROWSER)("closes the input when the output cannot be opened", async () => { + const input = join(dir, "in.wav"); + writeWav(input, new Float32Array(SR * 2), SR, 1, true); + openedFds.log.length = 0; + await expect( + applyAudioFxChain(input, chainOf("gain"), join(dir, "no-such-dir", "out.wav"), { + trackId: "t", + streaming: limits, + }), + ).rejects.toThrow(); + const onInput = openedFds.log.filter((entry) => entry.path === input); + expect(onInput.length).toBeGreaterThan(0); + expect(onInput.every((entry) => entry.closed)).toBe(true); + }); + + it("names the unsupported case when automation meets a file past 2 GiB", async () => { + const path = join(dir, "huge-auto.wav"); + writeWav(path, new Float32Array(0), SR, 2, true); + truncateSync(path, 2.5 * 1024 ** 3); + await expect( + applyAudioFxChain(path, chainOf("gain"), join(dir, "out.wav"), { + trackId: "t", + automation: { version: 1, lanes: [] }, + }), + ).rejects.toThrow(/parameter automation is not supported on inputs over 2 GiB/); + }); + + const hot = (seconds: number): Float32Array => { + const frames = Math.floor(SR * seconds); + const s = new Float32Array(frames * 2); + for (let i = 0; i < frames; i++) { + const swell = 0.5 + 0.9 * Math.abs(Math.sin((2 * Math.PI * i) / (SR * 1.7))); + s[i * 2] = swell * Math.sin((2 * Math.PI * 300 * i) / SR); + s[i * 2 + 1] = swell * Math.sin((2 * Math.PI * 450 * i) / SR); + } + return s; + }; + const limiter: HfAudioFxChain = { + version: 1, + nodes: [ + { + type: "truepeak", + id: "m1", + enabled: true, + params: { ceiling: -6, lookahead: 3, release: 80 }, + }, + ], + }; + + it.skipIf(!HAS_BROWSER)( + "renders a long mix window by window, never reading the file whole, and matches the whole render", + async () => { + const input = join(dir, "long.wav"); + writeWav(input, hot(4.4), SR, 2, true); + const whole = join(dir, "whole.wav"); + const windowed = join(dir, "windowed.wav"); + await applyAudioFxChain(input, limiter, whole, { trackId: "whole" }); + + wholeFileReads.paths.length = 0; + const result = await applyAudioFxChain(input, limiter, windowed, { + trackId: "windowed", + streaming: limits, + }); + + expect(result.path).toBe(windowed); + expect(wholeFileReads.paths).not.toContain(input); + const a = readWav(whole); + const b = readWav(windowed); + expect(b).toMatchObject({ channels: 2, sampleRate: SR, float: true }); + expect(b.samples.length).toBe(a.samples.length); + let worst = 0; + // The final 10 ms is left out: a lookahead limiter's last frames come back silent or not + // depending on the input's length, whole or windowed alike. + for (let i = 0; i < a.samples.length - SR / 50; i++) { + worst = Math.max(worst, Math.abs((a.samples[i] ?? 0) - (b.samples[i] ?? 0))); + } + expect(worst).toBeLessThan(0.01); + }, + 180_000, + ); + + /** A tone with a loud burst ending just ahead of each 5 s seam. */ + const bursts = (seconds: number): Float32Array => { + const frames = Math.floor(SR * seconds); + const s = new Float32Array(frames * 2); + for (let i = 0; i < frames; i++) { + const t = i / SR; + const loud = t % 5 >= 3.4 && t % 5 < 3.9; + const amp = loud ? 0.9 : 0.12; + s[i * 2] = amp * Math.sin((2 * Math.PI * 330 * i) / SR); + s[i * 2 + 1] = amp * Math.sin((2 * Math.PI * 520 * i) / SR); + } + return s; + }; + const memoryChain = (): HfAudioFxChain => ({ + version: 1, + nodes: [ + // A hold of 5 does not divide the windows' 128-aligned starts, so an unseeded counter drifts. + { type: "bitcrush", id: "b", enabled: true, params: { bits: 5, samples: 5, mix: 0.7 } }, + { + type: "chorus", + id: "c", + enabled: true, + params: { ...defaultAudioFxParams("chorus"), speed: 0.25 }, + }, + { + type: "pitchshift", + id: "p", + enabled: true, + params: { semitones: 3, mix: 1 }, + }, + { + type: "compressor", + id: "k", + enabled: true, + params: { + ...defaultAudioFxParams("compressor"), + threshold: -30, + ratio: 10, + makeup: 20, + release: 700, + }, + }, + { + type: "delay", + id: "d", + enabled: true, + params: { time: 400, feedback: 0.5, mix: 0.5 }, + }, + { + type: "reverb", + id: "r", + enabled: true, + params: { size: 0.5, wet: 0.3 }, + }, + ], + }); + const memoryLimits = { + ...limits, + segmentSeconds: 5, + prerollSeconds: 0.5, + maxPrerollSeconds: 12, + }; + + const renderBoth = async ( + streaming: typeof memoryLimits, + ): Promise<{ whole: Float32Array; windowed: Float32Array }> => { + const input = join(dir, "mem.wav"); + writeWav(input, bursts(22), SR, 2, true); + await applyAudioFxChain(input, memoryChain(), join(dir, "mem-whole.wav"), { + trackId: "w", + }); + await applyAudioFxChain(input, memoryChain(), join(dir, "mem-windowed.wav"), { + trackId: "s", + streaming, + }); + return { + whole: readWav(join(dir, "mem-whole.wav")).samples, + windowed: readWav(join(dir, "mem-windowed.wav")).samples, + }; + }; + const worstDifference = (a: Float32Array, b: Float32Array): number => { + let worst = 0; + for (let i = 0; i < a.length; i++) worst = Math.max(worst, Math.abs((a[i] ?? 0) - (b[i] ?? 0))); + return worst; + }; + + it.skipIf(!HAS_BROWSER)( + "carries bitcrush, modulation, delay, reverb and compressor state across window seams", + async () => { + const { whole, windowed } = await renderBoth(memoryLimits); + // The tail of the last window is kept: dropping it would shorten the file. + expect(windowed.length).toBe(whole.length); + expect(whole.length).toBeGreaterThan(22 * SR * 2); + expect(worstDifference(whole, windowed)).toBeLessThan(0.001); + }, + 240_000, + ); + + it.skipIf(!HAS_BROWSER)( + "cannot match the whole render without the pre-roll the chain asks for", + async () => { + const { whole, windowed } = await renderBoth({ + ...memoryLimits, + maxPrerollSeconds: 0.5, + }); + expect(worstDifference(whole, windowed)).toBeGreaterThan(0.05); + }, + 240_000, + ); + + it.skipIf(!HAS_BROWSER)( + "a lookahead limiter keeps the last of its input and the file's length", + async () => { + const input = join(dir, "tp.wav"); + const frames = Math.round(0.5 * SR); + const steady = new Float32Array(frames).fill(0.3); + writeWav(input, steady, SR, 1, true); + const out = join(dir, "tp-out.wav"); + await applyAudioFxChain(input, chainOf("truepeak"), out, { trackId: "tp" }); + const samples = readWav(out).samples; + expect(samples.length).toBe(frames); + // The lookahead is 7 ms by default; the final 5 ms must still carry the signal. + expect(sliceRms(samples, 0.495, 0.5)).toBeGreaterThan(0.25); + }, + 120_000, + ); + + it.skipIf(!HAS_BROWSER)( + "bakes a volume envelope on the file's own clock across window boundaries", + async () => { + const input = join(dir, "env.wav"); + writeWav(input, new Float32Array(SR * 3 * 2).fill(0.25), SR, 2, true); + const out = join(dir, "env-out.wav"); + const gain: HfAudioFxChain = { + version: 1, + nodes: [{ type: "gain", id: "g", enabled: true, params: { gain: 0 } }], + }; + const result = await applyAudioFxChain(input, gain, out, { + trackId: "env", + streaming: limits, + envelope: { + keyframes: [ + { time: 0, volume: 1 }, + { time: 1.5, volume: 1 }, + { time: 1.5001, volume: 0.5 }, + { time: 3, volume: 0.5 }, + ], + trackStart: 0, + baseVolume: 1, + }, + }); + expect(result.envelopeBaked).toBe(true); + const samples = readWav(out).samples; + expect(samples[Math.floor(1.2 * SR) * 2]).toBeCloseTo(0.25, 2); + expect(samples[Math.floor(2.2 * SR) * 2]).toBeCloseTo(0.125, 2); + }, + 180_000, + ); +}); diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.ts index ad1dc0edc2..597498af6e 100644 --- a/packages/engine/src/services/audioFxRender.ts +++ b/packages/engine/src/services/audioFxRender.ts @@ -12,7 +12,18 @@ * same way, so preview would quietly stop predicting the render. */ -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + closeSync, + existsSync, + mkdtempSync, + openSync, + readFileSync, + readSync, + rmSync, + statSync, + writeFileSync, + writeSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; @@ -22,6 +33,7 @@ import { serializeAutomation, type HfAutomation } from "@hyperframes/core/audio- import { acquireBrowser } from "./browserManager.js"; import { createEnvelopeWalker } from "./audioVolumeEnvelope.js"; import { riffChunks, wavFormatTag } from "./wavChunks.js"; +import { windowPrerollSeconds } from "./audioFxWarmup.js"; import type { AudioVolumeKeyframe } from "./audioMixer.types.js"; export class AudioFxRenderError extends Error { @@ -52,9 +64,13 @@ function readWavChunks(buf: Buffer): { sampleRate: number; bits: number; data?: Buffer; + dataBody?: number; + dataSize?: number; } { const head = { format: 1, channels: 1, sampleRate: 48000, bits: 16 }; let data: Buffer | undefined; + let dataBody: number | undefined; + let dataSize: number | undefined; for (const { id, body, size } of riffChunks(buf)) { if (id === "fmt ") { const format = wavFormatTag(buf, body, size); @@ -64,7 +80,9 @@ function readWavChunks(buf: Buffer): { head.sampleRate = buf.readUInt32LE(body + 4); head.bits = buf.readUInt16LE(body + 14); } else if (id === "data") { - data = buf.subarray(body, Math.min(buf.length, body + size)); + data = buf.subarray(body, body + resolveDataBytes(size, buf.length - body)); + dataBody = body; + dataSize = size; // The payload is the rest of the file for anything the mixer writes, and // reading past it buys nothing: `fmt ` precedes `data` in every WAV these // steps produce, and the alternative is walking a several-hundred-megabyte @@ -72,7 +90,55 @@ function readWavChunks(buf: Buffer): { break; } } - return { ...head, data }; + return { ...head, data, dataBody, dataSize }; +} + +/** Where a WAV's samples sit, read from its head alone. */ +export interface WavHeader { + format: number; + channels: number; + sampleRate: number; + bits: number; + float: boolean; + dataOffset: number; + dataBytes: number; +} + +/** The most `readFileSync` returns in one call. */ +const MAX_WHOLE_FILE_BYTES = 2 ** 31 - 1; + +const HEADER_PROBE_BYTES = 64 * 1024; + +/** A declared size of 0 or 0xFFFFFFFF (live-written, or past 4 GiB), or one past the end, means "to the end"; both WAV readers use it. */ +export function resolveDataBytes(declared: number, availableBytes: number): number { + return declared === 0 || declared === 0xffffffff + ? availableBytes + : Math.min(declared, availableBytes); +} + +/** The header of `path`, or null when the head holds no readable `fmt `/`data` pair. */ +export function readWavHeader(path: string): WavHeader | null { + const fd = openSync(path, "r"); + try { + const head = Buffer.alloc(HEADER_PROBE_BYTES); + const read = readSync(fd, head, 0, head.length, 0); + const buf = head.subarray(0, read); + if (buf.length < 44 || buf.toString("ascii", 0, 4) !== "RIFF") return null; + const { format, channels, sampleRate, bits, dataBody, dataSize } = readWavChunks(buf); + if (dataBody === undefined || dataSize === undefined) return null; + const dataBytes = resolveDataBytes(dataSize, statSync(path).size - dataBody); + return { + format, + channels, + sampleRate, + bits, + float: format === 3 && bits === 32, + dataOffset: dataBody, + dataBytes, + }; + } finally { + closeSync(fd); + } } export function readWav(path: string): WavData { @@ -111,6 +177,41 @@ function decodeSamples(data: Buffer, format: number, bits: number, path: string) throw new AudioFxRenderError(`Unsupported WAV format ${format}/${bits}-bit: ${path}`); } +function wavHeader(bytes: number, sampleRate: number, channels: number, float: boolean): Buffer { + const bytesPerSample = float ? 4 : 2; + const buf = Buffer.alloc(44); + buf.write("RIFF", 0, "ascii"); + buf.writeUInt32LE(Math.min(0xffffffff, 36 + bytes), 4); + buf.write("WAVE", 8, "ascii"); + buf.write("fmt ", 12, "ascii"); + buf.writeUInt32LE(16, 16); + buf.writeUInt16LE(float ? 3 : 1, 20); // IEEE_FLOAT / PCM + buf.writeUInt16LE(channels, 22); + buf.writeUInt32LE(sampleRate, 24); + buf.writeUInt32LE(sampleRate * channels * bytesPerSample, 28); + buf.writeUInt16LE(channels * bytesPerSample, 32); + buf.writeUInt16LE(float ? 32 : 16, 34); + buf.write("data", 36, "ascii"); + buf.writeUInt32LE(Math.min(0xffffffff, bytes), 40); + return buf; +} + +function encodeSamples(samples: Float32Array, float: boolean): Buffer { + const n = samples.length; + const buf = Buffer.alloc(n * (float ? 4 : 2)); + for (let i = 0; i < n; i++) { + if (float) { + buf.writeFloatLE(samples[i] ?? 0, i * 4); + continue; + } + // Clamp before scaling: a limiter set to 0 dB or a resonant filter can push + // past full scale, and wrapping would turn that into a click. + const v = Math.max(-1, Math.min(1, samples[i] ?? 0)); + buf.writeInt16LE(Math.round(v * 32767), i * 2); + } + return buf; +} + /** * Write interleaved samples, preserving the channel count. * @@ -135,33 +236,8 @@ export function writeWav( channels = 1, float = false, ): void { - const n = samples.length; - const bytesPerSample = float ? 4 : 2; - const bytes = n * bytesPerSample; - const buf = Buffer.alloc(44 + bytes); - buf.write("RIFF", 0, "ascii"); - buf.writeUInt32LE(36 + bytes, 4); - buf.write("WAVE", 8, "ascii"); - buf.write("fmt ", 12, "ascii"); - buf.writeUInt32LE(16, 16); - buf.writeUInt16LE(float ? 3 : 1, 20); // IEEE_FLOAT / PCM - buf.writeUInt16LE(channels, 22); - buf.writeUInt32LE(sampleRate, 24); - buf.writeUInt32LE(sampleRate * channels * bytesPerSample, 28); - buf.writeUInt16LE(channels * bytesPerSample, 32); - buf.writeUInt16LE(float ? 32 : 16, 34); - buf.write("data", 36, "ascii"); - buf.writeUInt32LE(bytes, 40); - for (let i = 0; i < n; i++) { - if (float) { - buf.writeFloatLE(samples[i] ?? 0, 44 + i * 4); - continue; - } - // Clamp before scaling: a limiter set to 0 dB or a resonant filter can push - // past full scale, and wrapping would turn that into a click. - const v = Math.max(-1, Math.min(1, samples[i] ?? 0)); - buf.writeInt16LE(Math.round(v * 32767), 44 + i * 2); - } + const body = encodeSamples(samples, float); + const buf = Buffer.concat([wavHeader(body.length, sampleRate, channels, float), body]); // lgtm[js/insecure-temporary-file] — `path` is always inside a directory the // caller made with `mkdtempSync`, never a name assembled directly under // `tmpdir()`. Both routes here are covered: the browser host page writes into @@ -226,7 +302,13 @@ interface AudioFxPageIo { interface AudioFxWindow { __HF_AUDIO_FX?: { - render(p: Float32Array[], r: number, c: string, a?: string): Promise; + render( + p: Float32Array[], + r: number, + c: string, + a?: string, + elapsed?: number, + ): Promise; }; __HF_FX_IO?: AudioFxPageIo; } @@ -242,14 +324,179 @@ function applyEnvelopeToPlanes( planes: readonly Float32Array[], sampleRate: number, gainAt: (time: number) => number, + startFrame = 0, ): void { const frames = planes[0]?.length ?? 0; for (let frame = 0; frame < frames; frame += 1) { - const gain = gainAt(frame / sampleRate); + const gain = gainAt((startFrame + frame) / sampleRate); for (const plane of planes) plane[frame] = (plane[frame] ?? 0) * gain; } } +/** Inputs over `aboveBytes` render in windows: Node reads no file past 2 GiB whole, and the page would hold it twice. */ +export interface AudioFxStreamingLimits { + aboveBytes: number; + segmentSeconds: number; + /** Audio rendered ahead of each window and discarded; the least, when the chain needs more. */ + prerollSeconds: number; + /** The most pre-roll a chain's own memory can ask for. */ + maxPrerollSeconds: number; + /** Audio rendered after each window and discarded, for look-ahead. */ + postrollSeconds: number; +} + +const DEFAULT_STREAMING_LIMITS: AudioFxStreamingLimits = { + aboveBytes: 512 * 1024 * 1024, + segmentSeconds: 300, + prerollSeconds: 10, + maxPrerollSeconds: 300, + postrollSeconds: 1, +}; + +export interface AudioFxSegment { + /** The frames this window contributes to the output. */ + keepFrom: number; + keepTo: number; + /** The frames rendered to produce them. */ + readFrom: number; + readTo: number; + last: boolean; +} + +/** Frames the audio graph processes at a time; a window starting on one sees the same blocks a whole render does. */ +const RENDER_QUANTUM = 128; + +/** Tile `totalFrames` into windows that each carry their pre/post-roll. */ +export function planAudioFxSegments( + totalFrames: number, + sampleRate: number, + limits: AudioFxStreamingLimits, +): AudioFxSegment[] { + const step = Math.max(1, Math.round(limits.segmentSeconds * sampleRate)); + const pre = Math.round(limits.prerollSeconds * sampleRate); + const post = Math.round(limits.postrollSeconds * sampleRate); + const segments: AudioFxSegment[] = []; + for (let keepFrom = 0; keepFrom < totalFrames; keepFrom += step) { + const keepTo = Math.min(totalFrames, keepFrom + step); + segments.push({ + keepFrom, + keepTo, + readFrom: Math.max(0, Math.floor((keepFrom - pre) / RENDER_QUANTUM) * RENDER_QUANTUM), + readTo: Math.min(totalFrames, keepTo + post), + last: keepTo === totalFrames, + }); + } + return segments; +} + +function readFrames(fd: number, header: WavHeader, from: number, to: number): Float32Array { + const frameBytes = (header.channels * header.bits) / 8; + const data = Buffer.alloc((to - from) * frameBytes); + for (let filled = 0; filled < data.length; ) { + const n = readSync( + fd, + data, + filled, + data.length - filled, + header.dataOffset + from * frameBytes + filled, + ); + if (n === 0) break; + filled += n; + } + return decodeSamples(data, header.format, header.bits, "streamed input"); +} + +/** A chain remembers longer than the floor: size the pre-roll to its slowest state, up to the cap. */ +function chainPrerollSeconds( + chain: HfAudioFxChain, + limits: AudioFxStreamingLimits, + trackId: string, +): number { + const { seconds, capped } = windowPrerollSeconds( + chain, + limits.prerollSeconds, + limits.maxPrerollSeconds, + ); + if (capped) { + console.warn( + `Audio FX for track ${trackId}: the chain remembers longer than the ${limits.maxPrerollSeconds}s pre-roll cap, so its oldest repeats are cut at window seams`, + ); + } + return seconds; +} + +async function renderInSegments( + openPage: () => Promise, + inputWav: string, + header: WavHeader, + chain: HfAudioFxChain, + outputWav: string, + options: { + trackId: string; + signal?: AbortSignal; + envelope?: AudioFxEnvelope; + }, + limits: AudioFxStreamingLimits, +): Promise { + const { sampleRate, channels, float } = header; + const totalFrames = Math.floor(header.dataBytes / ((channels * header.bits) / 8)); + const gainAt = envelopeWalkerFor(options.envelope); + const prerollSeconds = chainPrerollSeconds(chain, limits, options.trackId); + const inFd = openSync(inputWav, "r"); + let outFd: number | undefined; + try { + // Same directory rule as writeWav: the caller's mkdtemp work dir. + outFd = openSync(outputWav, "w"); + writeSync(outFd, wavHeader(0, sampleRate, channels, float)); + let written = 0; + for (const seg of planAudioFxSegments(totalFrames, sampleRate, { + ...limits, + prerollSeconds, + })) { + if (options.signal?.aborted) { + throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`); + } + const planes = deinterleave(readFrames(inFd, header, seg.readFrom, seg.readTo), channels); + // A fresh page per window, so the browser never holds more than one window of audio. + const page = await openPage(); + let outPlanes: Float32Array[]; + try { + await sendPlanesToPage(page, planes); + outPlanes = await readPlanesFromPage( + page, + await renderPlanesInPage(page, sampleRate, chain, "", seg.readFrom / sampleRate), + ); + } finally { + await page.close().catch(() => undefined); + } + const rendered = outPlanes[0]?.length ?? 0; + if (rendered === 0) { + throw new AudioFxRenderError(`Audio FX produced no samples for track ${options.trackId}`); + } + // The last window keeps the chain's tail (reverb, delay), as a whole-file render does. + const skip = seg.keepFrom - seg.readFrom; + const kept = outPlanes.map((plane) => + plane.subarray(skip, seg.last ? rendered : seg.keepTo - seg.readFrom), + ); + if (gainAt) applyEnvelopeToPlanes(kept, sampleRate, gainAt, seg.keepFrom); + const bytes = encodeSamples(interleave(kept), float); + writeFileSync(outFd, bytes); + written += bytes.length; + } + writeSync(outFd, wavHeader(written, sampleRate, channels, float), 0, 44, 0); + } finally { + closeSync(inFd); + if (outFd !== undefined) closeSync(outFd); + } + return gainAt !== null; +} + +type AudioFxEnvelope = { + keyframes: AudioVolumeKeyframe[]; + trackStart: number; + baseVolume: number; +}; + /** * Run a chain over `inputWav`, writing `outputWav`. Resolves to the path to use * downstream — `outputWav` when the chain did something, `inputWav` untouched @@ -275,7 +522,8 @@ export async function applyAudioFxChain( trackId: string; signal?: AbortSignal; automation?: HfAutomation; - envelope?: { keyframes: AudioVolumeKeyframe[]; trackStart: number; baseVolume: number }; + envelope?: AudioFxEnvelope; + streaming?: Partial; }, ): Promise<{ path: string; envelopeBaked: boolean }> { if (enabledAudioFxNodes(chain).length === 0) return { path: inputWav, envelopeBaked: false }; @@ -283,8 +531,34 @@ export async function applyAudioFxChain( throw new AudioFxRenderError(`Audio FX input is missing: ${inputWav}`); } - const { samples, sampleRate, channels, float } = readWav(inputWav); - const planes = deinterleave(samples, channels); + // Automation is clocked from the file's own start, so a windowed render leaves it to the whole path. + const limits = { ...DEFAULT_STREAMING_LIMITS, ...options.streaming }; + const header = options.automation ? null : readWavHeader(inputWav); + const streamed = header !== null && header.dataBytes > limits.aboveBytes; + + if (streamed) { + return withFxBrowser(options, async (openPage) => ({ + path: outputWav, + envelopeBaked: await renderInSegments( + openPage, + inputWav, + header, + chain, + outputWav, + options, + limits, + ), + })); + } + + if (statSync(inputWav).size > MAX_WHOLE_FILE_BYTES) { + const why = options.automation + ? "parameter automation is not supported on inputs over 2 GiB" + : "the WAV header of an input over 2 GiB could not be read"; + throw new AudioFxRenderError(`Audio FX for track ${options.trackId}: ${why} (${inputWav})`); + } + const input = readWav(inputWav); + const planes = deinterleave(input.samples, input.channels); // An empty track has nothing to process — and an OfflineAudioContext of zero // length throws, which is fatal for the WHOLE render rather than this track: // the error travels past the mixer's per-track failure collector. ffmpeg @@ -294,36 +568,13 @@ export async function applyAudioFxChain( // never costs a browser. if ((planes[0]?.length ?? 0) === 0) return { path: inputWav, envelopeBaked: false }; - // Both resources are taken INSIDE the try that releases them. The lease used - // to be acquired above it, with the mkdtemp between — so a failure there - // (a full disk, a read-only tmpdir) leaked a pooled browser, and a pool with - // no leases left hangs every later render rather than failing one. - const hostDir = mkdtempSync(join(tmpdir(), "hf-fx-host-")); - let lease: Awaited> | null = null; - try { - // Audio processing needs no GPU or special capture mode; a plain sandboxed - // browser is enough, and the lease pool reuses one across tracks. - // Checked before the lease, not after: an already-cancelled track has no - // reason to take a browser out of the pool just to hand it straight back. - if (options.signal?.aborted) { - throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`); - } - lease = await acquireBrowser(["--no-sandbox", "--autoplay-policy=no-user-gesture-required"]); - const page = await lease.browser.newPage(); + return withFxBrowser(options, async (openPage) => { + const page = await openPage(); try { - // AudioWorklet is only exposed in a secure context, and about:blank is - // not one — the module would fail with an opaque error. A file:// page - // qualifies and needs no listening socket. - const hostPage = join(hostDir, "audio-fx.html"); - writeFileSync(hostPage, "audio fx"); - await page.goto(pathToFileURL(hostPage).href, { waitUntil: "domcontentloaded" }); - await page.addScriptTag({ content: getAudioFxRuntimeScript() }); - await sendPlanesToPage(page, planes); - const outLengths = await renderPlanesInPage( page, - sampleRate, + input.sampleRate, chain, options.automation ? serializeAutomation(options.automation) : "", ); @@ -335,14 +586,35 @@ export async function applyAudioFxChain( // own paths still own this track's gain, so say so rather than claiming // a bake that never happened. const gainAt = envelopeWalkerFor(options.envelope); - if (gainAt) applyEnvelopeToPlanes(outPlanes, sampleRate, gainAt); + if (gainAt) applyEnvelopeToPlanes(outPlanes, input.sampleRate, gainAt); // Same format in as out: a float input is a group sub-mix whose headroom // must survive to its fader (see writeWav). - writeWav(outputWav, interleave(outPlanes), sampleRate, outPlanes.length, float); + writeWav(outputWav, interleave(outPlanes), input.sampleRate, outPlanes.length, input.float); return { path: outputWav, envelopeBaked: gainAt !== null }; } finally { await page.close().catch(() => undefined); } + }); +} + +/** Run `body` with a leased browser and a way to open pages on the runtime; both are released in the same try that takes them. */ +async function withFxBrowser( + options: { trackId: string; signal?: AbortSignal }, + body: (openPage: () => Promise) => Promise, +): Promise { + const hostDir = mkdtempSync(join(tmpdir(), "hf-fx-host-")); + let lease: Awaited> | null = null; + try { + // Audio processing needs no GPU or special capture mode; a plain sandboxed + // browser is enough, and the lease pool reuses one across tracks. + // Checked before the lease, not after: an already-cancelled track has no + // reason to take a browser out of the pool just to hand it straight back. + if (options.signal?.aborted) { + throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`); + } + lease = await acquireBrowser(["--no-sandbox", "--autoplay-policy=no-user-gesture-required"]); + const { browser } = lease; + return await body(() => openFxPage(browser, hostDir)); } catch (err) { if (err instanceof AudioFxRenderError) throw err; throw new AudioFxRenderError( @@ -357,6 +629,28 @@ export async function applyAudioFxChain( /** The page handle `acquireBrowser().browser.newPage()` returns. */ type FxPage = Awaited>["browser"]["newPage"]>>; +async function openFxPage( + browser: Awaited>["browser"], + hostDir: string, +): Promise { + const page = await browser.newPage(); + try { + // AudioWorklet is only exposed in a secure context, and about:blank is + // not one — the module would fail with an opaque error. A file:// page + // qualifies and needs no listening socket. + const hostPage = join(hostDir, "audio-fx.html"); + writeFileSync(hostPage, "audio fx"); + await page.goto(pathToFileURL(hostPage).href, { + waitUntil: "domcontentloaded", + }); + await page.addScriptTag({ content: getAudioFxRuntimeScript() }); + return page; + } catch (err) { + await page.close().catch(() => undefined); + throw err; + } +} + /** * Hand the input planes to the page a chunk at a time. * @@ -397,9 +691,10 @@ async function renderPlanesInPage( sampleRate: number, chain: HfAudioFxChain, automationJson: string, + elapsed = 0, ): Promise { return (await page.evaluate( - async ([rate, chainJson, automation]: [number, string, string]) => { + async ([rate, chainJson, automation, at]: [number, string, string, number]) => { const w = window as unknown as AudioFxWindow; const io = w.__HF_FX_IO; if (!w.__HF_AUDIO_FX || !io) throw new Error("audio FX runtime failed to load"); @@ -415,10 +710,15 @@ async function renderPlanesInPage( // Dropped before the render allocates its own buffers, so the page does // not hold two copies of the track at once. io.in = []; - io.out = await w.__HF_AUDIO_FX.render(inPlanes, rate, chainJson, automation || undefined); + io.out = await w.__HF_AUDIO_FX.render(inPlanes, rate, chainJson, automation || undefined, at); return io.out.map((plane) => plane.length); }, - [sampleRate, JSON.stringify(chain), automationJson] as [number, string, string], + [sampleRate, JSON.stringify(chain), automationJson, elapsed] as [ + number, + string, + string, + number, + ], )) as number[]; } @@ -470,7 +770,11 @@ async function readPlanesFromPage(page: FxPage, outLengths: number[]): Promise | null { if (!envelope) return null; diff --git a/packages/engine/src/services/audioFxWarmup.test.ts b/packages/engine/src/services/audioFxWarmup.test.ts new file mode 100644 index 0000000000..d8843ce58a --- /dev/null +++ b/packages/engine/src/services/audioFxWarmup.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import type { HfAudioFxChain } from "@hyperframes/core/audio-fx"; +import { chainWarmupSeconds, windowPrerollSeconds } from "./audioFxWarmup.js"; + +const chainOf = (...nodes: HfAudioFxChain["nodes"]): HfAudioFxChain => ({ version: 1, nodes }); + +describe("chainWarmupSeconds", () => { + it("is the time a delay's repeats take to fall to -60 dB", () => { + const delay = { + type: "delay", + enabled: true, + params: { time: 5000, feedback: 0.95, mix: 0.5 }, + }; + expect(chainWarmupSeconds(chainOf(delay))).toBeCloseTo(135 * 5, 5); + }); + + it("is seven release times for a compressor", () => { + const comp = { type: "compressor", enabled: true, params: { release: 9000 } }; + expect(chainWarmupSeconds(chainOf(comp))).toBeCloseTo(9 * Math.log(1000), 5); + }); + + it("adds a reverb's room and sums a serial chain", () => { + const reverb = { type: "reverb", enabled: true, params: { size: 0.5, wet: 0.3 } }; + const delay = { type: "delay", enabled: true, params: { time: 300, feedback: 0.4, mix: 0.4 } }; + expect(chainWarmupSeconds(chainOf(reverb))).toBeCloseTo(0.6 + 1.3, 5); + expect(chainWarmupSeconds(chainOf(delay, reverb))).toBeCloseTo(0.3 * 8 + 1.9, 5); + }); + + it("ignores nodes that are off or mixed out", () => { + const off = { type: "delay", enabled: false, params: { time: 5000, feedback: 0.9, mix: 0.5 } }; + const dry = { type: "reverb", enabled: true, params: { size: 1, wet: 0 } }; + const mixedOut = { + type: "delay", + enabled: true, + params: { time: 5000, feedback: 0.9, mix: 0 }, + }; + expect(chainWarmupSeconds(chainOf(off, dry, mixedOut))).toBe(0); + }); +}); + +describe("windowPrerollSeconds", () => { + const delay = (time: number, feedback: number): HfAudioFxChain => + chainOf({ type: "delay", enabled: true, params: { time, feedback, mix: 0.5 } }); + + it("never goes below the floor", () => { + expect(windowPrerollSeconds(delay(100, 0.1), 10, 300)).toEqual({ seconds: 10, capped: false }); + }); + + it("follows the chain between the floor and the cap", () => { + expect(windowPrerollSeconds(delay(400, 0.5), 1, 300)).toEqual({ seconds: 4, capped: false }); + }); + + it("stops at the cap and says so", () => { + expect(windowPrerollSeconds(delay(5000, 0.95), 10, 300)).toEqual({ + seconds: 300, + capped: true, + }); + }); +}); diff --git a/packages/engine/src/services/audioFxWarmup.ts b/packages/engine/src/services/audioFxWarmup.ts new file mode 100644 index 0000000000..b21a21695e --- /dev/null +++ b/packages/engine/src/services/audioFxWarmup.ts @@ -0,0 +1,58 @@ +import { delayTailSeconds, reverbRoomSeconds } from "@hyperframes/core/audio-fx-tail"; +import { + enabledAudioFxNodes, + normalizeAudioFxParams, + type HfAudioFxChain, + type HfAudioFxNode, +} from "@hyperframes/core/audio-fx"; + +/** Where state stops mattering: -60 dB, i.e. e^-6.9 of what it started as. */ +const DECAY_TO_FLOOR = Math.log(1000); + +const param = (node: HfAudioFxNode, key: string): number => + Number(normalizeAudioFxParams(node.type, node.params)[key] ?? 0); + +const reverbMemory = (node: HfAudioFxNode): number => + param(node, "wet") > 0 ? reverbRoomSeconds(param(node, "size")) : 0; + +const delayMemory = (node: HfAudioFxNode): number => + param(node, "mix") > 0 ? delayTailSeconds(param(node, "time"), param(node, "feedback")) : 0; + +const releaseMemory = (node: HfAudioFxNode): number => + (DECAY_TO_FLOOR * param(node, "release")) / 1000; + +/** Seconds of input a node still remembers, to -60 dB. */ +const MEMORY: Record number> = { + reverb: reverbMemory, + delay: delayMemory, + compressor: releaseMemory, + limiter: releaseMemory, + truepeak: releaseMemory, + gate: releaseMemory, +}; + +/** + * How far ahead of a window a chain must be fed for its state to match a whole render. + * + * Summed, since the chain is serial: a delay ahead of a reverb hands the reverb every repeat. + * Biquads, chorus and pitch shift settle well inside the caller's floor. + */ +export function chainWarmupSeconds(chain: HfAudioFxChain): number { + return enabledAudioFxNodes(chain).reduce( + (sum, node) => sum + (MEMORY[node.type]?.(node) ?? 0), + 0, + ); +} + +/** The pre-roll a chain gets: its own memory, between `floorSeconds` and `capSeconds`. */ +export function windowPrerollSeconds( + chain: HfAudioFxChain, + floorSeconds: number, + capSeconds: number, +): { seconds: number; capped: boolean } { + const needed = chainWarmupSeconds(chain); + return { + seconds: Math.min(capSeconds, Math.max(floorSeconds, needed)), + capped: needed > capSeconds, + }; +}