From c9448aec004628bae16b74984aaab9bc8bf29de0 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Sat, 8 Aug 2026 11:50:36 -0700 Subject: [PATCH] fix(engine): move a track's PCM across the wire in bounded chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole track crossed in one `page.evaluate` pair — ~184 MB of base64 for a 3-minute stereo 48 kHz clip, in a single WebSocket frame each way. puppeteer-core caps its frames at 256 MB (NodeWebSocketTransport, verified in the installed copy) and the browser pool does not pass `pipe: true`, so stereo 48 kHz past ~8.7 minutes raised AudioFxRenderError and failed the render. V8's max string length is a second wall not much beyond it. Both were fatal: an FX failure takes the whole mix down rather than dropping one track. Input and output now move 8 MiB at a time, staying separate byte arrays page-side rather than being concatenated into one string, so neither the frame cap nor the string limit ever sees the whole track. That also bounds the peak Node-side allocation: `processCompositionAudio` renders tracks through an unbounded `Promise.all`, so every track's payload used to be live at once. The page drops the input chunks before the render allocates its buffers, so it no longer holds two copies of the track. Also drops the `Array.from(u8.subarray(...))` in the encoder, which boxed every byte of each 32 KiB window into a JS array for nothing — `apply` takes array-likes. The 8.7-minute ceiling itself has no test: reproducing it needs a 256 MB payload. What is tested is the assembly, which is where chunking actually goes wrong — a 45 s stereo ramp spanning two chunks per plane, checked for length, for values at the seam, and for monotonicity across all 2.16 M frames. Falsified against three mutations: chunks reassembled in reverse (1.75 off at the seam), the trailing partial chunk dropped (short by 125,696 frames), and an unbounded input subarray (long by 125,696). Co-Authored-By: Claude Opus 5 (1M context) --- .../engine/src/services/audioFxRender.test.ts | 60 +++++++ packages/engine/src/services/audioFxRender.ts | 164 ++++++++++++------ 2 files changed, 175 insertions(+), 49 deletions(-) diff --git a/packages/engine/src/services/audioFxRender.test.ts b/packages/engine/src/services/audioFxRender.test.ts index cccd3d5328..bd78710c83 100644 --- a/packages/engine/src/services/audioFxRender.test.ts +++ b/packages/engine/src/services/audioFxRender.test.ts @@ -316,6 +316,66 @@ describe("browser render", () => { } }, 180_000); + it("round-trips a track larger than one transfer chunk", async () => { + // The PCM used to cross in a single evaluate pair, so a stereo track past + // ~8.7 minutes blew puppeteer's 256 MB frame cap and failed the render. + // It now goes a chunk at a time; this clip is 45 s stereo, so each plane + // spans two chunks and the seam is inside the audio rather than at its end. + // + // A ramp, not a tone: every sample is a unique position marker, so a chunk + // dropped, reordered or over-read shows up as a value at the wrong place + // instead of hiding inside a periodic signal. + const input = join(dir, "long-in.wav"); + const frames = SR * 45; + const at = (i: number): number => -0.9 + (1.8 * i) / (frames - 1); + const s = new Float32Array(frames * 2); + for (let i = 0; i < frames; i++) { + s[i * 2] = at(i); + s[i * 2 + 1] = -at(i); + } + writeWav(input, s, SR, 2); + + const outPath = join(dir, "long-out.wav"); + await applyAudioFxChain( + input, + // Transparent: a peaking band at 0 dB is unity, so the output is the + // input and any difference is the transfer's doing. + { + version: 1, + nodes: [{ type: "peaking", enabled: true, params: { frequency: 1000, gain: 0, q: 1 } }], + }, + outPath, + { trackId: "t" }, + ); + + const out = readWav(outPath); + expect(out.channels).toBe(2); + // A dropped tail chunk shows up here first. + expect(out.samples.length).toBe(frames * 2); + + // Probe across the whole clip and tightly around the 8 MiB seam. + const seam = (8 * 1024 * 1024) / 4; + const probes = [ + ...Array.from({ length: 60 }, (_, k) => Math.floor((k * (frames - 1)) / 59)), + ...[-2, -1, 0, 1, 2].map((d) => seam + d), + ].filter((i) => i >= 0 && i < frames); + for (const i of probes) { + // Two 16-bit steps of tolerance: the input was quantised on the way in + // and the output again on the way out. + expect(out.samples[i * 2]).toBeCloseTo(at(i), 3); + expect(out.samples[i * 2 + 1]).toBeCloseTo(-at(i), 3); + } + + // The ramp only ever rises, so this reads every sample and fails on a + // chunk reordered, duplicated or over-read anywhere in the file — not just + // at the points probed above. + let breaks = 0; + for (let i = 1; i < frames; i++) { + if ((out.samples[i * 2] ?? 0) < (out.samples[(i - 1) * 2] ?? 0)) breaks += 1; + } + expect(breaks).toBe(0); + }, 180_000); + it("renders a multi-effect chain including reverb", async () => { const input = join(dir, "in.wav"); tone(input); diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.ts index f1da45b093..bca39030b4 100644 --- a/packages/engine/src/services/audioFxRender.ts +++ b/packages/engine/src/services/audioFxRender.ts @@ -174,6 +174,35 @@ function interleave(planes: readonly Float32Array[]): Float32Array { return out; } +/** + * Bytes of PCM per CDP message. + * + * The whole track used to cross in a single `page.evaluate` pair — ~184 MB of + * base64 for a 3-minute stereo 48 kHz clip, in one WebSocket frame each way. + * puppeteer-core caps its frames at 256 MB and the browser pool does not use + * the pipe transport, so stereo past ~8.7 minutes failed the render outright; + * V8's max string length is a second wall not far beyond it. Chunking bounds + * both, and bounds the peak Node-side allocation with them: the mixer renders + * tracks concurrently, so every track's payload was live at once. + * + * 8 MiB encodes to ~11 MB of base64. A multiple of 4, so a chunk boundary + * never falls inside a float. + */ +const TRANSFER_BYTES = 8 * 1024 * 1024; + +/** The page-side handover buffers, named off `window` so each step can find them. */ +interface AudioFxPageIo { + in: Uint8Array[][]; + out: Float32Array[]; +} + +interface AudioFxWindow { + __HF_AUDIO_FX?: { + render(p: Float32Array[], r: number, c: string, a?: string): Promise; + }; + __HF_FX_IO?: AudioFxPageIo; +} + /** * Multiply every channel by the envelope, in place, one gain per frame. * @@ -262,66 +291,103 @@ export async function applyAudioFxChain( await page.goto(pathToFileURL(hostPage).href, { waitUntil: "domcontentloaded" }); await page.addScriptTag({ content: getAudioFxRuntimeScript() }); - const rendered = (await page.evaluate( - async ([channelB64, rate, chainJson, automationJson]: [ - string[], - number, - string, - string, - ]) => { - const decode = (b64: string): Float32Array => { - const bin = atob(b64); - const bytes = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); - return new Float32Array(bytes.buffer); - }; - const api = ( - window as unknown as { - __HF_AUDIO_FX?: { - render( - p: Float32Array[], - r: number, - c: string, - a?: string, - ): Promise; - }; + // Hand the input over a chunk at a time. The chunks stay separate byte + // arrays page-side rather than being concatenated into one string, so + // neither the frame cap nor V8's string limit sees the whole track. + await page.evaluate((count: number) => { + (window as unknown as AudioFxWindow).__HF_FX_IO = { + in: Array.from({ length: count }, (): Uint8Array[] => []), + out: [], + }; + }, planes.length); + + for (let p = 0; p < planes.length; p += 1) { + const plane = planes[p]; + if (!plane) continue; + const bytes = Buffer.from(plane.buffer, plane.byteOffset, plane.length * 4); + for (let at = 0; at < bytes.length; at += TRANSFER_BYTES) { + await page.evaluate( + ([index, b64]: [number, string]) => { + const bin = atob(b64); + const chunk = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) chunk[i] = bin.charCodeAt(i); + (window as unknown as AudioFxWindow).__HF_FX_IO?.in[index]?.push(chunk); + }, + [p, bytes.subarray(at, at + TRANSFER_BYTES).toString("base64")] as [number, string], + ); + } + } + + const outLengths = (await page.evaluate( + async ([rate, chainJson, automationJson]: [number, string, string]) => { + const w = window as unknown as AudioFxWindow; + const io = w.__HF_FX_IO; + if (!w.__HF_AUDIO_FX || !io) throw new Error("audio FX runtime failed to load"); + const inPlanes = io.in.map((chunks) => { + const bytes = new Uint8Array(chunks.reduce((n, c) => n + c.length, 0)); + let at = 0; + for (const chunk of chunks) { + bytes.set(chunk, at); + at += chunk.length; } - ).__HF_AUDIO_FX; - if (!api) throw new Error("audio FX runtime failed to load"); - const out = await api.render( - channelB64.map(decode), + return new Float32Array(bytes.buffer); + }); + // Dropped before the render allocates its own buffers, so the page + // does not hold two copies of the track at once. + io.in = []; + io.out = await w.__HF_AUDIO_FX.render( + inPlanes, rate, chainJson, automationJson || undefined, ); - const encode = (plane: Float32Array): string => { - const u8 = new Uint8Array(plane.buffer, plane.byteOffset, plane.length * 4); - let s = ""; - const CHUNK = 0x8000; - for (let i = 0; i < u8.length; i += CHUNK) { - s += String.fromCharCode.apply(null, Array.from(u8.subarray(i, i + CHUNK))); - } - return btoa(s); - }; - return out.map(encode); + return io.out.map((plane) => plane.length); }, [ - planes.map((plane) => - Buffer.from(plane.buffer, plane.byteOffset, plane.length * 4).toString("base64"), - ), sampleRate, JSON.stringify(chain), options.automation ? serializeAutomation(options.automation) : "", - ] as [string[], number, string, string], - )) as string[]; + ] as [number, string, string], + )) as number[]; - // byteOffset and byteLength matter: Node pools small allocations, so a - // short payload decodes into an 8 KiB pool and a view over the whole - // ArrayBuffer would read kilobytes of unrelated memory at the wrong length. - const outPlanes = rendered.map((b64) => { - const buf = Buffer.from(b64, "base64"); - return new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)); - }); + const outPlanes: Float32Array[] = []; + for (let p = 0; p < outLengths.length; p += 1) { + const byteLength = (outLengths[p] ?? 0) * 4; + const parts: Buffer[] = []; + for (let at = 0; at < byteLength; at += TRANSFER_BYTES) { + const b64 = (await page.evaluate( + ([index, offset, limit]: [number, number, number]) => { + const plane = (window as unknown as AudioFxWindow).__HF_FX_IO?.out[index]; + if (!plane) return ""; + const u8 = new Uint8Array( + plane.buffer, + plane.byteOffset + offset, + Math.min(limit, plane.length * 4 - offset), + ); + let s = ""; + const CHUNK = 0x8000; + for (let i = 0; i < u8.length; i += CHUNK) { + // `apply` takes array-likes, so the subarray goes in as it is; + // Array.from boxed every byte of a 32 KiB window for nothing. + s += String.fromCharCode.apply( + null, + u8.subarray(i, i + CHUNK) as unknown as number[], + ); + } + return btoa(s); + }, + [p, at, TRANSFER_BYTES] as [number, number, number], + )) as string; + parts.push(Buffer.from(b64, "base64")); + } + // byteOffset and byteLength matter: Node pools small allocations, so a + // short payload decodes into an 8 KiB pool and a view over the whole + // ArrayBuffer would read kilobytes of unrelated memory at the wrong length. + const buf = Buffer.concat(parts); + outPlanes.push( + new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)), + ); + } if (outPlanes.length === 0 || (outPlanes[0]?.length ?? 0) === 0) { throw new AudioFxRenderError(`Audio FX produced no samples for track ${options.trackId}`); }