From 7ae1ae64942f036a18c546bef024121d8df76ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 11 Nov 2026 05:17:46 +0000 Subject: [PATCH] fix(engine): offset clips from a WAV with a wrong byte rate play the right sound to their end (#7541) * fix(engine): offset clips from a WAV with a wrong byte rate play the right sound to their end A WAV header that states twice its real byte rate sent the mixer's input seek to twice the clip's in-point: the clip played later words, then silence. Such a source is now cut by decoded timestamps instead of by a byte seek. * refactor(engine): let the byte-rate flag's name carry the seek rule --- packages/engine/src/services/audioMixer.ts | 3 +- .../services/audioMixer.wavByteRate.test.ts | 93 +++++++++++++++++++ packages/engine/src/utils/ffprobe.ts | 15 +++ 3 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 packages/engine/src/services/audioMixer.wavByteRate.test.ts diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index c765872151..94131aff58 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -777,7 +777,8 @@ async function extractAudioSegment( } } const ffmpegStart = ffmpegSeekSeconds(metadata ?? {}, mediaStart); - const inputSeek = ffmpegStart >= (metadata?.latestStreamLeadSeconds ?? 0); + const inputSeek = + ffmpegStart >= (metadata?.latestStreamLeadSeconds ?? 0) && !metadata?.misstatedByteRate; const start = formatFilterNumber(ffmpegStart); const timestampTrim = inputSeek ? null diff --git a/packages/engine/src/services/audioMixer.wavByteRate.test.ts b/packages/engine/src/services/audioMixer.wavByteRate.test.ts new file mode 100644 index 0000000000..cff83b64e6 --- /dev/null +++ b/packages/engine/src/services/audioMixer.wavByteRate.test.ts @@ -0,0 +1,93 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { getFfmpegBinary } from "../utils/ffmpegBinaries.js"; +import { parseAudioElements, processCompositionAudio } from "./audioMixer.js"; + +const FFMPEG = getFfmpegBinary(); +const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"]).status === 0; +const SAMPLE_RATE = 48_000; + +function ffmpeg(args: string[]): Buffer { + const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args]); + if (result.status !== 0) throw new Error(result.stderr.toString()); + return result.stdout; +} + +function frequenciesAt(pcm: Buffer, times: number[]): Array { + return times.map((time) => { + const start = Math.round((time - 0.04) * SAMPLE_RATE); + const count = Math.round(0.08 * SAMPLE_RATE); + let energy = 0; + let crossings = 0; + let previous = 0; + for (let i = start; i < start + count; i += 1) { + const sample = pcm.readInt16LE(i * 2); + energy += sample * sample; + if (i > start && previous < 0 && sample >= 0) crossings += 1; + previous = sample; + } + return Math.sqrt(energy / count) < 30 ? null : Math.round(crossings / 0.08 / 100) * 100; + }); +} + +/** A copy of a canonical 44-byte-header WAV with its fmt chunk's byte rate and block align overwritten. */ +function withHeader(source: string, target: string, byteRate: number, blockAlign: number): void { + const wav = Buffer.from(readFileSync(source)); + expect(wav.toString("ascii", 36, 40)).toBe("data"); + wav.writeUInt32LE(byteRate, 28); + wav.writeUInt16LE(blockAlign, 32); + writeFileSync(target, wav); +} + +describe.skipIf(!HAS_FFMPEG)("a WAV whose header misstates its byte rate", () => { + let dir: string; + + beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), "hf-audio-byterate-")); + // 8 s of 24 kHz mono speech-rate PCM, a different pitch each second. + ffmpeg([ + "-f", + "lavfi", + "-i", + "aevalsrc=0.3*sin(2*PI*(400+400*floor(t))*t):s=24000:d=8", + "-ac", + "1", + "-c:a", + "pcm_s16le", + "-bitexact", + "-map_metadata", + "-1", + join(dir, "true.wav"), + ]); + // Written as if the mono samples were stereo: 96,000 bytes a second for 48,000. + withHeader(join(dir, "true.wav"), join(dir, "double-rate.wav"), 96_000, 2); + withHeader(join(dir, "true.wav"), join(dir, "stereo-header.wav"), 96_000, 4); + }); + + afterAll(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + }); + + async function render(src: string): Promise { + const work = mkdtempSync(join(dir, "work-")); + const output = `${work}.m4a`; + const html = ``; + const result = await processCompositionAudio(parseAudioElements(html), dir, work, output, 4); + expect(result, result.error).toMatchObject({ success: true, tracksProcessed: 1 }); + return ffmpeg(["-i", output, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", "-"]); + } + + const heardAt = [0.5, 1.5, 2.5, 3.5]; + + it.each(["double-rate.wav", "stereo-header.wav"])( + "%s plays the same words as the true file from its in-point, for the whole clip", + async (src) => { + const reference = frequenciesAt(await render("true.wav"), heardAt); + expect(reference).toEqual([1600, 2000, 2400, 2800]); + expect(frequenciesAt(await render(src), heardAt)).toEqual(reference); + }, + ); +}); diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts index b05d5616d5..efc039524d 100644 --- a/packages/engine/src/utils/ffprobe.ts +++ b/packages/engine/src/utils/ffprobe.ts @@ -248,6 +248,8 @@ export interface AudioMetadata { audioStreamLeadSeconds?: number; mediaTimeOffsetSeconds?: number; audioStreamCount?: number; + /** PCM whose header byte rate is not sample rate x channels x sample size: a seek by it lands at the wrong time. */ + misstatedByteRate?: boolean; /** The track read for these facts and for the mix, by its place among the audio streams (decodableAudioStream). */ audioStream: number; } @@ -270,6 +272,8 @@ interface FFProbeStream { avg_frame_rate?: string; sample_rate?: string; channels?: number; + bit_rate?: string; + bits_per_sample?: number; color_range?: string; color_transfer?: string; color_primaries?: string; @@ -1343,9 +1347,20 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi mediaTimeOffsetSeconds: mediaTimeOffsetSeconds(output), audioStreamCount: audioStreams.length, audioStream: audioStreamIndex, + misstatedByteRate: misstatesPcmByteRate(audioStream), }; } +function misstatesPcmByteRate(stream: FFProbeStream): boolean { + const declared = Number(stream.bit_rate); + const { channels, bits_per_sample: bits } = stream; + const rate = Number(stream.sample_rate); + if (!stream.codec_name?.startsWith("pcm_") || !declared || !channels || !bits || !rate) { + return false; + } + return declared !== rate * channels * bits; +} + /** A stream starting after the earliest of these plays that much after it. */ function audioVideoStreamStarts(output: FFProbeOutput | null): number[] { return (output?.streams ?? [])