fix(engine): offset clips from a WAV with a wrong byte rate play the right sound to their end (#7541)

* fix(engine): offset clips from a WAV with a wrong byte rate play the right sound to their end

A WAV header that states twice its real byte rate sent the mixer's input seek
to twice the clip's in-point: the clip played later words, then silence. Such a
source is now cut by decoded timestamps instead of by a byte seek.

* refactor(engine): let the byte-rate flag's name carry the seek rule
This commit is contained in:
Miguel Ángel
2026-11-11 05:17:46 +00:00
committed by GitHub
parent 8a15d9feed
commit 7ae1ae6494
3 changed files with 110 additions and 1 deletions
+2 -1
View File
@@ -777,7 +777,8 @@ async function extractAudioSegment(
}
}
const ffmpegStart = ffmpegSeekSeconds(metadata ?? {}, mediaStart);
const inputSeek = ffmpegStart >= (metadata?.latestStreamLeadSeconds ?? 0);
const inputSeek =
ffmpegStart >= (metadata?.latestStreamLeadSeconds ?? 0) && !metadata?.misstatedByteRate;
const start = formatFilterNumber(ffmpegStart);
const timestampTrim = inputSeek
? null
@@ -0,0 +1,93 @@
import { spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
import { parseAudioElements, processCompositionAudio } from "./audioMixer.js";
const FFMPEG = getFfmpegBinary();
const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"]).status === 0;
const SAMPLE_RATE = 48_000;
function ffmpeg(args: string[]): Buffer {
const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args]);
if (result.status !== 0) throw new Error(result.stderr.toString());
return result.stdout;
}
function frequenciesAt(pcm: Buffer, times: number[]): Array<number | null> {
return times.map((time) => {
const start = Math.round((time - 0.04) * SAMPLE_RATE);
const count = Math.round(0.08 * SAMPLE_RATE);
let energy = 0;
let crossings = 0;
let previous = 0;
for (let i = start; i < start + count; i += 1) {
const sample = pcm.readInt16LE(i * 2);
energy += sample * sample;
if (i > start && previous < 0 && sample >= 0) crossings += 1;
previous = sample;
}
return Math.sqrt(energy / count) < 30 ? null : Math.round(crossings / 0.08 / 100) * 100;
});
}
/** A copy of a canonical 44-byte-header WAV with its fmt chunk's byte rate and block align overwritten. */
function withHeader(source: string, target: string, byteRate: number, blockAlign: number): void {
const wav = Buffer.from(readFileSync(source));
expect(wav.toString("ascii", 36, 40)).toBe("data");
wav.writeUInt32LE(byteRate, 28);
wav.writeUInt16LE(blockAlign, 32);
writeFileSync(target, wav);
}
describe.skipIf(!HAS_FFMPEG)("a WAV whose header misstates its byte rate", () => {
let dir: string;
beforeAll(() => {
dir = mkdtempSync(join(tmpdir(), "hf-audio-byterate-"));
// 8 s of 24 kHz mono speech-rate PCM, a different pitch each second.
ffmpeg([
"-f",
"lavfi",
"-i",
"aevalsrc=0.3*sin(2*PI*(400+400*floor(t))*t):s=24000:d=8",
"-ac",
"1",
"-c:a",
"pcm_s16le",
"-bitexact",
"-map_metadata",
"-1",
join(dir, "true.wav"),
]);
// Written as if the mono samples were stereo: 96,000 bytes a second for 48,000.
withHeader(join(dir, "true.wav"), join(dir, "double-rate.wav"), 96_000, 2);
withHeader(join(dir, "true.wav"), join(dir, "stereo-header.wav"), 96_000, 4);
});
afterAll(() => {
if (dir) rmSync(dir, { recursive: true, force: true });
});
async function render(src: string): Promise<Buffer> {
const work = mkdtempSync(join(dir, "work-"));
const output = `${work}.m4a`;
const html = `<audio id="voice" src="${src}" data-start="0" data-duration="4" data-media-start="3"></audio>`;
const result = await processCompositionAudio(parseAudioElements(html), dir, work, output, 4);
expect(result, result.error).toMatchObject({ success: true, tracksProcessed: 1 });
return ffmpeg(["-i", output, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", "-"]);
}
const heardAt = [0.5, 1.5, 2.5, 3.5];
it.each(["double-rate.wav", "stereo-header.wav"])(
"%s plays the same words as the true file from its in-point, for the whole clip",
async (src) => {
const reference = frequenciesAt(await render("true.wav"), heardAt);
expect(reference).toEqual([1600, 2000, 2400, 2800]);
expect(frequenciesAt(await render(src), heardAt)).toEqual(reference);
},
);
});
+15
View File
@@ -248,6 +248,8 @@ export interface AudioMetadata {
audioStreamLeadSeconds?: number;
mediaTimeOffsetSeconds?: number;
audioStreamCount?: number;
/** PCM whose header byte rate is not sample rate x channels x sample size: a seek by it lands at the wrong time. */
misstatedByteRate?: boolean;
/** The track read for these facts and for the mix, by its place among the audio streams (decodableAudioStream). */
audioStream: number;
}
@@ -270,6 +272,8 @@ interface FFProbeStream {
avg_frame_rate?: string;
sample_rate?: string;
channels?: number;
bit_rate?: string;
bits_per_sample?: number;
color_range?: string;
color_transfer?: string;
color_primaries?: string;
@@ -1343,9 +1347,20 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi
mediaTimeOffsetSeconds: mediaTimeOffsetSeconds(output),
audioStreamCount: audioStreams.length,
audioStream: audioStreamIndex,
misstatedByteRate: misstatesPcmByteRate(audioStream),
};
}
function misstatesPcmByteRate(stream: FFProbeStream): boolean {
const declared = Number(stream.bit_rate);
const { channels, bits_per_sample: bits } = stream;
const rate = Number(stream.sample_rate);
if (!stream.codec_name?.startsWith("pcm_") || !declared || !channels || !bits || !rate) {
return false;
}
return declared !== rate * channels * bits;
}
/** A stream starting after the earliest of these plays that much after it. */
function audioVideoStreamStarts(output: FFProbeOutput | null): number[] {
return (output?.streams ?? [])