mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): check hears a WAV whose header misstates its byte rate, as render plays it (#7618)
* fix(cli): check hears a WAV whose header misstates its byte rate, as render plays it A PCM WAV whose header byte rate is wrong (some speech generators write 2x) sent the silence decode's input seek past the file's end, so every clip past the middle was reported audio_clip_silent. Render already cuts such a file by decoded time (#7541); the byte-rate rule now lives in parsers and the CLI decoder seeks after decoding starts for those files. * test(cli): keep the misstated byte-rate case with the other single-clip checks * style(cli): format the WAV test helper * style(parsers): keep the byte-rate rule's comment within the line limit
This commit is contained in:
@@ -1,7 +1,11 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
|
||||
import { decodableAudioStream, type FfprobeStreamCodec } from "@hyperframes/parsers/media-streams";
|
||||
import {
|
||||
decodableAudioStream,
|
||||
misstatesPcmByteRate,
|
||||
type FfprobePcmFields,
|
||||
} from "@hyperframes/parsers/media-streams";
|
||||
import type { AudioSignal } from "@hyperframes/core/audio-analysis";
|
||||
|
||||
const run = promisify(execFile);
|
||||
@@ -11,6 +15,7 @@ export interface AudioStreamInfo {
|
||||
sampleRate: number;
|
||||
channels: number;
|
||||
duration: number;
|
||||
misstatedByteRate?: boolean;
|
||||
}
|
||||
|
||||
function requiredBinary(name: "ffmpeg" | "ffprobe"): string {
|
||||
@@ -38,11 +43,7 @@ export async function probeAudio(file: string): Promise<AudioStreamInfo | null>
|
||||
{ maxBuffer: 8 * 1024 * 1024, windowsHide: true },
|
||||
);
|
||||
const parsed = JSON.parse(stdout) as {
|
||||
streams?: (FfprobeStreamCodec & {
|
||||
sample_rate?: string;
|
||||
channels?: number;
|
||||
duration?: string;
|
||||
})[];
|
||||
streams?: (FfprobePcmFields & { duration?: string })[];
|
||||
format?: { duration?: string };
|
||||
};
|
||||
const index = decodableAudioStream(parsed.streams ?? []);
|
||||
@@ -53,6 +54,7 @@ export async function probeAudio(file: string): Promise<AudioStreamInfo | null>
|
||||
sampleRate: Number(stream.sample_rate),
|
||||
channels: stream.channels ?? 1,
|
||||
duration: Number(stream.duration ?? parsed.format?.duration ?? 0),
|
||||
misstatedByteRate: misstatesPcmByteRate(stream),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -88,15 +90,17 @@ export async function decodeAudio(
|
||||
channels: format.mono ? 1 : probedInfo.channels,
|
||||
};
|
||||
const seek = window.from ? ["-ss", String(window.from)] : [];
|
||||
const [inputSeek, decodedSeek] = probedInfo.misstatedByteRate ? [[], seek] : [seek, []];
|
||||
const length = window.seconds === undefined ? [] : ["-t", String(window.seconds)];
|
||||
const { stdout } = await run(
|
||||
requiredBinary("ffmpeg"),
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
...seek,
|
||||
...inputSeek,
|
||||
"-i",
|
||||
file,
|
||||
...decodedSeek,
|
||||
...length,
|
||||
"-map",
|
||||
`0:a:${probedInfo.audioStream}`,
|
||||
|
||||
@@ -11,8 +11,13 @@ const RATE = 48_000;
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-check-audio-"));
|
||||
afterAll(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
/** A mono 16-bit WAV of `seconds`, each sample from `fill`. */
|
||||
function wav(name: string, seconds: number, fill: (t: number) => number): string {
|
||||
/** A mono 16-bit WAV of `seconds`, each sample from `fill`; `byteRate` is the header's claim. */
|
||||
function wav(
|
||||
name: string,
|
||||
seconds: number,
|
||||
fill: (t: number) => number,
|
||||
byteRate = RATE * 2,
|
||||
): string {
|
||||
const frames = Math.round(seconds * RATE);
|
||||
const data = Buffer.alloc(frames * 2);
|
||||
for (let i = 0; i < frames; i++) data.writeInt16LE(Math.round(fill(i / RATE) * 32767), i * 2);
|
||||
@@ -24,7 +29,7 @@ function wav(name: string, seconds: number, fill: (t: number) => number): string
|
||||
header.writeUInt16LE(1, 20);
|
||||
header.writeUInt16LE(1, 22);
|
||||
header.writeUInt32LE(RATE, 24);
|
||||
header.writeUInt32LE(RATE * 2, 28);
|
||||
header.writeUInt32LE(byteRate, 28);
|
||||
header.writeUInt16LE(2, 32);
|
||||
header.writeUInt16LE(16, 34);
|
||||
header.write("data", 36);
|
||||
@@ -72,6 +77,8 @@ describe.skipIf(!ffmpeg)("inspectAudio", () => {
|
||||
// Long enough to be sampled at spots first; sound only between the spots.
|
||||
wav("long-silent.wav", 20, silent);
|
||||
wav("long-gap.wav", 20, (t) => (t > 5 && t < 6 ? tone(t) : 0));
|
||||
// Its header claims twice the real byte rate, as some speech generators write it.
|
||||
wav("misstated-rate.wav", 8, tone, RATE * 4);
|
||||
// Not audio at all: the render's audio probe rejects it and the whole mix fails.
|
||||
writeFileSync(join(dir, "notes.wav"), "RIFF but not really a wave file");
|
||||
// A 1x1 PNG: it probes, but as a still image, so render refuses it as an <audio> source.
|
||||
@@ -176,6 +183,10 @@ describe.skipIf(!ffmpeg)("inspectAudio", () => {
|
||||
// Played at double speed, 2 s of film reach 4 s into the file, past the silent start.
|
||||
expect(await codes(media([clip("late.wav", { end: 2, playbackRate: 2 })]))).toEqual([]);
|
||||
});
|
||||
|
||||
it("hears a clip past the middle of a WAV whose header misstates its byte rate, as render plays it", async () => {
|
||||
expect(await codes(media([clip("misstated-rate.wav", { mediaStart: 5 })]))).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("inspectAudio on a project of many clips trimmed from few files", () => {
|
||||
|
||||
@@ -67,6 +67,7 @@ async function silentPart(
|
||||
sampleRate: probed.sampleRate,
|
||||
channels: probed.channels,
|
||||
duration: probed.durationSeconds,
|
||||
misstatedByteRate: probed.misstatedByteRate,
|
||||
},
|
||||
{ mono: true, sampleRate: SILENCE_SAMPLE_RATE },
|
||||
);
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
videoStreamHasAlpha,
|
||||
} from "@hyperframes/parsers/media-alpha";
|
||||
import { urlInputArgs } from "./urlInput.js";
|
||||
import { decodableAudioStream } from "@hyperframes/parsers/media-streams";
|
||||
import { decodableAudioStream, misstatesPcmByteRate } from "@hyperframes/parsers/media-streams";
|
||||
import { runFfmpeg } from "./runFfmpeg.js";
|
||||
import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js";
|
||||
import { ManagedChildProcess } from "./managedChildProcess.js";
|
||||
@@ -1351,16 +1351,6 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi
|
||||
};
|
||||
}
|
||||
|
||||
function misstatesPcmByteRate(stream: FFProbeStream): boolean {
|
||||
const declared = Number(stream.bit_rate);
|
||||
const { channels, bits_per_sample: bits } = stream;
|
||||
const rate = Number(stream.sample_rate);
|
||||
if (!stream.codec_name?.startsWith("pcm_") || !declared || !channels || !bits || !rate) {
|
||||
return false;
|
||||
}
|
||||
return declared !== rate * channels * bits;
|
||||
}
|
||||
|
||||
/** A stream starting after the earliest of these plays that much after it. */
|
||||
function audioVideoStreamStarts(output: FFProbeOutput | null): number[] {
|
||||
return (output?.streams ?? [])
|
||||
|
||||
@@ -3,6 +3,24 @@ export interface FfprobeStreamCodec {
|
||||
codec_name?: string;
|
||||
}
|
||||
|
||||
export interface FfprobePcmFields extends FfprobeStreamCodec {
|
||||
sample_rate?: string;
|
||||
channels?: number;
|
||||
bit_rate?: string;
|
||||
bits_per_sample?: number;
|
||||
}
|
||||
|
||||
/** PCM whose header byte rate is not sample rate x channels x sample size: a seek by it lands wrong. */
|
||||
export function misstatesPcmByteRate(stream: FfprobePcmFields): boolean {
|
||||
const declared = Number(stream.bit_rate);
|
||||
const { channels, bits_per_sample: bits } = stream;
|
||||
const rate = Number(stream.sample_rate);
|
||||
if (!stream.codec_name?.startsWith("pcm_") || !declared || !channels || !bits || !rate) {
|
||||
return false;
|
||||
}
|
||||
return declared !== rate * channels * bits;
|
||||
}
|
||||
|
||||
/** The audio track ffmpeg can decode, as N in `-map 0:a:N`; ffprobe names no codec for one it cannot. */
|
||||
export function decodableAudioStream(streams: readonly FfprobeStreamCodec[]): number | null {
|
||||
const index = streams
|
||||
|
||||
Reference in New Issue
Block a user