ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-10-30 17:27:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: phone videos with an undecodable first audio track check, preview and render (#6551)

Browse Source 
* fix: phone videos with an undecodable first audio track check, preview and render

* fix(engine): a rate-lane clip bakes its ramp from the decodable audio track

* test: phone-clip tests end ffprobe options before the input
This commit is contained in:
Miguel Ángel
2026-10-27 13:09:02 +00:00
committed by GitHub
parent 41a7bc98f7
commit cfef528d3a
35 changed files with 606 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli/src/audio/analysis/decode.ts
+15 -7
View File
@@ -1,11 +1,13 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { decodableAudioStream, type FfprobeStreamCodec } from "@hyperframes/parsers/media-streams";
import type { AudioSignal } from "@hyperframes/core/audio-analysis";
const run = promisify(execFile);
export interface AudioStreamInfo {
audioStream: number;
sampleRate: number;
channels: number;
duration: number;
@@ -17,7 +19,7 @@ function requiredBinary(name: "ffmpeg" | "ffprobe"): string {
return path;
}
/** The first audio stream's format, or `null` when the file has no audio. */
/** The format of the audio track ffmpeg can decode, or `null` when the file has none. */
export async function probeAudio(file: string): Promise<AudioStreamInfo | null> {
const { stdout } = await run(
requiredBinary("ffprobe"),
@@ -25,7 +27,7 @@ export async function probeAudio(file: string): Promise<AudioStreamInfo | null>
"-v",
"error",
"-select_streams",
"a:0",
"a",
"-show_streams",
"-show_format",
"-of",
@@ -36,12 +38,18 @@ export async function probeAudio(file: string): Promise<AudioStreamInfo | null>
{ maxBuffer: 8 * 1024 * 1024, windowsHide: true },
);
const parsed = JSON.parse(stdout) as {
streams?: { sample_rate?: string; channels?: number; duration?: string }[];
streams?: (FfprobeStreamCodec & {
sample_rate?: string;
channels?: number;
duration?: string;
})[];
format?: { duration?: string };
};
const stream = parsed.streams?.[0];
if (!stream) return null;
const index = decodableAudioStream(parsed.streams ?? []);
const stream = index === null ? undefined : parsed.streams?.[index];
if (index === null || !stream) return null;
return {
audioStream: index,
sampleRate: Number(stream.sample_rate),
channels: stream.channels ?? 1,
duration: Number(stream.duration ?? parsed.format?.duration ?? 0),
@@ -65,7 +73,7 @@ export interface Decoded {
stream: AudioStreamInfo;
}
/** Decode the first audio stream at its own sample rate, each channel kept apart; `null` when the file has none. */
/** Decode the decodable audio track at its own sample rate, channels kept apart; `null` when the file has none. */
export async function decodeAudio(
file: string,
window: DecodeWindow = {},
@@ -91,7 +99,7 @@ export async function decodeAudio(
file,
...length,
"-map",
"0:a:0",
`0:a:${probedInfo.audioStream}`,
...(format.mono ? ["-ac", "1"] : []),
...(format.sampleRate ? ["-ar", String(format.sampleRate)] : []),
"-f",
packages/cli/src/audio/read/signal.test.ts
+1 -1
View File
@@ -8,7 +8,7 @@ describe("read signal boundaries", () => {
it.each([NaN, Infinity, -Infinity])("rejects decoded non-finite PCM (%s)", async (value) => {
vi.mocked(decodeAudio).mockResolvedValue({
signal: { channels: [Float32Array.of(0, value, 0)], sampleRate: READ_RATE },
stream: { channels: 1, sampleRate: READ_RATE, duration: 3 / READ_RATE },
stream: { audioStream: 0, channels: 1, sampleRate: READ_RATE, duration: 3 / READ_RATE },
});
await expect(decodeMono("invalid.wav")).rejects.toThrow("non-finite audio samples");
});
packages/cli/src/audio/read/stems.ts
+7 -1
View File
@@ -14,6 +14,7 @@ import {
serviceFailure,
type ServiceAccess,
} from "../analysis/providers/shared.js";
import { probeAudio } from "../analysis/decode.js";
/**
* Upload 192 kb/s stereo MP3 through the service origin's direct-upload routes, then POST
@@ -87,10 +88,15 @@ async function job(response: Response): Promise<StemsJob> {
async function encoded(path: string, dir: string, signal: AbortSignal): Promise<string> {
const ffmpeg = findFfBinary("ffmpeg", { configuredMustExist: true });
if (!ffmpeg) throw new Error("ffmpeg is required to separate stems; run: npx hyperframes doctor");
const audio = await probeAudio(path);
if (!audio) throw new Error("the recording has no audio track ffmpeg can decode");
const out = join(dir, "upload.mp3");
await runFile(
ffmpeg,
["-v", "error", "-y", "-i", path, "-vn", "-ac", "2", "-ar", "44100", "-b:a", "192k", out],
[
...["-v", "error", "-y", "-i", path, "-map", `0:a:${audio.audioStream}`],
...["-vn", "-ac", "2", "-ar", "44100", "-b:a", "192k", out],
],
{ windowsHide: true, maxBuffer: 8 * 1024 * 1024, signal },
);
return out;
packages/cli/src/cloud/offload.ts
+37 -36
View File
@@ -27,6 +27,7 @@ import {
resolveLocalAssetCandidates,
} from "@hyperframes/parsers/asset-resolution";
import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
import { probeAudio } from "../audio/analysis/decode.js";
import { formatBytes } from "../ui/format.js";
import { type PlayedSegment, planTrim } from "./offloadTrim.js";
import {
@@ -293,22 +294,6 @@ function hdrColorArgs({ color }: VideoProbe): string[] {
];
}
async function hasAudioStream(ffprobe: string, path: string): Promise<boolean> {
const { stdout } = await execFileAsync(ffprobe, [
"-v",
"error",
"-select_streams",
"a",
"-show_entries",
"stream=index",
"-of",
"csv=p=0",
"--",
path,
]);
return stdout.trim() !== "";
}
// Seconds floored to the microsecond, so a frame-aligned cut never rounds past its first frame.
const seconds = (t: number) => (Math.floor(t * 1e6) / 1e6).toFixed(6);
@@ -317,32 +302,48 @@ async function segmentArgs(
sourcePath: string,
decoder: string[],
segments: PlayedSegment[] | undefined,
audioOnly: boolean,
media: "audio" | "image" | "video",
): Promise<string[]> {
if (!segments?.length) return [...decoder, "-i", sourcePath];
const inputs = segments.flatMap((s) => [
...decoder,
"-ss",
seconds(s.start),
"-t",
seconds(s.end - s.start),
"-i",
sourcePath,
]);
if (segments.length === 1) return inputs;
const audio = audioOnly || (await hasAudioStream(findFFprobe()!, sourcePath));
return [...inputs, ...concatArgs(segments.length, !audioOnly, audio)];
if (media === "image") return [...decoder, "-i", sourcePath];
const audio = await keptAudio(sourcePath, media);
const cuts = segments?.length ? segments : [undefined];
const inputs = cuts.flatMap((s) => [...decoder, ...seekArgs(s), "-i", sourcePath]);
const video = media === "video";
return [
...inputs,
...(cuts.length > 1 ? concatArgs(cuts.length, video, audio) : streamMaps(video, audio)),
];
}
function concatArgs(count: number, video: boolean, audio: boolean): string[] {
const pad = (i: number) => (video ? `[${i}:v]` : "") + (audio ? `[${i}:a]` : "");
async function keptAudio(sourcePath: string, media: "audio" | "video"): Promise<number | null> {
const audio = (await probeAudio(sourcePath))?.audioStream ?? null;
if (media === "audio" && audio === null) {
throw new Error("the file has no audio track ffmpeg can decode");
}
return audio;
}
const seekArgs = (s: PlayedSegment | undefined) =>
s ? ["-ss", seconds(s.start), "-t", seconds(s.end - s.start)] : [];
const streamMaps = (video: boolean, audio: number | null) => [
...(video ? ["-map", "0:V:0"] : []),
...(audio === null ? [] : ["-map", `0:a:${audio}`]),
];
const mediaOf = (preset: PresetName) =>
preset === "audio" || preset === "image" ? preset : ("video" as const);
function concatArgs(count: number, video: boolean, audio: number | null): string[] {
const pad = (i: number) =>
(video ? `[${i}:v]` : "") + (audio === null ? "" : `[${i}:a:${audio}]`);
const pads = Array.from({ length: count }, (_, i) => pad(i)).join("");
const flags = `n=${count}:v=${video ? 1 : 0}:a=${audio ? 1 : 0}`;
const flags = `n=${count}:v=${video ? 1 : 0}:a=${audio === null ? 0 : 1}`;
return [
"-filter_complex",
`${pads}concat=${flags}${video ? "[v]" : ""}${audio ? "[a]" : ""}`,
`${pads}concat=${flags}${video ? "[v]" : ""}${audio === null ? "" : "[a]"}`,
...(video ? ["-map", "[v]"] : []),
...(audio ? ["-map", "[a]"] : []),
...(audio === null ? [] : ["-map", "[a]"]),
];
}
@@ -366,7 +367,7 @@ async function ffmpegReencode(
"-v",
"error",
"-y",
...(await segmentArgs(sourcePath, decoder, segments, presetName === "audio")),
...(await segmentArgs(sourcePath, decoder, segments, mediaOf(presetName))),
...preset.args,
...color,
...preset.audio,
packages/cli/src/cloud/offload.undecodableTrack.test.ts
+84
View File
@@ -0,0 +1,84 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { decodeAudio } from "../audio/analysis/decode.js";
import { buildPublishFileMap } from "../utils/publishProject.js";
import { OFFLOAD_MIN_BYTES, offloadHeavyMedia, type UploadableFile } from "./offload.js";
const ffmpeg = findFfBinary("ffmpeg");
const ffprobe = findFfBinary("ffprobe");
const dir = mkdtempSync(join(tmpdir(), "hf-offload-phone-"));
/** A phone recording's shape: a 4-channel track whose codec ffmpeg cannot decode, then a mono tone. An mkv
* name (the demuxer reads the content) and a `free` box past the cutoff send it through a re-encode. */
function phoneClip(): Buffer {
const made = join(dir, "made.mov");
execFileSync(ffmpeg!, [
...["-v", "error", "-f", "lavfi", "-i", "testsrc2=s=160x90:d=1"],
...["-f", "lavfi", "-i", "sine=f=300:d=1,pan=4c|c0=c0|c1=c0|c2=c0|c3=c0"],
...["-f", "lavfi", "-i", "sine=f=1000:d=1", "-map", "0:v", "-map", "1:a", "-map", "2:a"],
...["-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "pcm_s16le", made],
]);
const bytes = readFileSync(made);
bytes.write("apac", bytes.indexOf("sowt", bytes.indexOf("moov")));
const free = Buffer.alloc(OFFLOAD_MIN_BYTES);
free.writeUInt32BE(free.length);
free.write("free", 4);
return Buffer.concat([bytes, free]);
}
const audioChannels = (path: string) =>
execFileSync(
ffprobe!,
[
"-v",
"error",
"-select_streams",
"a",
"-show_entries",
"stream=channels",
"-of",
"csv=p=0",
"--",
path,
],
{ encoding: "utf8" },
).trim();
afterAll(() => rmSync(dir, { recursive: true, force: true }));
describe.skipIf(!ffmpeg || !ffprobe)(
"a phone clip whose first audio track ffmpeg cannot decode",
() => {
it("decodes the track ffmpeg can", async () => {
writeFileSync(join(dir, "phone.mov"), phoneClip());
const decoded = await decodeAudio(join(dir, "phone.mov"));
expect(decoded?.stream).toMatchObject({ audioStream: 1, channels: 1 });
});
it("re-encodes it for upload with that track", async () => {
const project = mkdtempSync(join(dir, "project-"));
writeFileSync(
join(project, "index.html"),
`<html><body><video src="phone.mkv"></video></body></html>`,
);
writeFileSync(join(project, "phone.mkv"), phoneClip());
const uploads: UploadableFile[] = [];
await offloadHeavyMedia(project, buildPublishFileMap(project), {
apiBaseUrl: "https://api.example",
cachePath: join(project, "cache.json"),
upload: async (file) => {
uploads.push(file);
return { asset_id: "a", url: "https://assets.example/a.mp4" };
},
});
const [upload] = uploads;
if (!upload || !("bytes" in upload)) throw new Error("nothing re-encoded");
writeFileSync(join(project, upload.filename), upload.bytes);
expect(audioChannels(join(project, upload.filename))).toBe("1");
}, 60_000);
},
);
packages/cli/src/commands/init.transcodeToMp4.windowsHide.test.ts
+4 -1
View File
@@ -3,7 +3,10 @@ import { describe, expect, it, vi } from "vitest";
const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn() }));
vi.mock("node:child_process", () => ({ spawn: spawnMock }));
vi.mock("node:child_process", async (importOriginal) => ({
...(await importOriginal<typeof import("node:child_process")>()),
spawn: spawnMock,
}));
vi.mock("../browser/ffmpeg.js", () => ({
findFFmpeg: () => "/fake/bin/ffmpeg",
findFFprobe: () => "/fake/bin/ffprobe",
packages/cli/src/commands/init.ts
+2 -1
View File
@@ -57,6 +57,7 @@ import {
HTML_BODY_CSS_WIDTH_FIRST_RE,
VIEWPORT_META_SIZE_RE,
} from "@hyperframes/parsers";
import { decodableAudioStream } from "@hyperframes/parsers/media-streams";
function resolveScaffoldTemplateId(exampleFlag: string | undefined, hasMediaFile: boolean): string {
const example = exampleFlag === "agent" ? "blank" : exampleFlag;
@@ -144,7 +145,7 @@ function probeVideo(filePath: string): VideoMeta | undefined {
const videoStream = streams.find((s) => s.codec_type === "video");
if (!videoStream) return undefined;
const hasAudio = streams.some((s) => s.codec_type === "audio");
const hasAudio = decodableAudioStream(streams) !== null;
let fps = 30;
const fpsStr = videoStream.avg_frame_rate ?? videoStream.r_frame_rate;
packages/cli/src/commands/normalize-audio.test.ts
+2 -2
View File
@@ -73,14 +73,14 @@ describe("loudnessMeasureArgs", () => {
// bounding: ffmpeg keeps feeding the graph and ebur128 integrates audio the
// clip never plays. Measured on a fixture whose played window is -61.8 LUFS,
// output-side `-t` reported -33.8.
const args = loudnessMeasureArgs("clip.wav", { mediaStart: 2, duration: 4 });
const args = loudnessMeasureArgs("clip.wav", { mediaStart: 2, duration: 4 }, 0);
expect(args.indexOf("-ss")).toBeLessThan(args.indexOf("-i"));
expect(args.indexOf("-t")).toBeLessThan(args.indexOf("-i"));
expect(args.slice(args.indexOf("-ss"), args.indexOf("-i"))).toEqual(["-ss", "2", "-t", "4"]);
});
it("leaves an untrimmed clip unbounded", () => {
const args = loudnessMeasureArgs("clip.wav", { mediaStart: 0, duration: null });
const args = loudnessMeasureArgs("clip.wav", { mediaStart: 0, duration: null }, 0);
expect(args).not.toContain("-ss");
expect(args).not.toContain("-t");
});
packages/cli/src/utils/checkAudio.ts
+6 -1
View File
@@ -62,7 +62,12 @@ async function silentPart(
const decoded = await decodeAudio(
path,
{ from, seconds },
{ sampleRate: probed.sampleRate, channels: probed.channels, duration: probed.durationSeconds },
{
audioStream: probed.audioStream,
sampleRate: probed.sampleRate,
channels: probed.channels,
duration: probed.durationSeconds,
},
{ mono: true, sampleRate: SILENCE_SAMPLE_RATE },
);
const samples = decoded?.signal.channels[0]?.length ?? 0;
packages/cli/src/whisper/transcribe.ts
+20 -2
View File
@@ -24,6 +24,8 @@ import { TRANSCRIPT_FILE } from "./transcriptFile.js";
import { findWavChunk } from "./wav.js";
import type { Word } from "./normalize.js";
import { emitWords } from "./progress.js";
import { probeAudio } from "../audio/analysis/decode.js";
import { decodableAudioStream } from "@hyperframes/parsers/media-streams";
export { initialModelForLanguage };
@@ -351,10 +353,24 @@ async function extractAudio(videoPath: string, signal?: AbortSignal): Promise<st
`ffmpeg is required to extract audio from video. Install: ${getFFmpegInstallHint()}`,
);
}
const audio = await probeAudio(videoPath);
if (!audio) throw new Error("the file has no audio track ffmpeg can decode");
const wavPath = tempWavPath();
await runFfmpeg(
ffmpegPath,
["-i", videoPath, "-vn", "-ar", "16000", "-ac", "1", "-f", "wav"],
[
"-i",
videoPath,
"-map",
`0:a:${audio.audioStream}`,
"-vn",
"-ar",
"16000",
"-ac",
"1",
"-f",
"wav",
],
wavPath,
resolveAudioPreparationTimeoutMs(getMediaDurationSeconds(videoPath)),
signal,
@@ -389,7 +405,9 @@ function isWav16kMono(filePath: string): boolean {
{ encoding: "utf-8", timeout: 10_000 },
);
const parsed: { streams?: AudioStream[] } = JSON.parse(raw);
return isPcm16kMono(parsed.streams?.find((s) => s.codec_type === "audio"));
const audio = (parsed.streams ?? []).filter((s) => s.codec_type === "audio");
const index = decodableAudioStream(audio);
return isPcm16kMono(index === null ? undefined : audio[index]);
} catch {
return false;
}
packages/engine/src/services/audioMixer.order.test.ts
+1 -1
View File
@@ -24,7 +24,7 @@ vi.mock("./aacEncode.js", async (importOriginal) => ({
}));
vi.mock("../utils/ffprobe.js", () => ({
extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2 })),
extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2, audioStream: 0 })),
}));
import { processCompositionAudio, type AudioElement } from "./audioMixer.js";
packages/engine/src/services/audioMixer.test.ts
+2
View File
@@ -20,6 +20,7 @@ const { runFfmpegMock, capturedFilterScripts, extractAudioMetadataMock } = vi.ho
sampleRate: 48_000,
channels: 2,
audioCodec: "aac",
audioStream: 0,
})),
runFfmpegMock: vi.fn(async (args: string[]) => {
const legacyIdx = args.indexOf("-filter_complex_script");
@@ -162,6 +163,7 @@ describe("processCompositionAudio", () => {
sampleRate: 48_000,
channels: 2,
audioCodec: "aac",
audioStream: 0,
});
applyAudioFxChainMock.mockClear();
capturedFilterScripts.length = 0;
packages/engine/src/services/audioMixer.ts
+19 -4
View File
@@ -130,11 +130,12 @@ function buildRampFilterComplex(
duration: number,
headFilter: string | null,
tailFilter: string | null,
input: string,
): string {
const slices = Math.min(MAX_RAMP_SLICES, Math.max(1, Math.ceil(duration / RAMP_SLICE_SECONDS)));
const step = duration / slices;
const split = Array.from({ length: slices }, (_, i) => `[s${i}]`).join("");
const parts = [`[0:a]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`];
const parts = [`[${input}]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`];
for (let i = 0; i < slices; i += 1) {
const from = sourceTimeAt(lane, i * step);
const to = sourceTimeAt(lane, (i + 1) * step);
@@ -160,16 +161,23 @@ function preparedAudioOutputArgs(
playbackRate: RateSpec,
duration: number,
headFilter: string | null,
stream: string | null,
): string[] {
const channelFilter = metadata?.channels === 1 ? STEREO_CHANNEL_FILTER : null;
const outputArgs = channelFilter ? [] : ["-ac", "2"];
if (typeof playbackRate === "object") {
const graph = buildRampFilterComplex(playbackRate, duration, headFilter, channelFilter);
const graph = buildRampFilterComplex(
playbackRate,
duration,
headFilter,
channelFilter,
stream ?? "0:a",
);
return ["-filter_complex", graph, "-map", "[out]", ...outputArgs];
}
const filters = [headFilter, channelFilter, buildAtempoFilter(playbackRate)].filter(Boolean);
if (filters.length > 0) outputArgs.push("-af", filters.join(","));
return outputArgs;
return [...(stream ? ["-map", stream] : []), ...outputArgs];
}
function escapeExpressionCommas(expression: string): string {
@@ -719,7 +727,13 @@ async function extractAudioSegment(
{ padToDuration: true },
);
if (!cycle.success) return { ...cycle, outputPath };
const outputArgs = preparedAudioOutputArgs(metadata, normalizedPlaybackRate, duration, null);
const outputArgs = preparedAudioOutputArgs(
metadata,
normalizedPlaybackRate,
duration,
null,
null,
);
const result = await runFfmpeg(
[
"-stream_loop",
@@ -772,6 +786,7 @@ async function extractAudioSegment(
]
.filter(Boolean)
.join(",") || null,
metadata ? `0:a:${metadata.audioStream}` : null,
);
const readFrom = fromSidecar ? audioSidecarUrl : srcPath;
packages/engine/src/services/audioMixer.undecodableTrack.test.ts
+107
View File
@@ -0,0 +1,107 @@
import { spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
import { extractAudioMetadata, extractMediaMetadata } from "../utils/ffprobe.js";
import { parseAudioElements, processCompositionAudio } from "./audioMixer.js";
const FFMPEG = getFfmpegBinary();
const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"]).status === 0;
const SAMPLE_RATE = 48_000;
function ffmpeg(args: string[]): Buffer {
const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args]);
if (result.status !== 0) throw new Error(result.stderr.toString());
return result.stdout;
}
/** A phone recording's shape: its first audio track names a codec ffmpeg has no decoder for (an iPhone's
* spatial track), so ffmpeg's own pick of the 4-channel track fails; `tracks` are the sine tones after it. */
function phoneClip(path: string, tracks: number[]): void {
const tones = tracks.flatMap((hz) => ["-f", "lavfi", "-i", `sine=f=${hz}:r=${SAMPLE_RATE}:d=3`]);
ffmpeg([
...["-f", "lavfi", "-i", "color=size=64x36:rate=10:duration=3"],
...["-f", "lavfi", "-i", `sine=f=300:r=${SAMPLE_RATE}:d=3,pan=4c|c0=c0|c1=c0|c2=c0|c3=c0`],
...tones,
...["-map", "0:v", "-map", "1:a", ...tracks.flatMap((_, i) => ["-map", `${i + 2}:a`])],
...["-c:v", "mpeg4", "-c:a", "pcm_s16le", `${path}.tmp.mov`],
]);
// PCM carries no codec config, so its sample entry's fourcc alone names the codec.
const bytes = readFileSync(`${path}.tmp.mov`);
const at = bytes.indexOf("sowt", bytes.indexOf("moov"));
bytes.write("apac", at);
writeFileSync(path, bytes);
}
function loudestHz(pcm: Buffer): number {
let crossings = 0;
for (let i = 1; i < pcm.length / 2; i += 1) {
if (pcm.readInt16LE((i - 1) * 2) < 0 && pcm.readInt16LE(i * 2) >= 0) crossings += 1;
}
return Math.round(crossings / (pcm.length / 2 / SAMPLE_RATE) / 100) * 100;
}
describe.skipIf(!HAS_FFMPEG)("a source whose first audio track ffmpeg cannot decode", () => {
let dir: string;
beforeAll(() => {
dir = mkdtempSync(join(tmpdir(), "hf-audio-undecodable-"));
phoneClip(join(dir, "phone.mov"), [1000]);
phoneClip(join(dir, "spatial-only.mov"), []);
});
afterAll(() => {
if (dir) rmSync(dir, { recursive: true, force: true });
});
it("mixes the track ffmpeg can decode", async () => {
const work = mkdtempSync(join(dir, "work-"));
const output = `${work}.m4a`;
const html =
'<video id="clip" src="phone.mov" data-start="0" data-duration="2" data-has-audio="true"></video>';
const result = await processCompositionAudio(parseAudioElements(html), dir, work, output, 2);
expect(result, result.error).toMatchObject({ success: true, tracksProcessed: 1 });
const pcm = ffmpeg(["-i", output, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", "-"]);
expect(loudestHz(pcm.subarray(SAMPLE_RATE, SAMPLE_RATE * 3))).toBe(1000);
});
it("bakes a rate lane from that track", async () => {
const work = mkdtempSync(join(dir, "work-"));
const output = `${work}.m4a`;
const rate = {
target: "rate" as const,
points: [
{ t: 0, v: 1 },
{ t: 2, v: 1.5 },
],
};
const clip = {
id: "clip",
src: "phone.mov",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
};
const result = await processCompositionAudio(
[{ ...clip, playbackRate: rate, type: "video" }],
dir,
work,
output,
2,
);
expect(result, result.error).toMatchObject({ success: true, tracksProcessed: 1 });
const pcm = ffmpeg(["-i", output, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", "-"]);
expect(loudestHz(pcm.subarray(SAMPLE_RATE, SAMPLE_RATE * 3))).toBe(1000);
});
it("probes that track, and counts a clip with only the undecodable one as silent", async () => {
const audio = await extractAudioMetadata(join(dir, "phone.mov"));
expect([audio.audioStream, audio.channels, audio.audioStreamCount]).toEqual([1, 1, 2]);
expect((await extractMediaMetadata(join(dir, "phone.mov"))).hasAudio).toBe(true);
expect((await extractMediaMetadata(join(dir, "spatial-only.mov"))).hasAudio).toBe(false);
});
});
packages/engine/src/utils/ffprobe.test.ts
+3 -3
View File
@@ -317,7 +317,7 @@ describe("probeMediaProfile", () => {
kind: "exit",
code: 0,
stdout: JSON.stringify({
streams: [{ codec_type: "video" }, { codec_type: "audio" }],
streams: [{ codec_type: "video" }, { codec_type: "audio", codec_name: "opus" }],
format: { format_name: "matroska,webm" },
}),
},
@@ -360,7 +360,7 @@ describe("probeMediaProfile", () => {
stdout: JSON.stringify({
streams: [
{ codec_type: "video", disposition: { attached_pic: 1 } },
{ codec_type: "audio" },
{ codec_type: "audio", codec_name: "mp3" },
],
format: { format_name: "mp3" },
}),
@@ -494,7 +494,7 @@ describe("probeMediaProfile", () => {
streams: [
{ codec_type: "video", disposition: { attached_pic: 1 } },
{ codec_type: "video", codec_name: "h264" },
{ codec_type: "audio" },
{ codec_type: "audio", codec_name: "mp3" },
],
format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" },
}),
packages/engine/src/utils/ffprobe.ts
+14 -6
View File
@@ -12,6 +12,7 @@ import {
videoStreamHasAlpha,
} from "@hyperframes/parsers/media-alpha";
import { urlInputArgs } from "./urlInput.js";
import { decodableAudioStream } from "@hyperframes/parsers/media-streams";
import { runFfmpeg } from "./runFfmpeg.js";
import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js";
import { ManagedChildProcess } from "./managedChildProcess.js";
@@ -243,6 +244,8 @@ export interface AudioMetadata {
latestStreamLeadSeconds?: number;
audioStreamLeadSeconds?: number;
audioStreamCount?: number;
/** The track read for these facts and for the mix, by its place among the audio streams (decodableAudioStream). */
audioStream: number;
}
interface FFProbeStream {
@@ -634,7 +637,7 @@ export async function probeMediaProfile(
}
return {
hasVideoStream: videoStreams.length > 0,
hasAudioStream: output.streams.some((stream) => stream.codec_type === "audio"),
hasAudioStream: decodableAudioStream(output.streams) !== null,
visualKind: isStillImage ? "still" : hasMovingVideoStream ? "moving" : "none",
};
} catch (error) {
@@ -1002,7 +1005,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
height: videoStream.height || stillImage()?.height || 0,
fps,
videoCodec: videoStream.codec_name || "unknown",
hasAudio: output?.streams.some((s) => s.codec_type === "audio") ?? false,
hasAudio: decodableAudioStream(output?.streams ?? []) !== null,
isVFR,
hasAlpha,
...(alphaPlaneDecoded === undefined ? {} : { alphaPlaneDecoded }),
@@ -1209,8 +1212,10 @@ export async function extractAudioMetadata(
async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promise<AudioMetadata> {
const output = await probeMediaOutput(filePath, signal);
const audioStream = output.streams.find((s) => s.codec_type === "audio");
if (!audioStream) throw new Error("[FFmpeg] No audio stream found");
const audioStreams = output.streams.filter((s) => s.codec_type === "audio");
const audioStreamIndex = decodableAudioStream(output.streams);
const audioStream = audioStreamIndex === null ? undefined : audioStreams[audioStreamIndex];
if (audioStreamIndex === null || !audioStream) throw new Error("[FFmpeg] No audio stream found");
let durationSeconds = output.format.duration ? parseFloat(output.format.duration) : 0;
const streamDuration = audioStream.duration ? parseFloat(audioStream.duration) : undefined;
@@ -1221,6 +1226,7 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi
if (sampleRate > 0) {
const decoded = await probeDecodedAudioDuration(
filePath,
audioStreamIndex,
sampleRate,
audioStartSeconds(audioStream, output.format),
durationSeconds,
@@ -1240,7 +1246,8 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi
bitrate: output.format.bit_rate ? parseInt(output.format.bit_rate) : undefined,
latestStreamLeadSeconds: latestStreamLeadSeconds(output),
audioStreamLeadSeconds: streamLeadSeconds(output, audioStream),
audioStreamCount: output.streams.filter((s) => s.codec_type === "audio").length,
audioStreamCount: audioStreams.length,
audioStream: audioStreamIndex,
};
}
@@ -1291,6 +1298,7 @@ function reconcileAudioDuration(
*/
async function probeDecodedAudioDuration(
filePath: string,
audioStream: number,
sampleRate: number,
startSeconds: number,
containerDurationSeconds: number,
@@ -1301,7 +1309,7 @@ async function probeDecodedAudioDuration(
): Promise<{ timestamp: number; nbSamples: number } | undefined> => {
const args = [
"-select_streams",
"a:0",
`a:${audioStream}`,
"-show_entries",
"frame=best_effort_timestamp_time,nb_samples",
"-of",
packages/parsers/package-subpaths.json
+6
View File
@@ -122,6 +122,12 @@
"types": "./dist/mediaAlpha.d.ts",
"environments": ["browser", "bun", "node"]
},
"./media-streams": {
"source": "./src/mediaStreams.ts",
"runtime": "./dist/mediaStreams.js",
"types": "./dist/mediaStreams.d.ts",
"environments": ["browser", "bun", "node"]
},
"./asset-resolution": {
"source": "./src/assetResolution.ts",
"runtime": "./dist/assetResolution.js",
packages/parsers/package.json
+10
View File
@@ -130,6 +130,12 @@
"import": "./src/mediaAlpha.ts",
"types": "./src/mediaAlpha.ts"
},
"./media-streams": {
"bun": "./src/mediaStreams.ts",
"node": "./dist/mediaStreams.js",
"import": "./src/mediaStreams.ts",
"types": "./src/mediaStreams.ts"
},
"./asset-resolution": {
"bun": "./src/assetResolution.ts",
"node": "./dist/assetResolution.js",
@@ -247,6 +253,10 @@
"import": "./dist/mediaAlpha.js",
"types": "./dist/mediaAlpha.d.ts"
},
"./media-streams": {
"import": "./dist/mediaStreams.js",
"types": "./dist/mediaStreams.d.ts"
},
"./asset-resolution": {
"import": "./dist/assetResolution.js",
"types": "./dist/assetResolution.d.ts"
packages/parsers/src/mediaStreams.test.ts
+23
View File
@@ -0,0 +1,23 @@
import { describe, expect, it } from "vitest";
import { decodableAudioStream } from "./mediaStreams";
describe("decodableAudioStream", () => {
it("skips an audio track ffprobe names no codec for, counting only audio streams", () => {
const phone = [
{ codec_type: "video", codec_name: "hevc" },
{ codec_type: "audio" },
{ codec_type: "audio", codec_name: "aac" },
];
expect(decodableAudioStream(phone)).toBe(1);
});
it("takes the first decodable track, and none when no audio can be decoded", () => {
const two = [
{ codec_type: "audio", codec_name: "aac" },
{ codec_type: "audio", codec_name: "ac3" },
];
expect(decodableAudioStream(two)).toBe(0);
expect(decodableAudioStream([{ codec_type: "video", codec_name: "h264" }])).toBeNull();
expect(decodableAudioStream([{ codec_type: "audio" }])).toBeNull();
});
});
packages/parsers/src/mediaStreams.ts
+12
View File
@@ -0,0 +1,12 @@
export interface FfprobeStreamCodec {
codec_type?: string;
codec_name?: string;
}
/** The audio track ffmpeg can decode, as N in `-map 0:a:N`; ffprobe names no codec for one it cannot. */
export function decodableAudioStream(streams: readonly FfprobeStreamCodec[]): number | null {
const index = streams
.filter((stream) => stream.codec_type === "audio")
.findIndex((stream) => stream.codec_name !== undefined);
return index < 0 ? null : index;
}
packages/parsers/tsup.config.ts
+1
View File
@@ -22,6 +22,7 @@ export default defineConfig({
subCompositionValidity: "src/subCompositionValidity.ts",
ffBinaries: "src/ffBinaries.ts",
mediaAlpha: "src/mediaAlpha.ts",
mediaStreams: "src/mediaStreams.ts",
assetResolution: "src/assetResolution.ts",
htmlAttributeSpans: "src/htmlAttributeSpans.ts",
timeSpans: "src/timeSpans.ts",
packages/studio-server/src/helpers/childProcess.windowsHide.test.ts
+21 -5
View File
@@ -42,12 +42,28 @@ describe("Studio child-process options", () => {
it("hides the waveform FFmpeg console window", async () => {
const proc = new EventEmitter() as EventEmitter & { stdout: EventEmitter };
proc.stdout = new EventEmitter();
spawnMock.mockReturnValue(proc);
spawnMock.mockImplementation(() => {
queueMicrotask(() => {
proc.stdout.emit("data", Buffer.from(new Float32Array([0.5]).buffer));
proc.emit("close", 0);
});
return proc;
});
execFileMock.mockImplementation(
(
_command: string,
_args: string[],
_options: unknown,
callback: (...args: unknown[]) => void,
) =>
callback(
null,
JSON.stringify({ streams: [{ codec_type: "audio", codec_name: "aac" }] }),
"",
),
);
const peaks = decodeAudioPeaks("/tmp/audio.wav");
proc.stdout.emit("data", Buffer.from(new Float32Array([0.5]).buffer));
proc.emit("close", 0);
await peaks;
await decodeAudioPeaks("/tmp/audio.wav");
expect(spawnMock.mock.calls[0]?.[2]).toEqual(expect.objectContaining({ windowsHide: true }));
});
packages/studio-server/src/helpers/loudness.test.ts
+13 -2
View File
@@ -34,7 +34,7 @@ describe("audioTags playback offsets", () => {
const [tag] = audioTags(html);
expect(tag).toBeDefined();
if (!tag) throw new Error("missing video");
const args = loudnessMeasureArgs("talk.mp4", tag);
const args = loudnessMeasureArgs("talk.mp4", tag, 0);
expect(args.slice(args.indexOf("-ss"), args.indexOf("-i"))).toEqual(["-ss", "4", "-t", "4"]);
});
@@ -129,9 +129,20 @@ describe("audioTags with HTML character references", () => {
});
});
describe("loudnessMeasureArgs track", () => {
it("measures the audio track it is given, not ffmpeg's first", () => {
const args = loudnessMeasureArgs("phone.mov", { mediaStart: 0, duration: null }, 1);
expect(args.slice(args.indexOf("-map"), args.indexOf("-map") + 2)).toEqual(["-map", "0:a:1"]);
});
});
describe("loudnessMeasureArgs with a playback rate", () => {
it("measures the source span a sped-up clip actually plays", () => {
const args = loudnessMeasureArgs("clip.mp4", { mediaStart: 0, duration: 4, playbackRate: 2 });
const args = loudnessMeasureArgs(
"clip.mp4",
{ mediaStart: 0, duration: 4, playbackRate: 2 },
0,
);
expect(args.slice(args.indexOf("-t"), args.indexOf("-t") + 2)).toEqual(["-t", "8"]);
});
packages/studio-server/src/helpers/loudness.ts
+17 -3
View File
@@ -7,6 +7,7 @@ import { decodeAuthoredAttribute } from "@hyperframes/parsers";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { readMediaOffsetSeconds } from "@hyperframes/parsers/media-duration";
import { requestedProjectPath } from "./requestSubPath.js";
import { probeAudioTrack } from "./mediaMetadata.js";
const execFileAsync = promisify(execFile);
@@ -454,12 +455,23 @@ export function updateAudioVolume(html: string, id: string, volume: number): str
* -61.8 LUFS: output-side `-t 4` reported -33.8 LUFS, input-side reports -61.8,
* the same as physically cutting the file first.
*/
export function loudnessMeasureArgs(file: string, tag: MeasuredWindow): string[] {
export function loudnessMeasureArgs(file: string, tag: MeasuredWindow, stream: number): string[] {
const args = ["-hide_banner", "-nostats"];
if (tag.mediaStart > 0) args.push("-ss", String(tag.mediaStart));
const sourceSpan = tag.duration === null ? null : tag.duration * (tag.playbackRate ?? 1);
if (sourceSpan !== null && sourceSpan > 0) args.push("-t", String(sourceSpan));
args.push("-i", file, "-map", "0:a:0", "-vn", "-af", "ebur128=peak=true", "-f", "null", "-");
args.push(
"-i",
file,
"-map",
`0:a:${stream}`,
"-vn",
"-af",
"ebur128=peak=true",
"-f",
"null",
"-",
);
return args;
}
@@ -471,7 +483,9 @@ export async function measureAudio(
file: string,
tag: MeasuredWindow,
): Promise<LoudnessMeasurement> {
const result = await execFileAsync(ffmpegPath, loudnessMeasureArgs(file, tag), {
const track = await probeAudioTrack(file);
if (!track) throw new Error("no audio stream to measure");
const result = await execFileAsync(ffmpegPath, loudnessMeasureArgs(file, tag, track.stream), {
encoding: "utf8",
maxBuffer: 16 * 1024 * 1024,
timeout: FFMPEG_TIMEOUT_MS,
packages/studio-server/src/helpers/loudness.windowsHide.test.ts
+16 -3
View File
@@ -2,8 +2,20 @@ import { describe, expect, it, vi } from "vitest";
const { execFileMock } = vi.hoisted(() => ({
execFileMock: vi.fn(
(_file: string, _args: string[], _options: unknown, callback: (error: Error) => void) =>
callback(new Error("ffmpeg stub")),
(
_file: string,
args: string[],
_options: unknown,
callback: (error: Error | null, stdout?: string, stderr?: string) => void,
) =>
// ffprobe finds a track to measure; ffmpeg's measurement then fails.
args.includes("-select_streams")
? callback(
null,
JSON.stringify({ streams: [{ codec_type: "audio", codec_name: "aac" }] }),
"",
)
: callback(new Error("ffmpeg stub")),
),
}));
@@ -20,6 +32,7 @@ describe("measureAudio child-process options", () => {
measureAudio("/fake/ffmpeg", "/fake/a.wav", { mediaStart: 0, duration: 1 }),
).rejects.toThrow();
expect(execFileMock.mock.calls[0]?.[2]).toEqual(expect.objectContaining({ windowsHide: true }));
const ffmpeg = execFileMock.mock.calls.find(([file]) => file === "/fake/ffmpeg");
expect(ffmpeg?.[2]).toEqual(expect.objectContaining({ windowsHide: true }));
});
});
packages/studio-server/src/helpers/mediaMetadata.test.ts
+24 -1
View File
@@ -93,7 +93,30 @@ describe("probeMediaMetadata", () => {
}),
stderr: "",
}));
expect(withAudio).toMatchObject({ kind: "video", hasAudio: true });
expect(withAudio).toMatchObject({ kind: "video", hasAudio: true, audioStream: 0 });
// A phone's spatial track names no codec ffmpeg can decode; the AAC track after it is the one to play.
const phone = await probeMediaMetadata("/tmp/phone.mov", () => ({
status: 0,
stdout: JSON.stringify({
streams: [
{ codec_type: "video", codec_name: "hevc" },
{ codec_type: "audio" },
{ codec_type: "audio", codec_name: "aac" },
],
}),
stderr: "",
}));
expect(phone).toMatchObject({ kind: "video", hasAudio: true, audioStream: 1 });
const spatialOnly = await probeMediaMetadata("/tmp/spatial.mov", () => ({
status: 0,
stdout: JSON.stringify({
streams: [{ codec_type: "video", codec_name: "hevc" }, { codec_type: "audio" }],
}),
stderr: "",
}));
expect(spatialOnly).toMatchObject({ kind: "video", hasAudio: false });
expect(spatialOnly).not.toHaveProperty("audioStream");
const silent = await probeMediaMetadata("/tmp/silent.mp4", () => ({
status: 0,
packages/studio-server/src/helpers/mediaMetadata.ts
+29 -2
View File
@@ -8,6 +8,7 @@ import {
} from "@hyperframes/core";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { videoStreamHasAlpha } from "@hyperframes/parsers/media-alpha";
import { decodableAudioStream, type FfprobeStreamCodec } from "@hyperframes/parsers/media-streams";
export interface FfprobeRunResult {
status: number | null;
@@ -73,8 +74,9 @@ export interface MediaColorMetadata {
export interface MediaMetadata {
kind: "video" | "image" | "audio" | "unknown";
color: MediaColorMetadata;
/** Video audio stream, when probed. Absent means the drop path stays muted. */
/** Video audio stream ffmpeg can decode, when probed. Absent means the drop path stays muted. */
hasAudio?: boolean;
audioStream?: number;
/** Video stream carries alpha (pix_fmt or the VP8/VP9 alpha_mode tag). */
hasAlpha?: boolean;
durationSeconds?: number;
@@ -234,6 +236,11 @@ type ProbeOutput = {
format?: { duration?: string; bit_rate?: string };
};
function audioOf(streams: FfprobeStream[]): Pick<MediaMetadata, "hasAudio" | "audioStream"> {
const audioStream = decodableAudioStream(streams);
return audioStream === null ? { hasAudio: false } : { hasAudio: true, audioStream };
}
function metadataFromProbe(kind: MediaMetadata["kind"], parsed: ProbeOutput): MediaMetadata {
const stream = parsed.streams?.find((item) => {
if (kind === "image") return item.codec_type === "video";
@@ -245,7 +252,7 @@ function metadataFromProbe(kind: MediaMetadata["kind"], parsed: ProbeOutput): Me
streams: (parsed.streams ?? []).slice(0, STREAM_SUMMARY_MAX).map(summarizeStream),
};
if (kind === "video") {
metadata.hasAudio = (parsed.streams ?? []).some((item) => item.codec_type === "audio");
Object.assign(metadata, audioOf(parsed.streams ?? []));
metadata.hasAlpha = stream !== undefined && videoStreamHasAlpha(stream);
const duration = Number(parsed.format?.duration);
if (Number.isFinite(duration) && duration > 0) metadata.durationSeconds = duration;
@@ -254,6 +261,26 @@ function metadataFromProbe(kind: MediaMetadata["kind"], parsed: ProbeOutput): Me
return metadata;
}
/** The audio track ffmpeg can decode in a file, with its channel count; null when it has none. */
export async function probeAudioTrack(
filePath: string,
runner: FfprobeRunner = execFileRunner,
): Promise<{ stream: number; channels: number } | null> {
const result = await runner(
findFfBinary("ffprobe") ?? "ffprobe",
[
...["-v", "error", "-select_streams", "a"],
...["-show_entries", "stream=codec_type,codec_name,channels", "-of", "json", "--", filePath],
],
{ timeout: 30_000 },
);
if (result.status !== 0) return null;
type Probed = { streams?: (FfprobeStreamCodec & { channels?: number })[] };
const streams = (JSON.parse(String(result.stdout || "{}")) as Probed).streams ?? [];
const stream = decodableAudioStream(streams);
return stream === null ? null : { stream, channels: streams[stream]?.channels ?? 0 };
}
export async function probeMediaMetadata(
filePath: string,
runner: FfprobeRunner = execFileRunner,
packages/studio-server/src/helpers/mediaValidation.test.ts
+13 -8
View File
@@ -51,14 +51,19 @@ describe("validateUploadedMedia", () => {
},
);
it("accepts audio files with an audio stream", () => {
expect(
validateUploadedMedia("/tmp/test.wav", () => ({
status: 0,
stdout: JSON.stringify({ streams: [{ codec_type: "audio" }] }),
stderr: "",
})),
).toEqual({ ok: true });
it("accepts an audio file only with an audio track ffmpeg can decode", () => {
const probed = (streams: object[]) => () => ({
status: 0,
stdout: JSON.stringify({ streams }),
stderr: "",
});
const decodable = probed([{ codec_type: "audio", codec_name: "pcm_s16le" }]);
expect(validateUploadedMedia("/tmp/test.wav", decodable)).toEqual({ ok: true });
// ffprobe names no codec for a track it cannot decode.
expect(validateUploadedMedia("/tmp/test.m4a", probed([{ codec_type: "audio" }]))).toEqual({
ok: false,
reason: "no supported audio stream found",
});
});
it("accepts the upload but says it was not checked when ffprobe is unavailable", () => {
packages/studio-server/src/helpers/mediaValidation.ts
+4 -3
View File
@@ -3,6 +3,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import { FFPROBE_PATH_ENV, findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { decodableAudioStream, type FfprobeStreamCodec } from "@hyperframes/parsers/media-streams";
const VIDEO_EXT = /\.(mp4|webm|mov|mkv|avi|m4v|mxf|mts|m2ts|ts)$/i;
const AUDIO_EXT = /\.(mp3|wav|ogg|m4a|aac|flac)$/i;
@@ -38,7 +39,7 @@ export function validateUploadedMedia(
if (!ffprobe) return { ok: true, unchecked: FFPROBE_MISSING };
const result = runner(
ffprobe,
["-v", "error", "-show_entries", "stream=codec_type", "-of", "json", "--", filePath],
["-v", "error", "-show_entries", "stream=codec_type,codec_name", "-of", "json", "--", filePath],
{ windowsHide: true },
);
@@ -60,10 +61,10 @@ export function validateUploadedMedia(
function checkStreams(stdout: string, isVideo: boolean, isAudio: boolean): MediaCheck {
try {
const parsed = JSON.parse(stdout) as { streams?: Array<{ codec_type?: string }> };
const parsed = JSON.parse(stdout) as { streams?: FfprobeStreamCodec[] };
const streams = parsed.streams ?? [];
const hasVideo = streams.some((stream) => stream.codec_type === "video");
const hasAudio = streams.some((stream) => stream.codec_type === "audio");
const hasAudio = decodableAudioStream(streams) !== null;
if (isVideo && !hasVideo) {
return { ok: false, reason: "no supported video stream found" };
packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts
+4 -1
View File
@@ -21,7 +21,10 @@ describe("decodePeakMap, decode failure", () => {
writeFileSync(file, `#!/bin/sh\n${body}\n`);
chmodSync(file, 0o755);
};
script("ffprobe", "echo 2");
script(
"ffprobe",
`echo '{"streams":[{"codec_type":"audio","codec_name":"aac","channels":2}]}'`,
);
script("ffmpeg", "head -c 38400 /dev/zero; exit 1");
await expect(decodePeakMap("truncated.mov")).rejects.toThrow(/exit 1/);
},
packages/studio-server/src/helpers/peakMap.ts
+6 -27
View File
@@ -1,10 +1,9 @@
import { execFile, spawn } from "node:child_process";
import { promisify } from "node:util";
import { spawn } from "node:child_process";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { probeAudioTrack } from "./mediaMetadata.js";
export const PEAK_BIN_SECONDS = 0.05;
const PEAK_SAMPLE_RATE = 48_000;
const execFileAsync = promisify(execFile);
const PEAK_MAP_CACHE_VERSION = "peaks-v2";
/**
@@ -61,30 +60,10 @@ export function finishPeaks(acc: PeakAccumulator): number[] {
return acc.bins.map((peak) => Number(peak.toFixed(4)));
}
async function probeChannels(mediaPath: string): Promise<number> {
const { stdout } = await execFileAsync(
findFfBinary("ffprobe") ?? "ffprobe",
[
"-v",
"error",
"-select_streams",
"a:0",
"-show_entries",
"stream=channels",
"-of",
"csv=p=0",
"--",
mediaPath,
],
{ encoding: "utf8", timeout: 30_000, windowsHide: true },
);
const channels = Number.parseInt(stdout.trim(), 10);
if (!Number.isFinite(channels) || channels < 1) throw new Error("no audio stream");
return channels;
}
export async function decodePeakMap(mediaPath: string): Promise<number[]> {
const channels = await probeChannels(mediaPath);
const track = await probeAudioTrack(mediaPath);
if (!track || track.channels < 1) throw new Error("no audio stream");
const { stream, channels } = track;
return new Promise((resolvePromise, reject) => {
const proc = spawn(
findFfBinary("ffmpeg") ?? "ffmpeg",
@@ -94,7 +73,7 @@ export async function decodePeakMap(mediaPath: string): Promise<number[]> {
"-i",
mediaPath,
"-map",
"0:a:0",
`0:a:${stream}`,
"-vn",
"-ar",
String(PEAK_SAMPLE_RATE),
packages/studio-server/src/helpers/proxyTranscoder.ts
+5
View File
@@ -706,6 +706,10 @@ async function runFfmpeg(
const videoFilter = await buildVideoFilter(ffmpegPath, sourcePath, metadata, keepsAlpha, box);
const alphaDecoder = keepsAlpha ? alphaDecoderFor(metadata.color.codecName) : null;
const decoderArgs = alphaDecoder ? ["-c:v", alphaDecoder] : [];
const streamArgs = [
...["-map", "0:V:0"],
...(metadata.audioStream === undefined ? [] : ["-map", `0:a:${metadata.audioStream}`]),
];
const totalSeconds = metadata.durationSeconds;
// libvpx scales poorly across threads, so alpha pieces run side by side with the cores shared out.
const gate = keepsAlpha ? ALPHA_PIECE_GATE : UNGATED;
@@ -725,6 +729,7 @@ async function runFfmpeg(
...decoderArgs,
"-i",
sourcePath,
...streamArgs,
...(range ? ["-t", String(range.lengthSeconds)] : []),
"-vf",
videoFilter,
packages/studio-server/src/helpers/undecodableAudioTrack.test.ts
+69
View File
@@ -0,0 +1,69 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { afterAll, describe, expect, it } from "vitest";
import { measureAudio } from "./loudness.js";
import { decodePeakMap } from "./peakMap.js";
import { resolveProxy } from "./proxyTranscoder.js";
import { decodeAudioPeaks } from "./waveform.js";
const ffmpeg = findFfBinary("ffmpeg");
const ffprobe = findFfBinary("ffprobe");
const dir = mkdtempSync(join(tmpdir(), "hf-phone-audio-"));
/** A phone recording's shape: a 4-channel track whose codec ffmpeg cannot decode, then `tones` mono tracks.
* PCM carries no codec config, so renaming its sample entry is all it takes. */
function phoneClip(name: string, tones: number): string {
const made = join(dir, `made-${name}`);
const inputs = Array.from({ length: tones }, () => ["-f", "lavfi", "-i", "sine=f=1000:d=1"]);
execFileSync(ffmpeg!, [
...["-v", "error", "-f", "lavfi", "-i", "testsrc2=s=160x90:d=1"],
...["-f", "lavfi", "-i", "sine=f=300:d=1,pan=4c|c0=c0|c1=c0|c2=c0|c3=c0", ...inputs.flat()],
...["-map", "0:v", "-map", "1:a", ...inputs.flatMap((_, i) => ["-map", `${i + 2}:a`])],
...["-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "pcm_s16le", made],
]);
const bytes = readFileSync(made);
bytes.write("apac", bytes.indexOf("sowt", bytes.indexOf("moov")));
writeFileSync(join(dir, name), bytes);
return join(dir, name);
}
const audioChannels = (path: string) =>
execFileSync(
ffprobe!,
[
"-v",
"error",
"-select_streams",
"a",
"-show_entries",
"stream=channels",
"-of",
"csv=p=0",
"--",
path,
],
{ encoding: "utf8" },
).trim();
afterAll(() => rmSync(dir, { recursive: true, force: true }));
describe.skipIf(!ffmpeg || !ffprobe)(
"a phone clip whose first audio track ffmpeg cannot decode",
() => {
it("gets a preview copy with the track ffmpeg can decode, or none when there is none", async () => {
expect(audioChannels(await resolveProxy(dir, phoneClip("phone.mov", 1)))).toBe("1");
expect(audioChannels(await resolveProxy(dir, phoneClip("spatial.mov", 0)))).toBe("");
}, 60_000);
it("draws its waveform and peaks and measures its loudness from that track", async () => {
const phone = phoneClip("phone-read.mov", 1);
expect((await decodeAudioPeaks(phone)).some((peak) => peak > 0.05)).toBe(true);
expect((await decodePeakMap(phone)).some((peak) => peak > 0.05)).toBe(true);
const loudness = await measureAudio(ffmpeg!, phone, { mediaStart: 0, duration: null });
expect(loudness.integratedLufs).toBeGreaterThan(-40);
}, 60_000);
},
);
packages/studio-server/src/helpers/waveform.file-race.test.ts
+3
View File
@@ -35,6 +35,9 @@ vi.mock("node:child_process", async () => {
};
});
vi.mock("@hyperframes/parsers/ff-binaries", () => ({ findFfBinary: () => "ffmpeg" }));
vi.mock("./mediaMetadata.js", () => ({
probeAudioTrack: async () => ({ stream: 0, channels: 1 }),
}));
vi.mock("node:fs", async (importOriginal) => {
const fs = await importOriginal<typeof import("node:fs")>();
return {
packages/studio-server/src/helpers/waveform.ts
+6 -1
View File
@@ -11,6 +11,7 @@ import {
} from "node:fs";
import { dirname, join } from "node:path";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { probeAudioTrack } from "./mediaMetadata.js";
const SAMPLE_RATE = 4000;
const PEAK_COUNT = 4000;
@@ -58,13 +59,17 @@ function computePeaks(floats: Float32Array, count: number): number[] {
return peaks.map((p) => p / maxPeak);
}
export function decodeAudioPeaks(audioPath: string): Promise<number[]> {
export async function decodeAudioPeaks(audioPath: string): Promise<number[]> {
const track = await probeAudioTrack(audioPath);
if (!track) throw new Error("no audio stream");
return new Promise((resolvePromise, reject) => {
const proc = spawn(
findFfBinary("ffmpeg") ?? "ffmpeg",
[
"-i",
audioPath,
"-map",
`0:a:${track.stream}`,
"-af",
"atrim=start_sample=1152",
"-f",