fix(engine): preserve AAC start time during MP4 mux (#1615)

* fix(engine): copy mixed AAC during MP4 mux

* fix(producer): avoid AAC re-encode in distributed audio pad

* fix(engine): probe AAC sidecars before mux copy decision

* fix(producer): avoid temp concat file for audio padding
This commit is contained in:
Miguel Ángel
2026-06-20 17:22:15 -04:00
committed by GitHub
parent 8408a44745
commit 0473254bdd
7 changed files with 558 additions and 56 deletions
@@ -18,6 +18,7 @@ afterEach(() => {
} }
vi.resetModules(); vi.resetModules();
vi.doUnmock("child_process"); vi.doUnmock("child_process");
vi.doUnmock("../utils/ffprobe.js");
vi.useRealTimers(); vi.useRealTimers();
}); });
@@ -88,6 +89,11 @@ function emitClose(proc: FakeProc, code: number): void {
proc.emit("close", code); proc.emit("close", code);
} }
async function flushMuxCodecResolution(): Promise<void> {
await Promise.resolve();
await Promise.resolve();
}
describe("ENCODER_PRESETS", () => { describe("ENCODER_PRESETS", () => {
it("has draft, standard, and high presets", () => { it("has draft, standard, and high presets", () => {
expect(ENCODER_PRESETS).toHaveProperty("draft"); expect(ENCODER_PRESETS).toHaveProperty("draft");
@@ -355,6 +361,212 @@ describe("encodeFramesChunkedConcat ffmpegEncodeTimeout", () => {
}); });
}); });
describe("muxVideoWithAudio audio codec handling", () => {
it("copies HyperFrames AAC sidecars into MP4 instead of re-encoding", async () => {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { muxVideoWithAudio } = await import("./chunkEncoder.js");
const muxPromise = muxVideoWithAudio(
"/tmp/video-only.mp4",
"/tmp/audio.aac",
"/tmp/output.mp4",
undefined,
undefined,
{ num: 30, den: 1 },
);
await flushMuxCodecResolution();
expect(calls).toHaveLength(1);
expect(calls[0]!.args).toEqual([
"-i",
"/tmp/video-only.mp4",
"-i",
"/tmp/audio.aac",
"-c:v",
"copy",
"-c:a",
"copy",
"-movflags",
"+faststart",
"-avoid_negative_ts",
"make_zero",
"-r",
"30",
"-shortest",
"-y",
"/tmp/output.mp4",
]);
expect(calls[0]!.args).not.toContain("-use_editlist");
emitClose(calls[0]!.proc, 0);
await expect(muxPromise).resolves.toMatchObject({
success: true,
outputPath: "/tmp/output.mp4",
});
});
it("uses the caller-provided AAC codec contract instead of the sidecar extension", async () => {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { muxVideoWithAudio } = await import("./chunkEncoder.js");
const muxPromise = muxVideoWithAudio(
"/tmp/video-only.mp4",
"/tmp/audio-sidecar",
"/tmp/output.mp4",
undefined,
{ audioCodec: "aac" },
{ num: 30, den: 1 },
);
await flushMuxCodecResolution();
expect(calls).toHaveLength(1);
expect(calls[0]!.args).toContain("-c:a");
expect(calls[0]!.args[calls[0]!.args.indexOf("-c:a") + 1]).toBe("copy");
expect(calls[0]!.args).not.toContain("-b:a");
expect(calls[0]!.args).toContain("+faststart");
emitClose(calls[0]!.proc, 0);
await expect(muxPromise).resolves.toMatchObject({
success: true,
outputPath: "/tmp/output.mp4",
});
});
it("probes unknown-extension AAC sidecars before choosing the MP4 copy path", async () => {
const { spawn, calls } = createSpawnSpy();
const extractAudioMetadata = vi.fn(async () => ({
durationSeconds: 1,
sampleRate: 48000,
channels: 2,
audioCodec: "aac",
}));
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
vi.doMock("../utils/ffprobe.js", () => ({ extractAudioMetadata }));
const { muxVideoWithAudio } = await import("./chunkEncoder.js");
const muxPromise = muxVideoWithAudio(
"/tmp/video-only.mp4",
"/tmp/audio-sidecar",
"/tmp/output.mp4",
);
await flushMuxCodecResolution();
expect(extractAudioMetadata).toHaveBeenCalledWith("/tmp/audio-sidecar");
expect(calls).toHaveLength(1);
expect(calls[0]!.args).toContain("-c:a");
expect(calls[0]!.args[calls[0]!.args.indexOf("-c:a") + 1]).toBe("copy");
expect(calls[0]!.args).not.toContain("-b:a");
emitClose(calls[0]!.proc, 0);
await expect(muxPromise).resolves.toMatchObject({
success: true,
outputPath: "/tmp/output.mp4",
});
});
it("keeps probed non-AAC unknown-extension sidecars on the MP4 transcode path", async () => {
const { spawn, calls } = createSpawnSpy();
const extractAudioMetadata = vi.fn(async () => ({
durationSeconds: 1,
sampleRate: 48000,
channels: 2,
audioCodec: "mp3",
}));
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
vi.doMock("../utils/ffprobe.js", () => ({ extractAudioMetadata }));
const { muxVideoWithAudio } = await import("./chunkEncoder.js");
const muxPromise = muxVideoWithAudio(
"/tmp/video-only.mp4",
"/tmp/audio-sidecar",
"/tmp/output.mp4",
);
await flushMuxCodecResolution();
expect(extractAudioMetadata).toHaveBeenCalledWith("/tmp/audio-sidecar");
expect(calls).toHaveLength(1);
expect(calls[0]!.args).toContain("-c:a");
expect(calls[0]!.args[calls[0]!.args.indexOf("-c:a") + 1]).toBe("aac");
expect(calls[0]!.args).toContain("-b:a");
emitClose(calls[0]!.proc, 0);
await expect(muxPromise).resolves.toMatchObject({ success: true });
});
it("still transcodes non-AAC audio when muxing MP4", async () => {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { muxVideoWithAudio } = await import("./chunkEncoder.js");
const muxPromise = muxVideoWithAudio(
"/tmp/video-only.mp4",
"/tmp/audio.wav",
"/tmp/output.mp4",
);
await flushMuxCodecResolution();
expect(calls).toHaveLength(1);
expect(calls[0]!.args).toContain("-c:a");
expect(calls[0]!.args[calls[0]!.args.indexOf("-c:a") + 1]).toBe("aac");
expect(calls[0]!.args).toContain("-b:a");
expect(calls[0]!.args).toContain("+faststart");
emitClose(calls[0]!.proc, 0);
await expect(muxPromise).resolves.toMatchObject({ success: true });
});
it("copies HyperFrames AAC sidecars into MOV containers without MP4 faststart flags", async () => {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { muxVideoWithAudio } = await import("./chunkEncoder.js");
const muxPromise = muxVideoWithAudio(
"/tmp/video-only.mov",
"/tmp/audio.aac",
"/tmp/output.mov",
);
await flushMuxCodecResolution();
expect(calls).toHaveLength(1);
expect(calls[0]!.args).toContain("-c:a");
expect(calls[0]!.args[calls[0]!.args.indexOf("-c:a") + 1]).toBe("copy");
expect(calls[0]!.args).not.toContain("-b:a");
expect(calls[0]!.args).not.toContain("+faststart");
emitClose(calls[0]!.proc, 0);
await expect(muxPromise).resolves.toMatchObject({ success: true });
});
it("keeps WebM audio on the Opus transcode path", async () => {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { muxVideoWithAudio } = await import("./chunkEncoder.js");
const muxPromise = muxVideoWithAudio(
"/tmp/video-only.webm",
"/tmp/audio.aac",
"/tmp/output.webm",
);
expect(calls).toHaveLength(1);
expect(calls[0]!.args).toContain("-c:a");
expect(calls[0]!.args[calls[0]!.args.indexOf("-c:a") + 1]).toBe("libopus");
expect(calls[0]!.args).not.toContain("+faststart");
emitClose(calls[0]!.proc, 0);
await expect(muxPromise).resolves.toMatchObject({ success: true });
});
});
describe("getEncoderPreset", () => { describe("getEncoderPreset", () => {
it("returns h264 with yuv420p for mp4 format", () => { it("returns h264 with yuv420p for mp4 format", () => {
const preset = getEncoderPreset("standard", "mp4"); const preset = getEncoderPreset("standard", "mp4");
+63 -4
View File
@@ -8,7 +8,7 @@
import { spawn } from "child_process"; import { spawn } from "child_process";
import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from "fs"; import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from "fs";
import { join, dirname } from "path"; import { join, dirname, extname } from "path";
import { trackChildProcess } from "../utils/processTracker.js"; import { trackChildProcess } from "../utils/processTracker.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { import {
@@ -20,6 +20,7 @@ import {
import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js"; import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
import { formatFfmpegError, runFfmpeg } from "../utils/runFfmpeg.js"; import { formatFfmpegError, runFfmpeg } from "../utils/runFfmpeg.js";
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js"; import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
import { extractAudioMetadata } from "../utils/ffprobe.js";
import { type Fps, fpsToFfmpegArg } from "@hyperframes/core"; import { type Fps, fpsToFfmpegArg } from "@hyperframes/core";
import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js"; import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js";
import { appendVp9CpuUsedArg } from "./vp9Options.js"; import { appendVp9CpuUsedArg } from "./vp9Options.js";
@@ -45,6 +46,50 @@ function appendEncodeTimeoutMessage(error: string, timedOut: boolean, timeoutMs:
return `${error}\nFFmpeg killed after exceeding ffmpegEncodeTimeout (${timeoutMs} ms)`; return `${error}\nFFmpeg killed after exceeding ffmpegEncodeTimeout (${timeoutMs} ms)`;
} }
function isAacSidecar(audioPath: string): boolean {
return extname(audioPath).toLowerCase() === ".aac";
}
const KNOWN_NON_AAC_AUDIO_EXTENSIONS = new Set([
".flac",
".mp3",
".oga",
".ogg",
".opus",
".wav",
".webm",
]);
export interface MuxVideoWithAudioOptions extends Partial<
Pick<EngineConfig, "ffmpegProcessTimeout">
> {
/**
* Codec of the sidecar audio when the caller already knows it. HyperFrames
* render paths pass the mixed AAC sidecar by contract, so muxing should not
* depend on the file extension alone.
*/
audioCodec?: "aac";
}
async function shouldCopyAacSidecar(
audioPath: string,
options: MuxVideoWithAudioOptions | undefined,
) {
if (options?.audioCodec === "aac" || isAacSidecar(audioPath)) return true;
const audioExtension = extname(audioPath).toLowerCase();
if (KNOWN_NON_AAC_AUDIO_EXTENSIONS.has(audioExtension)) return false;
try {
const metadata = await extractAudioMetadata(audioPath);
return metadata.audioCodec === "aac";
} catch {
// Preserve the pre-existing fallback for invalid or unprobeable sidecars:
// let the final ffmpeg transcode path surface the actionable mux error.
return false;
}
}
/** /**
* Get encoder preset for a given quality and output format. * Get encoder preset for a given quality and output format.
* WebM uses VP9 with alpha-capable pixel format; MP4 uses h264 (or h265 for HDR); * WebM uses VP9 with alpha-capable pixel format; MP4 uses h264 (or h265 for HDR);
@@ -683,7 +728,7 @@ export async function muxVideoWithAudio(
audioPath: string, audioPath: string,
outputPath: string, outputPath: string,
signal?: AbortSignal, signal?: AbortSignal,
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>, config?: MuxVideoWithAudioOptions,
fps?: Fps, fps?: Fps,
): Promise<MuxResult> { ): Promise<MuxResult> {
const outputDir = dirname(outputPath); const outputDir = dirname(outputPath);
@@ -691,14 +736,28 @@ export async function muxVideoWithAudio(
const isWebm = outputPath.endsWith(".webm"); const isWebm = outputPath.endsWith(".webm");
const isMov = outputPath.endsWith(".mov"); const isMov = outputPath.endsWith(".mov");
const shouldCopyAudio = isWebm ? false : await shouldCopyAacSidecar(audioPath, config);
const args = ["-i", videoPath, "-i", audioPath, "-c:v", "copy"]; const args = ["-i", videoPath, "-i", audioPath, "-c:v", "copy"];
if (isWebm) { if (isWebm) {
args.push("-c:a", "libopus", "-b:a", "128k"); args.push("-c:a", "libopus", "-b:a", "128k");
} else if (isMov) { } else if (isMov) {
args.push("-c:a", "aac", "-b:a", "192k"); if (shouldCopyAudio) {
args.push("-c:a", "copy");
} else {
args.push("-c:a", "aac", "-b:a", "192k");
}
} else { } else {
args.push("-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"); // processCompositionAudio (audioMixer.ts) performs the AAC encode and
// owns the single encoder-priming interval. Copying that sidecar into
// MP4 preserves the correct priming metadata; re-encoding it during mux
// creates another priming interval that ffmpeg writes as an empty leading
// video edit list, which QuickTime/Safari render as a black first frame.
if (shouldCopyAudio) {
args.push("-c:a", "copy", "-movflags", "+faststart");
} else {
args.push("-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart");
}
} }
// PTS bases can diverge during mux and reintroduce negative DTS. See // PTS bases can diverge during mux and reintroduce negative DTS. See
// buildEncoderArgs for the full reasoning on why that breaks playback. // buildEncoderArgs for the full reasoning on why that breaks playback.
@@ -146,7 +146,7 @@ function probeStream(
"-select_streams", "-select_streams",
streamSelector, streamSelector,
"-show_entries", "-show_entries",
"stream=duration,nb_frames,nb_read_packets,codec_name,r_frame_rate", "stream=start_time,duration,nb_frames,nb_read_packets,codec_name,r_frame_rate",
"-count_packets", "-count_packets",
"-of", "-of",
"json", "json",
@@ -316,6 +316,10 @@ describe("assemble()", () => {
const audioStream = probeStream(outputPath, "a:0"); const audioStream = probeStream(outputPath, "a:0");
expect(audioStream).toBeDefined(); expect(audioStream).toBeDefined();
expect(audioStream?.codec_name).toBe("aac"); expect(audioStream?.codec_name).toBe("aac");
const videoStream = probeStream(outputPath, "v:0");
expect(videoStream).toBeDefined();
expect(Number(videoStream?.start_time ?? NaN)).toBeLessThan(0.001);
expect(Number(audioStream?.start_time ?? NaN)).toBeLessThan(0.001);
// Audio duration should be within ~25ms of `totalFrames / fps` after // Audio duration should be within ~25ms of `totalFrames / fps` after
// pad/trim. The 25ms tolerance absorbs AAC frame quantization (1024 // pad/trim. The 25ms tolerance absorbs AAC frame quantization (1024
// samples @ 48kHz = ~21ms). // samples @ 48kHz = ~21ms).
@@ -326,6 +330,48 @@ describe("assemble()", () => {
TIMEOUT_MS, TIMEOUT_MS,
); );
it(
"muxes padded short audio without shifting the first video frame",
async () => {
if (!hasFfmpeg) return;
const chunks: ChunkSliceJson[] = [
{ index: 0, startFrame: 0, endFrame: 6 },
{ index: 1, startFrame: 6, endFrame: 12 },
];
const totalFrames = 12;
const fps = 30;
const planDir = buildPlanDir("mp4", chunks, totalFrames, true);
const chunkAPath = join(planDir, "chunk-0.mp4");
const chunkBPath = join(planDir, "chunk-1.mp4");
const audioPath = join(planDir, "audio.aac");
makeMp4Chunk(chunkAPath, 6);
makeMp4Chunk(chunkBPath, 6);
// Audio is shorter than the video, forcing the distributed pad branch.
makeAacAudio(audioPath, totalFrames / fps - 0.2);
const outputPath = join(planDir, "output-audio-padded.mp4");
const result = await assemble(planDir, [chunkAPath, chunkBPath], audioPath, outputPath);
expect(existsSync(outputPath)).toBe(true);
expect(result.framesEncoded).toBe(totalFrames);
const audioStream = probeStream(outputPath, "a:0");
expect(audioStream).toBeDefined();
expect(audioStream?.codec_name).toBe("aac");
const videoStream = probeStream(outputPath, "v:0");
expect(videoStream).toBeDefined();
expect(Number(videoStream?.start_time ?? NaN)).toBeLessThan(0.001);
expect(Number(audioStream?.start_time ?? NaN)).toBeLessThan(0.001);
const audioDuration = Number(audioStream?.duration ?? 0);
const expected = totalFrames / fps;
expect(Math.abs(audioDuration - expected)).toBeLessThan(0.05);
},
TIMEOUT_MS,
);
it( it(
"cfr:true re-encodes for exact avg_frame_rate matching r_frame_rate", "cfr:true re-encodes for exact avg_frame_rate matching r_frame_rate",
async () => { async () => {
@@ -320,7 +320,7 @@ export async function assemble(
audioForMux, audioForMux,
muxOutputPath, muxOutputPath,
abortSignal, abortSignal,
undefined, { audioCodec: "aac" },
{ num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen }, { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
); );
if (!muxResult.success) { if (!muxResult.success) {
@@ -16,6 +16,7 @@
import { describe, expect, it } from "bun:test"; import { describe, expect, it } from "bun:test";
import { import {
buildPadTrimAudioArgs, buildPadTrimAudioArgs,
buildPadTrimAudioPlan,
padOrTrimAudioToVideoFrameCount, padOrTrimAudioToVideoFrameCount,
type AudioProbeInfo, type AudioProbeInfo,
type PadTrimAudioInput, type PadTrimAudioInput,
@@ -23,18 +24,45 @@ import {
} from "./audioPadTrim.js"; } from "./audioPadTrim.js";
describe("buildPadTrimAudioArgs", () => { describe("buildPadTrimAudioArgs", () => {
it("emits an apad filter when audio is shorter than target", () => { it("emits a concat-copy pad plan when audio is shorter than target", () => {
const plan = buildPadTrimAudioPlan("/tmp/in.aac", "/tmp/out.aac", 4.0, 5.0, {
sampleRate: 48000,
channels: 2,
});
expect(plan.operation).toBe("pad");
expect(plan.steps).toHaveLength(2);
const silenceArgs = plan.steps[0]!.args;
expect(plan.steps[0]!.kind).toBe("pad-silence");
expect(silenceArgs).not.toContain("/tmp/in.aac");
expect(silenceArgs[silenceArgs.indexOf("-i") + 1]).toBe(
"anullsrc=channel_layout=stereo:sample_rate=48000",
);
expect(silenceArgs[silenceArgs.indexOf("-t") + 1]).toBe("1.000000");
expect(silenceArgs[silenceArgs.indexOf("-c:a") + 1]).toBe("aac");
const concatArgs = plan.steps[1]!.args;
expect(plan.steps[1]!.kind).toBe("pad-concat");
expect(concatArgs).toContain("concat");
expect(concatArgs[concatArgs.indexOf("-i") + 1]).toBe("pipe:0");
expect(concatArgs[concatArgs.indexOf("-c:a") + 1]).toBe("copy");
expect(concatArgs[concatArgs.length - 1]).toBe("/tmp/out.aac");
expect(plan.steps[1]!.stdin).toContain("file 'file:///tmp/in.aac'");
expect(plan.steps[1]!.stdin).toContain("file 'file:///tmp/out.aac.pad-silence.aac'");
expect(plan.cleanupPaths).toEqual(["/tmp/out.aac.pad-silence.aac"]);
const reencodedSourceStep = plan.steps.find(
(step) =>
step.args.includes("/tmp/in.aac") && step.args[step.args.indexOf("-c:a") + 1] === "aac",
);
expect(reencodedSourceStep).toBeUndefined();
});
it("keeps the legacy args helper on the first pad materialization step", () => {
const { args, operation } = buildPadTrimAudioArgs("/tmp/in.aac", "/tmp/out.aac", 4.0, 5.0); const { args, operation } = buildPadTrimAudioArgs("/tmp/in.aac", "/tmp/out.aac", 4.0, 5.0);
expect(operation).toBe("pad"); expect(operation).toBe("pad");
const afIdx = args.indexOf("-af"); expect(args).not.toContain("/tmp/in.aac");
expect(afIdx).toBeGreaterThan(-1); expect(args[args.indexOf("-t") + 1]).toBe("1.000000");
expect(args[afIdx + 1]).toContain("apad=pad_dur=");
expect(args[afIdx + 1]).toMatch(/pad_dur=1\.0+/);
// Pad must re-encode — apad is a filter and filters can't combine with copy.
const codecIdx = args.indexOf("-c:a");
expect(args[codecIdx + 1]).toBe("aac");
expect(args[args.length - 1]).toBe("/tmp/out.aac");
expect(args.includes("-y")).toBe(true);
}); });
it("emits -t when audio is longer than target", () => { it("emits -t when audio is longer than target", () => {
@@ -57,14 +85,14 @@ describe("buildPadTrimAudioArgs", () => {
expect(args[codecIdx + 1]).toBe("copy"); expect(args[codecIdx + 1]).toBe("copy");
}); });
it("emits 6-decimal-place pad_dur (no scientific notation)", () => { it("emits 6-decimal-place pad duration (no scientific notation)", () => {
// 1.23ms — just over the AUDIO_DURATION_TOLERANCE_SECONDS=1ms threshold, // 1.23ms — just over the AUDIO_DURATION_TOLERANCE_SECONDS=1ms threshold,
// so we exercise the pad path with a tiny duration that would round to // so we exercise the pad path with a tiny duration that would round to
// exponent notation if we used `toString()` instead of `toFixed(6)`. // exponent notation if we used `toString()` instead of `toFixed(6)`.
const { args, operation } = buildPadTrimAudioArgs("/tmp/in.aac", "/tmp/out.aac", 0.0, 0.00123); const { args, operation } = buildPadTrimAudioArgs("/tmp/in.aac", "/tmp/out.aac", 0.0, 0.00123);
expect(operation).toBe("pad"); expect(operation).toBe("pad");
const afIdx = args.indexOf("-af"); const tIdx = args.indexOf("-t");
expect(args[afIdx + 1]).toBe("apad=pad_dur=0.001230"); expect(args[tIdx + 1]).toBe("0.001230");
}); });
it("flags ~1ms drift as a copy (below the tolerance threshold)", () => { it("flags ~1ms drift as a copy (below the tolerance threshold)", () => {
@@ -120,9 +148,11 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
expect(result.operation).toBe("pad"); expect(result.operation).toBe("pad");
expect(result.targetDurationSeconds).toBe(6); expect(result.targetDurationSeconds).toBe(6);
expect(result.sourceDurationSeconds).toBe(5.5); expect(result.sourceDurationSeconds).toBe(5.5);
expect(captured.args).toHaveLength(1); expect(captured.args).toHaveLength(2);
const afIdx = captured.args[0]!.indexOf("-af"); const tIdx = captured.args[0]!.indexOf("-t");
expect(captured.args[0]![afIdx + 1]).toBe("apad=pad_dur=0.500000"); expect(captured.args[0]![tIdx + 1]).toBe("0.500000");
expect(captured.args[0]).not.toContain("/tmp/a.aac");
expect(captured.args[1]![captured.args[1]!.indexOf("-c:a") + 1]).toBe("copy");
}); });
it("trims a video of N=120 frames at 30/1 fps with longer audio", async () => { it("trims a video of N=120 frames at 30/1 fps with longer audio", async () => {
@@ -162,8 +192,8 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
expect(result.success).toBe(true); expect(result.success).toBe(true);
expect(result.operation).toBe("pad"); expect(result.operation).toBe("pad");
expect(result.targetDurationSeconds).toBeCloseTo((120 * 1001) / 30000, 9); expect(result.targetDurationSeconds).toBeCloseTo((120 * 1001) / 30000, 9);
const afIdx = captured.args[0]!.indexOf("-af"); const tIdx = captured.args[0]!.indexOf("-t");
expect(captured.args[0]![afIdx + 1]).toMatch(/^apad=pad_dur=0\.004\d+$/); expect(captured.args[0]![tIdx + 1]).toMatch(/^0\.004\d+$/);
}); });
it("propagates video probe failure as success=false", async () => { it("propagates video probe failure as success=false", async () => {
@@ -14,13 +14,17 @@
* "audio cuts off early" or "video shows a frozen final frame" bugs. * "audio cuts off early" or "video shows a frozen final frame" bugs.
* *
* The fix: post-pad/trim audio to *exactly* `frameCount / fps` seconds at * The fix: post-pad/trim audio to *exactly* `frameCount / fps` seconds at
* assemble time. Pad with `apad=pad_dur=…` (silence fill), trim with `-t`. * assemble time. Pad by concat-copying a generated silence tail, trim with
* `-t`, and avoid re-encoding the already mixed source AAC in either case.
*/ */
import { spawn } from "node:child_process"; import { spawn } from "node:child_process";
import { rmSync } from "node:fs";
import { pathToFileURL } from "node:url";
import { import {
extractAudioMetadata, extractAudioMetadata,
formatFfmpegError, formatFfmpegError,
getFfmpegBinary,
getFfprobeBinary, getFfprobeBinary,
runFfmpeg, runFfmpeg,
type AudioMetadata, type AudioMetadata,
@@ -45,6 +49,12 @@ export interface ProbeVideoFrameInfo {
export interface AudioProbeInfo { export interface AudioProbeInfo {
/** Decoded duration in seconds. */ /** Decoded duration in seconds. */
durationSeconds: number; durationSeconds: number;
/** Audio sample rate in Hz. Used when generating pad silence. */
sampleRate?: number;
/** Audio channel count. Used when generating pad silence. */
channels?: number;
/** Codec name reported by ffprobe. */
audioCodec?: string;
} }
export interface PadTrimAudioInput { export interface PadTrimAudioInput {
@@ -60,7 +70,10 @@ export interface PadTrimAudioInput {
*/ */
probeVideoFrameInfo?: (videoPath: string) => Promise<ProbeVideoFrameInfo>; probeVideoFrameInfo?: (videoPath: string) => Promise<ProbeVideoFrameInfo>;
probeAudioInfo?: (audioPath: string) => Promise<AudioProbeInfo>; probeAudioInfo?: (audioPath: string) => Promise<AudioProbeInfo>;
runFfmpeg?: (args: string[]) => Promise<{ success: boolean; error?: string }>; runFfmpeg?: (
args: string[],
options?: { stdin?: string },
) => Promise<{ success: boolean; error?: string }>;
} }
export type PadTrimOperation = "pad" | "trim" | "copy"; export type PadTrimOperation = "pad" | "trim" | "copy";
@@ -78,49 +91,93 @@ export interface PadTrimAudioResult {
error?: string; error?: string;
} }
export type PadTrimAudioStepKind = "copy" | "trim" | "pad-silence" | "pad-concat";
export interface PadTrimAudioStep {
kind: PadTrimAudioStepKind;
args: string[];
stdin?: string;
}
export interface PadTrimAudioPlan {
operation: PadTrimOperation;
steps: PadTrimAudioStep[];
cleanupPaths: string[];
}
/** /**
* Pure helper: decide the pad/trim operation and build the ffmpeg argv list * Pure helper: decide the pad/trim operation and build the ffmpeg argv
* that materializes it. Exported separately so unit tests can pin both * sequence that materializes it. Exported separately so unit tests can pin
* branches without spawning ffmpeg. * every branch without spawning ffmpeg.
* *
* - `sourceDuration < targetDuration` pad with `apad=pad_dur=Δ`. * - `sourceDuration < targetDuration` generate only the missing silence
* Re-encode is required: `apad` is a filter and filters can't combine * tail, then concat-copy the source AAC plus that tail. This avoids
* with `-c:a copy`. * re-encoding the already mixed `audio.aac`; the pad branch remains the
* inverse of trim instead of becoming a second full-source AAC encode.
* - `sourceDuration > targetDuration` trim with `-t target`. `-c:a copy` * - `sourceDuration > targetDuration` trim with `-t target`. `-c:a copy`
* is preserved when the input is already AAC. * is preserved when the input is already AAC.
* - `|Δ| < AUDIO_DURATION_TOLERANCE_SECONDS` no-op `copy`, but we still * - `|Δ| < AUDIO_DURATION_TOLERANCE_SECONDS` no-op `copy`, but we still
* run ffmpeg with `-c:a copy` to materialize the output path. * run ffmpeg with `-c:a copy` to materialize the output path.
*/ */
export function buildPadTrimAudioArgs( export function buildPadTrimAudioPlan(
audioPath: string, audioPath: string,
outputPath: string, outputPath: string,
sourceDurationSeconds: number, sourceDurationSeconds: number,
targetDurationSeconds: number, targetDurationSeconds: number,
): { args: string[]; operation: PadTrimOperation } { audioInfo: Pick<AudioProbeInfo, "sampleRate" | "channels"> = {},
): PadTrimAudioPlan {
const delta = targetDurationSeconds - sourceDurationSeconds; const delta = targetDurationSeconds - sourceDurationSeconds;
const targetSec = formatSeconds(targetDurationSeconds); const targetSec = formatSeconds(targetDurationSeconds);
if (Math.abs(delta) < AUDIO_DURATION_TOLERANCE_SECONDS) { if (Math.abs(delta) < AUDIO_DURATION_TOLERANCE_SECONDS) {
return { return {
operation: "copy", operation: "copy",
args: ["-i", audioPath, "-c:a", "copy", "-y", outputPath], steps: [{ kind: "copy", args: ["-i", audioPath, "-c:a", "copy", "-y", outputPath] }],
cleanupPaths: [],
}; };
} }
if (delta > 0) { if (delta > 0) {
const padDur = formatSeconds(delta); const padDur = formatSeconds(delta);
const silencePath = `${outputPath}.pad-silence.aac`;
return { return {
operation: "pad", operation: "pad",
args: [ steps: [
"-i", {
audioPath, kind: "pad-silence",
"-af", args: [
`apad=pad_dur=${padDur}`, "-f",
"-c:a", "lavfi",
"aac", "-i",
"-b:a", `anullsrc=channel_layout=${channelLayoutForChannels(audioInfo.channels)}:sample_rate=${sampleRateForFilter(audioInfo.sampleRate)}`,
"192k", "-t",
"-y", padDur,
outputPath, "-c:a",
"aac",
"-b:a",
"192k",
"-y",
silencePath,
],
},
{
kind: "pad-concat",
args: [
"-f",
"concat",
"-safe",
"0",
"-protocol_whitelist",
"file,pipe,crypto,data",
"-i",
"pipe:0",
"-c:a",
"copy",
"-y",
outputPath,
],
stdin: `${concatFileLine(audioPath)}\n${concatFileLine(silencePath)}\n`,
},
], ],
cleanupPaths: [silencePath],
}; };
} }
// Trim. `-t` truncates AAC without re-encoding because AAC frames are // Trim. `-t` truncates AAC without re-encoding because AAC frames are
@@ -128,10 +185,28 @@ export function buildPadTrimAudioArgs(
// packet boundary, fine for the ±1ms tolerance we care about here. // packet boundary, fine for the ±1ms tolerance we care about here.
return { return {
operation: "trim", operation: "trim",
args: ["-i", audioPath, "-t", targetSec, "-c:a", "copy", "-y", outputPath], steps: [
{ kind: "trim", args: ["-i", audioPath, "-t", targetSec, "-c:a", "copy", "-y", outputPath] },
],
cleanupPaths: [],
}; };
} }
export function buildPadTrimAudioArgs(
audioPath: string,
outputPath: string,
sourceDurationSeconds: number,
targetDurationSeconds: number,
): { args: string[]; operation: PadTrimOperation } {
const plan = buildPadTrimAudioPlan(
audioPath,
outputPath,
sourceDurationSeconds,
targetDurationSeconds,
);
return { operation: plan.operation, args: plan.steps[0]?.args ?? [] };
}
/** /**
* Format a duration as a fixed-precision decimal string. ffmpeg parses * Format a duration as a fixed-precision decimal string. ffmpeg parses
* scientific notation inconsistently across versions (some treat `1e-3` as * scientific notation inconsistently across versions (some treat `1e-3` as
@@ -142,6 +217,24 @@ function formatSeconds(sec: number): string {
return sec.toFixed(6); return sec.toFixed(6);
} }
function sampleRateForFilter(sampleRate: number | undefined): number {
return sampleRate !== undefined && Number.isFinite(sampleRate) && sampleRate > 0
? Math.round(sampleRate)
: 48000;
}
function channelLayoutForChannels(channels: number | undefined): string {
if (channels === 1) return "mono";
if (channels === 6) return "5.1";
if (channels === 8) return "7.1";
return "stereo";
}
function concatFileLine(path: string): string {
const normalized = pathToFileURL(path).href;
return `file '${normalized.replace(/'/g, "'\\''")}'`;
}
/** /**
* Pad or trim `audio.aac` so its exact duration matches `frameCount / fps` * Pad or trim `audio.aac` so its exact duration matches `frameCount / fps`
* for the assembled video. * for the assembled video.
@@ -197,30 +290,48 @@ export async function padOrTrimAudioToVideoFrameCount(
} }
const targetDurationSeconds = (videoInfo.frameCount * videoInfo.fpsDen) / videoInfo.fpsNum; const targetDurationSeconds = (videoInfo.frameCount * videoInfo.fpsDen) / videoInfo.fpsNum;
const { args, operation } = buildPadTrimAudioArgs( const plan = buildPadTrimAudioPlan(
input.audioPath, input.audioPath,
input.outputPath, input.outputPath,
audioInfo.durationSeconds, audioInfo.durationSeconds,
targetDurationSeconds, targetDurationSeconds,
audioInfo,
); );
const ffmpegResult = await runner(args); try {
if (!ffmpegResult.success) { for (const step of plan.steps) {
const ffmpegResult = await runner(step.args, { stdin: step.stdin });
if (!ffmpegResult.success) {
return {
success: false,
outputPath: input.outputPath,
targetDurationSeconds,
sourceDurationSeconds: audioInfo.durationSeconds,
operation: plan.operation,
error: ffmpegResult.error,
};
}
}
} catch (err) {
return { return {
success: false, success: false,
outputPath: input.outputPath, outputPath: input.outputPath,
targetDurationSeconds, targetDurationSeconds,
sourceDurationSeconds: audioInfo.durationSeconds, sourceDurationSeconds: audioInfo.durationSeconds,
operation, operation: plan.operation,
error: ffmpegResult.error, error: `audioPadTrim: failed to materialize ${plan.operation}: ${
err instanceof Error ? err.message : String(err)
}`,
}; };
} finally {
for (const path of plan.cleanupPaths) rmSync(path, { force: true });
} }
return { return {
success: true, success: true,
outputPath: input.outputPath, outputPath: input.outputPath,
targetDurationSeconds, targetDurationSeconds,
sourceDurationSeconds: audioInfo.durationSeconds, sourceDurationSeconds: audioInfo.durationSeconds,
operation, operation: plan.operation,
}; };
} }
@@ -307,10 +418,20 @@ function parseFrameRate(rate: string): { fpsNum: number; fpsDen: number } {
async function defaultProbeAudioInfo(audioPath: string): Promise<AudioProbeInfo> { async function defaultProbeAudioInfo(audioPath: string): Promise<AudioProbeInfo> {
// extractAudioMetadata is the shared ffprobe wrapper (caches results). // extractAudioMetadata is the shared ffprobe wrapper (caches results).
const metadata: AudioMetadata = await extractAudioMetadata(audioPath); const metadata: AudioMetadata = await extractAudioMetadata(audioPath);
return { durationSeconds: metadata.durationSeconds }; return {
durationSeconds: metadata.durationSeconds,
sampleRate: metadata.sampleRate,
channels: metadata.channels,
audioCodec: metadata.audioCodec,
};
} }
async function defaultRunFfmpeg(args: string[]): Promise<{ success: boolean; error?: string }> { async function defaultRunFfmpeg(
args: string[],
options?: { stdin?: string },
): Promise<{ success: boolean; error?: string }> {
if (options?.stdin !== undefined) return runFfmpegWithStdin(args, options.stdin);
const result = await runFfmpeg(args); const result = await runFfmpeg(args);
if (result.success) return { success: true }; if (result.success) return { success: true };
return { return {
@@ -319,6 +440,40 @@ async function defaultRunFfmpeg(args: string[]): Promise<{ success: boolean; err
}; };
} }
async function runFfmpegWithStdin(
args: string[],
stdin: string,
): Promise<{ success: boolean; error?: string }> {
return new Promise((resolve) => {
const proc = spawn(getFfmpegBinary(), args);
let stderr = "";
proc.stderr.on("data", (data: Buffer) => {
stderr += data.toString();
});
proc.on("error", (err) => {
resolve({
success: false,
error: `[audioPadTrim] ${err instanceof Error ? err.message : String(err)}`,
});
});
proc.on("close", (code) => {
if (code === 0) {
resolve({ success: true });
return;
}
resolve({
success: false,
error: `[audioPadTrim] ${formatFfmpegError(code, stderr)}`,
});
});
proc.stdin.end(stdin);
});
}
// ── ffprobe JSON runner (shared between fast/slow video probe paths) ───── // ── ffprobe JSON runner (shared between fast/slow video probe paths) ─────
function runFfprobeJson<T>(args: string[]): Promise<T> { function runFfprobeJson<T>(args: string[]): Promise<T> {
@@ -60,7 +60,7 @@ export async function runAssembleStage(input: AssembleStageInput): Promise<Assem
audioOutputPath, audioOutputPath,
outputPath, outputPath,
abortSignal, abortSignal,
undefined, { audioCodec: "aac" },
job.config.fps, job.config.fps,
); );
assertNotAborted(); assertNotAborted();