fix(engine): exports of webm and mkv files place sound and picture where the browser plays them

This commit is contained in:
Miguel Ángel
2026-10-11 07:16:37 -07:00
parent b211fc7cb8
commit dd5b73305a
2 changed files with 92 additions and 5 deletions
@@ -10,6 +10,7 @@ import {
extractVideoFramesRange,
getFrameAtTime,
resolveFinalFrameExtractionWindow,
resolvePlayableVideoDuration,
resolveVideoExtractionWindow,
} from "./videoFrameExtractor.js";
@@ -91,6 +92,14 @@ describe.skipIf(!HAS_FFMPEG)(
ffmpeg(["-i", picture, "-itsoffset", lead, "-i", steps, ...mux, join(dir, "audio-late.mp4")]);
// MPEG-TS starts its clock at 1.4 s, so no stream starts at 0.
ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.ts")]);
// Matroska reports start_time 0 for both streams; FLV gives the video a start but no duration.
ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.mkv")]);
ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.flv")]);
ffmpeg([
...["-itsoffset", lead, "-i", picture, "-i", steps, "-map", "0:v", "-map", "1:a"],
...["-c:v", "libvpx-vp9", "-deadline", "realtime", "-cpu-used", "8", "-c:a", "libopus"],
join(dir, "video-late.webm"),
]);
}, 60_000);
afterAll(() => rmSync(dir, { recursive: true, force: true }));
@@ -107,6 +116,8 @@ describe.skipIf(!HAS_FFMPEG)(
{ source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3 },
{ source: "video-late.mp4", audioLead: 0, type: "audio", mediaStart: 0 },
{ source: "video-late.ts", audioLead: 0, type: "video", mediaStart: 0 },
{ source: "video-late.webm", audioLead: 0, type: "video", mediaStart: 0 },
{ source: "video-late.mkv", audioLead: 0, type: "video", mediaStart: 0 },
{ source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3, rate: flatLane },
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 0 },
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 3 },
@@ -242,5 +253,23 @@ describe.skipIf(!HAS_FFMPEG)(
);
expect(window.extractionMediaStart).toBeGreaterThan(LEAD_SECONDS + PICTURE_SECONDS - 0.1);
}, 30_000);
it.each(["video-late.webm", "video-late.mkv", "video-late.flv"])(
"%s: hides the late video's lead and ends it where the stream ends",
async (name) => {
const source = join(dir, name);
expect(resolvePlayableVideoDuration(await extractMediaMetadata(source))).toBeCloseTo(
LEAD_SECONDS + PICTURE_SECONDS,
0,
);
const extracted = await extractVideoFramesRange(source, name, 0, CLIP_SECONDS, {
fps: 10,
outputDir: join(dir, "frames"),
});
expect(getFrameAtTime(extracted, 1, 0)).toBeNull();
expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.5, 0)).not.toBeNull();
},
60_000,
);
},
);
+63 -5
View File
@@ -47,6 +47,57 @@ function sanitizeFfprobeDiagnostic(stderr: string, filePath: string): string {
return `…${redacted.slice(-(FFPROBE_ERROR_MAX_CHARS - 1))}`;
}
/**
* Matroska reports start_time 0 for every stream, so a video that starts 7 s after its audio looks
* aligned. Its first packets say when each stream starts; the stream's end stays where it was.
*/
async function withMeasuredMatroskaStarts(
filePath: string,
output: FFProbeOutput,
signal?: AbortSignal,
): Promise<FFProbeOutput> {
if (!/matroska|webm/.test(output.format.format_name ?? "")) return output;
const streams = await Promise.all(
output.streams.map(async (stream, index) => {
if (stream.codec_type !== "audio" && stream.codec_type !== "video") return stream;
const first = await firstPacketSeconds(filePath, index, signal);
if (first === undefined) return stream;
const end = parseFloat(stream.start_time ?? "0") + parseFloat(stream.duration ?? "");
return {
...stream,
start_time: String(first),
...(Number.isFinite(end) ? { duration: String(Math.max(0, end - first)) } : {}),
};
}),
);
return { ...output, streams };
}
/** Earliest pts of a stream's first packets (a B-frame stream's first packet is not always the earliest). */
async function firstPacketSeconds(
filePath: string,
streamIndex: number,
signal?: AbortSignal,
): Promise<number | undefined> {
// A failed refinement keeps the container's own start time, as before this probe existed.
const stdout = await runFfprobe(
filePath,
[
"-select_streams",
String(streamIndex),
"-show_entries",
"packet=pts_time",
"-read_intervals",
"%+#8",
"-of",
"csv=p=0",
],
signal,
).catch(() => "");
const times = stdout.split("\n").map(parseFloat).filter(Number.isFinite);
return times.length > 0 ? Math.min(...times) : undefined;
}
/** Spawn ffprobe with given args, return stdout. Throws on non-zero exit or missing binary. */
async function runFfprobe(
filePath: string,
@@ -392,7 +443,9 @@ async function probeMediaOutput(filePath: string, signal?: AbortSignal): Promise
filePath,
["-print_format", "json", "-show_format", "-show_streams"],
signal,
).then(parseProbeJson);
)
.then(parseProbeJson)
.then((output) => withMeasuredMatroskaStarts(filePath, output, signal));
if (identity !== null) {
cache.set(filePath, { identity, promise });
if (!signal && cache.size > MEDIA_PROBE_OUTPUT_CACHE_MAX_ENTRIES) {
@@ -806,25 +859,30 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
// (format.duration) includes the longest stream — often a longer audio
// track in stock/looping clips. Cross-check with nb_frames to bound the
// video-specific extent and avoid inflating the expected frame count.
// The container's span starts at media time 0, so it includes a late stream's lead.
const videoStreamLeadSeconds = streamLeadSeconds(output, videoStream);
const containerStreamSpan = Math.max(0, containerDuration - videoStreamLeadSeconds);
let effectiveStreamDuration: number;
if (streamDuration > 0) {
effectiveStreamDuration = streamDuration;
} else if (frames !== undefined && fps > 0 && containerDuration > 0) {
} else if (frames !== undefined && fps > 0 && containerStreamSpan > 0) {
const frameDerivedDuration = frames / fps;
// Only override when the frame-derived duration is meaningfully shorter
// (>10% gap). Within 10% the container value is close enough and may
// account for a trailing hold frame that nb_frames does not include.
effectiveStreamDuration =
frameDerivedDuration < containerDuration * 0.9 ? frameDerivedDuration : containerDuration;
frameDerivedDuration < containerStreamSpan * 0.9
? frameDerivedDuration
: containerStreamSpan;
} else {
effectiveStreamDuration = containerDuration;
effectiveStreamDuration = containerStreamSpan;
}
return {
durationSeconds: containerDuration,
videoStreamDurationSeconds: effectiveStreamDuration,
videoStreamStartSeconds: streamStart,
videoStreamLeadSeconds: streamLeadSeconds(output, videoStream),
videoStreamLeadSeconds,
width: videoStream.width || stillImage()?.width || 0,
height: videoStream.height || stillImage()?.height || 0,
fps,