diff --git a/packages/engine/src/services/streamStartOffset.test.ts b/packages/engine/src/services/streamStartOffset.test.ts index 2d87f1f7c7..88dc59da8b 100644 --- a/packages/engine/src/services/streamStartOffset.test.ts +++ b/packages/engine/src/services/streamStartOffset.test.ts @@ -10,6 +10,7 @@ import { extractVideoFramesRange, getFrameAtTime, resolveFinalFrameExtractionWindow, + resolvePlayableVideoDuration, resolveVideoExtractionWindow, } from "./videoFrameExtractor.js"; @@ -91,6 +92,14 @@ describe.skipIf(!HAS_FFMPEG)( ffmpeg(["-i", picture, "-itsoffset", lead, "-i", steps, ...mux, join(dir, "audio-late.mp4")]); // MPEG-TS starts its clock at 1.4 s, so no stream starts at 0. ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.ts")]); + // Matroska reports start_time 0 for both streams; FLV gives the video a start but no duration. + ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.mkv")]); + ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.flv")]); + ffmpeg([ + ...["-itsoffset", lead, "-i", picture, "-i", steps, "-map", "0:v", "-map", "1:a"], + ...["-c:v", "libvpx-vp9", "-deadline", "realtime", "-cpu-used", "8", "-c:a", "libopus"], + join(dir, "video-late.webm"), + ]); }, 60_000); afterAll(() => rmSync(dir, { recursive: true, force: true })); @@ -107,6 +116,8 @@ describe.skipIf(!HAS_FFMPEG)( { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3 }, { source: "video-late.mp4", audioLead: 0, type: "audio", mediaStart: 0 }, { source: "video-late.ts", audioLead: 0, type: "video", mediaStart: 0 }, + { source: "video-late.webm", audioLead: 0, type: "video", mediaStart: 0 }, + { source: "video-late.mkv", audioLead: 0, type: "video", mediaStart: 0 }, { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3, rate: flatLane }, { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 0 }, { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 3 }, @@ -242,5 +253,23 @@ describe.skipIf(!HAS_FFMPEG)( ); expect(window.extractionMediaStart).toBeGreaterThan(LEAD_SECONDS + PICTURE_SECONDS - 0.1); }, 30_000); + + it.each(["video-late.webm", "video-late.mkv", "video-late.flv"])( + "%s: hides the late video's lead and ends it where the stream ends", + async (name) => { + const source = join(dir, name); + expect(resolvePlayableVideoDuration(await extractMediaMetadata(source))).toBeCloseTo( + LEAD_SECONDS + PICTURE_SECONDS, + 0, + ); + const extracted = await extractVideoFramesRange(source, name, 0, CLIP_SECONDS, { + fps: 10, + outputDir: join(dir, "frames"), + }); + expect(getFrameAtTime(extracted, 1, 0)).toBeNull(); + expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.5, 0)).not.toBeNull(); + }, + 60_000, + ); }, ); diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts index 76eb27acff..902a32b209 100644 --- a/packages/engine/src/utils/ffprobe.ts +++ b/packages/engine/src/utils/ffprobe.ts @@ -47,6 +47,57 @@ function sanitizeFfprobeDiagnostic(stderr: string, filePath: string): string { return `…${redacted.slice(-(FFPROBE_ERROR_MAX_CHARS - 1))}`; } +/** + * Matroska reports start_time 0 for every stream, so a video that starts 7 s after its audio looks + * aligned. Its first packets say when each stream starts; the stream's end stays where it was. + */ +async function withMeasuredMatroskaStarts( + filePath: string, + output: FFProbeOutput, + signal?: AbortSignal, +): Promise { + if (!/matroska|webm/.test(output.format.format_name ?? "")) return output; + const streams = await Promise.all( + output.streams.map(async (stream, index) => { + if (stream.codec_type !== "audio" && stream.codec_type !== "video") return stream; + const first = await firstPacketSeconds(filePath, index, signal); + if (first === undefined) return stream; + const end = parseFloat(stream.start_time ?? "0") + parseFloat(stream.duration ?? ""); + return { + ...stream, + start_time: String(first), + ...(Number.isFinite(end) ? { duration: String(Math.max(0, end - first)) } : {}), + }; + }), + ); + return { ...output, streams }; +} + +/** Earliest pts of a stream's first packets (a B-frame stream's first packet is not always the earliest). */ +async function firstPacketSeconds( + filePath: string, + streamIndex: number, + signal?: AbortSignal, +): Promise { + // A failed refinement keeps the container's own start time, as before this probe existed. + const stdout = await runFfprobe( + filePath, + [ + "-select_streams", + String(streamIndex), + "-show_entries", + "packet=pts_time", + "-read_intervals", + "%+#8", + "-of", + "csv=p=0", + ], + signal, + ).catch(() => ""); + const times = stdout.split("\n").map(parseFloat).filter(Number.isFinite); + return times.length > 0 ? Math.min(...times) : undefined; +} + /** Spawn ffprobe with given args, return stdout. Throws on non-zero exit or missing binary. */ async function runFfprobe( filePath: string, @@ -392,7 +443,9 @@ async function probeMediaOutput(filePath: string, signal?: AbortSignal): Promise filePath, ["-print_format", "json", "-show_format", "-show_streams"], signal, - ).then(parseProbeJson); + ) + .then(parseProbeJson) + .then((output) => withMeasuredMatroskaStarts(filePath, output, signal)); if (identity !== null) { cache.set(filePath, { identity, promise }); if (!signal && cache.size > MEDIA_PROBE_OUTPUT_CACHE_MAX_ENTRIES) { @@ -806,25 +859,30 @@ export async function extractMediaMetadata(filePath: string): Promise 0) { effectiveStreamDuration = streamDuration; - } else if (frames !== undefined && fps > 0 && containerDuration > 0) { + } else if (frames !== undefined && fps > 0 && containerStreamSpan > 0) { const frameDerivedDuration = frames / fps; // Only override when the frame-derived duration is meaningfully shorter // (>10% gap). Within 10% the container value is close enough and may // account for a trailing hold frame that nb_frames does not include. effectiveStreamDuration = - frameDerivedDuration < containerDuration * 0.9 ? frameDerivedDuration : containerDuration; + frameDerivedDuration < containerStreamSpan * 0.9 + ? frameDerivedDuration + : containerStreamSpan; } else { - effectiveStreamDuration = containerDuration; + effectiveStreamDuration = containerStreamSpan; } return { durationSeconds: containerDuration, videoStreamDurationSeconds: effectiveStreamDuration, videoStreamStartSeconds: streamStart, - videoStreamLeadSeconds: streamLeadSeconds(output, videoStream), + videoStreamLeadSeconds, width: videoStream.width || stillImage()?.width || 0, height: videoStream.height || stillImage()?.height || 0, fps,