ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-10-30 17:27:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(engine): late-starting MP4 and WebM videos render every frame the preview shows (#6664)

Browse Source 
* fix(engine): late-starting MP4 and WebM videos render every frame the preview shows

The browser counts an MP4 or WebM from timestamp 0, but FFmpeg seeks from the file's
earliest stream start. A file whose first frame comes after 0 (a fragment cut from a
longer stream, an edit-list offset, a WebM with a start offset) was extracted early and
short, so renders tripped the video coverage gate. Every video seek now maps media time
to FFmpeg's timeline, and the video begins where the browser shows it.

* fix(engine): audio from late-starting MP4 and WebM plays when the preview plays it

The mixer seeked and trimmed audio by FFmpeg's clock, which starts at the file's earliest
stream. In an MP4 or WebM whose streams all start after 0, the browser plays that audio
later, so the render heard it early by the same amount. The mixer now maps media time onto
FFmpeg's clock with the same offset frame extraction uses.

* fix(engine): a late-starting file's picture shows on the clock its audio plays on

* test(producer): vfr-screen-recording baseline shows each frame at its timestamp
This commit is contained in:
Miguel Ángel
2026-10-28 13:09:53 +00:00
committed by GitHub
parent 1df31d8717
commit 437784038b
12 changed files with 254 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/engine/src/index.ts
+2
View File
@@ -230,6 +230,8 @@ export {
resolveFrameFormat,
resolveVideoExtractionDuration,
resolvePlayableVideoDuration,
videoStartOnMediaTimeline,
ffmpegSeekSeconds,
extractionFrameCountForDuration,
resolveProjectRelativeSrc,
getFrameAtTime,
packages/engine/src/services/audioMixer.order.test.ts
+2 -1
View File
@@ -23,7 +23,8 @@ vi.mock("./aacEncode.js", async (importOriginal) => ({
encodeAac: vi.fn(async () => ({ success: true, durationMs: 1, stderr: "", exitCode: 0 })),
}));
vi.mock("../utils/ffprobe.js", () => ({
vi.mock("../utils/ffprobe.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../utils/ffprobe.js")>()),
extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2, audioStream: 0 })),
}));
packages/engine/src/services/audioMixer.ts
+7 -6
View File
@@ -5,7 +5,7 @@ import { renameWithRetry } from "@hyperframes/core/rename-retry";
import { removeScratch } from "@hyperframes/core/remove-scratch";
import { join, dirname, isAbsolute, relative } from "path";
import { parseHTML } from "linkedom";
import { extractAudioMetadata, type AudioMetadata } from "../utils/ffprobe.js";
import { extractAudioMetadata, ffmpegSeekSeconds, type AudioMetadata } from "../utils/ffprobe.js";
import { isNotMediaPayload } from "../utils/notMediaPayload.js";
import { urlInputArgs } from "../utils/urlInput.js";
import { clampAudioGain, clipGain } from "@hyperframes/core/audio-gain";
@@ -769,13 +769,14 @@ async function extractAudioSegment(
removeScratch(cycleDir);
}
}
const inputSeek = mediaStart >= (metadata?.latestStreamLeadSeconds ?? 0);
const start = formatFilterNumber(mediaStart);
const ffmpegStart = ffmpegSeekSeconds(metadata ?? {}, mediaStart);
const inputSeek = ffmpegStart >= (metadata?.latestStreamLeadSeconds ?? 0);
const start = formatFilterNumber(ffmpegStart);
const timestampTrim = inputSeek
? null
: `atrim=start=${start}:end=${formatFilterNumber(mediaStart + sourceDuration)},asetpts=PTS-${start}/TB,aresample=async=1:first_pts=0`;
: `atrim=start=${start}:end=${formatFilterNumber(ffmpegStart + sourceDuration)},asetpts=PTS-${start}/TB,aresample=async=1:first_pts=0`;
const fromSidecar = audioSidecarUrl !== undefined && inputSeek && normalizedPlaybackRate === 1;
const warmUp = fromSidecar ? Math.min(SIDECAR_WARM_UP_SECONDS, mediaStart) : 0;
const warmUp = fromSidecar ? Math.min(SIDECAR_WARM_UP_SECONDS, ffmpegStart) : 0;
const outputArgs = preparedAudioOutputArgs(
metadata,
normalizedPlaybackRate,
@@ -794,7 +795,7 @@ async function extractAudioSegment(
...(readsUrl ? FAIL_ON_READ_ERROR_INSTEAD_OF_ENDING_THERE : []),
...urlInputArgs(readFrom),
...(inputSeek
? ["-ss", String(mediaStart - warmUp), "-t", String(sourceDuration + warmUp)]
? ["-ss", String(ffmpegStart - warmUp), "-t", String(sourceDuration + warmUp)]
: []),
"-i",
readFrom,
packages/engine/src/services/streamStartOffset.test.ts
+155 -2
View File
@@ -1,11 +1,11 @@
import { spawnSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
import { processCompositionAudio } from "./audioMixer.js";
import { extractMediaMetadata } from "../utils/ffprobe.js";
import { extractMediaMetadata, resolvePlayableVideoDuration } from "../utils/ffprobe.js";
import {
extractVideoFramesRange,
getFrameAtTime,
@@ -19,6 +19,7 @@ const SAMPLE_RATE = 48_000;
const LEAD_SECONDS = 7;
const CLIP_SECONDS = 12;
const PICTURE_SECONDS = 20;
const BOTH_LATE_SECONDS = 2;
function ffmpeg(args: string[]): void {
const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args], {
@@ -89,6 +90,17 @@ describe.skipIf(!HAS_FFMPEG)(
const mux = ["-map", "0:v", "-map", "1:a", "-c", "copy"];
ffmpeg(["-itsoffset", lead, "-i", picture, "-i", steps, ...mux, join(dir, "video-late.mp4")]);
ffmpeg(["-i", picture, "-itsoffset", lead, "-i", steps, ...mux, join(dir, "audio-late.mp4")]);
const both = String(BOTH_LATE_SECONDS);
ffmpeg([
"-i",
picture,
"-i",
steps,
...mux,
"-output_ts_offset",
both,
join(dir, "both-late.mp4"),
]);
// MPEG-TS starts its clock at 1.4 s, so no stream starts at 0.
ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.ts")]);
}, 60_000);
@@ -111,6 +123,9 @@ describe.skipIf(!HAS_FFMPEG)(
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 0 },
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 3 },
{ source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "audio", mediaStart: 0 },
{ source: "both-late.mp4", audioLead: BOTH_LATE_SECONDS, type: "audio", mediaStart: 0 },
{ source: "both-late.mp4", audioLead: BOTH_LATE_SECONDS, type: "video", mediaStart: 1 },
{ source: "both-late.mp4", audioLead: BOTH_LATE_SECONDS, type: "audio", mediaStart: 3 },
{
source: "audio-late.mp4",
audioLead: LEAD_SECONDS,
@@ -168,6 +183,26 @@ describe.skipIf(!HAS_FFMPEG)(
expect(getFrameAtTime(extracted, 8, 0)).not.toBeNull();
}, 30_000);
it("shows the picture of a file whose streams all start late on the clock its audio plays on", async () => {
const frames = join(dir, "frames");
const source = await extractVideoFramesRange(join(dir, "picture.mp4"), "picture", 0, 5, {
fps: 30,
outputDir: frames,
});
const late = await extractVideoFramesRange(
join(dir, "both-late.mp4"),
"both-late",
0,
5 + BOTH_LATE_SECONDS,
{ fps: 30, outputDir: frames },
);
for (const second of [0.5, 3.5]) {
expect(readFileSync(getFrameAtTime(late, BOTH_LATE_SECONDS + second, 0)!)).toEqual(
readFileSync(getFrameAtTime(source, second, 0)!),
);
}
}, 30_000);
it("keeps every frame of a video stream that starts first", async () => {
const extracted = await extractVideoFramesRange(
join(dir, "audio-late.mp4"),
@@ -244,3 +279,121 @@ describe.skipIf(!HAS_FFMPEG)(
}, 30_000);
},
);
/** Moves every fragment of a fragmented MP4 later, as cutting it from the middle of a longer stream does. */
function delayFragments(path: string, ticks: number): void {
const data = readFileSync(path);
const walk = (start: number, end: number): void => {
for (let at = start; at + 8 <= end; at += data.readUInt32BE(at)) {
const type = data.toString("latin1", at + 4, at + 8);
if (type === "moof" || type === "traf") walk(at + 8, at + data.readUInt32BE(at));
if (type !== "tfdt") continue;
if (data[at + 8] === 1) {
data.writeBigUInt64BE(data.readBigUInt64BE(at + 12) + BigInt(ticks), at + 12);
} else {
data.writeUInt32BE(data.readUInt32BE(at + 12) + ticks, at + 12);
}
}
};
walk(0, data.length);
writeFileSync(path, data);
}
describe.skipIf(!HAS_FFMPEG)("MP4 and WebM sources whose first timestamp is after 0", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-late-first-frame-"));
// Every frame a different grey, so two frames are equal only when they show the same source frame.
const picture = (seconds: number) => [
"-f",
"lavfi",
"-i",
`nullsrc=size=64x36:rate=24:duration=${seconds},geq=lum='mod(N*37\\,256)':cb=128:cr=128`,
];
const h264 = ["-c:v", "libx264", "-bf", "3", "-pix_fmt", "yuv420p"];
beforeAll(() => {
const fragmented = join(dir, "fragment-cut.mp4");
ffmpeg([
...picture(3.125),
...h264,
"-video_track_timescale",
"12288",
"-movflags",
"cmaf",
fragmented,
]);
delayFragments(fragmented, 2048);
ffmpeg([
...picture(3),
"-c:v",
"libvpx-vp9",
"-deadline",
"realtime",
"-output_ts_offset",
"0.52",
join(dir, "late.webm"),
]);
ffmpeg([...picture(3.125), ...h264, "-output_ts_offset", "1.5", join(dir, "edit-list.mp4")]);
}, 60_000);
afterAll(() => rmSync(dir, { recursive: true, force: true }), 30_000);
const frameBytes = (
extracted: Awaited<ReturnType<typeof extractVideoFramesRange>>,
time: number,
) => {
const path = getFrameAtTime(extracted, time, 0);
return path === null ? null : readFileSync(path);
};
// The browser counts these files from timestamp 0; FFmpeg counts from the first one.
it.each([
{ source: "fragment-cut.mp4", firstFrameAt: 0.25 },
{ source: "late.webm", firstFrameAt: 0.52 },
{ source: "edit-list.mp4", firstFrameAt: 1.5 },
])(
"$source extracts every frame the browser plays, each at its timestamp",
async ({ source, firstFrameAt }) => {
const path = join(dir, source);
const playable = resolvePlayableVideoDuration(await extractMediaMetadata(path));
const extracted = await extractVideoFramesRange(path, source, 0, playable, {
fps: 30,
outputDir: join(dir, `frames-${source}`),
});
expect(extracted.totalFrames).toBe(Math.round(playable * 30));
const sourceFrameAt = (slot: number) =>
Math.max(0, Math.floor((slot / 30 - firstFrameAt) * 24 + 1e-6));
const slots = Array.from({ length: extracted.totalFrames }, (_, slot) =>
readFileSync(extracted.framePaths.get(slot)!),
);
for (let slot = 1; slot < slots.length; slot += 1) {
expect(slots[slot]!.equals(slots[slot - 1]!), `slot ${slot}`).toBe(
sourceFrameAt(slot) === sourceFrameAt(slot - 1),
);
}
const first = frameBytes(extracted, firstFrameAt + 0.03);
if (firstFrameAt < 1) {
expect(frameBytes(extracted, 0)).toEqual(first);
} else {
expect(frameBytes(extracted, firstFrameAt - 1.1)).toBeNull();
expect(frameBytes(extracted, firstFrameAt - 0.9)).toEqual(first);
}
const ranged = await extractVideoFramesRange(path, `${source}-ranged`, 0, playable, {
fps: 30,
outputDir: join(dir, `ranged-${source}`),
frameRanges: [
{ firstFrame: 0, frames: 5 },
{ firstFrame: 40, frames: 10 },
],
});
expect(ranged.framePaths.size).toBe(15);
for (const [index, framePath] of ranged.framePaths) {
expect(readFileSync(framePath), `frame ${index}`).toEqual(
readFileSync(extracted.framePaths.get(index)!),
);
}
},
30_000,
);
});
packages/engine/src/services/videoFrameExtractor.ts
+27 -20
View File
@@ -44,11 +44,13 @@ import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
import {
extractFinalVideoFrameTimestamp,
extractMediaMetadata,
ffmpegSeekSeconds,
keyframeGapSeconds,
mediaFileIdentity,
readMetadataCache,
readStoredFinalFrame,
resolvePlayableVideoDuration,
videoStartOnMediaTimeline,
writeMetadataCache,
type MediaProbeCacheEntry,
type VideoMetadata,
@@ -980,6 +982,7 @@ const inTailProbeSlot = createConcurrencyLimit(TAIL_PROBE_SLOTS);
interface SegmentedExtraction {
decodeArgs: string[];
filterAndEncodeArgs: (seek: number) => string[];
ffmpegSeek: (mediaTime: number) => number;
videoPath: string;
startTime: number;
fps: number;
@@ -1026,7 +1029,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp
...decodeArgs,
"-noaccurate_seek",
"-ss",
String(seek),
String(job.ffmpegSeek(seek)),
"-i",
videoPath,
// Read one frame past the segment and keep exactly its own frames.
@@ -1167,16 +1170,17 @@ async function extractVideoFramesRangeUnredacted(
args.push("-c:v", decoderForCodec(metadata.videoCodec));
}
const decodeArgs = [...args];
const ffmpegSeek = (mediaTime: number) => ffmpegSeekSeconds(metadata, mediaTime);
if (options.finalFrameOnly) {
// Output-side seek decodes from the start before selecting the final
// sample. This is intentionally reserved for the one-frame path: input
// seeking is faster, but valid unindexed transports (notably MPEG-TS with
// a negative timestamp base) can seek to EOF and emit zero frames.
args.push("-i", videoPath, "-ss", String(startTime), "-frames:v", "1");
args.push("-i", videoPath, "-ss", String(Math.max(0, ffmpegSeek(startTime))), "-frames:v", "1");
} else {
// An accurate seek drops the frame on screen at startTime; the fps sampling below needs it.
if (sampleAtOutputFps) args.push("-noaccurate_seek");
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
args.push("-ss", String(ffmpegSeek(startTime)), "-i", videoPath, "-t", String(duration));
}
const vfFilters: string[] = [];
@@ -1191,7 +1195,7 @@ async function extractVideoFramesRangeUnredacted(
? [
...(metadata.isVFR
? []
: undoTickRoundingOnTheSourceFrameGrid(seek, metadata.videoStreamLeadSeconds ?? 0)),
: undoTickRoundingOnTheSourceFrameGrid(seek, videoStartOnMediaTimeline(metadata))),
"settb=intb/2",
"setpts=PTS-lte(TB\\,0.001)",
`fps=${ffmpegFps}:start_time=0:round=up`,
@@ -1240,6 +1244,7 @@ async function extractVideoFramesRangeUnredacted(
? runSegmentedExtraction({
decodeArgs: [...hwaccel, ...decodeArgs],
filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],
ffmpegSeek,
videoPath,
startTime,
fps,
@@ -1416,7 +1421,7 @@ function resolveSegmentDuration(
* the container duration when ffprobe omits the stream duration; keep the
* explicit fallback here for callers supplying older/manual metadata.
*/
export { resolvePlayableVideoDuration };
export { ffmpegSeekSeconds, resolvePlayableVideoDuration, videoStartOnMediaTimeline };
export interface TimelineExtractionWindow {
compositionStart: number;
@@ -1616,16 +1621,19 @@ export async function resolveFinalFrameExtractionWindow(
if (!window.ensureFinalFrame) return window;
const playableDuration = resolvePlayableVideoDuration(metadata);
const videoStreamLead = metadata.videoStreamLeadSeconds ?? 0;
const finalFrameTimestamp =
videoStreamLead +
(await extractFinalVideoFrameTimestamp(
videoPath,
{
videoStreamDurationSeconds: playableDuration - videoStreamLead,
videoStreamStartSeconds: metadata.videoStreamStartSeconds,
},
signal,
));
const lastInstantTheBrowserPlays = playableDuration - FINAL_FRAME_LOGICAL_DURATION_SECONDS;
const finalFrameTimestamp = Math.min(
lastInstantTheBrowserPlays,
videoStartOnMediaTimeline(metadata) +
(await extractFinalVideoFrameTimestamp(
videoPath,
{
videoStreamDurationSeconds: playableDuration - videoStreamLead,
videoStreamStartSeconds: metadata.videoStreamStartSeconds,
},
signal,
)),
);
if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;
const sourceRemaining = playableDuration - video.mediaStart;
@@ -2443,7 +2451,7 @@ export async function extractAllVideoFrames(
sourceFps: work.metadata.fps,
finalFrameSeconds: storedFinalFrame.seconds,
lastPacketSeconds: storedFinalFrame.lastPacketSeconds,
streamLeadSeconds: work.metadata.videoStreamLeadSeconds ?? 0,
streamLeadSeconds: videoStartOnMediaTimeline(work.metadata),
});
if (storedTailFrames !== null) {
const totalFrames = firstTailFrame + storedTailFrames;
@@ -2748,12 +2756,11 @@ export async function extractAllVideoFrames(
const BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS = 1;
export function isVideoHiddenBeforeStreamStart(
videoStreamLeadSeconds: number | undefined,
videoStartSeconds: number | undefined,
mediaTime: number,
): boolean {
return (
(videoStreamLeadSeconds ?? 0) - mediaTime >=
BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS - 1e-6
(videoStartSeconds ?? 0) - mediaTime >= BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS - 1e-6
);
}
@@ -2792,7 +2799,7 @@ function getFrameIndexAtTime(
if (frameIndex < 0 || extracted.totalFrames <= 0) return null;
if (
isVideoHiddenBeforeStreamStart(
extracted.metadata.videoStreamLeadSeconds,
videoStartOnMediaTimeline(extracted.metadata),
mediaStart + sourceTime,
)
) {
packages/engine/src/utils/ffprobe.ts
+26 -1
View File
@@ -207,6 +207,7 @@ export interface VideoMetadata {
videoStreamStartSeconds?: number;
/** Seconds after the file's media time 0 (its earliest audio/video stream start) at which video begins. */
videoStreamLeadSeconds?: number;
mediaTimeOffsetSeconds?: number;
width: number;
height: number;
fps: number;
@@ -243,6 +244,7 @@ export interface AudioMetadata {
/** Seconds after the file's media time 0 at which its last audio or video stream starts. */
latestStreamLeadSeconds?: number;
audioStreamLeadSeconds?: number;
mediaTimeOffsetSeconds?: number;
audioStreamCount?: number;
/** The track read for these facts and for the mix, by its place among the audio streams (decodableAudioStream). */
audioStream: number;
@@ -512,6 +514,19 @@ export async function buildProbeFactsText(filePath: string, signal?: AbortSignal
return JSON.stringify(facts);
}
export function videoStartOnMediaTimeline(
metadata: Pick<VideoMetadata, "videoStreamLeadSeconds" | "mediaTimeOffsetSeconds">,
): number {
return (metadata.videoStreamLeadSeconds ?? 0) + (metadata.mediaTimeOffsetSeconds ?? 0);
}
export function ffmpegSeekSeconds(
metadata: Pick<VideoMetadata, "mediaTimeOffsetSeconds">,
mediaTime: number,
): number {
return mediaTime - (metadata.mediaTimeOffsetSeconds ?? 0);
}
export function resolvePlayableVideoDuration(metadata: VideoMetadata): number {
return Number.isFinite(metadata.videoStreamDurationSeconds) &&
metadata.videoStreamDurationSeconds > 0
@@ -1004,6 +1019,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
videoStreamDurationSeconds: effectiveStreamDuration,
videoStreamStartSeconds: streamStart,
videoStreamLeadSeconds: streamLeadSeconds(output, videoStream),
mediaTimeOffsetSeconds: mediaTimeOffsetSeconds(output),
width: videoStream.width || stillImage()?.width || 0,
height: videoStream.height || stillImage()?.height || 0,
fps,
@@ -1249,12 +1265,13 @@ async function probeAudioMetadata(filePath: string, signal?: AbortSignal): Promi
bitrate: output.format.bit_rate ? parseInt(output.format.bit_rate) : undefined,
latestStreamLeadSeconds: latestStreamLeadSeconds(output),
audioStreamLeadSeconds: streamLeadSeconds(output, audioStream),
mediaTimeOffsetSeconds: mediaTimeOffsetSeconds(output),
audioStreamCount: audioStreams.length,
audioStream: audioStreamIndex,
};
}
/** Chrome's media time 0 is the earliest of these; a stream starting later plays that much in. */
/** A stream starting after the earliest of these plays that much after it. */
function audioVideoStreamStarts(output: FFProbeOutput | null): number[] {
return (output?.streams ?? [])
.filter(
@@ -1271,6 +1288,14 @@ function streamLeadSeconds(output: FFProbeOutput | null, stream: FFProbeStream):
return Number.isFinite(start) && starts.length > 0 ? Math.max(0, start - Math.min(...starts)) : 0;
}
const BROWSER_PLAYS_FROM_TIMESTAMP_ZERO = /\b(mp4|webm)\b/;
function mediaTimeOffsetSeconds(output: FFProbeOutput | null): number {
const starts = audioVideoStreamStarts(output);
const fromZero = BROWSER_PLAYS_FROM_TIMESTAMP_ZERO.test(output?.format.format_name ?? "");
return fromZero && starts.length > 0 ? Math.max(0, Math.min(...starts)) : 0;
}
function latestStreamLeadSeconds(output: FFProbeOutput): number {
const starts = audioVideoStreamStarts(output);
return starts.length > 0 ? Math.max(...starts) - Math.min(...starts) : 0;
packages/producer/src/services/distributed/shared.ts
+4
View File
@@ -243,6 +243,10 @@ function readVideoMetadata(
record.videoStreamLeadSeconds === undefined
? 0
: readFiniteNumber(record.videoStreamLeadSeconds, `${field}.videoStreamLeadSeconds`),
mediaTimeOffsetSeconds:
record.mediaTimeOffsetSeconds === undefined
? 0
: readFiniteNumber(record.mediaTimeOffsetSeconds, `${field}.mediaTimeOffsetSeconds`),
width: readPositiveInteger(record.width, `${field}.width`),
height: readPositiveInteger(record.height, `${field}.height`),
fps: readFiniteNumber(record.fps, `${field}.fps`),
packages/producer/src/services/distributed/videoMetadata.test.ts
+5 -1
View File
@@ -94,7 +94,7 @@ describe("distributed video metadata", () => {
expect(sourceDerived.videos[0]?.mediaStart).toBe(1);
});
it("round-trips a non-zero video stream start and lead and defaults legacy plans to zero", () => {
it("round-trips a non-zero video stream start, lead and media time offset and defaults legacy plans to zero", () => {
const withStart = buildPlanVideosJson({
videos: [video()],
extracted: [
@@ -103,6 +103,7 @@ describe("distributed video metadata", () => {
...extractedMetadata().metadata,
videoStreamStartSeconds: 5,
videoStreamLeadSeconds: 7,
mediaTimeOffsetSeconds: 0.25,
},
}),
],
@@ -111,13 +112,16 @@ describe("distributed video metadata", () => {
const parsed = parsePlanVideosJson(withStart).extracted[0]?.metadata;
expect(parsed?.videoStreamStartSeconds).toBe(5);
expect(parsed?.videoStreamLeadSeconds).toBe(7);
expect(parsed?.mediaTimeOffsetSeconds).toBe(0.25);
const legacy = structuredClone(withStart);
delete legacy.extracted[0]?.metadata.videoStreamStartSeconds;
delete legacy.extracted[0]?.metadata.videoStreamLeadSeconds;
delete legacy.extracted[0]?.metadata.mediaTimeOffsetSeconds;
const legacyParsed = parsePlanVideosJson(legacy).extracted[0]?.metadata;
expect(legacyParsed?.videoStreamStartSeconds).toBe(0);
expect(legacyParsed?.videoStreamLeadSeconds).toBe(0);
expect(legacyParsed?.mediaTimeOffsetSeconds).toBe(0);
});
it("keeps alphaPlaneDecoded when present and accepts plans without it", () => {
packages/producer/src/services/hdrCompositor.test.ts
+1 -1
View File
@@ -105,7 +105,7 @@ describe("blitHdrVideoLayer", () => {
frameCount: 2,
scratch: Buffer.alloc(frameSize),
mediaStart,
videoStreamLeadSeconds: lead,
videoStartSeconds: lead,
};
await blitHdrVideoLayer(
canvas,
packages/producer/src/services/hdrCompositor.ts
+2 -2
View File
@@ -143,7 +143,7 @@ export interface HdrVideoFrameSource {
/** The playable range is one source cycle and must wrap at its end. */
loop?: boolean;
mediaStart?: number;
videoStreamLeadSeconds?: number;
videoStartSeconds?: number;
}
export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: ProducerLogger): void {
@@ -201,7 +201,7 @@ export async function blitHdrVideoLayer(
);
if (effectiveIndex === null) return;
const mediaTime = (frameSource.mediaStart ?? 0) + effectiveIndex / fps;
if (isVideoHiddenBeforeStreamStart(frameSource.videoStreamLeadSeconds, mediaTime)) return;
if (isVideoHiddenBeforeStreamStart(frameSource.videoStartSeconds, mediaTime)) return;
if (hdrPerf) hdrPerf.hdrVideoLayerBlits += 1;
const lease = await frameSource.windows.lease(effectiveIndex);
packages/producer/src/services/render/stages/captureHdrResources.ts
+21 -12
View File
@@ -22,6 +22,7 @@ import {
type CaptureSession,
decodePngToRgb48le,
extractMediaMetadata,
ffmpegSeekSeconds,
getCgroupMemoryLimitMb,
normalizeObjectFit,
queryElementStacking,
@@ -32,6 +33,7 @@ import {
runFfmpeg,
type TimelineExtractionWindow,
type VideoMetadata,
videoStartOnMediaTimeline,
} from "@hyperframes/engine";
import { fpsToFfmpegArg, fpsToNumber } from "@hyperframes/core";
import type { ProducerLogger } from "../../../logger.js";
@@ -425,7 +427,7 @@ export async function extractHdrVideoFrames(args: {
let scratchBytes = 0;
const extractionWindows = new Map<
string,
{ window: HdrExtractionWindow; videoStreamLeadSeconds?: number }
{ window: HdrExtractionWindow; metadata: VideoMetadata }
>();
for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) {
const video = composition.videos.find((v) => v.id === videoId);
@@ -441,10 +443,7 @@ export async function extractHdrVideoFrames(args: {
initialWindow,
abortSignal,
);
extractionWindows.set(videoId, {
window,
videoStreamLeadSeconds: metadata.videoStreamLeadSeconds,
});
extractionWindows.set(videoId, { window, metadata });
prep.hdrVideoStartTimes.set(videoId, window.compositionStart);
const frameSize = dims.width * dims.height * 6;
const frames = window.finalFrameOnly ? 1 : Math.ceil(window.durationSeconds * fps);
@@ -463,17 +462,18 @@ export async function extractHdrVideoFrames(args: {
const video = composition.videos.find((v) => v.id === videoId);
const planned = extractionWindows.get(videoId);
if (!video || !planned) continue;
const { window } = planned;
const { window, metadata } = planned;
mkdirSync(framesDir, { recursive: true });
const frameDir = mkdtempSync(join(framesDir, `hdr_${tempDirSafePrefix(videoId)}-`));
createdFrameDirs.add(frameDir);
const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height };
const frameSize = dims.width * dims.height * 6;
const extractionMediaStart = window.extractionMediaStart ?? window.mediaStart;
const ffmpegSeek = ffmpegSeekSeconds(metadata, extractionMediaStart);
const scale = `scale=${dims.width}:${dims.height}:force_original_aspect_ratio=increase,crop=${dims.width}:${dims.height}`;
const firstTick = window.finalFrameOnly
? 0
: await probeFirstTick(srcPath, extractionMediaStart, fps, runFfmpegImpl, abortSignal);
: await probeFirstTick(srcPath, ffmpegSeek, fps, runFfmpegImpl, abortSignal);
const totalFrames = window.finalFrameOnly
? 1
: Math.max(1, Math.ceil(window.durationSeconds * fps - firstTick - 1e-9)) +
@@ -489,10 +489,19 @@ export async function extractHdrVideoFrames(args: {
const ffmpegArgs = window.finalFrameOnly
? // Decode before seeking for the one-frame path. Input-side seeking can
// return zero frames for valid unindexed/negative-base transports.
["-i", srcPath, "-ss", String(extractionMediaStart), "-frames:v", "1", "-vf", scale]
[
"-i",
srcPath,
"-ss",
String(Math.max(0, ffmpegSeek)),
"-frames:v",
"1",
"-vf",
scale,
]
: hdrWindowFfmpegArgs({
srcPath,
mediaStart: extractionMediaStart,
ffmpegSeek,
fps,
fpsArg: fpsToFfmpegArg(job.config.fps),
firstTick,
@@ -544,7 +553,7 @@ export async function extractHdrVideoFrames(args: {
scratch: Buffer.allocUnsafe(frameSize),
loop: video.loop,
mediaStart: extractionMediaStart,
videoStreamLeadSeconds: planned.videoStreamLeadSeconds,
videoStartSeconds: videoStartOnMediaTimeline(metadata),
});
}
return { sources: out, estimatedBytes, releaseReservation };
@@ -568,7 +577,7 @@ export async function extractHdrVideoFrames(args: {
*/
function hdrWindowFfmpegArgs(a: {
srcPath: string;
mediaStart: number;
ffmpegSeek: number;
fps: number;
fpsArg: string;
firstTick: number;
@@ -582,7 +591,7 @@ function hdrWindowFfmpegArgs(a: {
const filters = [`fps=${a.fpsArg}:start_time=${(a.firstTick / a.fps).toFixed(6)}`, a.scale];
if (trim > 0) filters.push(`trim=start_frame=${trim}`);
return [
...["-ss", (a.mediaStart + seekFrame / a.fps).toFixed(6), "-i", a.srcPath],
...["-ss", (a.ffmpegSeek + seekFrame / a.fps).toFixed(6), "-i", a.srcPath],
...["-vf", filters.join(","), "-frames:v", String(a.frames), "-fps_mode", "passthrough"],
];
}
packages/producer/tests/vfr-screen-recording/output/output.mp4
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:005a5c831c4c43d594d976581cd6ffac00c9ceb28e3020deef37fcdfda019529
size 455962
oid sha256:6f72fd79cf765e744a818ed89cd4601fe9f195dfbf6ef1f38aba504dbb0ca3e7
size 467779