Files
hyperframes/packages/engine/src/services/videoFrameExtractor.test.ts
T

4629 lines
160 KiB
TypeScript

// fallow-ignore-file code-duplication
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
statSync,
utimesSync,
writeFileSync,
} from "node:fs";
import { basename, join } from "node:path";
import { tmpdir } from "node:os";
import { spawnSync } from "node:child_process";
import {
parseVideoElements,
parseImageElements,
extractAllVideoFrames,
extractVideoFramesRange,
extractionFrameCountForDuration,
createFrameLookupTable,
FrameLookupTable,
rebaseVideoToWindow,
resolveFrameFormat,
codecMayHaveAlpha,
decoderForCodec,
resolveVideoExtractionWindow,
resolveVideoExtractionDuration,
getFrameAtTime,
analyzeClipMediaFit,
classifyVideoExtractionError,
runVideoExtractionWithRetry,
VideoSourceExtractionError,
type VideoElement,
type ExtractedFrames,
type ExtractionResult,
} from "./videoFrameExtractor.js";
describe("parseVideoElements strict literal timing", () => {
it.each(["", " ", "0s", "0abc", "0px", "-1s", "Infinity", "NaN", "0x10"])(
"does not drop hand-authored data-duration=%j as an explicit zero window",
(duration) => {
expect(
parseVideoElements(
`<video id="v" src="clip.mp4" data-start="0" data-duration="${duration}"></video>`,
),
).toHaveLength(1);
},
);
});
import {
extractFinalVideoFrameTimestamp,
extractVideoMetadata,
type VideoMetadata,
} from "../utils/ffprobe.js";
import { ffmpegStatsReader, runFfmpeg, type RunFfmpegOptions } from "../utils/runFfmpeg.js";
// Pass-through, so a test can tell whether extraction ran ffmpeg.
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
});
import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.js";
import { resolveRuntimeMediaClipDuration } from "../../../core/src/runtime/media.js";
import { compileTimingAttrs, sourceTimeAt } from "@hyperframes/core";
import { RATE_RANGE } from "@hyperframes/core/audio-automation";
// ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer
// regression test at packages/producer/tests/vfr-screen-recording/ runs inside
// Dockerfile.test (which does include ffmpeg) and is the primary CI signal
// for this bug. Locally and in any CI job with ffmpeg on PATH, the tests
// below run too — they exercise the extractor in isolation against a
// synthesized VFR fixture.
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
const HAS_ZSCALE =
HAS_FFMPEG &&
/\szscale\s/.test(spawnSync("ffmpeg", ["-hide_banner", "-filters"]).stdout.toString());
const FFMPEG_ENCODERS = HAS_FFMPEG
? spawnSync("ffmpeg", ["-hide_banner", "-encoders"]).stdout.toString()
: "";
// Homebrew's ffmpeg has libsvtav1 but not libaom-av1. Either one writes an untagged AV1 stream.
const AV1_ENCODER = ["libaom-av1", "libsvtav1"].find((name) =>
new RegExp(`\\s${name}\\s`).test(FFMPEG_ENCODERS),
);
describe("resolveVideoExtractionDuration", () => {
const metadata = (
durationSeconds: number,
videoStreamDurationSeconds = durationSeconds,
): VideoMetadata => ({
durationSeconds,
videoStreamDurationSeconds,
width: 1920,
height: 1080,
fps: 30,
videoCodec: "h264",
hasAudio: false,
isVFR: false,
hasAlpha: false,
colorSpace: null,
});
const video = (overrides: Partial<VideoElement> = {}): VideoElement => ({
id: "root-video",
src: "video.mp4",
start: 0,
end: Number.POSITIVE_INFINITY,
mediaStart: 0,
playbackRate: 1,
loop: false,
hasAudio: false,
...overrides,
});
it("caps an open 60-second root source to a two-second composition", () => {
expect(resolveVideoExtractionDuration(video(), metadata(60), 2)).toBe(2);
});
it("keeps a shorter natural source duration inside a longer composition", () => {
expect(resolveVideoExtractionDuration(video(), metadata(2), 10)).toBe(2);
});
it("falls back to container duration when stream duration is unavailable", () => {
expect(resolveVideoExtractionDuration(video(), metadata(2, 0), 10)).toBe(2);
});
it("preserves explicit bounds and loop flags while applying the timeline ceiling", () => {
const explicitLoop = video({ end: 8, loop: true });
expect(resolveVideoExtractionDuration(explicitLoop, metadata(60), 10)).toBe(8);
expect(explicitLoop.loop).toBe(true);
});
it("extracts the source span consumed by an explicit 2x timeline slot", () => {
expect(
resolveVideoExtractionWindow(
video({ end: 2, mediaStart: 1, playbackRate: 2 }),
metadata(8),
2,
),
).toMatchObject({
compositionStart: 0,
mediaStart: 1,
durationSeconds: 4,
timelineDurationSeconds: 2,
});
});
it("extracts the source span a ramped slot consumes: 2s of 1x to 3x reads 2*(3-1)/ln 3 source seconds", () => {
const rate = {
target: "rate",
points: [
{ t: 0, v: 1 },
{ t: 2, v: 3 },
],
};
expect(
resolveVideoExtractionWindow(video({ end: 2, playbackRate: rate }), metadata(8), 2),
).toMatchObject({ compositionStart: 0, mediaStart: 0, timelineDurationSeconds: 2 });
const { durationSeconds } = resolveVideoExtractionWindow(
video({ end: 2, playbackRate: rate }),
metadata(8),
2,
);
expect(durationSeconds).toBeCloseTo(3.6411, 3);
});
const RAMP = {
target: "rate",
points: [
{ t: 0, v: 1 },
{ t: 4, v: 3 },
],
};
it("starts a ramped clip that begins before the timeline at the integrated source time of the trimmed part", () => {
// geometric 1x to 3x over 4s: source(t) = 4(3^(t/4)-1)/ln 3; 1s trimmed reads 1.1508, all 4s read 7.2819
const window = resolveVideoExtractionWindow(
video({ start: -1, end: 3, playbackRate: RAMP }),
metadata(20),
3,
);
expect(window.mediaStart).toBeCloseTo(1.1508, 3);
expect(window.durationSeconds).toBeCloseTo(6.1311, 3);
});
it("rebases a ramped clip so lookup reads the same source second the untrimmed lane would", () => {
const clip = video({ start: -1, end: 3, playbackRate: RAMP });
rebaseVideoToWindow(clip, resolveVideoExtractionWindow(clip, metadata(20), 3));
// composition second 2 is 3s into the authored lane: source 4.6586
const seconds = clip.mediaStart + sourceTimeAt(clip.playbackRate ?? 1, 2 - clip.start);
expect(seconds).toBeCloseTo(4.6586, 3);
});
it("snaps a float-noise composition start to the timeline origin so the first frame is kept", () => {
const clip = video({ start: -1, end: 8, playbackRate: 0.7 });
rebaseVideoToWindow(clip, { compositionStart: 2.2e-16, mediaStart: 0.7, durationSeconds: 5 });
expect(clip.start).toBe(0);
});
it("reports the natural timeline duration of a ramped clip through the lane", () => {
const flat = {
target: "rate",
points: [
{ t: 0, v: 2 },
{ t: 1, v: 2 },
],
};
expect(resolveVideoExtractionWindow(video({ playbackRate: flat }), metadata(4))).toMatchObject({
durationSeconds: 4,
timelineDurationSeconds: 2,
});
});
it("reports natural timeline duration after constant playback-rate retiming", () => {
expect(
resolveVideoExtractionWindow(video({ mediaStart: 1, playbackRate: 2 }), metadata(5), 10),
).toMatchObject({
compositionStart: 0,
mediaStart: 1,
durationSeconds: 4,
timelineDurationSeconds: 2,
});
});
it("trims materially negative preroll and advances the source offset", () => {
const preroll = video({ start: -60, end: 120, mediaStart: 0 });
expect(resolveVideoExtractionWindow(preroll, metadata(120), 2)).toEqual({
compositionStart: 0,
mediaStart: 60,
durationSeconds: 2,
});
expect(resolveVideoExtractionDuration(preroll, metadata(120), 2)).toBe(2);
});
it("returns an empty window for a clip entirely before composition time zero", () => {
expect(resolveVideoExtractionWindow(video({ start: -60, end: -10 }), metadata(120), 2)).toEqual(
{ compositionStart: 0, mediaStart: 60, durationSeconds: 0 },
);
});
it("preserves a short source cycle when negative preroll crosses a loop boundary", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -5, end: 10, mediaStart: 0, loop: true }),
metadata(3),
2,
),
).toEqual({
compositionStart: -5,
mediaStart: 0,
durationSeconds: 3,
preserveTimelinePhase: true,
});
});
it("marks an entirely held interval for exact final-frame resolution", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -5, end: 10, mediaStart: 0, loop: false }),
metadata(3),
2,
),
).toEqual({
compositionStart: -2.000001,
mediaStart: 2.999999,
durationSeconds: 0.000001,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it.each([
{ loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" },
{
loop: false,
preservation: { preserveTimelineEnd: true, ensureFinalFrame: true },
label: "held tail",
},
])(
"caps a finite long slot to one short source range for $label playback",
({ loop, preservation }) => {
expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(3), 60)).toEqual({
compositionStart: 0,
mediaStart: 0,
durationSeconds: 3,
...preservation,
});
},
);
it.each([
{ loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" },
{
loop: false,
preservation: { preserveTimelineEnd: true, ensureFinalFrame: true },
label: "held tail",
},
])(
"uses the playable video-stream duration for a long-audio mux in $label playback",
({ loop, preservation }) => {
expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(60, 3), 60)).toEqual({
compositionStart: 0,
mediaStart: 0,
durationSeconds: 3,
...preservation,
});
},
);
it("preserves authored timing when the visible interval partially crosses a held tail", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -2, end: 10, mediaStart: 0, loop: false }),
metadata(3),
2,
),
).toEqual({
compositionStart: 0,
mediaStart: 2,
durationSeconds: 1,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it.each([
{ start: 0, expected: { compositionStart: 0, mediaStart: 0, durationSeconds: 3 } },
{ start: -2, expected: { compositionStart: 0, mediaStart: 2, durationSeconds: 1 } },
{ start: -5, expected: { compositionStart: 0, mediaStart: 5, durationSeconds: 0 } },
])(
"keeps an omitted-duration clip source-bounded like runtime (start=$start)",
({ start, expected }) => {
for (const loop of [false, true]) {
const parsed = parseVideoElements(
`<video id="natural" src="video.mp4"${loop ? " loop" : ""}></video>`,
)[0]!;
const planned = { ...parsed, start };
const runtimeDuration = resolveRuntimeMediaClipDuration({
isVideo: true,
sourceDuration: 3,
hostRemaining: 15 - start,
explicitDuration: null,
});
expect(runtimeDuration).toBe(3);
expect(parsed.end).toBe(Number.POSITIVE_INFINITY);
expect(resolveVideoExtractionWindow(planned, metadata(3), 15)).toEqual(expected);
}
},
);
it("bounds an entirely held long source to its final-frame sample", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -600, end: 10, mediaStart: 0, loop: false }),
metadata(120),
2,
),
).toEqual({
compositionStart: -480.000001,
mediaStart: 119.999999,
durationSeconds: 0.000001,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it("preserves a complete loop cycle when visibility ends exactly on a wrap boundary", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -1, end: 10, mediaStart: 0, loop: true }),
metadata(3),
2,
),
).toEqual({
compositionStart: -1,
mediaStart: 0,
durationSeconds: 3,
preserveTimelinePhase: true,
});
});
it("keeps an open-ended loop source-bounded instead of inventing a longer slot", () => {
expect(resolveVideoExtractionWindow(video({ loop: true }), metadata(3), 10)).toEqual({
compositionStart: 0,
mediaStart: 0,
durationSeconds: 3,
});
});
it("never plans more extraction than the playable source range", () => {
for (const loop of [false, true]) {
for (const sourceDuration of [0.5, 3, 120]) {
for (const mediaStart of [0, sourceDuration / 3]) {
for (const start of [-600, -5, -1, 0, 2]) {
const window = resolveVideoExtractionWindow(
video({ start, end: start + 60, mediaStart, loop }),
metadata(sourceDuration),
10,
);
expect(window.durationSeconds).toBeLessThanOrEqual(sourceDuration - mediaStart);
expect(window.durationSeconds).toBeGreaterThanOrEqual(0);
}
}
}
}
});
it("rejects a media start at source EOF before planning extraction", () => {
expect(() =>
resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(3), 10),
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
});
it("rejects a media start at video-stream EOF even when the container continues", () => {
expect(() =>
resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(60, 3), 10),
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
});
it("plans a one-frame held tail when an explicit non-looping slot starts past EOF", () => {
expect(resolveVideoExtractionWindow(video({ end: 6, mediaStart: 5 }), metadata(2))).toEqual({
compositionStart: 0,
mediaStart: 1.999999,
durationSeconds: 0.000001,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it("keeps the playable suffix when an explicit non-looping slot starts just inside EOF", () => {
expect(
resolveVideoExtractionWindow(video({ end: 6, mediaStart: 1.9 }), metadata(2), 6),
).toEqual({
compositionStart: 0,
mediaStart: 1.9,
durationSeconds: 0.10000000000000009,
preserveTimelineEnd: true,
ensureFinalFrame: true,
});
});
it("rejects a looping explicit slot that starts at source EOF", () => {
expect(() =>
resolveVideoExtractionWindow(video({ end: 6, mediaStart: 2, loop: true }), metadata(2), 6),
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
});
it("rebases a loop phase when the visible window stays within one cycle", () => {
expect(
resolveVideoExtractionWindow(
video({ start: -5, end: 10, mediaStart: 0, loop: true }),
metadata(3),
0.5,
),
).toEqual({ compositionStart: 0, mediaStart: 2, durationSeconds: 0.5 });
});
it("retains legacy behavior when no timeline end is supplied", () => {
expect(resolveVideoExtractionDuration(video(), metadata(60))).toBe(60);
});
});
describe("extractionFrameCountForDuration", () => {
it("uses the same VFR ceil and CFR nearest-boundary rules as FFmpeg", () => {
expect(extractionFrameCountForDuration(0.466666, 30, true)).toBe(14);
expect(extractionFrameCountForDuration(0.466666, 30, false)).toBe(14);
expect(extractionFrameCountForDuration(0.616666, 30, true)).toBe(19);
expect(extractionFrameCountForDuration(0.616666, 30, false)).toBe(18);
});
it.each([
[0.33 - 0.03, 30, 9],
[0.29 - 0.04, 24, 6],
[0.35 - 0.05, 60, 18],
[4.03 - 3.53, 30, 15],
[4.03 - 3.78, 24, 6],
])(
"snaps floating-point integral boundaries before VFR ceil (%s seconds at %i fps)",
(duration, fps, expectedFrames) => {
expect(extractionFrameCountForDuration(duration, fps, true)).toBe(expectedFrames);
},
);
it("still ceils a genuine fractional boundary beyond floating-point noise", () => {
expect(extractionFrameCountForDuration(0.300001, 30, true)).toBe(10);
});
it("matches FFmpeg's six-digit duration parsing", () => {
expect(extractionFrameCountForDuration(0.6000009, 30, true)).toBe(18);
expect(extractionFrameCountForDuration(0.600001, 30, true)).toBe(19);
expect(extractionFrameCountForDuration(2.05, 30, false)).toBe(62);
});
it("keeps exact NTSC rationals at short CFR and VFR boundaries", () => {
expect(extractionFrameCountForDuration(0.25025, { num: 30000, den: 1001 }, false)).toBe(8);
expect(extractionFrameCountForDuration(0.125125, { num: 24000, den: 1001 }, true)).toBe(3);
expect(extractionFrameCountForDuration(0.5005, { num: 24000, den: 1001 }, true)).toBe(12);
});
it("fails closed for invalid durations and emits one frame for positive sub-frame work", () => {
expect(extractionFrameCountForDuration(Number.NaN, 30, true)).toBe(0);
expect(extractionFrameCountForDuration(1, 0, true)).toBe(0);
expect(extractionFrameCountForDuration(0, 30, true)).toBe(0);
expect(extractionFrameCountForDuration(0.001, 30, false)).toBe(1);
});
});
describe("video extraction failure taxonomy and bounded retry", () => {
it("classifies missing and transient HTTP sources without exposing retry ambiguity", () => {
expect(classifyVideoExtractionError(new Error("HTTP 404: Not Found"))).toMatchObject({
kind: "download_not_found",
retryable: false,
});
expect(classifyVideoExtractionError(new Error("HTTP 503: Service Unavailable"))).toMatchObject({
kind: "download_transient",
retryable: true,
});
});
it("retries one transient failure, cleaning partial output before the retry", async () => {
const retryDir = mkdtempSync(join(tmpdir(), "hf-extract-retry-"));
const partialPath = join(retryDir, "frame-00001.jpg");
let attempts = 0;
try {
const outcome = await runVideoExtractionWithRetry(
async () => {
attempts += 1;
if (attempts === 1) {
writeFileSync(partialPath, "partial");
throw new VideoSourceExtractionError(
"ffmpeg_timeout",
true,
"Video frame extraction timed out",
);
}
expect(existsSync(partialPath)).toBe(false);
return "frames";
},
{
maxTransientRetries: 1,
onRetry: () => {
rmSync(retryDir, { recursive: true, force: true });
mkdirSync(retryDir, { recursive: true });
},
},
);
expect(outcome).toEqual({ result: "frames", retries: 1 });
expect(attempts).toBe(2);
} finally {
rmSync(retryDir, { recursive: true, force: true });
}
});
it("does not retry deterministic or caller-aborted failures", async () => {
let deterministicAttempts = 0;
await expect(
runVideoExtractionWithRetry(async () => {
deterministicAttempts += 1;
throw new VideoSourceExtractionError(
"zero_output",
false,
"Video source produced no decodable frames",
);
}),
).rejects.toMatchObject({ kind: "zero_output", retryable: false });
expect(deterministicAttempts).toBe(1);
const controller = new AbortController();
controller.abort();
let abortedAttempts = 0;
await expect(
runVideoExtractionWithRetry(
async () => {
abortedAttempts += 1;
throw new VideoSourceExtractionError(
"download_transient",
true,
"Video source download failed transiently",
);
},
{ signal: controller.signal },
),
).rejects.toMatchObject({ kind: "cancelled", retryable: false });
expect(abortedAttempts).toBe(0);
});
it("does not retry transient extraction failures unless the caller opts in", async () => {
let attempts = 0;
await expect(
runVideoExtractionWithRetry(async () => {
attempts += 1;
throw new VideoSourceExtractionError(
"ffmpeg_timeout",
true,
"Video frame extraction timed out",
);
}),
).rejects.toMatchObject({ kind: "ffmpeg_timeout", retryable: true });
expect(attempts).toBe(1);
});
it("fails closed to zero retries for a non-finite runtime retry budget", async () => {
let attempts = 0;
await expect(
runVideoExtractionWithRetry(
async () => {
attempts += 1;
throw new VideoSourceExtractionError(
"ffmpeg_timeout",
true,
"Video frame extraction timed out",
);
},
{ maxTransientRetries: Number.NaN },
),
).rejects.toMatchObject({ kind: "ffmpeg_timeout", retryable: true });
expect(attempts).toBe(1);
});
});
// Codec-based alpha defaulting replaces tag-based detection (the
// alpha_mode/ALPHA_MODE case bug — see ffprobe.test.ts for the regression
// pin on that). The extractor uses these helpers for two decisions:
// 1. whether to force the alpha-aware decoder (libvpx-vp9 for VP9, libvpx
// for VP8)
// 2. whether to default the cached frame format to PNG (with alpha) vs JPG
// The "default to capable" trade is small file-size growth on opaque VP9
// content for correctness on alpha-having content even when the sidecar tag
// is missing or muxed with the wrong case.
describe("codec alpha capability", () => {
it("flags VP9, VP8, and ProRes as alpha-capable", () => {
expect(codecMayHaveAlpha("vp9")).toBe(true);
expect(codecMayHaveAlpha("VP9")).toBe(true);
expect(codecMayHaveAlpha("vp8")).toBe(true);
expect(codecMayHaveAlpha("prores")).toBe(true);
});
it("does not flag h264 / h265 / mpeg4 (no alpha in their bitstreams)", () => {
expect(codecMayHaveAlpha("h264")).toBe(false);
expect(codecMayHaveAlpha("h265")).toBe(false);
expect(codecMayHaveAlpha("hevc")).toBe(false);
expect(codecMayHaveAlpha("mpeg4")).toBe(false);
});
it("treats undefined / empty input as non-alpha", () => {
expect(codecMayHaveAlpha(undefined)).toBe(false);
expect(codecMayHaveAlpha("")).toBe(false);
});
it("returns the alpha-aware decoder name for VP9 and VP8", () => {
expect(decoderForCodec("vp9")).toBe("libvpx-vp9");
expect(decoderForCodec("VP9")).toBe("libvpx-vp9");
expect(decoderForCodec("vp8")).toBe("libvpx");
});
});
describe("resolveFrameFormat", () => {
function metadata(overrides: Partial<VideoMetadata> = {}): VideoMetadata {
return {
durationSeconds: 1,
width: 320,
height: 180,
fps: 30,
hasAudio: false,
videoCodec: "h264",
colorSpace: {
colorTransfer: "bt709",
colorPrimaries: "bt709",
colorSpace: "bt709",
},
isVFR: false,
hasAlpha: false,
...overrides,
};
}
it("keeps opaque non-alpha sources on jpg by default", () => {
expect(resolveFrameFormat(metadata(), undefined)).toBe("jpg");
expect(resolveFrameFormat(metadata(), "auto")).toBe("jpg");
});
it("honors explicit png for opaque videos", () => {
expect(resolveFrameFormat(metadata(), "png")).toBe("png");
});
it("honors explicit jpg for opaque videos", () => {
expect(resolveFrameFormat(metadata(), "jpg")).toBe("jpg");
});
it("forces png when alpha is present or the codec can carry alpha", () => {
expect(resolveFrameFormat(metadata({ hasAlpha: true }), "jpg")).toBe("png");
expect(resolveFrameFormat(metadata({ videoCodec: "vp9" }), "jpg")).toBe("png");
});
it("keeps ProRes without an alpha plane on jpg and ProRes with one on png", () => {
expect(resolveFrameFormat(metadata({ videoCodec: "prores" }), undefined)).toBe("jpg");
expect(resolveFrameFormat(metadata({ videoCodec: "prores", hasAlpha: true }), "jpg")).toBe(
"png",
);
});
type FormatCase = Pick<VideoMetadata, "videoCodec" | "hasAlpha" | "alphaPlaneDecoded"> & {
requested: "auto" | "jpg" | "png";
};
const formatCases: FormatCase[] = ["vp9", "vp8", "h264", "prores"].flatMap((videoCodec) =>
[true, false].flatMap((hasAlpha) =>
[true, false, undefined].flatMap((alphaPlaneDecoded) =>
(["auto", "jpg", "png"] as const).map((requested) => ({
videoCodec,
hasAlpha,
alphaPlaneDecoded,
requested,
})),
),
),
);
function expectedFormat({ videoCodec, hasAlpha, alphaPlaneDecoded, requested }: FormatCase) {
const sidecar = videoCodec === "vp9" || videoCodec === "vp8";
const keepsPng = hasAlpha || (sidecar && alphaPlaneDecoded !== false);
return keepsPng || requested === "png" ? "png" : "jpg";
}
it("lets VP8/VP9 leave png only when a decode probe found no alpha plane", () => {
expect(formatCases).toHaveLength(72);
for (const { requested, ...fields } of formatCases) {
const actual = resolveFrameFormat(metadata(fields), requested);
expect(actual, JSON.stringify(fields)).toBe(expectedFormat({ requested, ...fields }));
}
expect(resolveFrameFormat(metadata({ videoCodec: "vp9", alphaPlaneDecoded: false }))).toBe(
"jpg",
);
expect(resolveFrameFormat(metadata({ videoCodec: "vp8", alphaPlaneDecoded: false }))).toBe(
"jpg",
);
});
});
describe("parseVideoElements", () => {
it.each([
{
label: "valueless playback-start",
attributes: 'data-playback-start data-media-start="1.5"',
expected: 1.5,
},
{
label: "empty playback-start",
attributes: 'data-playback-start="" data-media-start="1.5"',
expected: 1.5,
},
{
label: "whitespace playback-start",
attributes: 'data-playback-start=" " data-media-start="1.5"',
expected: 1.5,
},
{
label: "invalid playback-start",
attributes: 'data-playback-start="later" data-media-start="1.5"',
expected: 1.5,
},
{ label: "missing media-start", attributes: "", expected: 0 },
{ label: "invalid media-start", attributes: 'data-media-start="later"', expected: 0 },
{
label: "negative playback-start",
attributes: 'data-playback-start="-1" data-media-start="1.5"',
expected: 1.5,
},
{ label: "negative media-start", attributes: 'data-media-start="-1"', expected: 0 },
{
label: "finite playback-start",
attributes: 'data-playback-start="2.25" data-media-start="1.5"',
expected: 2.25,
},
{
label: "zero playback-start",
attributes: 'data-playback-start="0" data-media-start="1.5"',
expected: 0,
},
])("uses finite playback-start -> media-start -> 0 for $label", ({ attributes, expected }) => {
const [video] = parseVideoElements(`<video id="hero" src="clip.mp4" ${attributes}></video>`);
expect(video?.mediaStart).toBe(expected);
});
it("parses and normalizes constant playback rate for final rendering", () => {
const [fast, low, high, invalid] = parseVideoElements(
'<video id="fast" src="clip.mp4" data-playback-rate="2"></video>' +
'<video id="low" src="clip.mp4" data-playback-rate="0.01"></video>' +
'<video id="high" src="clip.mp4" data-playback-rate="20"></video>' +
'<video id="invalid" src="clip.mp4" data-playback-rate="nope"></video>',
);
expect(fast?.playbackRate).toBe(2);
expect(low?.playbackRate).toBe(RATE_RANGE.min);
expect(high?.playbackRate).toBe(RATE_RANGE.max);
expect(invalid?.playbackRate).toBe(1);
});
it("parses a rate lane from data-automation into the clip's rate", () => {
const automation = JSON.stringify({
version: 1,
lanes: [
{
target: "rate",
points: [
{ t: 0, v: 1 },
{ t: 2, v: 3 },
],
},
],
});
const [ramped] = parseVideoElements(
`<video id="ramped" src="clip.mp4" data-automation='${automation}'></video>`,
);
expect(ramped?.playbackRate).toMatchObject({ target: "rate" });
});
it("parses videos without an id or data-start attribute", () => {
const videos = parseVideoElements('<video src="clip.mp4"></video>');
expect(videos).toHaveLength(1);
expect(videos[0]).toMatchObject({
id: "hf-video-0",
src: "clip.mp4",
start: 0,
end: Infinity,
mediaStart: 0,
loop: false,
hasAudio: false,
});
});
it("preserves explicit ids and derives end from data-duration", () => {
const videos = parseVideoElements(
'<video id="hero" src="clip.mp4" data-start="2" data-duration="5" data-media-start="1.5" data-has-audio="true"></video>',
);
expect(videos).toHaveLength(1);
expect(videos[0]).toEqual({
id: "hero",
src: "clip.mp4",
start: 2,
end: 7,
mediaStart: 1.5,
playbackRate: 1,
loop: false,
hasAudio: true,
});
});
it("preserves looped timed video semantics for render frame lookup", () => {
const videos = parseVideoElements(
'<video id="hero" src="clip.webm" data-start="2" data-duration="5" loop></video>',
);
expect(videos[0]).toMatchObject({
id: "hero",
start: 2,
end: 7,
loop: true,
});
});
it("resolves a relative data-start reference to another clip's end", () => {
const videos = parseVideoElements(
'<video id="intro" src="a.mp4" data-start="0" data-duration="10"></video>' +
'<video id="main" src="b.mp4" data-start="intro" data-duration="20"></video>',
);
const main = videos.find((v) => v.id === "main");
// intro ends at 10, so main starts at 10 and ends at 30 — not NaN.
expect(main?.start).toBe(10);
expect(main?.end).toBe(30);
});
it("still resolves relative data-start after compileTimingAttrs", () => {
const raw =
'<video id="intro" src="a.mp4" data-start="0" data-duration="10"></video>' +
'<video id="main" src="b.mp4" data-start="intro" data-duration="20"></video>';
const { html } = compileTimingAttrs(raw);
const main = parseVideoElements(html).find((v) => v.id === "main");
expect(main?.start).toBe(10);
expect(main?.end).toBe(30);
});
it("applies + and - offsets on a relative reference", () => {
const videos = parseVideoElements(
'<video id="intro" src="a.mp4" data-start="0" data-duration="10"></video>' +
'<video id="gap" src="b.mp4" data-start="intro + 2" data-duration="5"></video>' +
'<video id="overlap" src="c.mp4" data-start="intro - 0.5" data-duration="5"></video>',
);
expect(videos.find((v) => v.id === "gap")?.start).toBe(12);
expect(videos.find((v) => v.id === "overlap")?.start).toBe(9.5);
});
it("resolves chained references (A -> B -> C)", () => {
const videos = parseVideoElements(
'<video id="a" src="a.mp4" data-start="0" data-duration="4"></video>' +
'<video id="b" src="b.mp4" data-start="a" data-duration="3"></video>' +
'<video id="c" src="c.mp4" data-start="b" data-duration="2"></video>',
);
expect(videos.find((v) => v.id === "b")?.start).toBe(4);
expect(videos.find((v) => v.id === "c")?.start).toBe(7); // 4 + 3
});
it("resolves a reference to a non-video timed element (div clip)", () => {
const videos = parseVideoElements(
'<div id="title" data-start="0" data-duration="6"></div>' +
'<video id="clip" src="b.mp4" data-start="title" data-duration="5"></video>',
);
expect(videos.find((v) => v.id === "clip")?.start).toBe(6);
});
it("derives a referenced clip's duration from data-end when data-duration is absent", () => {
const videos = parseVideoElements(
'<video id="intro" src="a.mp4" data-start="2" data-end="9"></video>' +
'<video id="main" src="b.mp4" data-start="intro" data-duration="5"></video>',
);
// intro: start 2, end 9 -> duration 7 -> main starts at 9.
expect(videos.find((v) => v.id === "main")?.start).toBe(9);
});
it("resolves a reference inside its own composition when another composition reuses the id", () => {
const videos = parseVideoElements(
'<div data-composition-id="main">' +
'<div data-composition-id="scene-a" data-start="0" data-duration="4">' +
'<div id="intro" data-start="0" data-duration="2"></div>' +
"</div>" +
'<div data-composition-id="scene-b" data-start="4" data-duration="4">' +
'<div id="intro" data-start="0" data-duration="3"></div>' +
'<video id="clip" src="b.mp4" data-start="intro" data-duration="1"></video>' +
"</div>" +
"</div>",
);
expect(videos.find((v) => v.id === "clip")?.start).toBe(3);
});
it("falls back to 0 (never NaN) for an unknown reference target", () => {
const videos = parseVideoElements(
'<video id="orphan" src="a.mp4" data-start="does-not-exist" data-duration="5"></video>',
);
const orphan = videos.find((v) => v.id === "orphan");
expect(orphan?.start).toBe(0);
expect(Number.isNaN(orphan?.start)).toBe(false);
expect(orphan?.end).toBe(5);
});
it("does not hang or NaN on a circular reference", () => {
const videos = parseVideoElements(
'<video id="a" src="a.mp4" data-start="b" data-duration="4"></video>' +
'<video id="b" src="b.mp4" data-start="a" data-duration="3"></video>',
);
for (const v of videos) {
expect(Number.isNaN(v.start)).toBe(false);
}
});
it("discovers <video> elements that use <source> children", () => {
const videos = parseVideoElements(
'<video id="rec" data-start="1" data-duration="4">' +
'<source src="https://cdn.example.com/rec.mp4" type="video/mp4">' +
'<source src="_remote_media/rec.webm" type="video/webm">' +
"</video>",
);
expect(videos).toHaveLength(1);
expect(videos[0]).toMatchObject({ id: "rec", src: "_remote_media/rec.webm", start: 1, end: 5 });
});
});
describe("FrameLookupTable", () => {
function fakeExtracted(totalFrames: number, fps: number): ExtractedFrames {
const framePaths = new Map<number, string>();
for (let i = 0; i < totalFrames; i += 1) {
framePaths.set(i, `frame-${i}.jpg`);
}
return {
videoId: "hero",
srcPath: "clip.webm",
outputDir: "/tmp/frames",
framePattern: "frame-%05d.jpg",
fps,
totalFrames,
metadata: {
durationSeconds: totalFrames / fps,
width: 320,
height: 180,
fps,
hasAudio: false,
videoCodec: "vp9",
colorSpace: {
colorTransfer: "bt709",
colorPrimaries: "bt709",
colorSpace: "bt709",
},
isVFR: false,
hasAlpha: false,
},
framePaths,
};
}
it("wraps active frame payloads for looped clips whose display window exceeds source frames", () => {
const table = createFrameLookupTable(
[
{
id: "hero",
src: "clip.webm",
start: 0,
end: 5,
mediaStart: 0,
loop: true,
hasAudio: false,
},
],
[fakeExtracted(30, 30)],
);
expect(table.getActiveFramePayloads(0.5).get("hero")?.frameIndex).toBe(15);
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(15);
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15);
});
it("shows a copy starting on a float sum at that instant, as the preview does", () => {
const copyStart = 0.1 + 0.2;
const clip = (id: string, start: number) => ({
id,
src: `${id}.webm`,
start,
end: start + 0.2,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const frames = (videoId: string) => ({ ...fakeExtracted(30, 30), videoId });
const table = () =>
createFrameLookupTable(
[clip("a", 0.1), clip("copy", copyStart)],
[frames("a"), frames("copy")],
);
expect(table().getActiveFramePayloads(0.3).get("copy")?.frameIndex).toBe(0);
const stepping = table();
stepping.getActiveFramePayloads(0.2);
expect(stepping.getActiveFramePayloads(0.3).get("copy")?.frameIndex).toBe(0);
expect(getFrameAtTime(frames("copy"), 0.3, copyStart)).toBe("frame-0.jpg");
});
it("selects source frames at the authored constant playback rate", () => {
const videos = parseVideoElements(
'<video id="hero" src="clip.webm" data-start="0" data-duration="2" data-playback-rate="2"></video>',
);
const table = createFrameLookupTable(videos, [fakeExtracted(120, 30)]);
expect(table.getActiveFramePayloads(1).get("hero")?.frameIndex).toBe(60);
});
it("wraps at video-stream EOF when a mux container has longer audio", () => {
const extracted = fakeExtracted(6, 2);
extracted.metadata.durationSeconds = 60;
extracted.metadata.videoStreamDurationSeconds = 3;
const table = createFrameLookupTable(
[
{
id: "hero",
src: "clip.webm",
start: 0,
end: 60,
mediaStart: 0,
loop: true,
hasAudio: false,
},
],
[extracted],
);
expect(table.getActiveFramePayloads(4).get("hero")?.frameIndex).toBe(2);
});
it("holds the last frame for a non-looping clip until its authored slot ends", () => {
const table = createFrameLookupTable(
[
{
id: "hero",
src: "clip.webm",
start: 0,
end: 5,
mediaStart: 0,
loop: false,
hasAudio: false,
},
],
[fakeExtracted(30, 30)],
);
expect(table.getActiveFramePayloads(0.5).has("hero")).toBe(true);
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(29);
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(29);
expect(table.getFrame("hero", 4.5)).toBeTruthy();
expect(table.getActiveFramePayloads(5.1).has("hero")).toBe(false);
expect(table.getFrame("hero", 5.1)).toBeNull();
});
it("does not invent a held frame when extraction produced no frames", () => {
const table = createFrameLookupTable(
[
{
id: "hero",
src: "clip.webm",
start: 0,
end: 5,
mediaStart: 0,
loop: false,
hasAudio: false,
},
],
[fakeExtracted(0, 30)],
);
expect(table.getActiveFramePayloads(4.5).has("hero")).toBe(false);
expect(table.getFrame("hero", 4.5)).toBeNull();
});
it("places a relative-reference video in its resolved window end-to-end (was blank)", () => {
// The reported bug: <video data-start="intro"> gave NaN start/end, so the
// active-window checks (start <= t <= end) were always false and the clip
// composited blank. With resolution, `main` is active across [10, 30].
const videos = parseVideoElements(
'<video id="intro" src="a.mp4" data-start="0" data-duration="10"></video>' +
'<video id="main" src="b.mp4" data-start="intro" data-duration="20"></video>',
);
const table = createFrameLookupTable(videos, [{ ...fakeExtracted(600, 30), videoId: "main" }]);
expect(table.getActiveFramePayloads(5).has("main")).toBe(false); // before resolved start (10)
expect(table.getActiveFramePayloads(15).has("main")).toBe(true); // within [10, 30]
expect(table.getActiveFramePayloads(29).has("main")).toBe(true);
expect(table.getActiveFramePayloads(31).has("main")).toBe(false); // after resolved end (30)
});
it("shows the last frame just before the end and leaves at t === end, as the runtime does", () => {
const table = createFrameLookupTable(
[
{
id: "hero",
src: "clip.webm",
start: 1,
end: 3,
mediaStart: 0,
loop: false,
hasAudio: false,
},
],
[fakeExtracted(60, 30)],
);
expect(table.getActiveFramePayloads(2.5).get("hero")?.frameIndex).toBe(45);
expect(table.getActiveFramePayloads(2.99).get("hero")?.frameIndex).toBe(59);
expect(table.getActiveFramePayloads(3.0).has("hero")).toBe(false);
expect(table.getFrame("hero", 3.0)).toBeNull();
});
it("holds the last frame across the tail when the source is shorter than the window", () => {
// clip [0,5] with only 1s of source (30 @ 30fps). The authored slot is the
// visibility contract, so the final source frame fills its remaining tail.
const table = createFrameLookupTable(
[
{
id: "hero",
src: "clip.webm",
start: 0,
end: 5,
mediaStart: 0,
loop: false,
hasAudio: false,
},
],
[fakeExtracted(30, 30)],
);
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(29);
expect(table.getActiveFramePayloads(4.99).get("hero")?.frameIndex).toBe(29);
});
it("holds the last frame when the source is a sub-frame shorter than the slot", () => {
// clip [2, 3.45] declares a 1.45s slot, but `ffmpeg -t 1.45` at 30fps emits
// 43 frames = 1.433s — a half-frame short. The tail between source
// exhaustion (~3.433) and the clip end (3.45) must hold the last frame
// rather than render the page background (a one-frame black flash at the
// cut). The held index is the final extracted frame (42).
const table = createFrameLookupTable(
[
{
id: "hero",
src: "clip.mp4",
start: 2,
end: 3.45,
mediaStart: 0,
loop: false,
hasAudio: false,
},
],
[fakeExtracted(43, 30)],
);
// last real frame
expect(table.getActiveFramePayloads(3.4).get("hero")?.frameIndex).toBe(42);
// source exhausted but within tolerance of the end → hold, don't blank
expect(table.getActiveFramePayloads(3.44).get("hero")?.frameIndex).toBe(42);
});
it("gives a frame a hair before two clips meet to the clip export shows there", () => {
const clip = (id: string, start: number, end: number) => ({
id,
src: `${id}.webm`,
start,
end,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const table = () =>
createFrameLookupTable(
[clip("a", 0, 1.00001), clip("b", 1.00001, 2)],
[
{ ...fakeExtracted(90, 30), videoId: "a" },
{ ...fakeExtracted(90, 30), videoId: "b" },
],
30,
);
const stepping = table();
stepping.getActiveFramePayloads(29 / 30);
const payloads = stepping.getActiveFramePayloads(1);
expect([...payloads.keys()]).toEqual(["b"]);
expect(payloads.get("b")?.frameIndex).toBe(0);
expect([...table().getActiveFramePayloads(1).keys()]).toEqual(["b"]);
expect(table().getFrame("a", 1)).toBeNull();
});
it.each([
["an exact", 0.1, 0.3, 0.3],
["a float-sum", 0.1, 0.1 + 0.2, 9 / 30],
])(
"hands %s shared boundary to the incoming clip only, as the runtime does",
(_, aStart, aEnd, t) => {
const clip = (id: string, start: number, end: number) => ({
id,
src: `${id}.webm`,
start,
end,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const table = () =>
createFrameLookupTable(
[clip("a", aStart, aEnd), clip("b", 0.3, 0.5)],
[
{ ...fakeExtracted(90, 30), videoId: "a" },
{ ...fakeExtracted(90, 30), videoId: "b" },
],
);
const stepping = table();
expect([...stepping.getActiveFramePayloads(8 / 30).keys()]).toEqual(["a"]);
expect([...stepping.getActiveFramePayloads(t).keys()]).toEqual(["b"]);
expect([...table().getActiveFramePayloads(t).keys()]).toEqual(["b"]);
expect(table().getFrame("a", t)).toBeNull();
},
);
});
describe("analyzeClipMediaFit", () => {
it("returns null for a sub-tolerance shortfall the compiler leaves unclamped", () => {
// 1.433s media in a 1.45s slot — a sub-frame shortfall (<0.05s) the renderer
// freezes seamlessly and the compiler never clamps. Not worth warning about.
expect(analyzeClipMediaFit({ slotSeconds: 1.45, mediaSeconds: 1.433 })).toBeNull();
});
it("returns null when media is longer than or equal to the slot", () => {
expect(analyzeClipMediaFit({ slotSeconds: 2, mediaSeconds: 2 })).toBeNull();
expect(analyzeClipMediaFit({ slotSeconds: 2, mediaSeconds: 5 })).toBeNull();
});
it("reports the shortfall when the slot exceeds media beyond the clamp epsilon", () => {
const fit = analyzeClipMediaFit({ slotSeconds: 5, mediaSeconds: 1 });
expect(fit).not.toBeNull();
expect(fit?.shortfallSeconds).toBeCloseTo(4, 5);
expect(fit?.toleranceSeconds).toBeCloseTo(0.05, 5);
});
it("never flags looping clips (they repeat to fill the slot)", () => {
expect(analyzeClipMediaFit({ slotSeconds: 5, mediaSeconds: 1, loop: true })).toBeNull();
});
it("returns null for unusable inputs (non-finite media, zero slot)", () => {
expect(analyzeClipMediaFit({ slotSeconds: 0, mediaSeconds: 1 })).toBeNull();
expect(analyzeClipMediaFit({ slotSeconds: 5, mediaSeconds: NaN })).toBeNull();
});
});
describe("parseImageElements", () => {
it("parses images with data-start and data-duration", () => {
const images = parseImageElements(
'<img id="photo" src="hdr-photo.png" data-start="0" data-duration="3" />',
);
expect(images).toHaveLength(1);
expect(images[0]).toEqual({
id: "photo",
src: "hdr-photo.png",
start: 0,
end: 3,
});
});
it("generates stable IDs for images without one", () => {
const images = parseImageElements(
'<img src="a.png" data-start="0" data-end="2" /><img src="b.png" data-start="1" data-end="4" />',
);
expect(images).toHaveLength(2);
expect(images[0]!.id).toBe("hf-img-0");
expect(images[1]!.id).toBe("hf-img-1");
});
it("defaults start to 0 and end to Infinity when attributes missing", () => {
const images = parseImageElements('<img src="photo.png" />');
expect(images).toHaveLength(1);
expect(images[0]).toMatchObject({
src: "photo.png",
start: 0,
end: Infinity,
});
});
it("ignores img elements without src", () => {
const images = parseImageElements('<img data-start="0" data-end="3" />');
expect(images).toHaveLength(0);
});
it("uses data-end over data-duration when both present", () => {
const images = parseImageElements(
'<img src="a.png" data-start="1" data-end="5" data-duration="10" />',
);
expect(images[0]!.end).toBe(5);
});
});
type Rgb = [number, number, number];
const UI_FIXTURE_WIDTH = 240;
const UI_FIXTURE_HEIGHT = 160;
const RED_SAMPLE_PIXELS = [
[70, 72],
[118, 82],
[178, 92],
] as const;
function pngChunkTypes(path: string): string[] {
const bytes = readFileSync(path);
const types: string[] = [];
for (let offset = 8; offset + 8 <= bytes.length; offset += 12 + bytes.readUInt32BE(offset)) {
types.push(bytes.toString("latin1", offset + 4, offset + 8));
}
return types;
}
function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb {
const result = spawnSync(
"ffmpeg",
[
"-v",
"error",
"-i",
mediaPath,
"-frames:v",
"1",
"-f",
"rawvideo",
"-pix_fmt",
"rgb24",
"pipe:1",
],
{ maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 },
);
if (result.status !== 0) {
throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`);
}
const offset = (y * UI_FIXTURE_WIDTH + x) * 3;
return [
result.stdout[offset] ?? 0,
result.stdout[offset + 1] ?? 0,
result.stdout[offset + 2] ?? 0,
];
}
function maxChannelDelta(a: Rgb, b: Rgb): number {
return Math.max(Math.abs(a[0] - b[0]), Math.abs(a[1] - b[1]), Math.abs(a[2] - b[2]));
}
// Regression for saturated UI recordings: default JPEG extraction can shift
// high-chroma reds before browser capture. Forcing PNG should keep extracted
// source-video frames effectively identical to the decoded source pixels.
describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-format-"));
const UI_FIXTURE = join(FIXTURE_DIR, "ui-red.mp4");
beforeAll(async () => {
const result = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`color=c=0xffe7ee:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
"-vf",
"drawbox=x=44:y=58:w=152:h=44:color=0xdd382e@1:t=fill,drawbox=x=64:y=75:w=112:h=10:color=0xfff0f0@1:t=fill",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"0",
"-pix_fmt",
"yuv420p",
"-color_primaries",
"bt709",
"-color_trc",
"bt709",
"-colorspace",
"bt709",
UI_FIXTURE,
]);
if (!result.success) {
throw new Error(`UI color fixture synthesis failed: ${result.stderr.slice(-400)}`);
}
}, 30_000);
afterAll(() => {
if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
}, 30_000);
function fixtureVideo(): VideoElement {
return {
id: "ui",
src: UI_FIXTURE,
start: 0,
end: 1,
mediaStart: 0,
loop: false,
hasAudio: false,
};
}
it("extracts into a work directory whose name holds a printf code", async () => {
const outDir = join(FIXTURE_DIR, "x 12% Up");
mkdirSync(outDir, { recursive: true });
const result = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
fps: 1,
outputDir: outDir,
});
expect(result.errors).toEqual([]);
const frame = result.extracted[0]!.framePaths.get(0)!;
expect(frame.startsWith(outDir)).toBe(true);
expect(existsSync(frame)).toBe(true);
});
it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) jpg extraction", async () => {
const source = join(FIXTURE_DIR, "white-1080.mp4");
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"color=c=white:s=1080x16:d=1:r=1",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"0",
"-pix_fmt",
"yuv420p",
source,
]);
expect(synth.success).toBe(true);
const outDir = join(FIXTURE_DIR, "out-white-1080");
mkdirSync(outDir, { recursive: true });
const result = await extractAllVideoFrames(
[{ ...fixtureVideo(), id: "white", src: source }],
FIXTURE_DIR,
{ fps: 1, outputDir: outDir },
);
expect(result.errors).toEqual([]);
const frame = result.extracted[0]!.framePaths.get(0)!;
expect(frame.endsWith(".jpg")).toBe(true);
const lastColumns = (file: string) => [
...spawnSync("ffmpeg", [
"-v",
"error",
"-i",
file,
"-frames:v",
"1",
"-vf",
"format=gray,crop=8:1:iw-8:8",
"-f",
"rawvideo",
"-",
]).stdout,
];
const delivered = lastColumns(frame);
const expected = lastColumns(source);
expect(delivered).toHaveLength(8);
for (const [i, v] of delivered.entries())
expect(Math.abs(v - expected[i]!)).toBeLessThanOrEqual(5);
}, 60_000);
async function proresFrame(name: string, pixFmt: string, profile: string, source: string) {
const clip = join(FIXTURE_DIR, `${name}.mov`);
const synth = await runFfmpeg(
["-y", "-loglevel", "error", "-f", "lavfi", "-i", source].concat([
"-c:v",
"prores_ks",
"-profile:v",
profile,
"-pix_fmt",
pixFmt,
clip,
]),
);
expect(synth.success).toBe(true);
const outDir = join(FIXTURE_DIR, `out-${name}`);
mkdirSync(outDir, { recursive: true });
const result = await extractAllVideoFrames(
[{ ...fixtureVideo(), id: name, src: clip }],
FIXTURE_DIR,
{ fps: 1, outputDir: outDir },
);
expect(result.errors).toEqual([]);
return result.extracted[0]!.framePaths.get(0)!;
}
it("extracts ProRes without an alpha plane to jpg", async () => {
const frame = await proresFrame("prores422", "yuv422p10le", "2", "testsrc2=s=320x180:d=1:r=1");
expect(frame.endsWith(".jpg")).toBe(true);
}, 60_000);
it("keeps the alpha of ProRes 4444 exact in png frames", async () => {
const frame = await proresFrame(
"prores4444",
"yuva444p10le",
"4",
"color=c=0x00ff00@0.5:s=320x180:d=1:r=1,format=yuva444p10le",
);
expect(frame.endsWith(".png")).toBe(true);
const alpha = spawnSync("ffmpeg", [
"-v",
"error",
"-i",
frame,
"-vf",
"alphaextract,crop=1:1:10:10",
"-f",
"rawvideo",
"-",
]);
expect(Math.abs(alpha.stdout[0]! - 128)).toBeLessThanOrEqual(1);
}, 60_000);
it("keeps color-sensitive UI reds closer to source when extraction is forced to png", async () => {
const defaultOut = join(FIXTURE_DIR, "out-default");
const pngOut = join(FIXTURE_DIR, "out-png");
mkdirSync(defaultOut, { recursive: true });
mkdirSync(pngOut, { recursive: true });
const defaultResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
fps: 1,
outputDir: defaultOut,
});
const pngResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
fps: 1,
outputDir: pngOut,
format: "png",
});
expect(defaultResult.errors).toEqual([]);
expect(pngResult.errors).toEqual([]);
const defaultFrame = defaultResult.extracted[0]!.framePaths.get(0)!;
const pngFrame = pngResult.extracted[0]!.framePaths.get(0)!;
expect(defaultFrame.endsWith(".jpg")).toBe(true);
expect(pngFrame.endsWith(".png")).toBe(true);
let worstDefaultDelta = 0;
let worstPngDelta = 0;
for (const [x, y] of RED_SAMPLE_PIXELS) {
const sourcePixel = readFirstFramePixel(UI_FIXTURE, x, y);
worstDefaultDelta = Math.max(
worstDefaultDelta,
maxChannelDelta(sourcePixel, readFirstFramePixel(defaultFrame, x, y)),
);
worstPngDelta = Math.max(
worstPngDelta,
maxChannelDelta(sourcePixel, readFirstFramePixel(pngFrame, x, y)),
);
}
expect(worstPngDelta).toBeLessThanOrEqual(5);
expect(worstPngDelta).toBeLessThanOrEqual(worstDefaultDelta);
}, 60_000);
// Chrome runs with --force-color-profile=srgb and colour-manages any frame not tagged sRGB;
// the SDR encoder writes the canvas code values as they are, so frames must reach it unconverted.
it.each([
["BT.709", "bt709"],
["BT.601", "smpte170m"],
])(
"declares png frames of a %s source as sRGB so Chrome shows their code values",
async (name, tag) => {
const fixture = join(FIXTURE_DIR, `${tag}-shadow.mp4`);
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`color=c=0x101010:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
"-vf",
`setparams=color_primaries=${tag}:color_trc=${tag}:colorspace=${tag}:range=tv`,
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"0",
"-pix_fmt",
"yuv420p",
fixture,
]);
if (!synth.success) {
throw new Error(`${name} fixture synthesis failed: ${synth.stderr.slice(-400)}`);
}
expect((await extractVideoMetadata(fixture)).colorSpace?.colorTransfer).toBe(tag);
const outputDir = join(FIXTURE_DIR, `out-png-transfer-${tag}`);
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[{ ...fixtureVideo(), id: tag, src: fixture }],
FIXTURE_DIR,
{ fps: 1, outputDir, format: "png" },
);
expect(result.errors).toEqual([]);
const frame = result.extracted[0]!.framePaths.get(0)!;
// Chrome reads cICP before sRGB, and ffmpeg < 6.0 writes no cICP, so only an sRGB chunk
// without cICP means "these are sRGB code values" on every ffmpeg.
const chunks = pngChunkTypes(frame);
expect(chunks).toContain("sRGB");
expect(chunks).not.toContain("cICP");
expect(readFirstFramePixel(frame, 10, 10)).toEqual(readFirstFramePixel(fixture, 10, 10));
},
60_000,
);
it("extracts jpg frames of a BT.709 source as the BT.601 full range JPEG readers assume", async () => {
const fixture = join(FIXTURE_DIR, "bt709-patch.mp4");
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1,format=rgb24,drawbox=x=31:y=0:w=64:h=${UI_FIXTURE_HEIGHT}:c=0x0000FE:t=fill`,
"-vf",
"scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"0",
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
const outputDir = join(FIXTURE_DIR, "out-jpg-matrix");
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[{ ...fixtureVideo(), id: "bt709-jpg", src: fixture }],
FIXTURE_DIR,
{ fps: 1, outputDir, format: "jpg" },
);
expect(result.errors).toEqual([]);
const frame = result.extracted[0]!.framePaths.get(0)!;
// x=10 is flat colour; x=32 sits one pixel inside the blue edge.
for (const x of [10, 32]) {
const source = readFirstFramePixel(fixture, x, 10);
const shown = readFirstFramePixel(frame, x, 10);
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
expect(worst, `x=${x}: source ${source} jpg ${shown}`).toBeLessThanOrEqual(3);
}
}, 60_000);
// Measured in Chrome 152: untagged H.264 plays as BT.601 at 1280x718 and BT.709 at 1280x720 or 406x720;
// untagged VP9 and AV1 play as BT.601 at every size.
it.each([
{ codec: "libx264", height: 718, matrix: "bt601", format: "png" },
{ codec: "libx264", height: 718, matrix: "bt601", format: "jpg" },
{ codec: "libx264", height: 720, matrix: "bt709", format: "png" },
{ codec: "libx264", height: 720, matrix: "bt709", format: "jpg" },
{ codec: "libvpx-vp9", height: 720, matrix: "bt601", format: "png" },
...(AV1_ENCODER
? [{ codec: AV1_ENCODER, height: 720, matrix: "bt601", format: "png" } as const]
: []),
] as const)(
"reads an untagged $height-line $codec source as $matrix like Chrome's own playback ($format)",
async ({ codec, height, matrix, format }) => {
const extension = codec === "libvpx-vp9" ? "webm" : "mp4";
const fixture = join(FIXTURE_DIR, `untagged-${codec}-${height}.${extension}`);
const synth = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
`color=c=0xC83C28:s=64x${height}:d=1:r=1`,
"-vf",
"scale=out_color_matrix=bt601:out_range=tv,format=yuv420p,setparams=colorspace=unknown:color_primaries=unknown:color_trc=unknown:range=tv",
"-c:v",
codec,
...(codec === "libx264" ? ["-qp", "0"] : ["-crf", "0", "-b:v", "0"]),
...(codec === "libaom-av1" ? ["-cpu-used", "8"] : []),
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
expect((await extractVideoMetadata(fixture)).colorSpace?.colorSpace ?? "unknown").toBe(
"unknown",
);
const result = await extractVideoFramesRange(
fixture,
`${basename(fixture)}-${format}`,
0,
1,
{
fps: 1,
outputDir: join(FIXTURE_DIR, "out-untagged"),
format,
},
);
const pixelAt = (file: string, decode: string): number[] => [
...spawnSync("ffmpeg", [
"-v",
"error",
"-i",
file,
"-vf",
`${decode}format=rgb24,crop=1:1:10:10`,
"-frames:v",
"1",
"-f",
"rawvideo",
"-",
]).stdout,
];
const expected = pixelAt(fixture, `scale=in_color_matrix=${matrix}:in_range=tv,`);
const shown = pixelAt(result.framePaths.get(0)!, "");
expect([expected.length, shown.length]).toEqual([3, 3]);
const worst = Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)));
expect(worst, `${matrix} decode ${expected}, ${format} ${shown}`).toBeLessThanOrEqual(
format === "png" ? 1 : 3,
);
},
60_000,
);
it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => {
// Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks.
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4");
const synth = await runFfmpeg([
...["-y", "-v", "error", "-f", "lavfi", "-i"],
"color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-ntsc-ms"),
format: "png",
});
const indexOf = (file: string): number => {
const row = spawnSync("ffmpeg", [
"-v",
"error",
"-i",
file,
"-vf",
"format=gray,crop=64:1:0:4",
"-f",
"rawvideo",
"-",
]).stdout;
return [...Array(8).keys()].reduce(
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
0,
);
};
const shown = [...result.framePaths.keys()]
.sort((a, b) => a - b)
.map((i) => indexOf(result.framePaths.get(i)!));
const steps = shown.slice(1).map((n, i) => n - shown[i]!);
expect(shown.length).toBeGreaterThanOrEqual(44);
expect(
steps.filter((step) => step > 1),
`source frames ${shown.join(",")}`,
).toEqual([]);
expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1);
}, 60_000);
it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => {
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4");
const synth = await runFfmpeg([
...["-y", "-v", "error", "-f", "lavfi", "-i"],
"color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
const indexOf = (file: string): number => {
const row = spawnSync("ffmpeg", [
"-v",
"error",
"-i",
file,
"-vf",
"format=gray,crop=64:1:0:4",
"-f",
"rawvideo",
"-",
]).stdout;
return [...Array(8).keys()].reduce(
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
0,
);
};
const options = {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-ntsc-seams"),
format: "png" as const,
};
const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options);
const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201];
const differing: string[] = [];
for (const firstFrame of seams) {
const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, {
...options,
frameRanges: [{ firstFrame, frames: 12 }],
});
for (const [slot, path] of segment.framePaths) {
const expected = indexOf(single.framePaths.get(slot)!);
const shown = indexOf(path);
if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`);
}
}
expect(differing).toEqual([]);
}, 120_000);
// ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour
// filters would cut the still that ends this window short.
it.each([
{
name: "full range BT.709",
format: "png",
codec: "libx264",
pixFmt: "yuvj420p",
probed: { colorTransfer: "bt709" },
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
},
{
name: "full range BT.709",
format: "jpg",
codec: "libx264",
pixFmt: "yuvj420p",
probed: { colorTransfer: "bt709" },
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
},
{
name: "BT.470BG gamma 2.8",
format: "png",
codec: "libx264",
pixFmt: "yuv420p",
probed: { colorTransfer: "bt470bg" },
tags: "color_primaries=bt470bg:color_trc=bt470bg:colorspace=bt470bg",
},
{
name: "RGB",
format: "png",
codec: "libx264rgb",
pixFmt: "rgb24",
probed: { colorSpace: "gbr" },
tags: "",
},
] as const)(
"keeps every frame and the colours of a $name VFR window that ends on a still ($format)",
async ({ name, format, codec, pixFmt, probed, tags }) => {
const fixture = join(FIXTURE_DIR, `vfr-still-${name.replace(/\W+/g, "-")}-${format}.mp4`);
const synth = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=2:r=60`,
"-vf",
`select='not(between(n\\,30\\,89))'${tags ? `,setparams=${tags}` : ""}`,
"-fps_mode",
"vfr",
"-c:v",
codec,
"-preset",
"ultrafast",
"-pix_fmt",
pixFmt,
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
const window = 0.616666;
const unfilteredDir = join(FIXTURE_DIR, `out-${basename(fixture)}-unfiltered`);
mkdirSync(unfilteredDir, { recursive: true });
const unfiltered = await runFfmpeg([
"-v",
"error",
"-ss",
"0",
"-i",
fixture,
"-t",
String(window),
"-fps_mode",
"cfr",
"-r",
"30",
join(unfilteredDir, "f_%05d.png"),
]);
expect(unfiltered.success).toBe(true);
const result = await extractVideoFramesRange(fixture, basename(fixture), 0, window, {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-vfr-still"),
format,
});
expect(result.metadata.isVFR).toBe(true);
expect(result.metadata.colorSpace).toMatchObject(probed);
expect(result.totalFrames).toBe(readdirSync(unfilteredDir).length);
const last = result.framePaths.get(result.totalFrames - 1)!;
const source = readFirstFramePixel(fixture, 10, 10);
const shown = readFirstFramePixel(last, 10, 10);
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
expect(worst, `source ${source} ${format} ${shown}`).toBeLessThanOrEqual(
format === "png" ? 0 : 3,
);
if (format === "png") expect(pngChunkTypes(last)).toContain("sRGB");
},
60_000,
);
it("leaves SDR-to-HDR frames in the HDR colours they were converted to", async () => {
const outputDir = join(FIXTURE_DIR, "out-sdr-to-hdr-gate");
mkdirSync(outputDir, { recursive: true });
const result = await extractVideoFramesRange(UI_FIXTURE, "sdr-to-hdr-gate", 0, 1, {
fps: 1,
outputDir,
format: "png",
sdrToHdrTransfer: "pq",
});
expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB");
}, 60_000);
it("keeps jpg and png extraction caches separate", async () => {
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-"));
try {
const defaultOut = join(FIXTURE_DIR, "cache-default");
const pngOut = join(FIXTURE_DIR, "cache-png");
const pngHitOut = join(FIXTURE_DIR, "cache-png-hit");
mkdirSync(defaultOut, { recursive: true });
mkdirSync(pngOut, { recursive: true });
mkdirSync(pngHitOut, { recursive: true });
const defaultResult = await extractAllVideoFrames(
[fixtureVideo()],
FIXTURE_DIR,
{ fps: 1, outputDir: defaultOut },
undefined,
{ extractCacheDir: cacheDir },
);
expect(defaultResult.errors).toEqual([]);
expect(defaultResult.phaseBreakdown.cacheHits).toBe(0);
expect(defaultResult.phaseBreakdown.cacheMisses).toBe(1);
expect(defaultResult.extracted[0]!.framePaths.get(0)!.endsWith(".jpg")).toBe(true);
const pngMiss = await extractAllVideoFrames(
[fixtureVideo()],
FIXTURE_DIR,
{ fps: 1, outputDir: pngOut, format: "png" },
undefined,
{ extractCacheDir: cacheDir },
);
expect(pngMiss.errors).toEqual([]);
expect(pngMiss.phaseBreakdown.cacheHits).toBe(0);
expect(pngMiss.phaseBreakdown.cacheMisses).toBe(1);
expect(pngMiss.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
const pngHit = await extractAllVideoFrames(
[fixtureVideo()],
FIXTURE_DIR,
{ fps: 1, outputDir: pngHitOut, format: "png" },
undefined,
{ extractCacheDir: cacheDir },
);
expect(pngHit.errors).toEqual([]);
expect(pngHit.phaseBreakdown.cacheHits).toBe(1);
expect(pngHit.phaseBreakdown.cacheMisses).toBe(0);
expect(pngHit.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
} finally {
rmSync(cacheDir, { recursive: true, force: true });
}
}, 60_000);
it("dedupes identical extractions within one render", async () => {
const outputDir = join(FIXTURE_DIR, "out-dedupe");
mkdirSync(outputDir, { recursive: true });
const videoA: VideoElement = { ...fixtureVideo(), id: "dupe-a" };
const videoB: VideoElement = { ...fixtureVideo(), id: "dupe-b" };
const result = await extractAllVideoFrames([videoA, videoB], FIXTURE_DIR, {
fps: 1,
outputDir,
});
expect(result.errors).toEqual([]);
expect(result.extracted).toHaveLength(2);
const first = result.extracted[0]!;
const second = result.extracted[1]!;
expect(first.videoId).toBe("dupe-a");
expect(second.videoId).toBe("dupe-b");
expect(second.outputDir).toBe(first.outputDir);
expect(Array.from(second.framePaths.entries())).toEqual(Array.from(first.framePaths.entries()));
const frameDirs = readdirSync(outputDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name);
expect(frameDirs).toEqual(["dupe-a"]);
expect(readdirSync(first.outputDir).filter((f) => f.endsWith(".jpg"))).toHaveLength(
first.totalFrames,
);
}, 60_000);
});
// Each output slot shows the frame on screen at its time, as the preview does.
describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-sampling-"));
const WIDTH = 32;
const HEIGHT = 16;
// Matroska stores whole-millisecond timestamps (a 30 fps frame at 66.67 ms reads 67 ms);
// a 1/30 timescale stores each frame exactly on a coarse tick.
const SOURCES = {
"mp4-60": { fps: 60, file: "index-60fps.mp4", muxer: [] },
"mkv-30": { fps: 30, file: "index-30fps.mkv", muxer: [] },
"mp4-30-timescale-30": {
fps: 30,
file: "index-30fps-ts30.mp4",
muxer: ["-video_track_timescale", "30"],
},
} as const;
type SourceName = keyof typeof SOURCES;
const sourcePath = (name: SourceName) => join(FIXTURE_DIR, SOURCES[name].file);
beforeAll(async () => {
for (const name of Object.keys(SOURCES) as SourceName[]) {
// Frame k carries luma 16 + 2k, so each extracted frame names its source index.
const result = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`nullsrc=s=${WIDTH}x${HEIGHT}:r=${SOURCES[name].fps}:d=1.5,geq=lum='16+2*N':cb=128:cr=128`,
"-c:v",
"libx264",
"-qp",
"0",
"-pix_fmt",
"yuv420p",
...SOURCES[name].muxer,
sourcePath(name),
]);
if (!result.success) throw new Error(`index fixture synthesis failed: ${result.stderr}`);
}
}, 30_000);
afterAll(() => {
rmSync(FIXTURE_DIR, { recursive: true, force: true });
}, 30_000);
function sourceIndexes(extracted: ExtractedFrames): number[] {
const decoded = spawnSync("ffmpeg", [
"-v",
"error",
"-i",
join(extracted.outputDir, extracted.framePattern),
"-f",
"rawvideo",
"-pix_fmt",
"gray",
"pipe:1",
]);
if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
const indexes: number[] = [];
for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) {
indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2));
}
return indexes;
}
it.each([
{ source: "mp4-60", fps: 24, startTime: 0, duration: 0.5 },
{ source: "mp4-60", fps: 10, startTime: 0.37, duration: 0.6 },
{ source: "mkv-30", fps: 30, startTime: 0, duration: 0.5 },
{ source: "mp4-30-timescale-30", fps: 60, startTime: 0, duration: 0.5 },
] as const)(
"$source: samples the frame on screen at each $fps fps slot from $startTime s",
async (c) => {
const extracted = await extractVideoFramesRange(
sourcePath(c.source),
`${c.source}-${c.fps}`,
c.startTime,
c.duration,
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "png" },
);
const onScreen = Array.from({ length: Math.round(c.duration * c.fps) }, (_, i) =>
Math.floor((c.startTime + i / c.fps) * SOURCES[c.source].fps + 1e-9),
);
expect(sourceIndexes(extracted)).toEqual(onScreen);
},
30_000,
);
// #5260: one ffmpeg process over a 26-minute clip hit ffmpegProcessTimeout.
it.each([
// 260 frames: one 240-frame segment, then 20 more.
{ startTime: 0.5, duration: 130, frames: 260 },
// 200.3 - 80.3 is 120.00000000000001 s: exactly one segment, not a second near-empty one.
{ startTime: 80.3, duration: 200.3 - 80.3, frames: 240 },
])(
"extracts $duration s from $startTime s at 2 fps across segments without a gap or repeat",
async (c) => {
const source = join(FIXTURE_DIR, "index-2fps-long.mp4");
if (!existsSync(source)) {
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`nullsrc=s=${WIDTH}x${HEIGHT}:r=2:d=205,geq=lum='16+2*mod(N\\,100)':cb=128:cr=128`,
"-c:v",
"libx264",
"-qp",
"0",
"-pix_fmt",
"yuv420p",
source,
]);
if (!synth.success) throw new Error(`long fixture synthesis failed: ${synth.stderr}`);
}
const extracted = await extractVideoFramesRange(
source,
`long-2fps-${c.startTime}`,
c.startTime,
c.duration,
{ fps: 2, outputDir: FIXTURE_DIR, format: "png" },
);
const onScreen = Array.from(
{ length: c.frames },
(_, i) => Math.floor((c.startTime + i / 2) * 2 + 1e-9) % 100,
);
expect(extracted.totalFrames).toBe(c.frames);
expect(sourceIndexes(extracted)).toEqual(onScreen);
},
60_000,
);
it("stops extraction only on a stall, and its ffmpeg reports each frame it writes", async () => {
const source = join(FIXTURE_DIR, "paced-2s.mp4");
const synth = await runFfmpeg([
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],
...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=2`, "-pix_fmt", "yuv420p", source],
]);
if (!synth.success) throw new Error(`paced fixture synthesis failed: ${synth.stderr}`);
const { runFfmpeg: real } =
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
let requested: RunFfmpegOptions | undefined;
const reported: number[] = [];
const readStats = ffmpegStatsReader(({ frames }) => {
if (frames) reported.push(frames);
});
// Real-time pacing spreads the stats over the run. The stall clock itself is runFfmpeg's to
// test; here it is lifted so a starved runner cannot race it.
vi.mocked(runFfmpeg).mockImplementation((args, opts) => {
requested = opts;
return real(["-stats_period", "0.1", "-re", ...args], {
...opts,
timeout: 60_000,
onStderr: readStats,
});
});
try {
const extracted = await extractVideoFramesRange(
source,
"paced-stall-only",
0,
2,
{ fps: 10, outputDir: FIXTURE_DIR, format: "png" },
undefined,
{ ffmpegProcessTimeout: 2000 },
);
expect(extracted.totalFrames).toBe(20);
expect(requested).toMatchObject({ timeout: 2000 });
expect(new Set(reported).size).toBeGreaterThan(1);
} finally {
vi.mocked(runFfmpeg).mockImplementation(real);
}
}, 30_000);
it.each([
{ name: "one keyframe", keyint: "infinite", gapSeconds: 8 },
{ name: "a keyframe every second", keyint: "30", gapSeconds: 0 },
])(
"gives the stall clock the time to decode up to the seek point ($name)",
async ({ keyint, gapSeconds }) => {
const source = join(FIXTURE_DIR, `gop-${keyint}.mp4`);
const synth = await runFfmpeg([
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],
...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=10`, "-pix_fmt", "yuv420p"],
...["-x264-params", `keyint=${keyint}:scenecut=0`, source],
]);
if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`);
const { runFfmpeg: real } =
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
let requested: RunFfmpegOptions | undefined;
vi.mocked(runFfmpeg).mockImplementation((args, opts) => {
requested = opts;
return real(args, opts);
});
try {
await extractVideoFramesRange(
source,
`stall-seek-${keyint}`,
8,
1,
{ fps: 10, outputDir: FIXTURE_DIR, format: "png" },
undefined,
{ ffmpegProcessTimeout: 2000 },
);
expect(requested?.timeout).toBe(2000 + gapSeconds * 1000);
} finally {
vi.mocked(runFfmpeg).mockImplementation(real);
}
},
30_000,
);
it("stops a run that writes no new frame for the stall timeout", async () => {
// stdin stays open and empty, so ffmpeg waits for its first frame forever.
const stalled = await runFfmpeg(
["-f", "rawvideo", "-pix_fmt", "gray", "-s", "2x2", "-i", "pipe:0", "-f", "null", "-"],
{ timeout: 500 },
);
expect(stalled.terminationReason).toBe("inactivity");
}, 30_000);
// A distributed chunk extracts only the frames it renders; they must be the full extraction's.
describe("partial extraction", () => {
const gopSource = join(FIXTURE_DIR, "gop-30fps.mp4");
beforeAll(async () => {
// Real motion and long GOPs, so range seeks land mid-GOP.
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"testsrc2=s=64x36:r=30:d=20",
"-c:v",
"libx264",
"-g",
"250",
"-pix_fmt",
"yuv420p",
gopSource,
]);
if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`);
}, 30_000);
function frameBytes(dir: string): Map<string, Buffer> {
return new Map(readdirSync(dir).map((name) => [name, readFileSync(join(dir, name))]));
}
it.each([
{ fps: 24, startTime: 1.3, duration: 17 },
{ fps: 60, startTime: 0, duration: 20 },
])(
"frameRanges at $fps fps write the same files as a full extraction",
async (c) => {
const full = await extractVideoFramesRange(
gopSource,
`full-${c.fps}`,
c.startTime,
c.duration,
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg" },
);
const last = full.totalFrames;
const ranges = [
{ firstFrame: 0, frames: 5 },
{ firstFrame: 97, frames: 40 },
{ firstFrame: last - 3, frames: 3 },
];
const partial = await extractVideoFramesRange(
gopSource,
`partial-${c.fps}`,
c.startTime,
c.duration,
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg", frameRanges: ranges },
);
const fullFrames = frameBytes(full.outputDir);
const partialFrames = frameBytes(partial.outputDir);
expect(partialFrames.size).toBe(48);
for (const [name, bytes] of partialFrames) {
expect(fullFrames.get(name)?.equals(bytes), name).toBe(true);
}
},
60_000,
);
it.each([
{ fps: 24, mediaStart: 3.3 },
{ fps: 30, mediaStart: 0 },
{ fps: 60, mediaStart: 7.77 },
])(
"a deferred window to the source end at $fps fps reports the full frame count",
async (c) => {
const video = (): VideoElement => ({
id: `defer-${c.fps}`,
src: gopSource,
start: 0,
end: Infinity,
mediaStart: c.mediaStart,
loop: false,
hasAudio: false,
});
const full = await extractAllVideoFrames([video()], FIXTURE_DIR, {
fps: c.fps,
outputDir: join(FIXTURE_DIR, `defer-full-${c.fps}`),
});
const deferred = await extractAllVideoFrames([video()], FIXTURE_DIR, {
fps: c.fps,
outputDir: join(FIXTURE_DIR, `defer-${c.fps}`),
deferRangeExtraction: true,
});
expect(deferred.errors).toEqual([]);
const result = deferred.extracted[0]!;
expect(result.totalFrames).toBe(full.extracted[0]!.totalFrames);
expect(result.framePaths.size).toBe(0);
expect(readdirSync(result.outputDir)).toEqual([]);
expect(result.deferredRange).toEqual({
startTime: c.mediaStart,
durationSeconds: expect.any(Number),
format: "jpg",
});
},
60_000,
);
it("halves frames while they still cover the box a video is shown in", async () => {
const firstFrameSize = async (
videoId: string,
shownBox: { width: number; height: number },
) => {
const result = await extractVideoFramesRange(gopSource, videoId, 0, 1, {
fps: 30,
outputDir: FIXTURE_DIR,
format: "jpg",
frameRanges: [{ firstFrame: 0, frames: 1 }],
shownBox,
});
const [frame] = result.framePaths.values();
return spawnSync("ffprobe", [
"-v",
"error",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0",
"--",
frame!,
])
.stdout.toString()
.trim();
};
// 64x36 source: halved as often as the box allows (even sides round up); never below the box.
expect(await firstFrameSize("shown-quarter", { width: 16, height: 9 })).toBe("16,10");
expect(await firstFrameSize("shown-third", { width: 20, height: 12 })).toBe("32,18");
expect(await firstFrameSize("shown-near-full", { width: 40, height: 20 })).toBe("64,36");
}, 30_000);
});
});
describe.skipIf(!HAS_FFMPEG)("held tails on sparse-timestamp sources", () => {
const fixtureDir = mkdtempSync(join(tmpdir(), "hf-sparse-held-tail-"));
const cfrFixture = join(fixtureDir, "sub-1fps-cfr.mp4");
const vfrFixture = join(fixtureDir, "sparse-vfr.mp4");
const nonZeroStartFixture = join(fixtureDir, "nonzero-start.mp4");
const negativeStartTransportFixture = join(fixtureDir, "negative-start.ts");
beforeAll(async () => {
const fixtures = [
{
path: cfrFixture,
input: "testsrc2=s=64x64:d=10:rate=1/5",
filters: [] as string[],
},
{
path: vfrFixture,
input: "testsrc2=s=64x64:d=10:rate=1/2",
filters: ["-vf", "select='eq(n,0)+eq(n,2)'", "-fps_mode", "vfr"],
},
{
path: nonZeroStartFixture,
input: "testsrc2=s=64x64:d=3:rate=1",
filters: ["-output_ts_offset", "5"],
},
{
path: negativeStartTransportFixture,
input: "testsrc2=s=64x64:d=3:rate=1",
filters: [
"-mpegts_copyts",
"1",
"-muxdelay",
"0",
"-avoid_negative_ts",
"disabled",
"-output_ts_offset",
"-2",
],
},
];
for (const fixture of fixtures) {
const result = await runFfmpeg([
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
fixture.input,
...fixture.filters,
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
"-y",
fixture.path,
]);
if (!result.success) {
throw new Error(`sparse fixture synthesis failed: ${result.stderr.slice(-400)}`);
}
}
}, 30_000);
afterAll(() => {
rmSync(fixtureDir, { recursive: true, force: true });
}, 30_000);
it.each([
{
label: "sub-1fps CFR",
src: cfrFixture,
expectedVfr: false,
finalTimestamp: 5,
streamStart: 0,
},
{
label: "sparse VFR",
src: vfrFixture,
expectedVfr: true,
finalTimestamp: 4,
streamStart: 0,
},
{
label: "non-zero stream start",
src: nonZeroStartFixture,
expectedVfr: false,
finalTimestamp: 2,
streamStart: 5,
},
{
label: "unindexed negative-base MPEG-TS",
src: negativeStartTransportFixture,
expectedVfr: false,
finalTimestamp: 2,
streamStart: -2,
},
])(
"extracts one real final SDR frame for $label",
async ({ src, expectedVfr, finalTimestamp, streamStart }) => {
const metadata = await extractVideoMetadata(src);
expect(metadata.fps).toBeLessThanOrEqual(1);
expect(metadata.isVFR).toBe(expectedVfr);
expect(metadata.videoStreamStartSeconds).toBeCloseTo(streamStart, 6);
await expect(extractFinalVideoFrameTimestamp(src, metadata)).resolves.toBe(finalTimestamp);
const outputDir = mkdtempSync(join(fixtureDir, "out-"));
const video: VideoElement = {
id: `held-${String(expectedVfr)}`,
src,
start: -15,
end: 5,
mediaStart: 0,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], fixtureDir, {
fps: 30,
format: "png",
outputDir,
timelineEnd: 2,
});
expect(result.errors).toEqual([]);
expect(result.extracted).toHaveLength(1);
expect(result.extracted[0]?.totalFrames).toBe(1);
expect(video).toMatchObject({ start: 0, end: 5, loop: false });
expect(video.mediaStart).toBeCloseTo(metadata.videoStreamDurationSeconds - 0.000001, 7);
},
30_000,
);
it("renders the same final decoded frame just inside and past EOF", async () => {
const metadata = await extractVideoMetadata(cfrFixture);
const sourceDuration = metadata.videoStreamDurationSeconds;
const outputDir = mkdtempSync(join(fixtureDir, "eof-out-"));
const videos: VideoElement[] = [
{
id: "just-inside-eof",
src: cfrFixture,
start: 0,
end: 5,
mediaStart: sourceDuration - 0.001,
loop: false,
hasAudio: false,
},
{
id: "past-eof",
src: cfrFixture,
start: 0,
end: 5,
mediaStart: sourceDuration + 1,
loop: false,
hasAudio: false,
},
];
const result = await extractAllVideoFrames(videos, fixtureDir, {
fps: 30,
format: "png",
outputDir,
timelineEnd: 5,
});
expect(result.errors).toEqual([]);
expect(result.extracted).toHaveLength(2);
const insideFrame = result.extracted[0]?.framePaths.get(0);
const pastFrame = result.extracted[1]?.framePaths.get(0);
expect(insideFrame).toBeDefined();
expect(pastFrame).toBeDefined();
if (!insideFrame || !pastFrame) throw new Error("expected both final-frame outputs");
expect(readFileSync(pastFrame)).toEqual(readFileSync(insideFrame));
}, 30_000);
});
// Regression test for the VFR (variable frame rate) freeze bug.
// Screen recordings and phone videos often have irregular timestamps.
// When such inputs hit `extractVideoFramesRange`'s `-ss <start> -i ... -t <dur>
// -vf fps=N` pipeline, the fps filter can emit fewer frames than requested —
// e.g. a 4-second segment at 30fps would produce ~90 frames instead of 120.
// FrameLookupTable.getFrameAtTime then returns null for out-of-range indices
// and the compositor holds the last valid frame, which the user perceives as
// the video freezing. extractAllVideoFrames now routes VFR sources through
// FFmpeg's one-pass `-fps_mode cfr -r` extraction path to fix this without a
// separate normalization encode.
describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-vfr-test-"));
const VFR_FIXTURE = join(FIXTURE_DIR, "vfr_screen.mp4");
beforeAll(async () => {
// 10s testsrc2 at 60fps, ~40% of frames dropped via select filter and
// encoded with -fps_mode vfr so timestamps are irregular. Declared fps 60,
// actual average ~36 — well over the 10% threshold used by isVFR.
// The select expression drops four 1-second windows (frames 30-89,
// 180-239, 330-389, 480-539) to simulate static segments in a screen
// recording where no pixels changed.
// -g/-keyint_min 600 forces a single keyframe so mid-segment seeks in the
// mediaStart=3 test don't snap to an intermediate IDR and drift the count.
const result = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"testsrc2=s=320x180:d=10:rate=60",
"-vf",
"select='not(between(n\\,30\\,89))*not(between(n\\,180\\,239))*not(between(n\\,330\\,389))*not(between(n\\,480\\,539))'",
"-fps_mode",
"vfr",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-pix_fmt",
"yuv420p",
"-g",
"600",
"-keyint_min",
"600",
VFR_FIXTURE,
]);
if (!result.success) {
throw new Error(
`ffmpeg fixture synthesis failed (${result.exitCode}): ${result.stderr.slice(-400)}`,
);
}
}, 30_000);
afterAll(() => {
if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
}, 30_000);
it("skips a clip entirely before time zero without reporting an extraction error", async () => {
const outputDir = join(FIXTURE_DIR, "out-before-timeline");
mkdirSync(outputDir, { recursive: true });
const video: VideoElement = {
id: "before-timeline",
src: VFR_FIXTURE,
start: -2,
end: -1,
mediaStart: 0,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 1,
outputDir,
timelineEnd: 2,
});
expect(result).toMatchObject({ success: true, extracted: [], errors: [] });
});
it("preserves loop phase when negative preroll crosses the source boundary", async () => {
const outputDir = join(FIXTURE_DIR, "out-negative-loop");
mkdirSync(outputDir, { recursive: true });
const video: VideoElement = {
id: "negative-loop",
src: VFR_FIXTURE,
start: -19,
end: 5,
mediaStart: 0,
loop: true,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 1,
outputDir,
timelineEnd: 2,
});
expect(result.errors).toEqual([]);
const extracted = result.extracted[0];
if (!extracted) throw new Error("expected loop source frames");
const lookup = createFrameLookupTable([video], result.extracted);
expect(video).toMatchObject({ start: -19, mediaStart: 0, loop: true });
expect(lookup.getFrame("negative-loop", 0)).toBe(
extracted.framePaths.get(extracted.totalFrames - 1),
);
}, 30_000);
it("preserves the held final frame after negative preroll exhausts a source", async () => {
const outputDir = join(FIXTURE_DIR, "out-negative-held-tail");
mkdirSync(outputDir, { recursive: true });
const video: VideoElement = {
id: "negative-held-tail",
src: VFR_FIXTURE,
start: -15,
end: 5,
mediaStart: 0,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 1,
outputDir,
timelineEnd: 2,
});
expect(result.errors).toEqual([]);
const extracted = result.extracted[0];
if (!extracted) throw new Error("expected held-tail source frames");
const lookup = createFrameLookupTable([video], result.extracted);
// The authored slot remains active through end=5, but lookup is rebased to
// one exact final frame instead of assuming the last second contains a
// timestamp or materializing the full source.
expect(video).toMatchObject({ start: 0, end: 5, mediaStart: 9.999999, loop: false });
expect(extracted.totalFrames).toBe(1);
expect(lookup.getFrame("negative-held-tail", 0)).toBe(
extracted.framePaths.get(extracted.totalFrames - 1),
);
}, 30_000);
it("detects the synthesized fixture as VFR", async () => {
const md = await extractVideoMetadata(VFR_FIXTURE);
expect(md.isVFR).toBe(true);
});
it.each([
["compiled EOF", 4, 1],
["compiled past EOF", 5, 2],
["ordinary explicit zero", 6, 0],
])(
"drops a known zero timeline window without extraction or error: %s",
async (label, start, mediaStart) => {
const src = await synthCfrClip(`zero-window-${label.replaceAll(" ", "-")}.mp4`, 1);
const videos = parseVideoElements(
`<video id="zero-window" src="${src}" data-start="${start}" data-duration="0" data-end="${start}" data-media-start="${mediaStart}" muted></video>`,
);
const outputDir = join(FIXTURE_DIR, `out-zero-window-${label.replaceAll(" ", "-")}`);
const result = await extractAllVideoFrames(videos, FIXTURE_DIR, { fps: 30, outputDir });
expect(result.errors).toEqual([]);
expect(result.extracted).toEqual([]);
expect(result.totalFramesExtracted).toBe(0);
expect(videos).toEqual([]);
},
);
it("passes an exact 24000/1001 rate through VFR normalization", async () => {
const outputDir = join(FIXTURE_DIR, "out-vfr-ntsc-boundary");
mkdirSync(outputDir, { recursive: true });
const result = await extractVideoFramesRange(VFR_FIXTURE, "vfr-ntsc-boundary", 0, 0.125125, {
fps: { num: 24000, den: 1001 },
outputDir,
format: "jpg",
});
expect(result.metadata.isVFR).toBe(true);
expect(result.totalFrames).toBe(3);
}, 60_000);
// With B-frames, mp4 packet durations are decode-order gaps, so a frame can claim 0.5s while
// the next one starts 17ms later. Each slot must still show the frame on screen at its time:
// `-fps_mode cfr` on ffmpeg 8 held such a frame for 14 slots and cut the still before a gap short.
it("samples a B-frame VFR source by timestamp, not by packet duration", async () => {
const source = join(FIXTURE_DIR, "vfr_bframes.mp4");
const synth = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
"testsrc2=s=320x180:d=4:rate=60",
"-vf",
"select='not(between(n\\,30\\,89))*not(between(n\\,150\\,179))'",
"-fps_mode",
"vfr",
"-c:v",
"libx264",
"-preset",
"medium",
"-bf",
"3",
"-g",
"600",
"-pix_fmt",
"yuv420p",
source,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
const distinctFrames = (paths: Iterable<string>) =>
new Set([...paths].map((path) => readFileSync(path).toString("base64"))).size;
// 2.5s to 3s is a still; every 30fps slot after it lands on its own 60fps source frame.
const afterGap = await extractVideoFramesRange(source, "vfr-bframes-after-gap", 2.5, 1.5, {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-vfr-bframes-after-gap"),
format: "png",
});
expect(afterGap.metadata.isVFR).toBe(true);
expect(afterGap.totalFrames).toBe(45);
const slots = [...afterGap.framePaths.entries()].sort(([a], [b]) => a - b);
expect(distinctFrames(slots.slice(0, 15).map(([, path]) => path))).toBe(1);
expect(distinctFrames(slots.slice(15).map(([, path]) => path))).toBe(30);
const endsOnStill = await extractVideoFramesRange(source, "vfr-bframes-still", 0, 3, {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-vfr-bframes-still"),
format: "png",
});
expect(endsOnStill.totalFrames).toBe(90);
}, 60_000);
it("produces the expected frame count for a mid-file segment", async () => {
const outputDir = join(FIXTURE_DIR, "out-mid-segment");
mkdirSync(outputDir, { recursive: true });
const video: VideoElement = {
id: "v1",
src: VFR_FIXTURE,
start: 0,
end: 4,
mediaStart: 3,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 30,
outputDir,
});
expect(result.errors).toEqual([]);
expect(result.extracted).toHaveLength(1);
const frames = readdirSync(join(outputDir, "v1")).filter((f) => f.endsWith(".jpg"));
// Pre-fix behavior produced ~90 frames (a 25% shortfall).
// ±3 tolerance: FFmpeg's one-pass VFR→CFR extraction yields slightly
// different frame counts across versions (timestamp rounding).
expect(frames.length).toBeGreaterThanOrEqual(117);
expect(frames.length).toBeLessThanOrEqual(123);
expect(result.phaseBreakdown).toBeDefined();
expect(result.phaseBreakdown.extractMs).toBeGreaterThan(0);
expect(result.phaseBreakdown.vfrPreflightCount).toBe(1);
expect(result.phaseBreakdown.vfrPreflightMs).toBeGreaterThanOrEqual(0);
}, 60_000);
// Shared fixture helpers for the cache tests below. All synthesize clean
// CFR SDR clips — keeps VFR preflight count at zero so cache keys are
// stable across runs within a test.
async function synthCfrClip(name: string, durationSeconds: number): Promise<string> {
const src = join(FIXTURE_DIR, name);
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`testsrc2=s=320x180:d=${durationSeconds}:rate=30`,
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-pix_fmt",
"yuv420p",
src,
]);
if (!synth.success) {
throw new Error(`Fixture synthesis failed (${name}): ${synth.stderr.slice(-400)}`);
}
return src;
}
it("rejects a media start beyond source duration before invoking FFmpeg", async () => {
const src = await synthCfrClip("zero-output-src.mp4", 1);
const outputDir = join(FIXTURE_DIR, "out-zero-output");
await expect(
extractVideoFramesRange(src, "past-eof", 2, 1, { fps: 30, outputDir }),
).rejects.toMatchObject({
kind: "media_start_out_of_range",
retryable: false,
});
}, 60_000);
it("preserves legacy metadata rejection unless typed aggregation is explicitly enabled", async () => {
const src = join(FIXTURE_DIR, "invalid-probe.mp4");
writeFileSync(src, "not a media container");
const video = cfrClipElement("invalid-probe", src, 1);
await expect(
extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-invalid-probe-legacy"),
}),
).rejects.toThrow();
const collected = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-invalid-probe-typed"),
collectProbeFailures: true,
});
expect(collected.success).toBe(false);
expect(collected.extracted).toEqual([]);
expect(collected.errors).toEqual([
expect.objectContaining({
videoId: "invalid-probe",
kind: "invalid_media",
retryable: false,
}),
]);
}, 60_000);
async function synthHdrTaggedClip(name: string, durationSeconds: number): Promise<string> {
const src = join(FIXTURE_DIR, name);
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`testsrc2=s=320x180:d=${durationSeconds}:rate=30`,
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-pix_fmt",
"yuv420p",
"-color_primaries",
"bt2020",
"-color_trc",
"smpte2084",
"-colorspace",
"bt2020nc",
// The -color_* flags above only tag the container/encoder context;
// whether they reach the H.264 VUI depends on the ffmpeg build (the
// pinned Windows CI build drops the transfer). Write the VUI directly
// so probing reports smpte2084 on every build (9/16/9 = bt2020/PQ/bt2020nc).
"-bsf:v",
"h264_metadata=colour_primaries=9:transfer_characteristics=16:matrix_coefficients=9",
src,
]);
if (!synth.success) {
throw new Error(`HDR fixture synthesis failed (${name}): ${synth.stderr.slice(-400)}`);
}
return src;
}
function cfrClipElement(
id: string,
src: string,
endSeconds: number,
mediaStart = 0,
): VideoElement {
return {
id,
src,
start: 0,
end: endSeconds,
mediaStart,
loop: false,
hasAudio: false,
};
}
async function extractWithCache(
video: VideoElement,
outName: string,
cacheDir: string,
fps = 30,
): Promise<ExtractionResult> {
const outputDir = join(FIXTURE_DIR, outName);
mkdirSync(outputDir, { recursive: true });
return extractAllVideoFrames([video], FIXTURE_DIR, { fps, outputDir }, undefined, {
extractCacheDir: cacheDir,
});
}
function cacheEntryNames(cacheDir: string): string[] {
return readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX));
}
function supersetDirNames(outputDir: string): string[] {
if (!existsSync(outputDir)) return [];
return readdirSync(outputDir).filter((name) => name.startsWith("__superset-"));
}
function extractedFor(result: ExtractionResult, videoId: string): ExtractedFrames {
const extracted = result.extracted.find((item) => item.videoId === videoId);
if (!extracted) throw new Error(`missing extraction result for ${videoId}`);
return extracted;
}
function framePath(result: ExtractionResult, videoId: string, frameIndex: number): string {
const extracted = extractedFor(result, videoId);
const frame = extracted?.framePaths.get(frameIndex);
if (!frame) throw new Error(`missing frame ${frameIndex} for ${videoId}`);
return frame;
}
it("reuses extracted frames on a warm cache hit", async () => {
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
const SRC = await synthCfrClip("cache-src.mp4", 2);
const video = cfrClipElement("cv1", SRC, 2);
const miss = await extractWithCache(video, "out-cache-miss", CACHE_DIR);
expect(miss.errors).toEqual([]);
expect(miss.phaseBreakdown.cacheHits).toBe(0);
expect(miss.phaseBreakdown.cacheMisses).toBe(1);
const ffmpegRunsBeforeHit = vi.mocked(runFfmpeg).mock.calls.length;
const hit = await extractWithCache(video, "out-cache-hit", CACHE_DIR);
expect(hit.errors).toEqual([]);
expect(hit.phaseBreakdown.cacheHits).toBe(1);
expect(hit.phaseBreakdown.cacheMisses).toBe(0);
expect(vi.mocked(runFfmpeg).mock.calls.length).toBe(ffmpegRunsBeforeHit);
expect(hit.extracted).toHaveLength(1);
expect(hit.extracted[0]!.totalFrames).toBe(miss.extracted[0]!.totalFrames);
rmSync(CACHE_DIR, { recursive: true, force: true });
}, 60_000);
it("does not reuse a decimal-rate VFR cache entry for the exact rational rate", async () => {
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-cache-ntsc-rate-test-"));
const decimalOutputDir = join(FIXTURE_DIR, "out-cache-vfr-ntsc-decimal");
const rationalOutputDir = join(FIXTURE_DIR, "out-cache-vfr-ntsc-rational");
mkdirSync(decimalOutputDir, { recursive: true });
mkdirSync(rationalOutputDir, { recursive: true });
try {
const decimal = await extractAllVideoFrames(
[cfrClipElement("vfr-ntsc-cache", VFR_FIXTURE, 0.125125)],
FIXTURE_DIR,
{ fps: 24000 / 1001, outputDir: decimalOutputDir },
undefined,
{ extractCacheDir: cacheDir },
);
expect(decimal.errors).toEqual([]);
expect(decimal.phaseBreakdown.cacheMisses).toBe(1);
expect(decimal.extracted[0]?.totalFrames).toBe(3);
// Model the warm v3 entry from before exact-rate extraction: the
// decimal path could persist one extra frame at this boundary. If the
// rational lookup collides, rehydration below will observe all four.
writeFileSync(
join(decimal.extracted[0]!.outputDir, "frame_00004.jpg"),
"stale-decimal-boundary-frame",
"utf-8",
);
const rational = await extractAllVideoFrames(
[cfrClipElement("vfr-ntsc-cache", VFR_FIXTURE, 0.125125)],
FIXTURE_DIR,
{ fps: { num: 24000, den: 1001 }, outputDir: rationalOutputDir },
undefined,
{ extractCacheDir: cacheDir },
);
expect(rational.errors).toEqual([]);
expect(rational.phaseBreakdown.cacheHits).toBe(0);
expect(rational.phaseBreakdown.cacheMisses).toBe(1);
expect(rational.extracted[0]?.totalFrames).toBe(3);
expect(cacheEntryNames(cacheDir)).toHaveLength(2);
} finally {
rmSync(cacheDir, { recursive: true, force: true });
}
}, 60_000);
it("reuses one-cycle loop extraction across different authored starts", async () => {
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-loop-phase-cache-test-"));
const src = await synthCfrClip("cache-loop-phase-src.mp4", 3);
try {
const firstOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-first");
const secondOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-second");
mkdirSync(firstOutputDir, { recursive: true });
mkdirSync(secondOutputDir, { recursive: true });
const first = await extractAllVideoFrames(
[
{
...cfrClipElement("loop-cache-first", src, 60),
loop: true,
},
],
FIXTURE_DIR,
{ fps: 30, outputDir: firstOutputDir, timelineEnd: 60 },
undefined,
{ extractCacheDir: cacheDir },
);
expect(first.errors).toEqual([]);
expect(first.phaseBreakdown.cacheMisses).toBe(1);
const second = await extractAllVideoFrames(
[
{
...cfrClipElement("loop-cache-second", src, 66),
start: -6,
loop: true,
},
],
FIXTURE_DIR,
{ fps: 30, outputDir: secondOutputDir, timelineEnd: 60 },
undefined,
{ extractCacheDir: cacheDir },
);
expect(second.errors).toEqual([]);
expect(second.phaseBreakdown.cacheHits).toBe(1);
expect(second.phaseBreakdown.cacheMisses).toBe(0);
expect(second.extracted[0]?.totalFrames).toBe(first.extracted[0]?.totalFrames);
} finally {
rmSync(cacheDir, { recursive: true, force: true });
}
}, 60_000);
it("updates the cache sentinel mtime on a hit", async () => {
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-touch-test-"));
const SRC = await synthCfrClip("cache-touch-src.mp4", 1);
const video = cfrClipElement("touch", SRC, 1);
const miss = await extractWithCache(video, "out-cache-touch-miss", CACHE_DIR);
expect(miss.errors).toEqual([]);
expect(miss.phaseBreakdown.cacheMisses).toBe(1);
const cacheEntryNames = readdirSync(CACHE_DIR).filter((name) => name.startsWith(SCHEMA_PREFIX));
expect(cacheEntryNames).toHaveLength(1);
const sentinel = join(CACHE_DIR, cacheEntryNames[0]!, COMPLETE_SENTINEL);
const old = new Date(Date.now() - 120_000);
utimesSync(sentinel, old, old);
const before = statSync(sentinel).mtimeMs;
const hit = await extractWithCache(video, "out-cache-touch-hit", CACHE_DIR);
expect(hit.errors).toEqual([]);
expect(hit.phaseBreakdown.cacheHits).toBe(1);
expect(statSync(sentinel).mtimeMs).toBeGreaterThan(before);
rmSync(CACHE_DIR, { recursive: true, force: true });
}, 60_000);
it("skips cache GC on all-hit renders", async () => {
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-gc-skip-test-"));
const SRC = await synthCfrClip("cache-gc-skip-src.mp4", 1);
const video = cfrClipElement("gc-skip", SRC, 1);
const miss = await extractWithCache(video, "out-cache-gc-skip-miss", CACHE_DIR);
expect(miss.errors).toEqual([]);
expect(miss.phaseBreakdown.cacheMisses).toBe(1);
const agedPartial = join(CACHE_DIR, `${SCHEMA_PREFIX}aged.partial-1234-deadbeef`);
mkdirSync(agedPartial, { recursive: true });
writeFileSync(join(agedPartial, "frame_00001.jpg"), "stale", "utf-8");
const old = new Date(Date.now() - 2 * 60 * 60 * 1000);
utimesSync(agedPartial, old, old);
const hit = await extractWithCache(video, "out-cache-gc-skip-hit", CACHE_DIR);
expect(hit.errors).toEqual([]);
expect(hit.phaseBreakdown.cacheHits).toBe(1);
expect(hit.phaseBreakdown.cacheMisses).toBe(0);
expect(existsSync(agedPartial)).toBe(true);
rmSync(CACHE_DIR, { recursive: true, force: true });
}, 60_000);
it("disables caching for this render when the cache dir is not writable", async () => {
const CACHE_FILE = join(FIXTURE_DIR, "cache-dir-is-a-file");
writeFileSync(CACHE_FILE, "not a directory", "utf-8");
const SRC = await synthCfrClip("cache-disabled-src.mp4", 1);
const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
try {
const result = await extractWithCache(
cfrClipElement("uncached", SRC, 1),
"out-cache-disabled",
CACHE_FILE,
);
expect(result.errors).toEqual([]);
expect(result.extracted).toHaveLength(1);
expect(result.phaseBreakdown.cacheHits).toBe(0);
expect(result.phaseBreakdown.cacheMisses).toBe(0);
expect(stderr).toHaveBeenCalledTimes(1);
expect(String(stderr.mock.calls[0]?.[0])).toContain("extraction cache dir");
expect(String(stderr.mock.calls[0]?.[0])).toContain("caching disabled for this render");
} finally {
stderr.mockRestore();
}
}, 60_000);
it("invalidates the cache when fps changes", async () => {
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
const SRC = await synthCfrClip("cache-fps-src.mp4", 1);
const video = cfrClipElement("cv2", SRC, 1);
const first = await extractWithCache(video, "out-cache-fps-30", CACHE_DIR);
expect(first.phaseBreakdown.cacheMisses).toBe(1);
const second = await extractWithCache(video, "out-cache-fps-60", CACHE_DIR, 60);
expect(second.phaseBreakdown.cacheMisses).toBe(1);
expect(second.phaseBreakdown.cacheHits).toBe(0);
rmSync(CACHE_DIR, { recursive: true, force: true });
}, 60_000);
it("applies SDR→HDR conversion during extraction without normalized intermediates", async () => {
const SDR_LONG = await synthCfrClip("sdr-long.mp4", 10);
const HDR_SHORT = await synthHdrTaggedClip("hdr-short.mp4", 2);
const outputDir = join(FIXTURE_DIR, "out-hdr-segment");
const plainOutputDir = join(FIXTURE_DIR, "out-hdr-plain-sdr");
mkdirSync(outputDir, { recursive: true });
mkdirSync(plainOutputDir, { recursive: true });
const videos: VideoElement[] = [
{ id: "sdr", src: SDR_LONG, start: 0, end: 2, mediaStart: 0, loop: false, hasAudio: false },
{
id: "hdr",
src: HDR_SHORT,
start: 2,
end: 4,
mediaStart: 0,
loop: false,
hasAudio: false,
},
];
const result = await extractAllVideoFrames(videos, FIXTURE_DIR, {
fps: 30,
outputDir,
});
expect(result.errors).toEqual([]);
expect(result.phaseBreakdown.hdrPreflightCount).toBe(1);
expect(existsSync(join(outputDir, "_hdr_normalized"))).toBe(false);
const sdrFrames = result.extracted.find((item) => item.videoId === "sdr");
expect(sdrFrames?.totalFrames).toBe(60);
const plain = await extractVideoFramesRange(SDR_LONG, "plain-sdr", 0, 2, {
fps: 30,
outputDir: plainOutputDir,
format: "jpg",
});
expect(plain.totalFrames).toBe(60);
expect(
readFileSync(framePath(result, "sdr", 0)).equals(readFileSync(plain.framePaths.get(0)!)),
).toBe(false);
}, 60_000);
it("leaves SDR clips alone in a mixed timeline that tone-maps to SDR", async () => {
const SDR = await synthCfrClip("sdr-tonemap-peer.mp4", 1);
const HDR = await synthHdrTaggedClip("hdr-tonemap-peer.mp4", 1);
const outputDir = join(FIXTURE_DIR, "out-hdr-tonemap-mixed");
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[cfrClipElement("sdr", SDR, 1), { ...cfrClipElement("hdr", HDR, 1), start: 1, end: 2 }],
FIXTURE_DIR,
{ fps: 30, outputDir, toneMapHdrToSdr: true },
);
expect(result.errors).toEqual([]);
expect(result.phaseBreakdown.hdrPreflightCount).toBe(0);
}, 60_000);
it("keeps a finite SDR past-EOF slot in a mixed HDR timeline", async () => {
const SDR_SHORT = await synthCfrClip("sdr-past-eof.mp4", 1);
const HDR_SHORT = await synthHdrTaggedClip("hdr-past-eof-peer.mp4", 1);
const outputDir = join(FIXTURE_DIR, "out-hdr-past-eof");
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[
cfrClipElement("sdr-past-eof", SDR_SHORT, 4, 5),
{ ...cfrClipElement("hdr-peer", HDR_SHORT, 1), start: 1, end: 2 },
],
FIXTURE_DIR,
{ fps: 30, outputDir },
);
// The SDR slot must survive the mixed-HDR preflight and use the held-tail
// path. Reverting its guard to `mediaStart >= playableDuration` records an
// out-of-range error here and drops the slot before extraction.
expect(result.errors).toEqual([]);
expect(result.phaseBreakdown.hdrPreflightCount).toBe(1);
expect(extractedFor(result, "sdr-past-eof").totalFrames).toBe(1);
expect(extractedFor(result, "hdr-peer").totalFrames).toBeGreaterThan(0);
}, 60_000);
it("keeps SDR→HDR cache entries distinct from plain SDR entries", async () => {
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-hdr-cache-test-"));
const SDR = await synthCfrClip("cache-hdr-sdr.mp4", 1);
const HDR = await synthHdrTaggedClip("cache-hdr-hdr.mp4", 1);
try {
const mixedOutputDir = join(FIXTURE_DIR, "out-cache-hdr-mixed");
mkdirSync(mixedOutputDir, { recursive: true });
const mixed = await extractAllVideoFrames(
[
cfrClipElement("sdr-transform", SDR, 1),
{ ...cfrClipElement("hdr-peer", HDR, 1), start: 1, end: 2 },
],
FIXTURE_DIR,
{ fps: 30, outputDir: mixedOutputDir },
undefined,
{ extractCacheDir: CACHE_DIR },
);
expect(mixed.errors).toEqual([]);
expect(mixed.phaseBreakdown.hdrPreflightCount).toBe(1);
expect(mixed.phaseBreakdown.cacheHits).toBe(0);
expect(mixed.phaseBreakdown.cacheMisses).toBe(2);
const plain = await extractWithCache(
cfrClipElement("sdr-plain", SDR, 1),
"out-cache-hdr-plain",
CACHE_DIR,
);
expect(plain.errors).toEqual([]);
expect(plain.phaseBreakdown.cacheHits).toBe(0);
expect(plain.phaseBreakdown.cacheMisses).toBe(1);
expect(cacheEntryNames(CACHE_DIR)).toHaveLength(3);
// Cross-render poisoning regression: the plain-SDR render must not be
// served the BT.2020-converted frames the mixed render cached for the
// SAME source+trim. Compare actual frame bytes across the cache
// boundary, not just entry counts.
expect(
readFileSync(framePath(plain, "sdr-plain", 0)).equals(
readFileSync(framePath(mixed, "sdr-transform", 0)),
),
).toBe(false);
// And a repeat plain render must HIT the plain entry and serve
// byte-identical plain frames (proves the hit path keys correctly too).
const plainAgain = await extractWithCache(
cfrClipElement("sdr-plain-again", SDR, 1),
"out-cache-hdr-plain-again",
CACHE_DIR,
);
expect(plainAgain.phaseBreakdown.cacheHits).toBe(1);
expect(
readFileSync(framePath(plainAgain, "sdr-plain-again", 0)).equals(
readFileSync(framePath(plain, "sdr-plain", 0)),
),
).toBe(true);
} finally {
rmSync(CACHE_DIR, { recursive: true, force: true });
}
}, 60_000);
it("clusters overlap components so a disjoint outlier does not break the group", async () => {
const SRC = await synthCfrClip("superset-cluster-src.mp4", 12);
const outputDir = join(FIXTURE_DIR, "out-superset-cluster");
mkdirSync(outputDir, { recursive: true });
// Three overlapping trims [0..4], [2..6], [4..8] plus one disjoint trim
// [10..12] of the same source. Pre-clustering, the outlier failed the
// union<=sum check for the whole bucket and ALL FOUR fell back to direct
// extraction; the overlapping three must still share one superset.
const result = await extractAllVideoFrames(
[
cfrClipElement("cl-a", SRC, 4, 0),
cfrClipElement("cl-b", SRC, 4, 2),
cfrClipElement("cl-c", SRC, 4, 4),
cfrClipElement("cl-out", SRC, 2, 10),
],
FIXTURE_DIR,
{ fps: 30, outputDir },
);
expect(result.errors).toEqual([]);
// Overlap region t=2..4 of the source: cl-a frame 60 and cl-b frame 0
// must be the SAME inode (shared superset extraction).
expect(statSync(framePath(result, "cl-a", 60)).ino).toBe(
statSync(framePath(result, "cl-b", 0)).ino,
);
expect(statSync(framePath(result, "cl-b", 60)).ino).toBe(
statSync(framePath(result, "cl-c", 0)).ino,
);
// The outlier extracted directly: its frames share no inode with the
// cluster (frame at source t=10 exists only in its own extraction).
expect(extractedFor(result, "cl-out").totalFrames).toBe(60);
expect(supersetDirNames(outputDir)).toEqual([]);
}, 60_000);
it("runs the GC staleness fallback sweep on all-hit renders with a stale marker", async () => {
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-gc-stale-test-"));
const SRC = await synthCfrClip("cache-gc-stale-src.mp4", 1);
const video = cfrClipElement("gc-stale", SRC, 1);
const miss = await extractWithCache(video, "out-cache-gc-stale-miss", CACHE_DIR);
expect(miss.phaseBreakdown.cacheMisses).toBe(1);
const agedPartial = join(CACHE_DIR, `${SCHEMA_PREFIX}aged.partial-1234-cafef00d`);
mkdirSync(agedPartial, { recursive: true });
writeFileSync(join(agedPartial, "frame_00001.jpg"), "stale", "utf-8");
const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000);
utimesSync(agedPartial, twoHoursAgo, twoHoursAgo);
// Age the sweep marker past the 24h staleness window: the next all-hit
// render must sweep anyway and clear the aged partial.
const twoDaysAgo = new Date(Date.now() - 48 * 60 * 60 * 1000);
utimesSync(join(CACHE_DIR, GC_MARKER), twoDaysAgo, twoDaysAgo);
const hit = await extractWithCache(video, "out-cache-gc-stale-hit", CACHE_DIR);
expect(hit.phaseBreakdown.cacheHits).toBe(1);
expect(hit.phaseBreakdown.cacheMisses).toBe(0);
expect(existsSync(agedPartial)).toBe(false);
expect(hit.phaseBreakdown.cacheAgedPartialsCleared).toBe(1);
rmSync(CACHE_DIR, { recursive: true, force: true });
}, 60_000);
it("hardlinks overlapping aligned trims from one superset extraction", async () => {
const SRC = await synthCfrClip("superset-overlap-src.mp4", 10);
const outputDir = join(FIXTURE_DIR, "out-superset-overlap");
const directOutputDir = join(FIXTURE_DIR, "out-superset-direct");
mkdirSync(outputDir, { recursive: true });
mkdirSync(directOutputDir, { recursive: true });
const result = await extractAllVideoFrames(
[cfrClipElement("trim-a", SRC, 4, 0), cfrClipElement("trim-b", SRC, 4, 2)],
FIXTURE_DIR,
{ fps: 30, outputDir },
);
expect(result.errors).toEqual([]);
expect(extractedFor(result, "trim-a").totalFrames).toBe(120);
expect(extractedFor(result, "trim-b").totalFrames).toBe(120);
expect(statSync(framePath(result, "trim-a", 60)).ino).toBe(
statSync(framePath(result, "trim-b", 0)).ino,
);
const direct = await extractVideoFramesRange(SRC, "direct-trim-b", 2, 4, {
fps: 30,
outputDir: directOutputDir,
format: "jpg",
});
expect(
readFileSync(framePath(result, "trim-b", 0)).equals(readFileSync(direct.framePaths.get(0)!)),
).toBe(true);
expect(supersetDirNames(outputDir)).toEqual([]);
}, 60_000);
it.each([
{ label: "decimal underflow half-frame", duration: 2.05, offset: 1, expectedFrames: 62 },
{
label: "non-half-integer boundary",
duration: 0.616666,
offset: 0.1,
expectedFrames: 18,
},
])(
"keeps direct and superset CFR extraction equal at a $label",
async ({ label, duration, offset, expectedFrames }) => {
const fixtureKey = label.replaceAll(" ", "-");
const src = await synthCfrClip(`superset-cfr-${fixtureKey}.mp4`, 4);
const groupedOutputDir = join(FIXTURE_DIR, `out-superset-cfr-${fixtureKey}`);
const directOutputDir = join(FIXTURE_DIR, `out-direct-cfr-${fixtureKey}`);
mkdirSync(groupedOutputDir, { recursive: true });
mkdirSync(directOutputDir, { recursive: true });
const grouped = await extractAllVideoFrames(
[
cfrClipElement(`${fixtureKey}-base`, src, duration, 0),
cfrClipElement(`${fixtureKey}-member`, src, duration, offset),
],
FIXTURE_DIR,
{ fps: 30, outputDir: groupedOutputDir },
);
const direct = await extractVideoFramesRange(src, `${fixtureKey}-direct`, offset, duration, {
fps: 30,
outputDir: directOutputDir,
format: "jpg",
});
expect(grouped.errors).toEqual([]);
expect(direct.totalFrames).toBe(expectedFrames);
expect(extractedFor(grouped, `${fixtureKey}-base`).totalFrames).toBe(expectedFrames);
expect(extractedFor(grouped, `${fixtureKey}-member`).totalFrames).toBe(expectedFrames);
expect(statSync(framePath(grouped, `${fixtureKey}-base`, Math.round(offset * 30))).ino).toBe(
statSync(framePath(grouped, `${fixtureKey}-member`, 0)).ino,
);
for (let frame = 0; frame < expectedFrames; frame += 1) {
expect(
readFileSync(framePath(grouped, `${fixtureKey}-member`, frame)).equals(
readFileSync(direct.framePaths.get(frame)!),
),
).toBe(true);
}
expect(supersetDirNames(groupedOutputDir)).toEqual([]);
},
60_000,
);
it("keeps direct and superset CFR extraction equal at 30000/1001", async () => {
const src = await synthCfrClip("superset-cfr-ntsc.mp4", 4);
const groupedOutputDir = join(FIXTURE_DIR, "out-superset-cfr-ntsc");
const directOutputDir = join(FIXTURE_DIR, "out-direct-cfr-ntsc");
const fps = { num: 30000, den: 1001 };
const duration = 0.25025;
mkdirSync(groupedOutputDir, { recursive: true });
mkdirSync(directOutputDir, { recursive: true });
const grouped = await extractAllVideoFrames(
[
cfrClipElement("ntsc-base", src, 0.5005, 0),
cfrClipElement("ntsc-member", src, duration, 0),
],
FIXTURE_DIR,
{ fps, outputDir: groupedOutputDir },
);
const direct = await extractVideoFramesRange(src, "ntsc-direct", 0, duration, {
fps,
outputDir: directOutputDir,
format: "jpg",
});
expect(grouped.errors).toEqual([]);
expect(direct.totalFrames).toBe(8);
expect(extractedFor(grouped, "ntsc-base").totalFrames).toBe(15);
expect(extractedFor(grouped, "ntsc-member").totalFrames).toBe(8);
expect(statSync(framePath(grouped, "ntsc-base", 0)).ino).toBe(
statSync(framePath(grouped, "ntsc-member", 0)).ino,
);
for (let frame = 0; frame < 8; frame += 1) {
expect(
readFileSync(framePath(grouped, "ntsc-member", frame)).equals(
readFileSync(direct.framePaths.get(frame)!),
),
).toBe(true);
}
expect(supersetDirNames(groupedOutputDir)).toEqual([]);
}, 60_000);
it("does not superset disjoint trims", async () => {
const SRC = await synthCfrClip("superset-disjoint-src.mp4", 10);
const outputDir = join(FIXTURE_DIR, "out-superset-disjoint");
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[cfrClipElement("trim-a", SRC, 2, 0), cfrClipElement("trim-b", SRC, 2, 8)],
FIXTURE_DIR,
{ fps: 30, outputDir },
);
expect(result.errors).toEqual([]);
expect(statSync(framePath(result, "trim-a", 0)).ino).not.toBe(
statSync(framePath(result, "trim-b", 0)).ino,
);
expect(supersetDirNames(outputDir)).toEqual([]);
}, 60_000);
it("does not superset trims whose offsets are not frame-aligned", async () => {
const SRC = await synthCfrClip("superset-misaligned-src.mp4", 2);
const outputDir = join(FIXTURE_DIR, "out-superset-misaligned");
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[cfrClipElement("trim-a", SRC, 1, 0), cfrClipElement("trim-b", SRC, 1, 0.017)],
FIXTURE_DIR,
{ fps: 30, outputDir },
);
expect(result.errors).toEqual([]);
expect(statSync(framePath(result, "trim-a", 0)).ino).not.toBe(
statSync(framePath(result, "trim-b", 0)).ino,
);
expect(supersetDirNames(outputDir)).toEqual([]);
}, 60_000);
it("keeps overlapping VFR trims direct because CFR resampling phase resets per seek", async () => {
const outputDir = join(FIXTURE_DIR, "out-vfr-superset-short");
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[
cfrClipElement("vfr-short-a", VFR_FIXTURE, 0.616666, 0),
cfrClipElement("vfr-short-b", VFR_FIXTURE, 0.616666, 0.1),
],
FIXTURE_DIR,
{ fps: 30, outputDir },
);
expect(result.errors).toEqual([]);
expect(extractedFor(result, "vfr-short-a").metadata.isVFR).toBe(true);
expect(extractedFor(result, "vfr-short-a").totalFrames).toBe(19);
expect(extractedFor(result, "vfr-short-b").totalFrames).toBe(19);
expect(statSync(framePath(result, "vfr-short-a", 3)).ino).not.toBe(
statSync(framePath(result, "vfr-short-b", 0)).ino,
);
expect(supersetDirNames(outputDir)).toEqual([]);
}, 60_000);
it("keeps batched and direct VFR extraction equal at a floating integral boundary", async () => {
const groupedOutputDir = join(FIXTURE_DIR, "out-vfr-superset-integral-boundary");
const directOutputDir = join(FIXTURE_DIR, "out-vfr-direct-integral-boundary");
mkdirSync(groupedOutputDir, { recursive: true });
mkdirSync(directOutputDir, { recursive: true });
const first: VideoElement = {
id: "vfr-integral-a",
src: VFR_FIXTURE,
start: 0.03,
end: 0.33,
mediaStart: 0.03,
loop: false,
hasAudio: false,
};
const second: VideoElement = {
...first,
id: "vfr-integral-b",
mediaStart: 0.13,
};
const direct = await extractAllVideoFrames([{ ...second }], FIXTURE_DIR, {
fps: 30,
outputDir: directOutputDir,
});
const grouped = await extractAllVideoFrames([{ ...first }, second], FIXTURE_DIR, {
fps: 30,
outputDir: groupedOutputDir,
});
expect(direct.errors).toEqual([]);
expect(grouped.errors).toEqual([]);
expect(extractedFor(direct, second.id).totalFrames).toBe(9);
expect(extractedFor(grouped, first.id).totalFrames).toBe(9);
expect(extractedFor(grouped, second.id).totalFrames).toBe(9);
for (let frame = 0; frame < 9; frame += 1) {
expect(
readFileSync(framePath(grouped, second.id, frame)).equals(
readFileSync(framePath(direct, second.id, frame)),
),
).toBe(true);
}
expect(statSync(framePath(grouped, first.id, 3)).ino).not.toBe(
statSync(framePath(grouped, second.id, 0)).ino,
);
expect(supersetDirNames(groupedOutputDir)).toEqual([]);
}, 60_000);
it("publishes overlapping superset slices to cache entries and hits them on the next render", async () => {
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-superset-cache-test-"));
const SRC = await synthCfrClip("superset-cache-src.mp4", 10);
try {
const firstOutputDir = join(FIXTURE_DIR, "out-superset-cache-first");
const secondOutputDir = join(FIXTURE_DIR, "out-superset-cache-second");
mkdirSync(firstOutputDir, { recursive: true });
mkdirSync(secondOutputDir, { recursive: true });
const videos = [cfrClipElement("trim-a", SRC, 4, 0), cfrClipElement("trim-b", SRC, 4, 2)];
const first = await extractAllVideoFrames(
videos,
FIXTURE_DIR,
{ fps: 30, outputDir: firstOutputDir },
undefined,
{ extractCacheDir: CACHE_DIR },
);
expect(first.errors).toEqual([]);
expect(first.phaseBreakdown.cacheHits).toBe(0);
expect(first.phaseBreakdown.cacheMisses).toBe(2);
expect(cacheEntryNames(CACHE_DIR)).toHaveLength(2);
expect(statSync(framePath(first, "trim-a", 60)).ino).toBe(
statSync(framePath(first, "trim-b", 0)).ino,
);
const second = await extractAllVideoFrames(
videos,
FIXTURE_DIR,
{ fps: 30, outputDir: secondOutputDir },
undefined,
{ extractCacheDir: CACHE_DIR },
);
expect(second.errors).toEqual([]);
expect(second.phaseBreakdown.cacheHits).toBe(2);
expect(second.phaseBreakdown.cacheMisses).toBe(0);
expect(supersetDirNames(secondOutputDir)).toEqual([]);
} finally {
rmSync(CACHE_DIR, { recursive: true, force: true });
}
}, 60_000);
it("clamps loop-past-EOF superset slices to available source frames", async () => {
const SRC = await synthCfrClip("superset-eof-src.mp4", 10);
const outputDir = join(FIXTURE_DIR, "out-superset-eof");
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[cfrClipElement("covered", SRC, 4, 6), cfrClipElement("past-eof", SRC, 6, 8)],
FIXTURE_DIR,
{ fps: 30, outputDir },
);
expect(result.errors).toEqual([]);
expect(extractedFor(result, "covered").totalFrames).toBe(120);
expect(extractedFor(result, "past-eof").totalFrames).toBe(60);
expect(statSync(framePath(result, "covered", 60)).ino).toBe(
statSync(framePath(result, "past-eof", 0)).ino,
);
expect(supersetDirNames(outputDir)).toEqual([]);
}, 60_000);
// Asserts frame-count correctness for a full VFR file. One-pass CFR image
// extraction may repeat held source frames across timestamp gaps; the freeze
// regression is missing frames, which leaves late timeline lookups null.
it("produces the full frame count on the full VFR file", async () => {
const outputDir = join(FIXTURE_DIR, "out-full");
mkdirSync(outputDir, { recursive: true });
const video: VideoElement = {
id: "vfull",
src: VFR_FIXTURE,
start: 0,
end: 10,
mediaStart: 0,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 30,
outputDir,
});
expect(result.errors).toEqual([]);
const frameDir = join(outputDir, "vfull");
const frames = readdirSync(frameDir)
.filter((f) => f.endsWith(".jpg"))
.sort();
// ±3 tolerance: same FFmpeg one-pass VFR→CFR rounding variance as the
// mid-segment test.
expect(frames.length).toBeGreaterThanOrEqual(297);
expect(frames.length).toBeLessThanOrEqual(303);
}, 60_000);
});
// Release builds print "version 8.1.3" or "n8.1.3"; master builds print "N-<number>" and are newer.
const FFPROBE_VERSION = spawnSync("ffprobe", ["-version"]).stdout?.toString() ?? "";
const FFPROBE_RELEASE = FFPROBE_VERSION.match(/version n?(\d+)\.(\d+)/)
?.slice(1)
.map(Number);
const FFPROBE_READS_CONTAINER_TRANSFER =
/version N-/.test(FFPROBE_VERSION) ||
(FFPROBE_RELEASE !== undefined &&
(FFPROBE_RELEASE[0]! > 8 || (FFPROBE_RELEASE[0] === 8 && FFPROBE_RELEASE[1]! >= 1)));
describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
let fixtureDir = "";
beforeAll(() => {
fixtureDir = mkdtempSync(join(tmpdir(), "hf-forced-sdr-tonemap-test-"));
});
afterAll(() => {
rmSync(fixtureDir, { recursive: true, force: true });
}, 30_000);
async function synthesizeHlgClip(
path: string,
vui = "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9",
transfer = "arib-std-b67",
): Promise<void> {
const synthesized = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"color=c=0xe0b080:s=64x64:r=1:d=1",
"-vf",
`zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=${transfer}:m=bt2020nc:r=tv,format=yuv420p`,
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-color_primaries",
"bt2020",
"-color_trc",
transfer,
"-colorspace",
"bt2020nc",
"-bsf:v",
`h264_metadata=${vui}`,
path,
]);
if (!synthesized.success) {
throw new Error(`HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`);
}
}
it.each([
[
"HLG",
"matrix",
"arib-std-b67",
"colour_primaries=9:transfer_characteristics=18:matrix_coefficients=2",
],
[
"HLG",
"primaries",
"arib-std-b67",
"colour_primaries=2:transfer_characteristics=18:matrix_coefficients=9",
],
[
"HLG",
"matrix and primaries",
"arib-std-b67",
"colour_primaries=2:transfer_characteristics=18:matrix_coefficients=2",
],
[
"PQ",
"matrix and primaries",
"smpte2084",
"colour_primaries=2:transfer_characteristics=16:matrix_coefficients=2",
],
[
"HLG",
"transfer",
"arib-std-b67",
"colour_primaries=9:transfer_characteristics=2:matrix_coefficients=9",
],
])(
"tone-maps %s footage whose video stream has no %s tag like the fully tagged clip",
async (name, missing, transfer, vui) => {
const slug = `${name}-no-${missing}`.replace(/\W+/g, "-");
const tagged = join(fixtureDir, `${name}-fully-tagged.mp4`);
const untagged = join(fixtureDir, `${slug}.mp4`);
const vuiTransfer = transfer === "smpte2084" ? 16 : 18;
await synthesizeHlgClip(
tagged,
`colour_primaries=9:transfer_characteristics=${vuiTransfer}:matrix_coefficients=9`,
transfer,
);
await synthesizeHlgClip(untagged, vui, transfer);
const extract = (source: string, id: string) =>
extractVideoFramesRange(source, id, 0, 1, {
fps: 1,
outputDir: join(fixtureDir, `out-${id}`),
format: "png",
toneMapHdrToSdr: true,
});
if (missing === "transfer") {
// ffprobe before 8.1 reads the stream's unspecified transfer, so the clip is SDR there, as on main.
if (!FFPROBE_READS_CONTAINER_TRANSFER) {
await expect(extract(untagged, slug)).resolves.toBeDefined();
return;
}
expect((await extractVideoMetadata(untagged)).colorSpace?.colorTransfer).toBe(transfer);
}
const reference = await extract(tagged, `ref-${slug}`);
const result = await extract(untagged, slug);
expect(readFileSync(result.framePaths.get(0)!)).toEqual(
readFileSync(reference.framePaths.get(0)!),
);
},
60_000,
);
it("probes each source's first frame once, however many cuts of it extract together", async () => {
const sources = ["a", "b"].map((name) => join(fixtureDir, `cuts-${name}.mp4`));
for (const source of sources) await synthesizeHlgClip(source);
const probes = () =>
vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;
const before = probes();
await Promise.all(
Array.from({ length: 12 }, (_, cut) =>
extractVideoFramesRange(sources[cut % 2]!, `cut-${cut}`, 0, 1, {
fps: 1,
outputDir: join(fixtureDir, "cuts"),
format: "png",
toneMapHdrToSdr: true,
}),
),
);
expect(probes() - before).toBe(2);
}, 60_000);
it("probes a source again after its first-frame probe failed", async () => {
const source = join(fixtureDir, "probe-retry.mp4");
await synthesizeHlgClip(source);
const probes = () =>
vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;
const { runFfmpeg: real } =
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
vi.mocked(runFfmpeg).mockImplementation((args, opts) =>
args.includes("showinfo")
? Promise.resolve({
success: false,
exitCode: 1,
stderr: "",
durationMs: 0,
terminationReason: "exit",
})
: real(args, opts),
);
const extract = (id: string) =>
extractVideoFramesRange(source, id, 0, 1, {
fps: 1,
outputDir: join(fixtureDir, "retry"),
format: "png",
toneMapHdrToSdr: true,
});
const before = probes();
try {
await extract("retry-failed");
} finally {
vi.mocked(runFfmpeg).mockImplementation(real);
}
await extract("retry-again");
expect(probes() - before).toBe(2);
}, 60_000);
// One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer.
it("tone-maps each frame of a mixed HLG and PQ stream with its own transfer", async () => {
const segment = async (name: string, transfer: number) => {
const path = join(fixtureDir, `mixed-${name}.h264`);
const result = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
"testsrc2=s=160x90:r=25:d=0.2",
"-c:v",
"libx264",
// No B-frames: genpts stamps a raw stream in decode order, so reordered frames get wrong times.
"-bf",
"0",
"-pix_fmt",
"yuv420p",
"-bsf:v",
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
"-f",
"h264",
path,
]);
if (!result.success)
throw new Error(`segment synthesis failed: ${result.stderr.slice(-400)}`);
return readFileSync(path);
};
const mux = async (name: string, stream: Buffer) => {
const raw = join(fixtureDir, `${name}.h264`);
const path = join(fixtureDir, `${name}.mp4`);
writeFileSync(raw, stream);
const result = await runFfmpeg([
...["-y", "-v", "error", "-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw],
...["-c", "copy", path],
]);
if (!result.success) throw new Error(`mux failed: ${result.stderr.slice(-400)}`);
return path;
};
const hlg = await segment("hlg", 18);
const pq = await segment("pq", 16);
const mixed = await mux("mixed-hlg-pq", Buffer.concat([hlg, pq]));
const pqOnly = await mux("mixed-pq-only", pq);
const extract = (source: string, id: string, duration: number) =>
extractVideoFramesRange(source, id, 0, duration, {
fps: 25,
outputDir: join(fixtureDir, `out-${id}`),
format: "png",
toneMapHdrToSdr: true,
});
const mixedFrames = await extract(mixed, "mixed-hlg-pq", 0.4);
const pqFrames = await extract(pqOnly, "mixed-pq-only", 0.2);
expect(readFileSync(mixedFrames.framePaths.get(5)!)).toEqual(
readFileSync(pqFrames.framePaths.get(0)!),
);
}, 60_000);
// A stream-copy trim opens on edit-list pre-roll, which ffprobe counts as packets without frames.
it("keeps per-frame tags in a mixed HLG and PQ file trimmed with a stream copy", async () => {
const run = async (args: string[]) => {
const result = await runFfmpeg(["-y", "-v", "error", ...args]);
if (!result.success) throw new Error(`fixture failed: ${result.stderr.slice(-400)}`);
};
const segment = async (name: string, transfer: number) => {
const path = join(fixtureDir, `trim-${name}.h264`);
await run([
...["-f", "lavfi", "-i", "testsrc2=s=160x90:r=25:d=2"],
...["-c:v", "libx264", "-g", "50", "-bf", "0"],
...["-pix_fmt", "yuv420p", "-bsf:v"],
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
...["-f", "h264", path],
]);
return readFileSync(path);
};
const mux = async (name: string, stream: Buffer) => {
const raw = join(fixtureDir, `${name}.h264`);
const path = join(fixtureDir, `${name}.mp4`);
writeFileSync(raw, stream);
await run(["-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw, "-c", "copy", path]);
return path;
};
const pq = await segment("pq", 16);
const mixed = await mux("trim-hlg-then-pq", Buffer.concat([await segment("hlg", 18), pq]));
const pqOnly = await mux("trim-pq-only", pq);
const trimmed = join(fixtureDir, "trim-from-hlg.mp4");
await run(["-ss", "1", "-i", mixed, "-c", "copy", trimmed]);
const extract = (source: string, id: string, start: number) =>
extractVideoFramesRange(source, id, start, 0.04, {
fps: 25,
outputDir: join(fixtureDir, `out-${id}`),
format: "png",
toneMapHdrToSdr: true,
});
// Trimmed 1.6 s is the mixed file's 2.6 s: the PQ segment's 0.6 s.
const fromTrim = await extract(trimmed, "trim-from-hlg", 1.6);
const fromPq = await extract(pqOnly, "trim-pq-only", 0.6);
const shown = readFirstFramePixel(fromTrim.framePaths.get(0)!, 40, 40);
const expected = readFirstFramePixel(fromPq.framePaths.get(0)!, 40, 40);
expect(Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)))).toBeLessThanOrEqual(2);
}, 60_000);
it("matches Studio's HLG tone map and isolates transformed cache entries", async () => {
const source = join(fixtureDir, "hlg-warm.mp4");
await synthesizeHlgClip(source);
const reference = join(fixtureDir, "studio-reference.png");
const referenceResult = await runFfmpeg([
"-y",
"-ss",
"0",
"-i",
source,
"-t",
"1",
"-vf",
"fps=1,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv",
"-q:v",
"0",
"-compression_level",
"1",
reference,
]);
if (!referenceResult.success) {
throw new Error(`Studio reference extraction failed: ${referenceResult.stderr.slice(-400)}`);
}
const cacheDir = join(fixtureDir, "cache");
const video = (id: string): VideoElement => ({
id,
src: source,
start: 0,
end: 1,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const extract = (id: string, toneMapHdrToSdr = false) =>
extractAllVideoFrames(
[video(id)],
fixtureDir,
{
fps: 1,
outputDir: join(fixtureDir, id),
format: "png",
toneMapHdrToSdr,
},
undefined,
{ extractCacheDir: cacheDir },
);
const plain = await extract("plain");
const toneMapped = await extract("tone-mapped", true);
const toneMappedAgain = await extract("tone-mapped-again", true);
expect(plain.errors).toEqual([]);
expect(toneMapped.errors).toEqual([]);
expect(toneMapped.phaseBreakdown.cacheHits).toBe(0);
expect(toneMapped.phaseBreakdown.cacheMisses).toBe(1);
expect(toneMappedAgain.phaseBreakdown.cacheHits).toBe(1);
expect(readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX))).toHaveLength(2);
const frame = (result: ExtractionResult): Buffer => {
const path = result.extracted[0]?.framePaths.get(0);
if (!path) throw new Error("expected extracted frame");
return readFileSync(path);
};
// Same pixels as Studio's tone map, declared as sRGB so Chrome shows them unconverted.
const rgb = (path: string): Buffer =>
spawnSync("ffmpeg", ["-v", "error", "-i", path, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"])
.stdout;
const toneMappedPath = toneMapped.extracted[0]!.framePaths.get(0)!;
expect(rgb(toneMappedPath)).toEqual(rgb(reference));
expect(pngChunkTypes(toneMappedPath)).toContain("sRGB");
expect(pngChunkTypes(toneMappedPath)).not.toContain("cICP");
expect(frame(plain)).not.toEqual(frame(toneMapped));
const toneMappedJpg = await extractAllVideoFrames([video("tone-mapped-jpg")], fixtureDir, {
fps: 1,
outputDir: join(fixtureDir, "tone-mapped-jpg"),
format: "jpg",
toneMapHdrToSdr: true,
});
expect(toneMappedJpg.errors).toEqual([]);
const jpgPixels = rgb(toneMappedJpg.extracted[0]!.framePaths.get(0)!);
const referencePixels = rgb(reference);
const worst = Math.max(...[...jpgPixels].map((v, i) => Math.abs(v - referencePixels[i]!)));
expect(worst, "tone-mapped jpg against Studio's tone map").toBeLessThanOrEqual(3);
expect(frame(toneMappedAgain)).toEqual(frame(toneMapped));
// A macOS ffmpeg that has zscale uses the same tone map, not VideoToolbox.
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
Object.defineProperty(process, "platform", { value: "darwin" });
try {
const mac = await extractAllVideoFrames([video("tone-mapped-mac")], fixtureDir, {
fps: 1,
outputDir: join(fixtureDir, "tone-mapped-mac"),
format: "png",
toneMapHdrToSdr: true,
});
expect(mac.errors).toEqual([]);
expect(rgb(mac.extracted[0]!.framePaths.get(0)!)).toEqual(rgb(reference));
} finally {
Object.defineProperty(process, "platform", platform);
}
}, 60_000);
it("warns once and keeps VideoToolbox, under its own cache key, when a macOS ffmpeg has no zscale", async () => {
const source = join(fixtureDir, "hlg-no-zscale.mp4");
await synthesizeHlgClip(source);
const cacheDir = join(fixtureDir, "no-zscale-cache");
const clip = (id: string): VideoElement => ({
id,
src: source,
start: 0,
end: 1,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const options = (id: string) => ({
fps: 1,
outputDir: join(fixtureDir, id),
format: "png" as const,
toneMapHdrToSdr: true,
});
const zscale = await extractAllVideoFrames(
[clip("zscale")],
fixtureDir,
options("zscale"),
undefined,
{
extractCacheDir: cacheDir,
},
);
expect(zscale.errors).toEqual([]);
const realPlatform = process.platform;
vi.resetModules();
vi.doMock("../utils/psnrFilterAvailability.js", () => ({
isFfmpegFilterAvailable: async () => false,
isPsnrFilterAvailable: async () => false,
}));
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
Object.defineProperty(process, "platform", { value: "darwin" });
const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
try {
const { extractAllVideoFrames: extractOnMac } = await import("./videoFrameExtractor.js");
for (const [i, id] of ["no-zscale-1", "no-zscale-2"].entries()) {
const result = await extractOnMac([clip(id)], fixtureDir, options(id), undefined, {
extractCacheDir: cacheDir,
});
// The zscale frames above must not be served for the VideoToolbox path.
if (i === 0) expect(result.phaseBreakdown.cacheHits).toBe(0);
// Off macOS the VideoToolbox decode itself fails, which proves it was attempted.
if (realPlatform !== "darwin")
expect(JSON.stringify(result.errors)).toMatch(/videotoolbox/i);
}
const zscaleWarnings = stderr.mock.calls.filter(([message]) =>
String(message).includes("no zscale filter"),
);
expect(zscaleWarnings).toHaveLength(1);
} finally {
stderr.mockRestore();
Object.defineProperty(process, "platform", platform);
vi.doUnmock("../utils/psnrFilterAvailability.js");
vi.resetModules();
}
}, 60_000);
// A second process would read the frames back without the HDR10 light-level metadata the tone map uses.
it("tone-maps a VFR HDR10 window in one process, as the reference tone map would", async () => {
const source = join(fixtureDir, "pq-vfr-4000nit.mp4");
const synthesized = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
"color=c=0xf0e0c0:s=64x64:r=60:d=2",
"-vf",
"select='not(between(n\\,30\\,89))',zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=smpte2084:m=bt2020nc:r=tv:npl=1000,format=yuv420p10le",
"-fps_mode",
"vfr",
"-c:v",
"libx265",
"-x265-params",
"log-level=error:hdr10=1:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:max-cll=4000,400:master-display=G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(40000000,50)",
source,
]);
if (!synthesized.success)
throw new Error(`PQ fixture synthesis failed: ${synthesized.stderr.slice(-400)}`);
const referenceDir = join(fixtureDir, "pq-vfr-reference");
mkdirSync(referenceDir, { recursive: true });
const reference = await runFfmpeg([
"-v",
"error",
"-ss",
"0",
"-i",
source,
"-t",
"0.616666",
"-vf",
"zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv,setparams=color_primaries=bt709:color_trc=iec61966-2-1",
"-fps_mode",
"cfr",
"-r",
"30",
"-q:v",
"0",
"-compression_level",
"1",
join(referenceDir, "frame_%05d.png"),
]);
expect(reference.success, reference.stderr).toBe(true);
const result = await extractVideoFramesRange(source, "pq-vfr", 0, 0.616666, {
fps: 30,
outputDir: join(fixtureDir, "pq-vfr-out"),
format: "png",
toneMapHdrToSdr: true,
});
expect(result.metadata.isVFR).toBe(true);
const referenceFrames = readdirSync(referenceDir).sort();
// One slot per 30fps tick before 0.616666s; x265 packet durations cut the reference's still short.
expect(result.totalFrames).toBe(19);
expect(readFileSync(result.framePaths.get(0)!)).toEqual(
readFileSync(join(referenceDir, referenceFrames[0]!)),
);
}, 60_000);
});
describe("getFrameAtTime — IEEE 754 boundary precision", () => {
function makeExtracted(fps: number, totalFrames: number): ExtractedFrames {
const framePaths = new Map<number, string>();
for (let i = 0; i < totalFrames; i++) framePaths.set(i, `frame-${i}.jpg`);
return {
fps,
totalFrames,
framePaths,
metadata: {
durationSeconds: totalFrames / fps,
width: 1920,
height: 1080,
codec: "h264",
hasAudio: false,
fps,
},
} as ExtractedFrames;
}
it("indexes frames by integrated source time for a rate lane", () => {
const extracted = { ...makeExtracted(25, 351), videoId: "ramped" } as ExtractedFrames;
const rate = {
target: "rate",
points: [
{ t: 0, v: 1 },
{ t: 2, v: 3 },
],
};
const table = new FrameLookupTable();
table.addVideo(extracted, 0, 10, 0, false, rate);
// 2s * (3-1)/ln 3 = 3.6411 source seconds * 25 fps = frame 91
expect(table.getFrame("ramped", 2)).toBe("frame-91.jpg");
expect(table.getFrame("ramped", 0)).toBe("frame-0.jpg");
});
it("wraps a ramped loop in source space so the lane keeps running across cycles", () => {
const extracted = { ...makeExtracted(25, 100), videoId: "looped" } as ExtractedFrames;
const rate = {
target: "rate",
points: [
{ t: 0, v: 1 },
{ t: 4, v: 3 },
],
};
const table = new FrameLookupTable();
table.addVideo(extracted, 0, 10, 0, true, rate);
// source(3) = 4.6586 on a 4s source wraps to 0.6586 * 25 fps = frame 16
expect(table.getFrame("looped", 3)).toBe("frame-16.jpg");
});
it("does not produce duplicate frames when data-start is grid-aligned", () => {
const extracted = makeExtracted(25, 351);
const videoStart = 0;
const seen: string[] = [];
let duplicates = 0;
for (let i = 0; i < 351; i++) {
const globalTime = i / 25;
const frame = getFrameAtTime(extracted, globalTime, videoStart);
if (frame && seen.length > 0 && frame === seen[seen.length - 1]) duplicates++;
if (frame) seen.push(frame);
}
expect(duplicates).toBe(0);
});
it("returns monotonically increasing frame indices", () => {
const extracted = makeExtracted(25, 100);
let lastIndex = -1;
for (let i = 0; i < 100; i++) {
const globalTime = i / 25;
const frame = getFrameAtTime(extracted, globalTime, 0);
const idx = frame ? parseInt(frame.split("-")[1]!) : -1;
expect(idx).toBeGreaterThan(lastIndex);
lastIndex = idx;
}
});
it("handles the 0.28 * 25 boundary case (6.999999 vs 7)", () => {
const extracted = makeExtracted(25, 10);
const frame = getFrameAtTime(extracted, 0.28, 0);
expect(frame).toBe("frame-7.jpg");
});
it("picks the frame the preview shows when a clip starts a hair after a frame instant", () => {
// 449/30 written with six decimals starts the clip a third of a microsecond late. The preview's
// <video> still shows the take's second frame at output frame 450, so the render must too.
const extracted = makeExtracted(30, 100);
expect(getFrameAtTime(extracted, 450 / 30, 14.966667, false, 4.3)).toBe("frame-1.jpg");
expect(getFrameAtTime(extracted, 460 / 30, 14.966667, false, 4.3)).toBe("frame-11.jpg");
});
it("mediaStart does not offset frame index (extractor handles trim via -ss)", () => {
const extracted = makeExtracted(25, 100);
const frame = getFrameAtTime(extracted, 0, 0, false, 1.0);
expect(frame).toBe("frame-0.jpg");
});
it("returns null after source exhaustion without an authored slot boundary", () => {
const extracted = makeExtracted(25, 25);
expect(getFrameAtTime(extracted, 3, 0)).toBeNull();
});
});
describe("parseVideoElements hidden flag", () => {
it("marks a video hidden on itself or through a hidden ancestor", () => {
const videos = parseVideoElements(
`<div data-hidden><video id="a" src="a.mp4" data-has-audio="true"></video></div>` +
`<video id="b" src="b.mp4" data-has-audio="true" data-hidden></video>` +
`<video id="c" src="c.mp4" data-has-audio="true"></video>`,
);
expect(videos.map((v) => [v.id, v.hidden])).toEqual([
["a", true],
["b", true],
["c", undefined],
]);
});
it("marks only an audible video hidden when its audio group is hidden", () => {
const videos = parseVideoElements(
`<hf-audio-group id="g" data-hidden></hf-audio-group>` +
`<video id="a" src="a.mp4" data-has-audio="true" data-audio-group="g"></video>` +
`<video id="b" src="b.mp4" muted data-audio-group="g"></video>`,
);
expect(videos.map((v) => [v.id, v.hidden])).toEqual([
["a", true],
["b", undefined],
]);
});
});
describe.skipIf(!HAS_FFMPEG)("a clip stored on a non-square pixel grid", () => {
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-sar-"));
const SOURCE = join(FIXTURE_DIR, "anamorphic.mp4");
beforeAll(async () => {
const result = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
"color=c=red:s=64x64:d=1:r=10",
"-vf",
"setsar=1/2",
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
SOURCE,
]);
if (!result.success) throw new Error(`fixture synthesis failed: ${result.stderr.slice(-400)}`);
}, 30_000);
afterAll(() => rmSync(FIXTURE_DIR, { recursive: true, force: true }), 30_000);
it("keeps the display shape of a rotated clip, whose autorotate swaps the pixel aspect", async () => {
const rotated = join(FIXTURE_DIR, "rotated.mp4");
// `-display_rotation` is an ffmpeg 6 input option; older builds take the same rotation as metadata.
const rotateWith = (args: string[]) =>
runFfmpeg(["-y", "-v", "error", ...args, "-c", "copy", rotated]);
let synth = await rotateWith(["-display_rotation:v:0", "90", "-i", SOURCE]);
if (!synth.success) synth = await rotateWith(["-i", SOURCE, "-metadata:s:v:0", "rotate=90"]);
expect(synth.success).toBe(true);
const outDir = join(FIXTURE_DIR, "out-rot");
mkdirSync(outDir, { recursive: true });
const video: VideoElement = {
id: "r",
src: rotated,
start: 0,
end: 1,
mediaStart: 0,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 10,
outputDir: outDir,
});
expect(result.errors).toEqual([]);
const probe = spawnSync("ffprobe", [
"-v",
"error",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0",
"--",
result.extracted[0]!.framePaths.get(0)!,
]);
expect(probe.stdout.toString().trim()).toBe("128,64");
});
it("extracts frames at the shape Chrome displays, not the stored grid", async () => {
const outDir = join(FIXTURE_DIR, "out");
mkdirSync(outDir, { recursive: true });
const video: VideoElement = {
id: "v",
src: SOURCE,
start: 0,
end: 1,
mediaStart: 0,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: 10,
outputDir: outDir,
});
expect(result.errors).toEqual([]);
const probe = spawnSync("ffprobe", [
"-v",
"error",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0",
"--",
result.extracted[0]!.framePaths.get(0)!,
]);
expect(probe.stdout.toString().trim()).toBe("32,64");
});
});
// Each source frame N is a flat gray of 3N, so the frame a render shows reads straight off a pixel.
describe.skipIf(!HAS_FFMPEG)("a clip whose data-start is off the frame grid", () => {
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-offgrid-start-"));
const SOURCE = join(FIXTURE_DIR, "numbered.mp4");
const FPS = 30;
beforeAll(async () => {
const result = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
"color=c=black:s=64x64:d=6:r=30",
"-vf",
"format=gray,geq=lum='N*3'",
"-c:v",
"libx264",
"-crf",
"0",
"-g",
"30",
"-pix_fmt",
"yuv420p",
SOURCE,
]);
if (!result.success) throw new Error(`fixture synthesis failed: ${result.stderr.slice(-400)}`);
}, 30_000);
afterAll(() => rmSync(FIXTURE_DIR, { recursive: true, force: true }), 30_000);
async function sourceFramesShown(
start: number,
outputFrames: number[],
): Promise<(number | null)[]> {
const outDir = join(FIXTURE_DIR, `out-${start}`);
mkdirSync(outDir, { recursive: true });
const video: VideoElement = {
id: "v",
src: SOURCE,
start,
end: start + 1.9,
mediaStart: start,
loop: false,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: FPS,
outputDir: outDir,
});
expect(result.errors).toEqual([]);
const table = createFrameLookupTable([video], result.extracted, FPS);
return outputFrames.map((n) => {
const path = table.getFrame("v", n / FPS);
return path ? Math.round(readFirstFramePixel(path, 0, 0)[0] / 3) : null;
});
}
it("keeps a looped clip on the same source frame after every wrap", async () => {
const loopSource = join(FIXTURE_DIR, "loop.mp4");
const synth = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
"color=c=black:s=64x64:d=2:r=30",
"-vf",
"format=gray,geq=lum='N*4'",
"-c:v",
"libx264",
"-crf",
"0",
"-pix_fmt",
"yuv420p",
loopSource,
]);
expect(synth.success).toBe(true);
const outDir = join(FIXTURE_DIR, "out-loop");
mkdirSync(outDir, { recursive: true });
const video: VideoElement = {
id: "v",
src: loopSource,
start: 0.35,
end: 6.35,
mediaStart: 1,
loop: true,
hasAudio: false,
};
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
fps: FPS,
outputDir: outDir,
timelineEnd: 6,
});
expect(result.errors).toEqual([]);
const table = createFrameLookupTable([video], result.extracted, FPS);
const shown = [0, 1, 2, 3, 4].map((wrap) => {
const path = table.getFrame("v", (12 + wrap * FPS + 1) / FPS);
return path ? Math.round(readFirstFramePixel(path, 0, 0)[0] / 4) : null;
});
expect(new Set(shown).size).toBe(1);
}, 60_000);
it.each([1.95, 2.0001, 1.999999])(
"shows source frame N at output frame N for data-start=data-media-start=%s",
async (start) => {
const shown = await sourceFramesShown(start, [61, 62, 63, 64, 65]);
expect(shown).toEqual([61, 62, 63, 64, 65]);
},
60_000,
);
});