mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 19:36:04 +00:00
* fix(producer): a failed video extraction names which clip files failed * fix(producer): the extraction source label hides scheme payloads and drive paths * style(producer): format the extraction source label
4707 lines
163 KiB
TypeScript
4707 lines
163 KiB
TypeScript
// fallow-ignore-file code-duplication
|
|
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
|
import {
|
|
existsSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
readdirSync,
|
|
rmSync,
|
|
statSync,
|
|
utimesSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { basename, join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { spawnSync } from "node:child_process";
|
|
import {
|
|
parseVideoElements,
|
|
hasGradedCanvas,
|
|
parseImageElements,
|
|
extractAllVideoFrames,
|
|
extractVideoFramesRange,
|
|
extractionFrameCountForDuration,
|
|
createFrameLookupTable,
|
|
FrameLookupTable,
|
|
rebaseVideoToWindow,
|
|
alignClipStartToFirstOutputFrame,
|
|
resolveFrameFormat,
|
|
codecMayHaveAlpha,
|
|
decoderForCodec,
|
|
resolveVideoExtractionWindow,
|
|
resolveVideoExtractionDuration,
|
|
getFrameAtTime,
|
|
analyzeClipMediaFit,
|
|
classifyVideoExtractionError,
|
|
videoExtractionSourceLabel,
|
|
runVideoExtractionWithRetry,
|
|
VideoSourceExtractionError,
|
|
type VideoElement,
|
|
type ExtractedFrames,
|
|
type ExtractionResult,
|
|
} from "./videoFrameExtractor.js";
|
|
|
|
describe("parseVideoElements strict literal timing", () => {
|
|
it.each(["", " ", "0s", "0abc", "0px", "-1s", "Infinity", "NaN", "0x10"])(
|
|
"does not drop hand-authored data-duration=%j as an explicit zero window",
|
|
(duration) => {
|
|
expect(
|
|
parseVideoElements(
|
|
`<video id="v" src="clip.mp4" data-start="0" data-duration="${duration}"></video>`,
|
|
),
|
|
).toHaveLength(1);
|
|
},
|
|
);
|
|
});
|
|
import {
|
|
extractFinalVideoFrameTimestamp,
|
|
extractVideoMetadata,
|
|
type VideoMetadata,
|
|
} from "../utils/ffprobe.js";
|
|
import { ffmpegStatsReader, runFfmpeg, type RunFfmpegOptions } from "../utils/runFfmpeg.js";
|
|
|
|
// Pass-through, so a test can tell whether extraction ran ffmpeg.
|
|
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
|
|
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
|
|
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
|
|
});
|
|
import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.js";
|
|
import { resolveRuntimeMediaClipDuration } from "../../../core/src/runtime/media.js";
|
|
import { compileTimingAttrs, sourceTimeAt } from "@hyperframes/core";
|
|
import { RATE_RANGE } from "@hyperframes/core/audio-automation";
|
|
|
|
// ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer
|
|
// regression test at packages/producer/tests/vfr-screen-recording/ runs inside
|
|
// Dockerfile.test (which does include ffmpeg) and is the primary CI signal
|
|
// for this bug. Locally and in any CI job with ffmpeg on PATH, the tests
|
|
// below run too — they exercise the extractor in isolation against a
|
|
// synthesized VFR fixture.
|
|
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
|
|
const HAS_ZSCALE =
|
|
HAS_FFMPEG &&
|
|
/\szscale\s/.test(spawnSync("ffmpeg", ["-hide_banner", "-filters"]).stdout.toString());
|
|
const FFMPEG_ENCODERS = HAS_FFMPEG
|
|
? spawnSync("ffmpeg", ["-hide_banner", "-encoders"]).stdout.toString()
|
|
: "";
|
|
// Homebrew's ffmpeg has libsvtav1 but not libaom-av1. Either one writes an untagged AV1 stream.
|
|
const AV1_ENCODER = ["libaom-av1", "libsvtav1"].find((name) =>
|
|
new RegExp(`\\s${name}\\s`).test(FFMPEG_ENCODERS),
|
|
);
|
|
|
|
describe("resolveVideoExtractionDuration", () => {
|
|
const metadata = (
|
|
durationSeconds: number,
|
|
videoStreamDurationSeconds = durationSeconds,
|
|
): VideoMetadata => ({
|
|
durationSeconds,
|
|
videoStreamDurationSeconds,
|
|
width: 1920,
|
|
height: 1080,
|
|
fps: 30,
|
|
videoCodec: "h264",
|
|
hasAudio: false,
|
|
isVFR: false,
|
|
hasAlpha: false,
|
|
colorSpace: null,
|
|
});
|
|
const video = (overrides: Partial<VideoElement> = {}): VideoElement => ({
|
|
id: "root-video",
|
|
src: "video.mp4",
|
|
start: 0,
|
|
end: Number.POSITIVE_INFINITY,
|
|
mediaStart: 0,
|
|
playbackRate: 1,
|
|
loop: false,
|
|
hasAudio: false,
|
|
...overrides,
|
|
});
|
|
|
|
it("caps an open 60-second root source to a two-second composition", () => {
|
|
expect(resolveVideoExtractionDuration(video(), metadata(60), 2)).toBe(2);
|
|
});
|
|
|
|
it("keeps a shorter natural source duration inside a longer composition", () => {
|
|
expect(resolveVideoExtractionDuration(video(), metadata(2), 10)).toBe(2);
|
|
});
|
|
|
|
it("falls back to container duration when stream duration is unavailable", () => {
|
|
expect(resolveVideoExtractionDuration(video(), metadata(2, 0), 10)).toBe(2);
|
|
});
|
|
|
|
it("preserves explicit bounds and loop flags while applying the timeline ceiling", () => {
|
|
const explicitLoop = video({ end: 8, loop: true });
|
|
expect(resolveVideoExtractionDuration(explicitLoop, metadata(60), 10)).toBe(8);
|
|
expect(explicitLoop.loop).toBe(true);
|
|
});
|
|
|
|
it("extracts the source span consumed by an explicit 2x timeline slot", () => {
|
|
expect(
|
|
resolveVideoExtractionWindow(
|
|
video({ end: 2, mediaStart: 1, playbackRate: 2 }),
|
|
metadata(8),
|
|
2,
|
|
),
|
|
).toMatchObject({
|
|
compositionStart: 0,
|
|
mediaStart: 1,
|
|
durationSeconds: 4,
|
|
timelineDurationSeconds: 2,
|
|
});
|
|
});
|
|
|
|
it("extracts the source span a ramped slot consumes: 2s of 1x to 3x reads 2*(3-1)/ln 3 source seconds", () => {
|
|
const rate = {
|
|
target: "rate",
|
|
points: [
|
|
{ t: 0, v: 1 },
|
|
{ t: 2, v: 3 },
|
|
],
|
|
};
|
|
expect(
|
|
resolveVideoExtractionWindow(video({ end: 2, playbackRate: rate }), metadata(8), 2),
|
|
).toMatchObject({ compositionStart: 0, mediaStart: 0, timelineDurationSeconds: 2 });
|
|
const { durationSeconds } = resolveVideoExtractionWindow(
|
|
video({ end: 2, playbackRate: rate }),
|
|
metadata(8),
|
|
2,
|
|
);
|
|
expect(durationSeconds).toBeCloseTo(3.6411, 3);
|
|
});
|
|
|
|
const RAMP = {
|
|
target: "rate",
|
|
points: [
|
|
{ t: 0, v: 1 },
|
|
{ t: 4, v: 3 },
|
|
],
|
|
};
|
|
|
|
it("starts a ramped clip that begins before the timeline at the integrated source time of the trimmed part", () => {
|
|
// geometric 1x to 3x over 4s: source(t) = 4(3^(t/4)-1)/ln 3; 1s trimmed reads 1.1508, all 4s read 7.2819
|
|
const window = resolveVideoExtractionWindow(
|
|
video({ start: -1, end: 3, playbackRate: RAMP }),
|
|
metadata(20),
|
|
3,
|
|
);
|
|
expect(window.mediaStart).toBeCloseTo(1.1508, 3);
|
|
expect(window.durationSeconds).toBeCloseTo(6.1311, 3);
|
|
});
|
|
|
|
it("rebases a ramped clip so lookup reads the same source second the untrimmed lane would", () => {
|
|
const clip = video({ start: -1, end: 3, playbackRate: RAMP });
|
|
rebaseVideoToWindow(clip, resolveVideoExtractionWindow(clip, metadata(20), 3));
|
|
// composition second 2 is 3s into the authored lane: source 4.6586
|
|
const seconds = clip.mediaStart + sourceTimeAt(clip.playbackRate ?? 1, 2 - clip.start);
|
|
expect(seconds).toBeCloseTo(4.6586, 3);
|
|
});
|
|
|
|
it("leaves media start untouched for a nested clip whose start sums to a frame boundary with float dust", () => {
|
|
// 3.3 + 1.3666666666666667 = 4.666666666666666, frame 140 at 30 fps
|
|
const clip = video({ start: 3.3 + 1.3666666666666667, end: 8, mediaStart: 1.35 });
|
|
alignClipStartToFirstOutputFrame(clip, 30);
|
|
expect(clip.mediaStart).toBe(1.35);
|
|
expect(clip.start).toBe(140 / 30);
|
|
});
|
|
|
|
it("snaps a float-noise composition start to the timeline origin so the first frame is kept", () => {
|
|
const clip = video({ start: -1, end: 8, playbackRate: 0.7 });
|
|
rebaseVideoToWindow(clip, { compositionStart: 2.2e-16, mediaStart: 0.7, durationSeconds: 5 });
|
|
expect(clip.start).toBe(0);
|
|
});
|
|
|
|
it("reports the natural timeline duration of a ramped clip through the lane", () => {
|
|
const flat = {
|
|
target: "rate",
|
|
points: [
|
|
{ t: 0, v: 2 },
|
|
{ t: 1, v: 2 },
|
|
],
|
|
};
|
|
expect(resolveVideoExtractionWindow(video({ playbackRate: flat }), metadata(4))).toMatchObject({
|
|
durationSeconds: 4,
|
|
timelineDurationSeconds: 2,
|
|
});
|
|
});
|
|
|
|
it("reports natural timeline duration after constant playback-rate retiming", () => {
|
|
expect(
|
|
resolveVideoExtractionWindow(video({ mediaStart: 1, playbackRate: 2 }), metadata(5), 10),
|
|
).toMatchObject({
|
|
compositionStart: 0,
|
|
mediaStart: 1,
|
|
durationSeconds: 4,
|
|
timelineDurationSeconds: 2,
|
|
});
|
|
});
|
|
|
|
it("trims materially negative preroll and advances the source offset", () => {
|
|
const preroll = video({ start: -60, end: 120, mediaStart: 0 });
|
|
expect(resolveVideoExtractionWindow(preroll, metadata(120), 2)).toEqual({
|
|
compositionStart: 0,
|
|
mediaStart: 60,
|
|
durationSeconds: 2,
|
|
});
|
|
expect(resolveVideoExtractionDuration(preroll, metadata(120), 2)).toBe(2);
|
|
});
|
|
|
|
it("returns an empty window for a clip entirely before composition time zero", () => {
|
|
expect(resolveVideoExtractionWindow(video({ start: -60, end: -10 }), metadata(120), 2)).toEqual(
|
|
{ compositionStart: 0, mediaStart: 60, durationSeconds: 0 },
|
|
);
|
|
});
|
|
|
|
it("preserves a short source cycle when negative preroll crosses a loop boundary", () => {
|
|
expect(
|
|
resolveVideoExtractionWindow(
|
|
video({ start: -5, end: 10, mediaStart: 0, loop: true }),
|
|
metadata(3),
|
|
2,
|
|
),
|
|
).toEqual({
|
|
compositionStart: -5,
|
|
mediaStart: 0,
|
|
durationSeconds: 3,
|
|
preserveTimelinePhase: true,
|
|
});
|
|
});
|
|
|
|
it("marks an entirely held interval for exact final-frame resolution", () => {
|
|
expect(
|
|
resolveVideoExtractionWindow(
|
|
video({ start: -5, end: 10, mediaStart: 0, loop: false }),
|
|
metadata(3),
|
|
2,
|
|
),
|
|
).toEqual({
|
|
compositionStart: -2.000001,
|
|
mediaStart: 2.999999,
|
|
durationSeconds: 0.000001,
|
|
preserveTimelineEnd: true,
|
|
ensureFinalFrame: true,
|
|
});
|
|
});
|
|
|
|
it.each([
|
|
{ loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" },
|
|
{
|
|
loop: false,
|
|
preservation: { preserveTimelineEnd: true, ensureFinalFrame: true },
|
|
label: "held tail",
|
|
},
|
|
])(
|
|
"caps a finite long slot to one short source range for $label playback",
|
|
({ loop, preservation }) => {
|
|
expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(3), 60)).toEqual({
|
|
compositionStart: 0,
|
|
mediaStart: 0,
|
|
durationSeconds: 3,
|
|
...preservation,
|
|
});
|
|
},
|
|
);
|
|
|
|
it.each([
|
|
{ loop: true, preservation: { preserveTimelinePhase: true }, label: "loop" },
|
|
{
|
|
loop: false,
|
|
preservation: { preserveTimelineEnd: true, ensureFinalFrame: true },
|
|
label: "held tail",
|
|
},
|
|
])(
|
|
"uses the playable video-stream duration for a long-audio mux in $label playback",
|
|
({ loop, preservation }) => {
|
|
expect(resolveVideoExtractionWindow(video({ end: 60, loop }), metadata(60, 3), 60)).toEqual({
|
|
compositionStart: 0,
|
|
mediaStart: 0,
|
|
durationSeconds: 3,
|
|
...preservation,
|
|
});
|
|
},
|
|
);
|
|
|
|
it("preserves authored timing when the visible interval partially crosses a held tail", () => {
|
|
expect(
|
|
resolveVideoExtractionWindow(
|
|
video({ start: -2, end: 10, mediaStart: 0, loop: false }),
|
|
metadata(3),
|
|
2,
|
|
),
|
|
).toEqual({
|
|
compositionStart: 0,
|
|
mediaStart: 2,
|
|
durationSeconds: 1,
|
|
preserveTimelineEnd: true,
|
|
ensureFinalFrame: true,
|
|
});
|
|
});
|
|
|
|
it.each([
|
|
{ start: 0, expected: { compositionStart: 0, mediaStart: 0, durationSeconds: 3 } },
|
|
{ start: -2, expected: { compositionStart: 0, mediaStart: 2, durationSeconds: 1 } },
|
|
{ start: -5, expected: { compositionStart: 0, mediaStart: 5, durationSeconds: 0 } },
|
|
])(
|
|
"keeps an omitted-duration clip source-bounded like runtime (start=$start)",
|
|
({ start, expected }) => {
|
|
for (const loop of [false, true]) {
|
|
const parsed = parseVideoElements(
|
|
`<video id="natural" src="video.mp4"${loop ? " loop" : ""}></video>`,
|
|
)[0]!;
|
|
const planned = { ...parsed, start };
|
|
const runtimeDuration = resolveRuntimeMediaClipDuration({
|
|
isVideo: true,
|
|
sourceDuration: 3,
|
|
hostRemaining: 15 - start,
|
|
explicitDuration: null,
|
|
});
|
|
expect(runtimeDuration).toBe(3);
|
|
expect(parsed.end).toBe(Number.POSITIVE_INFINITY);
|
|
expect(resolveVideoExtractionWindow(planned, metadata(3), 15)).toEqual(expected);
|
|
}
|
|
},
|
|
);
|
|
|
|
it("bounds an entirely held long source to its final-frame sample", () => {
|
|
expect(
|
|
resolveVideoExtractionWindow(
|
|
video({ start: -600, end: 10, mediaStart: 0, loop: false }),
|
|
metadata(120),
|
|
2,
|
|
),
|
|
).toEqual({
|
|
compositionStart: -480.000001,
|
|
mediaStart: 119.999999,
|
|
durationSeconds: 0.000001,
|
|
preserveTimelineEnd: true,
|
|
ensureFinalFrame: true,
|
|
});
|
|
});
|
|
|
|
it("preserves a complete loop cycle when visibility ends exactly on a wrap boundary", () => {
|
|
expect(
|
|
resolveVideoExtractionWindow(
|
|
video({ start: -1, end: 10, mediaStart: 0, loop: true }),
|
|
metadata(3),
|
|
2,
|
|
),
|
|
).toEqual({
|
|
compositionStart: -1,
|
|
mediaStart: 0,
|
|
durationSeconds: 3,
|
|
preserveTimelinePhase: true,
|
|
});
|
|
});
|
|
|
|
it("keeps an open-ended loop source-bounded instead of inventing a longer slot", () => {
|
|
expect(resolveVideoExtractionWindow(video({ loop: true }), metadata(3), 10)).toEqual({
|
|
compositionStart: 0,
|
|
mediaStart: 0,
|
|
durationSeconds: 3,
|
|
});
|
|
});
|
|
|
|
it("never plans more extraction than the playable source range", () => {
|
|
for (const loop of [false, true]) {
|
|
for (const sourceDuration of [0.5, 3, 120]) {
|
|
for (const mediaStart of [0, sourceDuration / 3]) {
|
|
for (const start of [-600, -5, -1, 0, 2]) {
|
|
const window = resolveVideoExtractionWindow(
|
|
video({ start, end: start + 60, mediaStart, loop }),
|
|
metadata(sourceDuration),
|
|
10,
|
|
);
|
|
expect(window.durationSeconds).toBeLessThanOrEqual(sourceDuration - mediaStart);
|
|
expect(window.durationSeconds).toBeGreaterThanOrEqual(0);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
it("rejects a media start at source EOF before planning extraction", () => {
|
|
expect(() =>
|
|
resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(3), 10),
|
|
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
|
|
});
|
|
|
|
it("rejects a media start at video-stream EOF even when the container continues", () => {
|
|
expect(() =>
|
|
resolveVideoExtractionWindow(video({ mediaStart: 3 }), metadata(60, 3), 10),
|
|
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
|
|
});
|
|
|
|
it("plans a one-frame held tail when an explicit non-looping slot starts past EOF", () => {
|
|
expect(resolveVideoExtractionWindow(video({ end: 6, mediaStart: 5 }), metadata(2))).toEqual({
|
|
compositionStart: 0,
|
|
mediaStart: 1.999999,
|
|
durationSeconds: 0.000001,
|
|
preserveTimelineEnd: true,
|
|
ensureFinalFrame: true,
|
|
});
|
|
});
|
|
|
|
it("keeps the playable suffix when an explicit non-looping slot starts just inside EOF", () => {
|
|
expect(
|
|
resolveVideoExtractionWindow(video({ end: 6, mediaStart: 1.9 }), metadata(2), 6),
|
|
).toEqual({
|
|
compositionStart: 0,
|
|
mediaStart: 1.9,
|
|
durationSeconds: 0.10000000000000009,
|
|
preserveTimelineEnd: true,
|
|
ensureFinalFrame: true,
|
|
});
|
|
});
|
|
|
|
it("rejects a looping explicit slot that starts at source EOF", () => {
|
|
expect(() =>
|
|
resolveVideoExtractionWindow(video({ end: 6, mediaStart: 2, loop: true }), metadata(2), 6),
|
|
).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
|
|
});
|
|
|
|
it("rebases a loop phase when the visible window stays within one cycle", () => {
|
|
expect(
|
|
resolveVideoExtractionWindow(
|
|
video({ start: -5, end: 10, mediaStart: 0, loop: true }),
|
|
metadata(3),
|
|
0.5,
|
|
),
|
|
).toEqual({ compositionStart: 0, mediaStart: 2, durationSeconds: 0.5 });
|
|
});
|
|
|
|
it("retains legacy behavior when no timeline end is supplied", () => {
|
|
expect(resolveVideoExtractionDuration(video(), metadata(60))).toBe(60);
|
|
});
|
|
});
|
|
|
|
describe("extractionFrameCountForDuration", () => {
|
|
it("uses the same VFR ceil and CFR nearest-boundary rules as FFmpeg", () => {
|
|
expect(extractionFrameCountForDuration(0.466666, 30, true)).toBe(14);
|
|
expect(extractionFrameCountForDuration(0.466666, 30, false)).toBe(14);
|
|
expect(extractionFrameCountForDuration(0.616666, 30, true)).toBe(19);
|
|
expect(extractionFrameCountForDuration(0.616666, 30, false)).toBe(18);
|
|
});
|
|
|
|
it.each([
|
|
[0.33 - 0.03, 30, 9],
|
|
[0.29 - 0.04, 24, 6],
|
|
[0.35 - 0.05, 60, 18],
|
|
[4.03 - 3.53, 30, 15],
|
|
[4.03 - 3.78, 24, 6],
|
|
])(
|
|
"snaps floating-point integral boundaries before VFR ceil (%s seconds at %i fps)",
|
|
(duration, fps, expectedFrames) => {
|
|
expect(extractionFrameCountForDuration(duration, fps, true)).toBe(expectedFrames);
|
|
},
|
|
);
|
|
|
|
it("still ceils a genuine fractional boundary beyond floating-point noise", () => {
|
|
expect(extractionFrameCountForDuration(0.300001, 30, true)).toBe(10);
|
|
});
|
|
|
|
it("matches FFmpeg's six-digit duration parsing", () => {
|
|
expect(extractionFrameCountForDuration(0.6000009, 30, true)).toBe(18);
|
|
expect(extractionFrameCountForDuration(0.600001, 30, true)).toBe(19);
|
|
expect(extractionFrameCountForDuration(2.05, 30, false)).toBe(62);
|
|
});
|
|
|
|
it("keeps exact NTSC rationals at short CFR and VFR boundaries", () => {
|
|
expect(extractionFrameCountForDuration(0.25025, { num: 30000, den: 1001 }, false)).toBe(8);
|
|
expect(extractionFrameCountForDuration(0.125125, { num: 24000, den: 1001 }, true)).toBe(3);
|
|
expect(extractionFrameCountForDuration(0.5005, { num: 24000, den: 1001 }, true)).toBe(12);
|
|
});
|
|
|
|
it("fails closed for invalid durations and emits one frame for positive sub-frame work", () => {
|
|
expect(extractionFrameCountForDuration(Number.NaN, 30, true)).toBe(0);
|
|
expect(extractionFrameCountForDuration(1, 0, true)).toBe(0);
|
|
expect(extractionFrameCountForDuration(0, 30, true)).toBe(0);
|
|
expect(extractionFrameCountForDuration(0.001, 30, false)).toBe(1);
|
|
});
|
|
});
|
|
|
|
describe("video extraction failure taxonomy and bounded retry", () => {
|
|
it("classifies missing and transient HTTP sources without exposing retry ambiguity", () => {
|
|
expect(classifyVideoExtractionError(new Error("HTTP 404: Not Found"))).toMatchObject({
|
|
kind: "download_not_found",
|
|
retryable: false,
|
|
});
|
|
expect(classifyVideoExtractionError(new Error("HTTP 503: Service Unavailable"))).toMatchObject({
|
|
kind: "download_transient",
|
|
retryable: true,
|
|
});
|
|
});
|
|
|
|
it("retries one transient failure, cleaning partial output before the retry", async () => {
|
|
const retryDir = mkdtempSync(join(tmpdir(), "hf-extract-retry-"));
|
|
const partialPath = join(retryDir, "frame-00001.jpg");
|
|
let attempts = 0;
|
|
try {
|
|
const outcome = await runVideoExtractionWithRetry(
|
|
async () => {
|
|
attempts += 1;
|
|
if (attempts === 1) {
|
|
writeFileSync(partialPath, "partial");
|
|
throw new VideoSourceExtractionError(
|
|
"ffmpeg_timeout",
|
|
true,
|
|
"Video frame extraction timed out",
|
|
);
|
|
}
|
|
expect(existsSync(partialPath)).toBe(false);
|
|
return "frames";
|
|
},
|
|
{
|
|
maxTransientRetries: 1,
|
|
onRetry: () => {
|
|
rmSync(retryDir, { recursive: true, force: true });
|
|
mkdirSync(retryDir, { recursive: true });
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(outcome).toEqual({ result: "frames", retries: 1 });
|
|
expect(attempts).toBe(2);
|
|
} finally {
|
|
rmSync(retryDir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("does not retry deterministic or caller-aborted failures", async () => {
|
|
let deterministicAttempts = 0;
|
|
await expect(
|
|
runVideoExtractionWithRetry(async () => {
|
|
deterministicAttempts += 1;
|
|
throw new VideoSourceExtractionError(
|
|
"zero_output",
|
|
false,
|
|
"Video source produced no decodable frames",
|
|
);
|
|
}),
|
|
).rejects.toMatchObject({ kind: "zero_output", retryable: false });
|
|
expect(deterministicAttempts).toBe(1);
|
|
|
|
const controller = new AbortController();
|
|
controller.abort();
|
|
let abortedAttempts = 0;
|
|
await expect(
|
|
runVideoExtractionWithRetry(
|
|
async () => {
|
|
abortedAttempts += 1;
|
|
throw new VideoSourceExtractionError(
|
|
"download_transient",
|
|
true,
|
|
"Video source download failed transiently",
|
|
);
|
|
},
|
|
{ signal: controller.signal },
|
|
),
|
|
).rejects.toMatchObject({ kind: "cancelled", retryable: false });
|
|
expect(abortedAttempts).toBe(0);
|
|
});
|
|
|
|
it("does not retry transient extraction failures unless the caller opts in", async () => {
|
|
let attempts = 0;
|
|
await expect(
|
|
runVideoExtractionWithRetry(async () => {
|
|
attempts += 1;
|
|
throw new VideoSourceExtractionError(
|
|
"ffmpeg_timeout",
|
|
true,
|
|
"Video frame extraction timed out",
|
|
);
|
|
}),
|
|
).rejects.toMatchObject({ kind: "ffmpeg_timeout", retryable: true });
|
|
expect(attempts).toBe(1);
|
|
});
|
|
|
|
it("fails closed to zero retries for a non-finite runtime retry budget", async () => {
|
|
let attempts = 0;
|
|
await expect(
|
|
runVideoExtractionWithRetry(
|
|
async () => {
|
|
attempts += 1;
|
|
throw new VideoSourceExtractionError(
|
|
"ffmpeg_timeout",
|
|
true,
|
|
"Video frame extraction timed out",
|
|
);
|
|
},
|
|
{ maxTransientRetries: Number.NaN },
|
|
),
|
|
).rejects.toMatchObject({ kind: "ffmpeg_timeout", retryable: true });
|
|
expect(attempts).toBe(1);
|
|
});
|
|
|
|
it("names a failed clip by its project-relative file, never a host, query or absolute path", () => {
|
|
expect(videoExtractionSourceLabel("assets/clips/a.mp4", "/proj")).toBe("assets/clips/a.mp4");
|
|
expect(videoExtractionSourceLabel("/proj/assets/a.mp4?x=1", "/proj")).toBe("assets/a.mp4");
|
|
expect(videoExtractionSourceLabel("/elsewhere/private/b.mp4", "/proj")).toBe("b.mp4");
|
|
expect(videoExtractionSourceLabel("https://cdn.example/v/c%201.mp4?Signature=s", "/proj")).toBe(
|
|
"c 1.mp4",
|
|
);
|
|
expect(videoExtractionSourceLabel("HTTPS://cdn.example/v/d.mp4?Signature=s", "/proj")).toBe(
|
|
"d.mp4",
|
|
);
|
|
});
|
|
|
|
it("never prints a scheme payload, username or drive path for a source that is not a project file", () => {
|
|
expect(videoExtractionSourceLabel("file:///Users/me/private/a.mp4", "/proj")).toBe("a.mp4");
|
|
expect(videoExtractionSourceLabel("data:video/mp4;base64,AAAA", "/proj")).toBe("inline video");
|
|
expect(videoExtractionSourceLabel("blob:https://cdn.example/1f3a", "/proj")).toBe(
|
|
"inline video",
|
|
);
|
|
expect(videoExtractionSourceLabel("ftp://host.example/pub/e.mp4", "/proj")).toBe("e.mp4");
|
|
expect(videoExtractionSourceLabel("C:\\Users\\me\\private\\f.mp4", "/proj")).toBe("f.mp4");
|
|
expect(videoExtractionSourceLabel("C:\\proj\\assets\\g.mp4", "C:\\proj")).toBe("assets/g.mp4");
|
|
});
|
|
|
|
it("reports which clip failed when a source file is missing", async () => {
|
|
const dir = mkdtempSync(join(tmpdir(), "hf-extract-names-"));
|
|
try {
|
|
const result = await extractAllVideoFrames(
|
|
[
|
|
{
|
|
id: "v1",
|
|
src: "assets/missing.mp4",
|
|
start: 0,
|
|
end: 1,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
},
|
|
],
|
|
dir,
|
|
{ fps: 30, outputDir: join(dir, "out") },
|
|
);
|
|
expect(result.errors).toHaveLength(1);
|
|
expect(result.errors[0]).toMatchObject({ videoId: "v1", source: "assets/missing.mp4" });
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|
|
|
|
// Codec-based alpha defaulting replaces tag-based detection (the
|
|
// alpha_mode/ALPHA_MODE case bug — see ffprobe.test.ts for the regression
|
|
// pin on that). The extractor uses these helpers for two decisions:
|
|
// 1. whether to force the alpha-aware decoder (libvpx-vp9 for VP9, libvpx
|
|
// for VP8)
|
|
// 2. whether to default the cached frame format to PNG (with alpha) vs JPG
|
|
// The "default to capable" trade is small file-size growth on opaque VP9
|
|
// content for correctness on alpha-having content even when the sidecar tag
|
|
// is missing or muxed with the wrong case.
|
|
describe("codec alpha capability", () => {
|
|
it("flags VP9, VP8, and ProRes as alpha-capable", () => {
|
|
expect(codecMayHaveAlpha("vp9")).toBe(true);
|
|
expect(codecMayHaveAlpha("VP9")).toBe(true);
|
|
expect(codecMayHaveAlpha("vp8")).toBe(true);
|
|
expect(codecMayHaveAlpha("prores")).toBe(true);
|
|
});
|
|
|
|
it("does not flag h264 / h265 / mpeg4 (no alpha in their bitstreams)", () => {
|
|
expect(codecMayHaveAlpha("h264")).toBe(false);
|
|
expect(codecMayHaveAlpha("h265")).toBe(false);
|
|
expect(codecMayHaveAlpha("hevc")).toBe(false);
|
|
expect(codecMayHaveAlpha("mpeg4")).toBe(false);
|
|
});
|
|
|
|
it("treats undefined / empty input as non-alpha", () => {
|
|
expect(codecMayHaveAlpha(undefined)).toBe(false);
|
|
expect(codecMayHaveAlpha("")).toBe(false);
|
|
});
|
|
|
|
it("returns the alpha-aware decoder name for VP9 and VP8", () => {
|
|
expect(decoderForCodec("vp9")).toBe("libvpx-vp9");
|
|
expect(decoderForCodec("VP9")).toBe("libvpx-vp9");
|
|
expect(decoderForCodec("vp8")).toBe("libvpx");
|
|
});
|
|
});
|
|
|
|
describe("resolveFrameFormat", () => {
|
|
function metadata(overrides: Partial<VideoMetadata> = {}): VideoMetadata {
|
|
return {
|
|
durationSeconds: 1,
|
|
width: 320,
|
|
height: 180,
|
|
fps: 30,
|
|
hasAudio: false,
|
|
videoCodec: "h264",
|
|
colorSpace: {
|
|
colorTransfer: "bt709",
|
|
colorPrimaries: "bt709",
|
|
colorSpace: "bt709",
|
|
},
|
|
isVFR: false,
|
|
hasAlpha: false,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
it("keeps opaque non-alpha sources on jpg by default", () => {
|
|
expect(resolveFrameFormat(metadata(), undefined)).toBe("jpg");
|
|
expect(resolveFrameFormat(metadata(), "auto")).toBe("jpg");
|
|
});
|
|
|
|
it("honors explicit png for opaque videos", () => {
|
|
expect(resolveFrameFormat(metadata(), "png")).toBe("png");
|
|
});
|
|
|
|
it("honors explicit jpg for opaque videos", () => {
|
|
expect(resolveFrameFormat(metadata(), "jpg")).toBe("jpg");
|
|
});
|
|
|
|
it("forces png when alpha is present or the codec can carry alpha", () => {
|
|
expect(resolveFrameFormat(metadata({ hasAlpha: true }), "jpg")).toBe("png");
|
|
expect(resolveFrameFormat(metadata({ videoCodec: "vp9" }), "jpg")).toBe("png");
|
|
});
|
|
|
|
it("keeps ProRes without an alpha plane on jpg and ProRes with one on png", () => {
|
|
expect(resolveFrameFormat(metadata({ videoCodec: "prores" }), undefined)).toBe("jpg");
|
|
expect(resolveFrameFormat(metadata({ videoCodec: "prores", hasAlpha: true }), "jpg")).toBe(
|
|
"png",
|
|
);
|
|
});
|
|
|
|
type FormatCase = Pick<VideoMetadata, "videoCodec" | "hasAlpha" | "alphaPlaneDecoded"> & {
|
|
requested: "auto" | "jpg" | "png";
|
|
};
|
|
const formatCases: FormatCase[] = ["vp9", "vp8", "h264", "prores"].flatMap((videoCodec) =>
|
|
[true, false].flatMap((hasAlpha) =>
|
|
[true, false, undefined].flatMap((alphaPlaneDecoded) =>
|
|
(["auto", "jpg", "png"] as const).map((requested) => ({
|
|
videoCodec,
|
|
hasAlpha,
|
|
alphaPlaneDecoded,
|
|
requested,
|
|
})),
|
|
),
|
|
),
|
|
);
|
|
function expectedFormat({ videoCodec, hasAlpha, alphaPlaneDecoded, requested }: FormatCase) {
|
|
const sidecar = videoCodec === "vp9" || videoCodec === "vp8";
|
|
const keepsPng = hasAlpha || (sidecar && alphaPlaneDecoded !== false);
|
|
return keepsPng || requested === "png" ? "png" : "jpg";
|
|
}
|
|
|
|
it("lets VP8/VP9 leave png only when a decode probe found no alpha plane", () => {
|
|
expect(formatCases).toHaveLength(72);
|
|
for (const { requested, ...fields } of formatCases) {
|
|
const actual = resolveFrameFormat(metadata(fields), requested);
|
|
expect(actual, JSON.stringify(fields)).toBe(expectedFormat({ requested, ...fields }));
|
|
}
|
|
expect(resolveFrameFormat(metadata({ videoCodec: "vp9", alphaPlaneDecoded: false }))).toBe(
|
|
"jpg",
|
|
);
|
|
expect(resolveFrameFormat(metadata({ videoCodec: "vp8", alphaPlaneDecoded: false }))).toBe(
|
|
"jpg",
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("parseVideoElements", () => {
|
|
it("marks a graded video and image, and only those", () => {
|
|
const html = `<video id="g" src="g.mp4" data-color-grading='{"preset":"clean-studio"}'></video>
|
|
<video id="p" src="p.mp4"></video>
|
|
<img id="gi" src="gi.png" data-color-grading='{"adjust":{"exposure":0.2}}'>
|
|
<img id="pi" src="pi.png">`;
|
|
expect(parseVideoElements(html).map(({ id, graded }) => [id, graded])).toEqual([
|
|
["g", true],
|
|
["p", undefined],
|
|
]);
|
|
expect(parseImageElements(html).map(({ id, graded }) => [id, graded])).toEqual([
|
|
["gi", true],
|
|
["pi", undefined],
|
|
]);
|
|
expect(hasGradedCanvas(html)).toBe(false);
|
|
expect(
|
|
hasGradedCanvas(`<canvas data-color-grading='{"preset":"clean-studio"}'></canvas>`),
|
|
).toBe(true);
|
|
});
|
|
|
|
it.each([
|
|
{
|
|
label: "valueless playback-start",
|
|
attributes: 'data-playback-start data-media-start="1.5"',
|
|
expected: 1.5,
|
|
},
|
|
{
|
|
label: "empty playback-start",
|
|
attributes: 'data-playback-start="" data-media-start="1.5"',
|
|
expected: 1.5,
|
|
},
|
|
{
|
|
label: "whitespace playback-start",
|
|
attributes: 'data-playback-start=" " data-media-start="1.5"',
|
|
expected: 1.5,
|
|
},
|
|
{
|
|
label: "invalid playback-start",
|
|
attributes: 'data-playback-start="later" data-media-start="1.5"',
|
|
expected: 1.5,
|
|
},
|
|
{ label: "missing media-start", attributes: "", expected: 0 },
|
|
{ label: "invalid media-start", attributes: 'data-media-start="later"', expected: 0 },
|
|
{
|
|
label: "negative playback-start",
|
|
attributes: 'data-playback-start="-1" data-media-start="1.5"',
|
|
expected: 1.5,
|
|
},
|
|
{ label: "negative media-start", attributes: 'data-media-start="-1"', expected: 0 },
|
|
{
|
|
label: "finite playback-start",
|
|
attributes: 'data-playback-start="2.25" data-media-start="1.5"',
|
|
expected: 2.25,
|
|
},
|
|
{
|
|
label: "zero playback-start",
|
|
attributes: 'data-playback-start="0" data-media-start="1.5"',
|
|
expected: 0,
|
|
},
|
|
])("uses finite playback-start -> media-start -> 0 for $label", ({ attributes, expected }) => {
|
|
const [video] = parseVideoElements(`<video id="hero" src="clip.mp4" ${attributes}></video>`);
|
|
|
|
expect(video?.mediaStart).toBe(expected);
|
|
});
|
|
|
|
it("parses and normalizes constant playback rate for final rendering", () => {
|
|
const [fast, low, high, invalid] = parseVideoElements(
|
|
'<video id="fast" src="clip.mp4" data-playback-rate="2"></video>' +
|
|
'<video id="low" src="clip.mp4" data-playback-rate="0.01"></video>' +
|
|
'<video id="high" src="clip.mp4" data-playback-rate="20"></video>' +
|
|
'<video id="invalid" src="clip.mp4" data-playback-rate="nope"></video>',
|
|
);
|
|
|
|
expect(fast?.playbackRate).toBe(2);
|
|
expect(low?.playbackRate).toBe(RATE_RANGE.min);
|
|
expect(high?.playbackRate).toBe(RATE_RANGE.max);
|
|
expect(invalid?.playbackRate).toBe(1);
|
|
});
|
|
|
|
it("parses a rate lane from data-automation into the clip's rate", () => {
|
|
const automation = JSON.stringify({
|
|
version: 1,
|
|
lanes: [
|
|
{
|
|
target: "rate",
|
|
points: [
|
|
{ t: 0, v: 1 },
|
|
{ t: 2, v: 3 },
|
|
],
|
|
},
|
|
],
|
|
});
|
|
const [ramped] = parseVideoElements(
|
|
`<video id="ramped" src="clip.mp4" data-automation='${automation}'></video>`,
|
|
);
|
|
|
|
expect(ramped?.playbackRate).toMatchObject({ target: "rate" });
|
|
});
|
|
|
|
it("parses videos without an id or data-start attribute", () => {
|
|
const videos = parseVideoElements('<video src="clip.mp4"></video>');
|
|
|
|
expect(videos).toHaveLength(1);
|
|
expect(videos[0]).toMatchObject({
|
|
id: "hf-video-0",
|
|
src: "clip.mp4",
|
|
start: 0,
|
|
end: Infinity,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
});
|
|
});
|
|
|
|
it("preserves explicit ids and derives end from data-duration", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="hero" src="clip.mp4" data-start="2" data-duration="5" data-media-start="1.5" data-has-audio="true"></video>',
|
|
);
|
|
|
|
expect(videos).toHaveLength(1);
|
|
expect(videos[0]).toEqual({
|
|
id: "hero",
|
|
src: "clip.mp4",
|
|
start: 2,
|
|
end: 7,
|
|
mediaStart: 1.5,
|
|
playbackRate: 1,
|
|
loop: false,
|
|
hasAudio: true,
|
|
});
|
|
});
|
|
|
|
it("preserves looped timed video semantics for render frame lookup", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="hero" src="clip.webm" data-start="2" data-duration="5" loop></video>',
|
|
);
|
|
|
|
expect(videos[0]).toMatchObject({
|
|
id: "hero",
|
|
start: 2,
|
|
end: 7,
|
|
loop: true,
|
|
});
|
|
});
|
|
|
|
it("resolves a relative data-start reference to another clip's end", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="intro" src="a.mp4" data-start="0" data-duration="10"></video>' +
|
|
'<video id="main" src="b.mp4" data-start="intro" data-duration="20"></video>',
|
|
);
|
|
const main = videos.find((v) => v.id === "main");
|
|
// intro ends at 10, so main starts at 10 and ends at 30 — not NaN.
|
|
expect(main?.start).toBe(10);
|
|
expect(main?.end).toBe(30);
|
|
});
|
|
|
|
it("still resolves relative data-start after compileTimingAttrs", () => {
|
|
const raw =
|
|
'<video id="intro" src="a.mp4" data-start="0" data-duration="10"></video>' +
|
|
'<video id="main" src="b.mp4" data-start="intro" data-duration="20"></video>';
|
|
const { html } = compileTimingAttrs(raw);
|
|
const main = parseVideoElements(html).find((v) => v.id === "main");
|
|
expect(main?.start).toBe(10);
|
|
expect(main?.end).toBe(30);
|
|
});
|
|
|
|
it("applies + and - offsets on a relative reference", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="intro" src="a.mp4" data-start="0" data-duration="10"></video>' +
|
|
'<video id="gap" src="b.mp4" data-start="intro + 2" data-duration="5"></video>' +
|
|
'<video id="overlap" src="c.mp4" data-start="intro - 0.5" data-duration="5"></video>',
|
|
);
|
|
expect(videos.find((v) => v.id === "gap")?.start).toBe(12);
|
|
expect(videos.find((v) => v.id === "overlap")?.start).toBe(9.5);
|
|
});
|
|
|
|
it("resolves chained references (A -> B -> C)", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="a" src="a.mp4" data-start="0" data-duration="4"></video>' +
|
|
'<video id="b" src="b.mp4" data-start="a" data-duration="3"></video>' +
|
|
'<video id="c" src="c.mp4" data-start="b" data-duration="2"></video>',
|
|
);
|
|
expect(videos.find((v) => v.id === "b")?.start).toBe(4);
|
|
expect(videos.find((v) => v.id === "c")?.start).toBe(7); // 4 + 3
|
|
});
|
|
|
|
it("resolves a reference to a non-video timed element (div clip)", () => {
|
|
const videos = parseVideoElements(
|
|
'<div id="title" data-start="0" data-duration="6"></div>' +
|
|
'<video id="clip" src="b.mp4" data-start="title" data-duration="5"></video>',
|
|
);
|
|
expect(videos.find((v) => v.id === "clip")?.start).toBe(6);
|
|
});
|
|
|
|
it("derives a referenced clip's duration from data-end when data-duration is absent", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="intro" src="a.mp4" data-start="2" data-end="9"></video>' +
|
|
'<video id="main" src="b.mp4" data-start="intro" data-duration="5"></video>',
|
|
);
|
|
// intro: start 2, end 9 -> duration 7 -> main starts at 9.
|
|
expect(videos.find((v) => v.id === "main")?.start).toBe(9);
|
|
});
|
|
|
|
it("resolves a reference inside its own composition when another composition reuses the id", () => {
|
|
const videos = parseVideoElements(
|
|
'<div data-composition-id="main">' +
|
|
'<div data-composition-id="scene-a" data-start="0" data-duration="4">' +
|
|
'<div id="intro" data-start="0" data-duration="2"></div>' +
|
|
"</div>" +
|
|
'<div data-composition-id="scene-b" data-start="4" data-duration="4">' +
|
|
'<div id="intro" data-start="0" data-duration="3"></div>' +
|
|
'<video id="clip" src="b.mp4" data-start="intro" data-duration="1"></video>' +
|
|
"</div>" +
|
|
"</div>",
|
|
);
|
|
expect(videos.find((v) => v.id === "clip")?.start).toBe(3);
|
|
});
|
|
|
|
it("falls back to 0 (never NaN) for an unknown reference target", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="orphan" src="a.mp4" data-start="does-not-exist" data-duration="5"></video>',
|
|
);
|
|
const orphan = videos.find((v) => v.id === "orphan");
|
|
expect(orphan?.start).toBe(0);
|
|
expect(Number.isNaN(orphan?.start)).toBe(false);
|
|
expect(orphan?.end).toBe(5);
|
|
});
|
|
|
|
it("does not hang or NaN on a circular reference", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="a" src="a.mp4" data-start="b" data-duration="4"></video>' +
|
|
'<video id="b" src="b.mp4" data-start="a" data-duration="3"></video>',
|
|
);
|
|
for (const v of videos) {
|
|
expect(Number.isNaN(v.start)).toBe(false);
|
|
}
|
|
});
|
|
|
|
it("discovers <video> elements that use <source> children", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="rec" data-start="1" data-duration="4">' +
|
|
'<source src="https://cdn.example.com/rec.mp4" type="video/mp4">' +
|
|
'<source src="_remote_media/rec.webm" type="video/webm">' +
|
|
"</video>",
|
|
);
|
|
expect(videos).toHaveLength(1);
|
|
expect(videos[0]).toMatchObject({ id: "rec", src: "_remote_media/rec.webm", start: 1, end: 5 });
|
|
});
|
|
});
|
|
|
|
describe("FrameLookupTable", () => {
|
|
function fakeExtracted(totalFrames: number, fps: number): ExtractedFrames {
|
|
const framePaths = new Map<number, string>();
|
|
for (let i = 0; i < totalFrames; i += 1) {
|
|
framePaths.set(i, `frame-${i}.jpg`);
|
|
}
|
|
return {
|
|
videoId: "hero",
|
|
srcPath: "clip.webm",
|
|
outputDir: "/tmp/frames",
|
|
framePattern: "frame-%05d.jpg",
|
|
fps,
|
|
totalFrames,
|
|
metadata: {
|
|
durationSeconds: totalFrames / fps,
|
|
width: 320,
|
|
height: 180,
|
|
fps,
|
|
hasAudio: false,
|
|
videoCodec: "vp9",
|
|
colorSpace: {
|
|
colorTransfer: "bt709",
|
|
colorPrimaries: "bt709",
|
|
colorSpace: "bt709",
|
|
},
|
|
isVFR: false,
|
|
hasAlpha: false,
|
|
},
|
|
framePaths,
|
|
};
|
|
}
|
|
|
|
it("wraps active frame payloads for looped clips whose display window exceeds source frames", () => {
|
|
const table = createFrameLookupTable(
|
|
[
|
|
{
|
|
id: "hero",
|
|
src: "clip.webm",
|
|
start: 0,
|
|
end: 5,
|
|
mediaStart: 0,
|
|
loop: true,
|
|
hasAudio: false,
|
|
},
|
|
],
|
|
[fakeExtracted(30, 30)],
|
|
);
|
|
|
|
expect(table.getActiveFramePayloads(0.5).get("hero")?.frameIndex).toBe(15);
|
|
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(15);
|
|
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15);
|
|
});
|
|
|
|
it("shows a copy starting on a float sum at that instant, as the preview does", () => {
|
|
const copyStart = 0.1 + 0.2;
|
|
const clip = (id: string, start: number) => ({
|
|
id,
|
|
src: `${id}.webm`,
|
|
start,
|
|
end: start + 0.2,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
});
|
|
const frames = (videoId: string) => ({ ...fakeExtracted(30, 30), videoId });
|
|
const table = () =>
|
|
createFrameLookupTable(
|
|
[clip("a", 0.1), clip("copy", copyStart)],
|
|
[frames("a"), frames("copy")],
|
|
);
|
|
|
|
expect(table().getActiveFramePayloads(0.3).get("copy")?.frameIndex).toBe(0);
|
|
const stepping = table();
|
|
stepping.getActiveFramePayloads(0.2);
|
|
expect(stepping.getActiveFramePayloads(0.3).get("copy")?.frameIndex).toBe(0);
|
|
expect(getFrameAtTime(frames("copy"), 0.3, copyStart)).toBe("frame-0.jpg");
|
|
});
|
|
|
|
it("selects source frames at the authored constant playback rate", () => {
|
|
const videos = parseVideoElements(
|
|
'<video id="hero" src="clip.webm" data-start="0" data-duration="2" data-playback-rate="2"></video>',
|
|
);
|
|
const table = createFrameLookupTable(videos, [fakeExtracted(120, 30)]);
|
|
|
|
expect(table.getActiveFramePayloads(1).get("hero")?.frameIndex).toBe(60);
|
|
});
|
|
|
|
it("wraps at video-stream EOF when a mux container has longer audio", () => {
|
|
const extracted = fakeExtracted(6, 2);
|
|
extracted.metadata.durationSeconds = 60;
|
|
extracted.metadata.videoStreamDurationSeconds = 3;
|
|
const table = createFrameLookupTable(
|
|
[
|
|
{
|
|
id: "hero",
|
|
src: "clip.webm",
|
|
start: 0,
|
|
end: 60,
|
|
mediaStart: 0,
|
|
loop: true,
|
|
hasAudio: false,
|
|
},
|
|
],
|
|
[extracted],
|
|
);
|
|
|
|
expect(table.getActiveFramePayloads(4).get("hero")?.frameIndex).toBe(2);
|
|
});
|
|
|
|
it("holds the last frame for a non-looping clip until its authored slot ends", () => {
|
|
const table = createFrameLookupTable(
|
|
[
|
|
{
|
|
id: "hero",
|
|
src: "clip.webm",
|
|
start: 0,
|
|
end: 5,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
},
|
|
],
|
|
[fakeExtracted(30, 30)],
|
|
);
|
|
|
|
expect(table.getActiveFramePayloads(0.5).has("hero")).toBe(true);
|
|
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(29);
|
|
expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(29);
|
|
expect(table.getFrame("hero", 4.5)).toBeTruthy();
|
|
expect(table.getActiveFramePayloads(5.1).has("hero")).toBe(false);
|
|
expect(table.getFrame("hero", 5.1)).toBeNull();
|
|
});
|
|
|
|
it("does not invent a held frame when extraction produced no frames", () => {
|
|
const table = createFrameLookupTable(
|
|
[
|
|
{
|
|
id: "hero",
|
|
src: "clip.webm",
|
|
start: 0,
|
|
end: 5,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
},
|
|
],
|
|
[fakeExtracted(0, 30)],
|
|
);
|
|
|
|
expect(table.getActiveFramePayloads(4.5).has("hero")).toBe(false);
|
|
expect(table.getFrame("hero", 4.5)).toBeNull();
|
|
});
|
|
|
|
it("places a relative-reference video in its resolved window end-to-end (was blank)", () => {
|
|
// The reported bug: <video data-start="intro"> gave NaN start/end, so the
|
|
// active-window checks (start <= t <= end) were always false and the clip
|
|
// composited blank. With resolution, `main` is active across [10, 30].
|
|
const videos = parseVideoElements(
|
|
'<video id="intro" src="a.mp4" data-start="0" data-duration="10"></video>' +
|
|
'<video id="main" src="b.mp4" data-start="intro" data-duration="20"></video>',
|
|
);
|
|
const table = createFrameLookupTable(videos, [{ ...fakeExtracted(600, 30), videoId: "main" }]);
|
|
expect(table.getActiveFramePayloads(5).has("main")).toBe(false); // before resolved start (10)
|
|
expect(table.getActiveFramePayloads(15).has("main")).toBe(true); // within [10, 30]
|
|
expect(table.getActiveFramePayloads(29).has("main")).toBe(true);
|
|
expect(table.getActiveFramePayloads(31).has("main")).toBe(false); // after resolved end (30)
|
|
});
|
|
|
|
it("shows the last frame just before the end and leaves at t === end, as the runtime does", () => {
|
|
const table = createFrameLookupTable(
|
|
[
|
|
{
|
|
id: "hero",
|
|
src: "clip.webm",
|
|
start: 1,
|
|
end: 3,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
},
|
|
],
|
|
[fakeExtracted(60, 30)],
|
|
);
|
|
expect(table.getActiveFramePayloads(2.5).get("hero")?.frameIndex).toBe(45);
|
|
expect(table.getActiveFramePayloads(2.99).get("hero")?.frameIndex).toBe(59);
|
|
expect(table.getActiveFramePayloads(3.0).has("hero")).toBe(false);
|
|
expect(table.getFrame("hero", 3.0)).toBeNull();
|
|
});
|
|
|
|
it("holds the last frame across the tail when the source is shorter than the window", () => {
|
|
// clip [0,5] with only 1s of source (30 @ 30fps). The authored slot is the
|
|
// visibility contract, so the final source frame fills its remaining tail.
|
|
const table = createFrameLookupTable(
|
|
[
|
|
{
|
|
id: "hero",
|
|
src: "clip.webm",
|
|
start: 0,
|
|
end: 5,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
},
|
|
],
|
|
[fakeExtracted(30, 30)],
|
|
);
|
|
expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(29);
|
|
expect(table.getActiveFramePayloads(4.99).get("hero")?.frameIndex).toBe(29);
|
|
});
|
|
|
|
it("holds the last frame when the source is a sub-frame shorter than the slot", () => {
|
|
// clip [2, 3.45] declares a 1.45s slot, but `ffmpeg -t 1.45` at 30fps emits
|
|
// 43 frames = 1.433s — a half-frame short. The tail between source
|
|
// exhaustion (~3.433) and the clip end (3.45) must hold the last frame
|
|
// rather than render the page background (a one-frame black flash at the
|
|
// cut). The held index is the final extracted frame (42).
|
|
const table = createFrameLookupTable(
|
|
[
|
|
{
|
|
id: "hero",
|
|
src: "clip.mp4",
|
|
start: 2,
|
|
end: 3.45,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
},
|
|
],
|
|
[fakeExtracted(43, 30)],
|
|
);
|
|
// last real frame
|
|
expect(table.getActiveFramePayloads(3.4).get("hero")?.frameIndex).toBe(42);
|
|
// source exhausted but within tolerance of the end → hold, don't blank
|
|
expect(table.getActiveFramePayloads(3.44).get("hero")?.frameIndex).toBe(42);
|
|
});
|
|
|
|
it("gives a frame a hair before two clips meet to the clip export shows there", () => {
|
|
const clip = (id: string, start: number, end: number) => ({
|
|
id,
|
|
src: `${id}.webm`,
|
|
start,
|
|
end,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
});
|
|
const table = () =>
|
|
createFrameLookupTable(
|
|
[clip("a", 0, 1.00001), clip("b", 1.00001, 2)],
|
|
[
|
|
{ ...fakeExtracted(90, 30), videoId: "a" },
|
|
{ ...fakeExtracted(90, 30), videoId: "b" },
|
|
],
|
|
30,
|
|
);
|
|
const stepping = table();
|
|
stepping.getActiveFramePayloads(29 / 30);
|
|
const payloads = stepping.getActiveFramePayloads(1);
|
|
|
|
expect([...payloads.keys()]).toEqual(["b"]);
|
|
expect(payloads.get("b")?.frameIndex).toBe(0);
|
|
expect([...table().getActiveFramePayloads(1).keys()]).toEqual(["b"]);
|
|
expect(table().getFrame("a", 1)).toBeNull();
|
|
});
|
|
|
|
it.each([
|
|
["an exact", 0.1, 0.3, 0.3],
|
|
["a float-sum", 0.1, 0.1 + 0.2, 9 / 30],
|
|
])(
|
|
"hands %s shared boundary to the incoming clip only, as the runtime does",
|
|
(_, aStart, aEnd, t) => {
|
|
const clip = (id: string, start: number, end: number) => ({
|
|
id,
|
|
src: `${id}.webm`,
|
|
start,
|
|
end,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
});
|
|
const table = () =>
|
|
createFrameLookupTable(
|
|
[clip("a", aStart, aEnd), clip("b", 0.3, 0.5)],
|
|
[
|
|
{ ...fakeExtracted(90, 30), videoId: "a" },
|
|
{ ...fakeExtracted(90, 30), videoId: "b" },
|
|
],
|
|
);
|
|
const stepping = table();
|
|
expect([...stepping.getActiveFramePayloads(8 / 30).keys()]).toEqual(["a"]);
|
|
expect([...stepping.getActiveFramePayloads(t).keys()]).toEqual(["b"]);
|
|
expect([...table().getActiveFramePayloads(t).keys()]).toEqual(["b"]);
|
|
expect(table().getFrame("a", t)).toBeNull();
|
|
},
|
|
);
|
|
});
|
|
|
|
describe("analyzeClipMediaFit", () => {
|
|
it("returns null for a sub-tolerance shortfall the compiler leaves unclamped", () => {
|
|
// 1.433s media in a 1.45s slot — a sub-frame shortfall (<0.05s) the renderer
|
|
// freezes seamlessly and the compiler never clamps. Not worth warning about.
|
|
expect(analyzeClipMediaFit({ slotSeconds: 1.45, mediaSeconds: 1.433 })).toBeNull();
|
|
});
|
|
|
|
it("returns null when media is longer than or equal to the slot", () => {
|
|
expect(analyzeClipMediaFit({ slotSeconds: 2, mediaSeconds: 2 })).toBeNull();
|
|
expect(analyzeClipMediaFit({ slotSeconds: 2, mediaSeconds: 5 })).toBeNull();
|
|
});
|
|
|
|
it("reports the shortfall when the slot exceeds media beyond the clamp epsilon", () => {
|
|
const fit = analyzeClipMediaFit({ slotSeconds: 5, mediaSeconds: 1 });
|
|
expect(fit).not.toBeNull();
|
|
expect(fit?.shortfallSeconds).toBeCloseTo(4, 5);
|
|
expect(fit?.toleranceSeconds).toBeCloseTo(0.05, 5);
|
|
});
|
|
|
|
it("never flags looping clips (they repeat to fill the slot)", () => {
|
|
expect(analyzeClipMediaFit({ slotSeconds: 5, mediaSeconds: 1, loop: true })).toBeNull();
|
|
});
|
|
|
|
it("returns null for unusable inputs (non-finite media, zero slot)", () => {
|
|
expect(analyzeClipMediaFit({ slotSeconds: 0, mediaSeconds: 1 })).toBeNull();
|
|
expect(analyzeClipMediaFit({ slotSeconds: 5, mediaSeconds: NaN })).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe("parseImageElements", () => {
|
|
it("parses images with data-start and data-duration", () => {
|
|
const images = parseImageElements(
|
|
'<img id="photo" src="hdr-photo.png" data-start="0" data-duration="3" />',
|
|
);
|
|
|
|
expect(images).toHaveLength(1);
|
|
expect(images[0]).toEqual({
|
|
id: "photo",
|
|
src: "hdr-photo.png",
|
|
start: 0,
|
|
end: 3,
|
|
});
|
|
});
|
|
|
|
it("generates stable IDs for images without one", () => {
|
|
const images = parseImageElements(
|
|
'<img src="a.png" data-start="0" data-end="2" /><img src="b.png" data-start="1" data-end="4" />',
|
|
);
|
|
|
|
expect(images).toHaveLength(2);
|
|
expect(images[0]!.id).toBe("hf-img-0");
|
|
expect(images[1]!.id).toBe("hf-img-1");
|
|
});
|
|
|
|
it("defaults start to 0 and end to Infinity when attributes missing", () => {
|
|
const images = parseImageElements('<img src="photo.png" />');
|
|
|
|
expect(images).toHaveLength(1);
|
|
expect(images[0]).toMatchObject({
|
|
src: "photo.png",
|
|
start: 0,
|
|
end: Infinity,
|
|
});
|
|
});
|
|
|
|
it("ignores img elements without src", () => {
|
|
const images = parseImageElements('<img data-start="0" data-end="3" />');
|
|
expect(images).toHaveLength(0);
|
|
});
|
|
|
|
it("uses data-end over data-duration when both present", () => {
|
|
const images = parseImageElements(
|
|
'<img src="a.png" data-start="1" data-end="5" data-duration="10" />',
|
|
);
|
|
expect(images[0]!.end).toBe(5);
|
|
});
|
|
});
|
|
|
|
type Rgb = [number, number, number];
|
|
|
|
const UI_FIXTURE_WIDTH = 240;
|
|
const UI_FIXTURE_HEIGHT = 160;
|
|
const RED_SAMPLE_PIXELS = [
|
|
[70, 72],
|
|
[118, 82],
|
|
[178, 92],
|
|
] as const;
|
|
|
|
function pngChunkTypes(path: string): string[] {
|
|
const bytes = readFileSync(path);
|
|
const types: string[] = [];
|
|
for (let offset = 8; offset + 8 <= bytes.length; offset += 12 + bytes.readUInt32BE(offset)) {
|
|
types.push(bytes.toString("latin1", offset + 4, offset + 8));
|
|
}
|
|
return types;
|
|
}
|
|
|
|
function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb {
|
|
const result = spawnSync(
|
|
"ffmpeg",
|
|
[
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
mediaPath,
|
|
"-frames:v",
|
|
"1",
|
|
"-f",
|
|
"rawvideo",
|
|
"-pix_fmt",
|
|
"rgb24",
|
|
"pipe:1",
|
|
],
|
|
{ maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 },
|
|
);
|
|
if (result.status !== 0) {
|
|
throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`);
|
|
}
|
|
|
|
const offset = (y * UI_FIXTURE_WIDTH + x) * 3;
|
|
return [
|
|
result.stdout[offset] ?? 0,
|
|
result.stdout[offset + 1] ?? 0,
|
|
result.stdout[offset + 2] ?? 0,
|
|
];
|
|
}
|
|
|
|
function maxChannelDelta(a: Rgb, b: Rgb): number {
|
|
return Math.max(Math.abs(a[0] - b[0]), Math.abs(a[1] - b[1]), Math.abs(a[2] - b[2]));
|
|
}
|
|
|
|
// Regression for saturated UI recordings: default JPEG extraction can shift
|
|
// high-chroma reds before browser capture. Forcing PNG should keep extracted
|
|
// source-video frames effectively identical to the decoded source pixels.
|
|
describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
|
|
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-format-"));
|
|
const UI_FIXTURE = join(FIXTURE_DIR, "ui-red.mp4");
|
|
|
|
beforeAll(async () => {
|
|
const result = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xffe7ee:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
|
|
"-vf",
|
|
"drawbox=x=44:y=58:w=152:h=44:color=0xdd382e@1:t=fill,drawbox=x=64:y=75:w=112:h=10:color=0xfff0f0@1:t=fill",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
"-color_primaries",
|
|
"bt709",
|
|
"-color_trc",
|
|
"bt709",
|
|
"-colorspace",
|
|
"bt709",
|
|
UI_FIXTURE,
|
|
]);
|
|
if (!result.success) {
|
|
throw new Error(`UI color fixture synthesis failed: ${result.stderr.slice(-400)}`);
|
|
}
|
|
}, 30_000);
|
|
|
|
afterAll(() => {
|
|
if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
|
|
}, 30_000);
|
|
|
|
function fixtureVideo(): VideoElement {
|
|
return {
|
|
id: "ui",
|
|
src: UI_FIXTURE,
|
|
start: 0,
|
|
end: 1,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
}
|
|
|
|
it("extracts into a work directory whose name holds a printf code", async () => {
|
|
const outDir = join(FIXTURE_DIR, "x 12% Up");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir: outDir,
|
|
});
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
expect(frame.startsWith(outDir)).toBe(true);
|
|
expect(existsSync(frame)).toBe(true);
|
|
});
|
|
|
|
it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) jpg extraction", async () => {
|
|
const source = join(FIXTURE_DIR, "white-1080.mp4");
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=white:s=1080x16:d=1:r=1",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
source,
|
|
]);
|
|
expect(synth.success).toBe(true);
|
|
const outDir = join(FIXTURE_DIR, "out-white-1080");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: "white", src: source }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: outDir },
|
|
);
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
expect(frame.endsWith(".jpg")).toBe(true);
|
|
const lastColumns = (file: string) => [
|
|
...spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-frames:v",
|
|
"1",
|
|
"-vf",
|
|
"format=gray,crop=8:1:iw-8:8",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout,
|
|
];
|
|
const delivered = lastColumns(frame);
|
|
const expected = lastColumns(source);
|
|
expect(delivered).toHaveLength(8);
|
|
for (const [i, v] of delivered.entries())
|
|
expect(Math.abs(v - expected[i]!)).toBeLessThanOrEqual(5);
|
|
}, 60_000);
|
|
|
|
async function proresFrame(name: string, pixFmt: string, profile: string, source: string) {
|
|
const clip = join(FIXTURE_DIR, `${name}.mov`);
|
|
const synth = await runFfmpeg(
|
|
["-y", "-loglevel", "error", "-f", "lavfi", "-i", source].concat([
|
|
"-c:v",
|
|
"prores_ks",
|
|
"-profile:v",
|
|
profile,
|
|
"-pix_fmt",
|
|
pixFmt,
|
|
clip,
|
|
]),
|
|
);
|
|
expect(synth.success).toBe(true);
|
|
const outDir = join(FIXTURE_DIR, `out-${name}`);
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: name, src: clip }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: outDir },
|
|
);
|
|
expect(result.errors).toEqual([]);
|
|
return result.extracted[0]!.framePaths.get(0)!;
|
|
}
|
|
|
|
it("extracts ProRes without an alpha plane to jpg", async () => {
|
|
const frame = await proresFrame("prores422", "yuv422p10le", "2", "testsrc2=s=320x180:d=1:r=1");
|
|
expect(frame.endsWith(".jpg")).toBe(true);
|
|
}, 60_000);
|
|
|
|
it("keeps the alpha of ProRes 4444 exact in png frames", async () => {
|
|
const frame = await proresFrame(
|
|
"prores4444",
|
|
"yuva444p10le",
|
|
"4",
|
|
"color=c=0x00ff00@0.5:s=320x180:d=1:r=1,format=yuva444p10le",
|
|
);
|
|
expect(frame.endsWith(".png")).toBe(true);
|
|
const alpha = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
frame,
|
|
"-vf",
|
|
"alphaextract,crop=1:1:10:10",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]);
|
|
expect(Math.abs(alpha.stdout[0]! - 128)).toBeLessThanOrEqual(1);
|
|
}, 60_000);
|
|
|
|
it("keeps color-sensitive UI reds closer to source when extraction is forced to png", async () => {
|
|
const defaultOut = join(FIXTURE_DIR, "out-default");
|
|
const pngOut = join(FIXTURE_DIR, "out-png");
|
|
mkdirSync(defaultOut, { recursive: true });
|
|
mkdirSync(pngOut, { recursive: true });
|
|
|
|
const defaultResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir: defaultOut,
|
|
});
|
|
const pngResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir: pngOut,
|
|
format: "png",
|
|
});
|
|
|
|
expect(defaultResult.errors).toEqual([]);
|
|
expect(pngResult.errors).toEqual([]);
|
|
const defaultFrame = defaultResult.extracted[0]!.framePaths.get(0)!;
|
|
const pngFrame = pngResult.extracted[0]!.framePaths.get(0)!;
|
|
expect(defaultFrame.endsWith(".jpg")).toBe(true);
|
|
expect(pngFrame.endsWith(".png")).toBe(true);
|
|
|
|
let worstDefaultDelta = 0;
|
|
let worstPngDelta = 0;
|
|
for (const [x, y] of RED_SAMPLE_PIXELS) {
|
|
const sourcePixel = readFirstFramePixel(UI_FIXTURE, x, y);
|
|
worstDefaultDelta = Math.max(
|
|
worstDefaultDelta,
|
|
maxChannelDelta(sourcePixel, readFirstFramePixel(defaultFrame, x, y)),
|
|
);
|
|
worstPngDelta = Math.max(
|
|
worstPngDelta,
|
|
maxChannelDelta(sourcePixel, readFirstFramePixel(pngFrame, x, y)),
|
|
);
|
|
}
|
|
|
|
expect(worstPngDelta).toBeLessThanOrEqual(5);
|
|
expect(worstPngDelta).toBeLessThanOrEqual(worstDefaultDelta);
|
|
}, 60_000);
|
|
|
|
// Chrome runs with --force-color-profile=srgb and colour-manages any frame not tagged sRGB;
|
|
// the SDR encoder writes the canvas code values as they are, so frames must reach it unconverted.
|
|
it.each([
|
|
["BT.709", "bt709"],
|
|
["BT.601", "smpte170m"],
|
|
])(
|
|
"declares png frames of a %s source as sRGB so Chrome shows their code values",
|
|
async (name, tag) => {
|
|
const fixture = join(FIXTURE_DIR, `${tag}-shadow.mp4`);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0x101010:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
|
|
"-vf",
|
|
`setparams=color_primaries=${tag}:color_trc=${tag}:colorspace=${tag}:range=tv`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
fixture,
|
|
]);
|
|
if (!synth.success) {
|
|
throw new Error(`${name} fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
}
|
|
expect((await extractVideoMetadata(fixture)).colorSpace?.colorTransfer).toBe(tag);
|
|
const outputDir = join(FIXTURE_DIR, `out-png-transfer-${tag}`);
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: tag, src: fixture }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir, format: "png" },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
// Chrome reads cICP before sRGB, and ffmpeg < 6.0 writes no cICP, so only an sRGB chunk
|
|
// without cICP means "these are sRGB code values" on every ffmpeg.
|
|
const chunks = pngChunkTypes(frame);
|
|
expect(chunks).toContain("sRGB");
|
|
expect(chunks).not.toContain("cICP");
|
|
expect(readFirstFramePixel(frame, 10, 10)).toEqual(readFirstFramePixel(fixture, 10, 10));
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("extracts jpg frames of a BT.709 source as the BT.601 full range JPEG readers assume", async () => {
|
|
const fixture = join(FIXTURE_DIR, "bt709-patch.mp4");
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1,format=rgb24,drawbox=x=31:y=0:w=64:h=${UI_FIXTURE_HEIGHT}:c=0x0000FE:t=fill`,
|
|
"-vf",
|
|
"scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const outputDir = join(FIXTURE_DIR, "out-jpg-matrix");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: "bt709-jpg", src: fixture }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir, format: "jpg" },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
// x=10 is flat colour; x=32 sits one pixel inside the blue edge.
|
|
for (const x of [10, 32]) {
|
|
const source = readFirstFramePixel(fixture, x, 10);
|
|
const shown = readFirstFramePixel(frame, x, 10);
|
|
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
|
|
expect(worst, `x=${x}: source ${source} jpg ${shown}`).toBeLessThanOrEqual(3);
|
|
}
|
|
}, 60_000);
|
|
|
|
// Measured in Chrome 152: untagged H.264 plays as BT.601 at 1280x718 and BT.709 at 1280x720 or 406x720;
|
|
// untagged VP9 and AV1 play as BT.601 at every size.
|
|
it.each([
|
|
{ codec: "libx264", height: 718, matrix: "bt601", format: "png" },
|
|
{ codec: "libx264", height: 718, matrix: "bt601", format: "jpg" },
|
|
{ codec: "libx264", height: 720, matrix: "bt709", format: "png" },
|
|
{ codec: "libx264", height: 720, matrix: "bt709", format: "jpg" },
|
|
{ codec: "libvpx-vp9", height: 720, matrix: "bt601", format: "png" },
|
|
...(AV1_ENCODER
|
|
? [{ codec: AV1_ENCODER, height: 720, matrix: "bt601", format: "png" } as const]
|
|
: []),
|
|
] as const)(
|
|
"reads an untagged $height-line $codec source as $matrix like Chrome's own playback ($format)",
|
|
async ({ codec, height, matrix, format }) => {
|
|
const extension = codec === "libvpx-vp9" ? "webm" : "mp4";
|
|
const fixture = join(FIXTURE_DIR, `untagged-${codec}-${height}.${extension}`);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xC83C28:s=64x${height}:d=1:r=1`,
|
|
"-vf",
|
|
"scale=out_color_matrix=bt601:out_range=tv,format=yuv420p,setparams=colorspace=unknown:color_primaries=unknown:color_trc=unknown:range=tv",
|
|
"-c:v",
|
|
codec,
|
|
...(codec === "libx264" ? ["-qp", "0"] : ["-crf", "0", "-b:v", "0"]),
|
|
...(codec === "libaom-av1" ? ["-cpu-used", "8"] : []),
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
expect((await extractVideoMetadata(fixture)).colorSpace?.colorSpace ?? "unknown").toBe(
|
|
"unknown",
|
|
);
|
|
|
|
const result = await extractVideoFramesRange(
|
|
fixture,
|
|
`${basename(fixture)}-${format}`,
|
|
0,
|
|
1,
|
|
{
|
|
fps: 1,
|
|
outputDir: join(FIXTURE_DIR, "out-untagged"),
|
|
format,
|
|
},
|
|
);
|
|
|
|
const pixelAt = (file: string, decode: string): number[] => [
|
|
...spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-vf",
|
|
`${decode}format=rgb24,crop=1:1:10:10`,
|
|
"-frames:v",
|
|
"1",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout,
|
|
];
|
|
const expected = pixelAt(fixture, `scale=in_color_matrix=${matrix}:in_range=tv,`);
|
|
const shown = pixelAt(result.framePaths.get(0)!, "");
|
|
expect([expected.length, shown.length]).toEqual([3, 3]);
|
|
const worst = Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)));
|
|
expect(worst, `${matrix} decode ${expected}, ${format} ${shown}`).toBeLessThanOrEqual(
|
|
format === "png" ? 1 : 3,
|
|
);
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => {
|
|
// Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks.
|
|
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4");
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-v", "error", "-f", "lavfi", "-i"],
|
|
"color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
|
|
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
|
|
const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-ntsc-ms"),
|
|
format: "png",
|
|
});
|
|
const indexOf = (file: string): number => {
|
|
const row = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-vf",
|
|
"format=gray,crop=64:1:0:4",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout;
|
|
return [...Array(8).keys()].reduce(
|
|
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
|
|
0,
|
|
);
|
|
};
|
|
const shown = [...result.framePaths.keys()]
|
|
.sort((a, b) => a - b)
|
|
.map((i) => indexOf(result.framePaths.get(i)!));
|
|
const steps = shown.slice(1).map((n, i) => n - shown[i]!);
|
|
|
|
expect(shown.length).toBeGreaterThanOrEqual(44);
|
|
expect(
|
|
steps.filter((step) => step > 1),
|
|
`source frames ${shown.join(",")}`,
|
|
).toEqual([]);
|
|
expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1);
|
|
}, 60_000);
|
|
|
|
it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => {
|
|
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4");
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-v", "error", "-f", "lavfi", "-i"],
|
|
"color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
|
|
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const indexOf = (file: string): number => {
|
|
const row = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-vf",
|
|
"format=gray,crop=64:1:0:4",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout;
|
|
return [...Array(8).keys()].reduce(
|
|
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
|
|
0,
|
|
);
|
|
};
|
|
const options = {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-ntsc-seams"),
|
|
format: "png" as const,
|
|
};
|
|
const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options);
|
|
|
|
const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201];
|
|
const differing: string[] = [];
|
|
for (const firstFrame of seams) {
|
|
const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, {
|
|
...options,
|
|
frameRanges: [{ firstFrame, frames: 12 }],
|
|
});
|
|
for (const [slot, path] of segment.framePaths) {
|
|
const expected = indexOf(single.framePaths.get(slot)!);
|
|
const shown = indexOf(path);
|
|
if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`);
|
|
}
|
|
}
|
|
expect(differing).toEqual([]);
|
|
}, 120_000);
|
|
|
|
// ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour
|
|
// filters would cut the still that ends this window short.
|
|
it.each([
|
|
{
|
|
name: "full range BT.709",
|
|
format: "png",
|
|
codec: "libx264",
|
|
pixFmt: "yuvj420p",
|
|
probed: { colorTransfer: "bt709" },
|
|
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
|
|
},
|
|
{
|
|
name: "full range BT.709",
|
|
format: "jpg",
|
|
codec: "libx264",
|
|
pixFmt: "yuvj420p",
|
|
probed: { colorTransfer: "bt709" },
|
|
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
|
|
},
|
|
{
|
|
name: "BT.470BG gamma 2.8",
|
|
format: "png",
|
|
codec: "libx264",
|
|
pixFmt: "yuv420p",
|
|
probed: { colorTransfer: "bt470bg" },
|
|
tags: "color_primaries=bt470bg:color_trc=bt470bg:colorspace=bt470bg",
|
|
},
|
|
{
|
|
name: "RGB",
|
|
format: "png",
|
|
codec: "libx264rgb",
|
|
pixFmt: "rgb24",
|
|
probed: { colorSpace: "gbr" },
|
|
tags: "",
|
|
},
|
|
] as const)(
|
|
"keeps every frame and the colours of a $name VFR window that ends on a still ($format)",
|
|
async ({ name, format, codec, pixFmt, probed, tags }) => {
|
|
const fixture = join(FIXTURE_DIR, `vfr-still-${name.replace(/\W+/g, "-")}-${format}.mp4`);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=2:r=60`,
|
|
"-vf",
|
|
`select='not(between(n\\,30\\,89))'${tags ? `,setparams=${tags}` : ""}`,
|
|
"-fps_mode",
|
|
"vfr",
|
|
"-c:v",
|
|
codec,
|
|
"-preset",
|
|
"ultrafast",
|
|
"-pix_fmt",
|
|
pixFmt,
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const window = 0.616666;
|
|
const unfilteredDir = join(FIXTURE_DIR, `out-${basename(fixture)}-unfiltered`);
|
|
mkdirSync(unfilteredDir, { recursive: true });
|
|
const unfiltered = await runFfmpeg([
|
|
"-v",
|
|
"error",
|
|
"-ss",
|
|
"0",
|
|
"-i",
|
|
fixture,
|
|
"-t",
|
|
String(window),
|
|
"-fps_mode",
|
|
"cfr",
|
|
"-r",
|
|
"30",
|
|
join(unfilteredDir, "f_%05d.png"),
|
|
]);
|
|
expect(unfiltered.success).toBe(true);
|
|
|
|
const result = await extractVideoFramesRange(fixture, basename(fixture), 0, window, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-vfr-still"),
|
|
format,
|
|
});
|
|
|
|
expect(result.metadata.isVFR).toBe(true);
|
|
expect(result.metadata.colorSpace).toMatchObject(probed);
|
|
expect(result.totalFrames).toBe(readdirSync(unfilteredDir).length);
|
|
const last = result.framePaths.get(result.totalFrames - 1)!;
|
|
const source = readFirstFramePixel(fixture, 10, 10);
|
|
const shown = readFirstFramePixel(last, 10, 10);
|
|
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
|
|
expect(worst, `source ${source} ${format} ${shown}`).toBeLessThanOrEqual(
|
|
format === "png" ? 0 : 3,
|
|
);
|
|
if (format === "png") expect(pngChunkTypes(last)).toContain("sRGB");
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("leaves SDR-to-HDR frames in the HDR colours they were converted to", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-sdr-to-hdr-gate");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractVideoFramesRange(UI_FIXTURE, "sdr-to-hdr-gate", 0, 1, {
|
|
fps: 1,
|
|
outputDir,
|
|
format: "png",
|
|
sdrToHdrTransfer: "pq",
|
|
});
|
|
|
|
expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB");
|
|
}, 60_000);
|
|
|
|
it("keeps jpg and png extraction caches separate", async () => {
|
|
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-"));
|
|
try {
|
|
const defaultOut = join(FIXTURE_DIR, "cache-default");
|
|
const pngOut = join(FIXTURE_DIR, "cache-png");
|
|
const pngHitOut = join(FIXTURE_DIR, "cache-png-hit");
|
|
mkdirSync(defaultOut, { recursive: true });
|
|
mkdirSync(pngOut, { recursive: true });
|
|
mkdirSync(pngHitOut, { recursive: true });
|
|
|
|
const defaultResult = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: defaultOut },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(defaultResult.errors).toEqual([]);
|
|
expect(defaultResult.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(defaultResult.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(defaultResult.extracted[0]!.framePaths.get(0)!.endsWith(".jpg")).toBe(true);
|
|
|
|
const pngMiss = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: pngOut, format: "png" },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(pngMiss.errors).toEqual([]);
|
|
expect(pngMiss.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(pngMiss.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(pngMiss.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
|
|
|
|
const pngHit = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: pngHitOut, format: "png" },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(pngHit.errors).toEqual([]);
|
|
expect(pngHit.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(pngHit.phaseBreakdown.cacheMisses).toBe(0);
|
|
expect(pngHit.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
|
|
} finally {
|
|
rmSync(cacheDir, { recursive: true, force: true });
|
|
}
|
|
}, 60_000);
|
|
|
|
it("dedupes identical extractions within one render", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-dedupe");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const videoA: VideoElement = { ...fixtureVideo(), id: "dupe-a" };
|
|
const videoB: VideoElement = { ...fixtureVideo(), id: "dupe-b" };
|
|
|
|
const result = await extractAllVideoFrames([videoA, videoB], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir,
|
|
});
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.extracted).toHaveLength(2);
|
|
const first = result.extracted[0]!;
|
|
const second = result.extracted[1]!;
|
|
expect(first.videoId).toBe("dupe-a");
|
|
expect(second.videoId).toBe("dupe-b");
|
|
expect(second.outputDir).toBe(first.outputDir);
|
|
expect(Array.from(second.framePaths.entries())).toEqual(Array.from(first.framePaths.entries()));
|
|
|
|
const frameDirs = readdirSync(outputDir, { withFileTypes: true })
|
|
.filter((entry) => entry.isDirectory())
|
|
.map((entry) => entry.name);
|
|
expect(frameDirs).toEqual(["dupe-a"]);
|
|
expect(readdirSync(first.outputDir).filter((f) => f.endsWith(".jpg"))).toHaveLength(
|
|
first.totalFrames,
|
|
);
|
|
}, 60_000);
|
|
});
|
|
|
|
// Each output slot shows the frame on screen at its time, as the preview does.
|
|
describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {
|
|
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-sampling-"));
|
|
const WIDTH = 32;
|
|
const HEIGHT = 16;
|
|
// Matroska stores whole-millisecond timestamps (a 30 fps frame at 66.67 ms reads 67 ms);
|
|
// a 1/30 timescale stores each frame exactly on a coarse tick.
|
|
const SOURCES = {
|
|
"mp4-60": { fps: 60, file: "index-60fps.mp4", muxer: [] },
|
|
"mkv-30": { fps: 30, file: "index-30fps.mkv", muxer: [] },
|
|
"mp4-30-timescale-30": {
|
|
fps: 30,
|
|
file: "index-30fps-ts30.mp4",
|
|
muxer: ["-video_track_timescale", "30"],
|
|
},
|
|
} as const;
|
|
type SourceName = keyof typeof SOURCES;
|
|
const sourcePath = (name: SourceName) => join(FIXTURE_DIR, SOURCES[name].file);
|
|
|
|
beforeAll(async () => {
|
|
for (const name of Object.keys(SOURCES) as SourceName[]) {
|
|
// Frame k carries luma 16 + 2k, so each extracted frame names its source index.
|
|
const result = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`nullsrc=s=${WIDTH}x${HEIGHT}:r=${SOURCES[name].fps}:d=1.5,geq=lum='16+2*N':cb=128:cr=128`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-qp",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
...SOURCES[name].muxer,
|
|
sourcePath(name),
|
|
]);
|
|
if (!result.success) throw new Error(`index fixture synthesis failed: ${result.stderr}`);
|
|
}
|
|
}, 30_000);
|
|
|
|
afterAll(() => {
|
|
rmSync(FIXTURE_DIR, { recursive: true, force: true });
|
|
}, 30_000);
|
|
|
|
function sourceIndexes(extracted: ExtractedFrames): number[] {
|
|
const decoded = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
join(extracted.outputDir, extracted.framePattern),
|
|
"-f",
|
|
"rawvideo",
|
|
"-pix_fmt",
|
|
"gray",
|
|
"pipe:1",
|
|
]);
|
|
if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
|
|
const indexes: number[] = [];
|
|
for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) {
|
|
indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2));
|
|
}
|
|
return indexes;
|
|
}
|
|
|
|
it.each([
|
|
{ source: "mp4-60", fps: 24, startTime: 0, duration: 0.5 },
|
|
{ source: "mp4-60", fps: 10, startTime: 0.37, duration: 0.6 },
|
|
{ source: "mkv-30", fps: 30, startTime: 0, duration: 0.5 },
|
|
{ source: "mp4-30-timescale-30", fps: 60, startTime: 0, duration: 0.5 },
|
|
] as const)(
|
|
"$source: samples the frame on screen at each $fps fps slot from $startTime s",
|
|
async (c) => {
|
|
const extracted = await extractVideoFramesRange(
|
|
sourcePath(c.source),
|
|
`${c.source}-${c.fps}`,
|
|
c.startTime,
|
|
c.duration,
|
|
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "png" },
|
|
);
|
|
const onScreen = Array.from({ length: Math.round(c.duration * c.fps) }, (_, i) =>
|
|
Math.floor((c.startTime + i / c.fps) * SOURCES[c.source].fps + 1e-9),
|
|
);
|
|
expect(sourceIndexes(extracted)).toEqual(onScreen);
|
|
},
|
|
30_000,
|
|
);
|
|
|
|
// #5260: one ffmpeg process over a 26-minute clip hit ffmpegProcessTimeout.
|
|
it.each([
|
|
// 260 frames: one 240-frame segment, then 20 more.
|
|
{ startTime: 0.5, duration: 130, frames: 260 },
|
|
// 200.3 - 80.3 is 120.00000000000001 s: exactly one segment, not a second near-empty one.
|
|
{ startTime: 80.3, duration: 200.3 - 80.3, frames: 240 },
|
|
])(
|
|
"extracts $duration s from $startTime s at 2 fps across segments without a gap or repeat",
|
|
async (c) => {
|
|
const source = join(FIXTURE_DIR, "index-2fps-long.mp4");
|
|
if (!existsSync(source)) {
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`nullsrc=s=${WIDTH}x${HEIGHT}:r=2:d=205,geq=lum='16+2*mod(N\\,100)':cb=128:cr=128`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-qp",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
source,
|
|
]);
|
|
if (!synth.success) throw new Error(`long fixture synthesis failed: ${synth.stderr}`);
|
|
}
|
|
const extracted = await extractVideoFramesRange(
|
|
source,
|
|
`long-2fps-${c.startTime}`,
|
|
c.startTime,
|
|
c.duration,
|
|
{ fps: 2, outputDir: FIXTURE_DIR, format: "png" },
|
|
);
|
|
const onScreen = Array.from(
|
|
{ length: c.frames },
|
|
(_, i) => Math.floor((c.startTime + i / 2) * 2 + 1e-9) % 100,
|
|
);
|
|
expect(extracted.totalFrames).toBe(c.frames);
|
|
expect(sourceIndexes(extracted)).toEqual(onScreen);
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("stops extraction only on a stall, and its ffmpeg reports each frame it writes", async () => {
|
|
const source = join(FIXTURE_DIR, "paced-2s.mp4");
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],
|
|
...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=2`, "-pix_fmt", "yuv420p", source],
|
|
]);
|
|
if (!synth.success) throw new Error(`paced fixture synthesis failed: ${synth.stderr}`);
|
|
const { runFfmpeg: real } =
|
|
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
|
|
let requested: RunFfmpegOptions | undefined;
|
|
const reported: number[] = [];
|
|
const readStats = ffmpegStatsReader(({ frames }) => {
|
|
if (frames) reported.push(frames);
|
|
});
|
|
// Real-time pacing spreads the stats over the run. The stall clock itself is runFfmpeg's to
|
|
// test; here it is lifted so a starved runner cannot race it.
|
|
vi.mocked(runFfmpeg).mockImplementation((args, opts) => {
|
|
requested = opts;
|
|
return real(["-stats_period", "0.1", "-re", ...args], {
|
|
...opts,
|
|
timeout: 60_000,
|
|
onStderr: readStats,
|
|
});
|
|
});
|
|
try {
|
|
const extracted = await extractVideoFramesRange(
|
|
source,
|
|
"paced-stall-only",
|
|
0,
|
|
2,
|
|
{ fps: 10, outputDir: FIXTURE_DIR, format: "png" },
|
|
undefined,
|
|
{ ffmpegProcessTimeout: 2000 },
|
|
);
|
|
expect(extracted.totalFrames).toBe(20);
|
|
expect(requested).toMatchObject({ timeout: 2000 });
|
|
expect(new Set(reported).size).toBeGreaterThan(1);
|
|
} finally {
|
|
vi.mocked(runFfmpeg).mockImplementation(real);
|
|
}
|
|
}, 30_000);
|
|
|
|
it.each([
|
|
{ name: "one keyframe", keyint: "infinite", gapSeconds: 8 },
|
|
{ name: "a keyframe every second", keyint: "30", gapSeconds: 0 },
|
|
])(
|
|
"gives the stall clock the time to decode up to the seek point ($name)",
|
|
async ({ keyint, gapSeconds }) => {
|
|
const source = join(FIXTURE_DIR, `gop-${keyint}.mp4`);
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],
|
|
...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=10`, "-pix_fmt", "yuv420p"],
|
|
...["-x264-params", `keyint=${keyint}:scenecut=0`, source],
|
|
]);
|
|
if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`);
|
|
const { runFfmpeg: real } =
|
|
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
|
|
let requested: RunFfmpegOptions | undefined;
|
|
vi.mocked(runFfmpeg).mockImplementation((args, opts) => {
|
|
requested = opts;
|
|
return real(args, opts);
|
|
});
|
|
try {
|
|
await extractVideoFramesRange(
|
|
source,
|
|
`stall-seek-${keyint}`,
|
|
8,
|
|
1,
|
|
{ fps: 10, outputDir: FIXTURE_DIR, format: "png" },
|
|
undefined,
|
|
{ ffmpegProcessTimeout: 2000 },
|
|
);
|
|
expect(requested?.timeout).toBe(2000 + gapSeconds * 1000);
|
|
} finally {
|
|
vi.mocked(runFfmpeg).mockImplementation(real);
|
|
}
|
|
},
|
|
30_000,
|
|
);
|
|
|
|
it("stops a run that writes no new frame for the stall timeout", async () => {
|
|
// stdin stays open and empty, so ffmpeg waits for its first frame forever.
|
|
const stalled = await runFfmpeg(
|
|
["-f", "rawvideo", "-pix_fmt", "gray", "-s", "2x2", "-i", "pipe:0", "-f", "null", "-"],
|
|
{ timeout: 500 },
|
|
);
|
|
expect(stalled.terminationReason).toBe("inactivity");
|
|
}, 30_000);
|
|
|
|
// A distributed chunk extracts only the frames it renders; they must be the full extraction's.
|
|
describe("partial extraction", () => {
|
|
const gopSource = join(FIXTURE_DIR, "gop-30fps.mp4");
|
|
|
|
beforeAll(async () => {
|
|
// Real motion and long GOPs, so range seeks land mid-GOP.
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"testsrc2=s=64x36:r=30:d=20",
|
|
"-c:v",
|
|
"libx264",
|
|
"-g",
|
|
"250",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
gopSource,
|
|
]);
|
|
if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`);
|
|
}, 30_000);
|
|
|
|
function frameBytes(dir: string): Map<string, Buffer> {
|
|
return new Map(readdirSync(dir).map((name) => [name, readFileSync(join(dir, name))]));
|
|
}
|
|
|
|
it.each([
|
|
{ fps: 24, startTime: 1.3, duration: 17 },
|
|
{ fps: 60, startTime: 0, duration: 20 },
|
|
])(
|
|
"frameRanges at $fps fps write the same files as a full extraction",
|
|
async (c) => {
|
|
const full = await extractVideoFramesRange(
|
|
gopSource,
|
|
`full-${c.fps}`,
|
|
c.startTime,
|
|
c.duration,
|
|
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg" },
|
|
);
|
|
const last = full.totalFrames;
|
|
const ranges = [
|
|
{ firstFrame: 0, frames: 5 },
|
|
{ firstFrame: 97, frames: 40 },
|
|
{ firstFrame: last - 3, frames: 3 },
|
|
];
|
|
const partial = await extractVideoFramesRange(
|
|
gopSource,
|
|
`partial-${c.fps}`,
|
|
c.startTime,
|
|
c.duration,
|
|
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg", frameRanges: ranges },
|
|
);
|
|
const fullFrames = frameBytes(full.outputDir);
|
|
const partialFrames = frameBytes(partial.outputDir);
|
|
expect(partialFrames.size).toBe(48);
|
|
for (const [name, bytes] of partialFrames) {
|
|
expect(fullFrames.get(name)?.equals(bytes), name).toBe(true);
|
|
}
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it.each([
|
|
{ fps: 24, mediaStart: 3.3 },
|
|
{ fps: 30, mediaStart: 0 },
|
|
{ fps: 60, mediaStart: 7.77 },
|
|
])(
|
|
"a deferred window to the source end at $fps fps reports the full frame count",
|
|
async (c) => {
|
|
const video = (): VideoElement => ({
|
|
id: `defer-${c.fps}`,
|
|
src: gopSource,
|
|
start: 0,
|
|
end: Infinity,
|
|
mediaStart: c.mediaStart,
|
|
loop: false,
|
|
hasAudio: false,
|
|
});
|
|
const full = await extractAllVideoFrames([video()], FIXTURE_DIR, {
|
|
fps: c.fps,
|
|
outputDir: join(FIXTURE_DIR, `defer-full-${c.fps}`),
|
|
});
|
|
const deferred = await extractAllVideoFrames([video()], FIXTURE_DIR, {
|
|
fps: c.fps,
|
|
outputDir: join(FIXTURE_DIR, `defer-${c.fps}`),
|
|
deferRangeExtraction: true,
|
|
});
|
|
expect(deferred.errors).toEqual([]);
|
|
const result = deferred.extracted[0]!;
|
|
expect(result.totalFrames).toBe(full.extracted[0]!.totalFrames);
|
|
expect(result.framePaths.size).toBe(0);
|
|
expect(readdirSync(result.outputDir)).toEqual([]);
|
|
expect(result.deferredRange).toEqual({
|
|
startTime: c.mediaStart,
|
|
durationSeconds: expect.any(Number),
|
|
format: "jpg",
|
|
});
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("halves frames while they still cover the box a video is shown in", async () => {
|
|
const firstFrameSize = async (
|
|
videoId: string,
|
|
shownBox: { width: number; height: number },
|
|
) => {
|
|
const result = await extractVideoFramesRange(gopSource, videoId, 0, 1, {
|
|
fps: 30,
|
|
outputDir: FIXTURE_DIR,
|
|
format: "jpg",
|
|
frameRanges: [{ firstFrame: 0, frames: 1 }],
|
|
shownBox,
|
|
});
|
|
const [frame] = result.framePaths.values();
|
|
return spawnSync("ffprobe", [
|
|
"-v",
|
|
"error",
|
|
"-show_entries",
|
|
"stream=width,height",
|
|
"-of",
|
|
"csv=p=0",
|
|
"--",
|
|
frame!,
|
|
])
|
|
.stdout.toString()
|
|
.trim();
|
|
};
|
|
// 64x36 source: halved as often as the box allows (even sides round up); never below the box.
|
|
expect(await firstFrameSize("shown-quarter", { width: 16, height: 9 })).toBe("16,10");
|
|
expect(await firstFrameSize("shown-third", { width: 20, height: 12 })).toBe("32,18");
|
|
expect(await firstFrameSize("shown-near-full", { width: 40, height: 20 })).toBe("64,36");
|
|
}, 30_000);
|
|
});
|
|
});
|
|
|
|
describe.skipIf(!HAS_FFMPEG)("held tails on sparse-timestamp sources", () => {
|
|
const fixtureDir = mkdtempSync(join(tmpdir(), "hf-sparse-held-tail-"));
|
|
const cfrFixture = join(fixtureDir, "sub-1fps-cfr.mp4");
|
|
const vfrFixture = join(fixtureDir, "sparse-vfr.mp4");
|
|
const nonZeroStartFixture = join(fixtureDir, "nonzero-start.mp4");
|
|
const negativeStartTransportFixture = join(fixtureDir, "negative-start.ts");
|
|
|
|
beforeAll(async () => {
|
|
const fixtures = [
|
|
{
|
|
path: cfrFixture,
|
|
input: "testsrc2=s=64x64:d=10:rate=1/5",
|
|
filters: [] as string[],
|
|
},
|
|
{
|
|
path: vfrFixture,
|
|
input: "testsrc2=s=64x64:d=10:rate=1/2",
|
|
filters: ["-vf", "select='eq(n,0)+eq(n,2)'", "-fps_mode", "vfr"],
|
|
},
|
|
{
|
|
path: nonZeroStartFixture,
|
|
input: "testsrc2=s=64x64:d=3:rate=1",
|
|
filters: ["-output_ts_offset", "5"],
|
|
},
|
|
{
|
|
path: negativeStartTransportFixture,
|
|
input: "testsrc2=s=64x64:d=3:rate=1",
|
|
filters: [
|
|
"-mpegts_copyts",
|
|
"1",
|
|
"-muxdelay",
|
|
"0",
|
|
"-avoid_negative_ts",
|
|
"disabled",
|
|
"-output_ts_offset",
|
|
"-2",
|
|
],
|
|
},
|
|
];
|
|
for (const fixture of fixtures) {
|
|
const result = await runFfmpeg([
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
fixture.input,
|
|
...fixture.filters,
|
|
"-c:v",
|
|
"libx264",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
"-y",
|
|
fixture.path,
|
|
]);
|
|
if (!result.success) {
|
|
throw new Error(`sparse fixture synthesis failed: ${result.stderr.slice(-400)}`);
|
|
}
|
|
}
|
|
}, 30_000);
|
|
|
|
afterAll(() => {
|
|
rmSync(fixtureDir, { recursive: true, force: true });
|
|
}, 30_000);
|
|
|
|
it.each([
|
|
{
|
|
label: "sub-1fps CFR",
|
|
src: cfrFixture,
|
|
expectedVfr: false,
|
|
finalTimestamp: 5,
|
|
streamStart: 0,
|
|
},
|
|
{
|
|
label: "sparse VFR",
|
|
src: vfrFixture,
|
|
expectedVfr: true,
|
|
finalTimestamp: 4,
|
|
streamStart: 0,
|
|
},
|
|
{
|
|
label: "non-zero stream start",
|
|
src: nonZeroStartFixture,
|
|
expectedVfr: false,
|
|
finalTimestamp: 2,
|
|
streamStart: 5,
|
|
},
|
|
{
|
|
label: "unindexed negative-base MPEG-TS",
|
|
src: negativeStartTransportFixture,
|
|
expectedVfr: false,
|
|
finalTimestamp: 2,
|
|
streamStart: -2,
|
|
},
|
|
])(
|
|
"extracts one real final SDR frame for $label",
|
|
async ({ src, expectedVfr, finalTimestamp, streamStart }) => {
|
|
const metadata = await extractVideoMetadata(src);
|
|
expect(metadata.fps).toBeLessThanOrEqual(1);
|
|
expect(metadata.isVFR).toBe(expectedVfr);
|
|
expect(metadata.videoStreamStartSeconds).toBeCloseTo(streamStart, 6);
|
|
await expect(extractFinalVideoFrameTimestamp(src, metadata)).resolves.toBe(finalTimestamp);
|
|
const outputDir = mkdtempSync(join(fixtureDir, "out-"));
|
|
const video: VideoElement = {
|
|
id: `held-${String(expectedVfr)}`,
|
|
src,
|
|
start: -15,
|
|
end: 5,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
|
|
const result = await extractAllVideoFrames([video], fixtureDir, {
|
|
fps: 30,
|
|
format: "png",
|
|
outputDir,
|
|
timelineEnd: 2,
|
|
});
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.extracted).toHaveLength(1);
|
|
expect(result.extracted[0]?.totalFrames).toBe(1);
|
|
expect(video).toMatchObject({ start: 0, end: 5, loop: false });
|
|
expect(video.mediaStart).toBeCloseTo(metadata.videoStreamDurationSeconds - 0.000001, 7);
|
|
},
|
|
30_000,
|
|
);
|
|
|
|
it("renders the same final decoded frame just inside and past EOF", async () => {
|
|
const metadata = await extractVideoMetadata(cfrFixture);
|
|
const sourceDuration = metadata.videoStreamDurationSeconds;
|
|
const outputDir = mkdtempSync(join(fixtureDir, "eof-out-"));
|
|
const videos: VideoElement[] = [
|
|
{
|
|
id: "just-inside-eof",
|
|
src: cfrFixture,
|
|
start: 0,
|
|
end: 5,
|
|
mediaStart: sourceDuration - 0.001,
|
|
loop: false,
|
|
hasAudio: false,
|
|
},
|
|
{
|
|
id: "past-eof",
|
|
src: cfrFixture,
|
|
start: 0,
|
|
end: 5,
|
|
mediaStart: sourceDuration + 1,
|
|
loop: false,
|
|
hasAudio: false,
|
|
},
|
|
];
|
|
|
|
const result = await extractAllVideoFrames(videos, fixtureDir, {
|
|
fps: 30,
|
|
format: "png",
|
|
outputDir,
|
|
timelineEnd: 5,
|
|
});
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.extracted).toHaveLength(2);
|
|
const insideFrame = result.extracted[0]?.framePaths.get(0);
|
|
const pastFrame = result.extracted[1]?.framePaths.get(0);
|
|
expect(insideFrame).toBeDefined();
|
|
expect(pastFrame).toBeDefined();
|
|
if (!insideFrame || !pastFrame) throw new Error("expected both final-frame outputs");
|
|
expect(readFileSync(pastFrame)).toEqual(readFileSync(insideFrame));
|
|
}, 30_000);
|
|
});
|
|
|
|
// Regression test for the VFR (variable frame rate) freeze bug.
|
|
// Screen recordings and phone videos often have irregular timestamps.
|
|
// When such inputs hit `extractVideoFramesRange`'s `-ss <start> -i ... -t <dur>
|
|
// -vf fps=N` pipeline, the fps filter can emit fewer frames than requested —
|
|
// e.g. a 4-second segment at 30fps would produce ~90 frames instead of 120.
|
|
// FrameLookupTable.getFrameAtTime then returns null for out-of-range indices
|
|
// and the compositor holds the last valid frame, which the user perceives as
|
|
// the video freezing. extractAllVideoFrames now routes VFR sources through
|
|
// FFmpeg's one-pass `-fps_mode cfr -r` extraction path to fix this without a
|
|
// separate normalization encode.
|
|
describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
|
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-vfr-test-"));
|
|
const VFR_FIXTURE = join(FIXTURE_DIR, "vfr_screen.mp4");
|
|
|
|
beforeAll(async () => {
|
|
// 10s testsrc2 at 60fps, ~40% of frames dropped via select filter and
|
|
// encoded with -fps_mode vfr so timestamps are irregular. Declared fps 60,
|
|
// actual average ~36 — well over the 10% threshold used by isVFR.
|
|
// The select expression drops four 1-second windows (frames 30-89,
|
|
// 180-239, 330-389, 480-539) to simulate static segments in a screen
|
|
// recording where no pixels changed.
|
|
// -g/-keyint_min 600 forces a single keyframe so mid-segment seeks in the
|
|
// mediaStart=3 test don't snap to an intermediate IDR and drift the count.
|
|
const result = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"testsrc2=s=320x180:d=10:rate=60",
|
|
"-vf",
|
|
"select='not(between(n\\,30\\,89))*not(between(n\\,180\\,239))*not(between(n\\,330\\,389))*not(between(n\\,480\\,539))'",
|
|
"-fps_mode",
|
|
"vfr",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
"-g",
|
|
"600",
|
|
"-keyint_min",
|
|
"600",
|
|
VFR_FIXTURE,
|
|
]);
|
|
if (!result.success) {
|
|
throw new Error(
|
|
`ffmpeg fixture synthesis failed (${result.exitCode}): ${result.stderr.slice(-400)}`,
|
|
);
|
|
}
|
|
}, 30_000);
|
|
|
|
afterAll(() => {
|
|
if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
|
|
}, 30_000);
|
|
|
|
it("skips a clip entirely before time zero without reporting an extraction error", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-before-timeline");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
const video: VideoElement = {
|
|
id: "before-timeline",
|
|
src: VFR_FIXTURE,
|
|
start: -2,
|
|
end: -1,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
|
|
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir,
|
|
timelineEnd: 2,
|
|
});
|
|
|
|
expect(result).toMatchObject({ success: true, extracted: [], errors: [] });
|
|
});
|
|
|
|
it("preserves loop phase when negative preroll crosses the source boundary", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-negative-loop");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
const video: VideoElement = {
|
|
id: "negative-loop",
|
|
src: VFR_FIXTURE,
|
|
start: -19,
|
|
end: 5,
|
|
mediaStart: 0,
|
|
loop: true,
|
|
hasAudio: false,
|
|
};
|
|
|
|
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir,
|
|
timelineEnd: 2,
|
|
});
|
|
|
|
expect(result.errors).toEqual([]);
|
|
const extracted = result.extracted[0];
|
|
if (!extracted) throw new Error("expected loop source frames");
|
|
const lookup = createFrameLookupTable([video], result.extracted);
|
|
expect(video).toMatchObject({ start: -19, mediaStart: 0, loop: true });
|
|
expect(lookup.getFrame("negative-loop", 0)).toBe(
|
|
extracted.framePaths.get(extracted.totalFrames - 1),
|
|
);
|
|
}, 30_000);
|
|
|
|
it("preserves the held final frame after negative preroll exhausts a source", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-negative-held-tail");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
const video: VideoElement = {
|
|
id: "negative-held-tail",
|
|
src: VFR_FIXTURE,
|
|
start: -15,
|
|
end: 5,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
|
|
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir,
|
|
timelineEnd: 2,
|
|
});
|
|
|
|
expect(result.errors).toEqual([]);
|
|
const extracted = result.extracted[0];
|
|
if (!extracted) throw new Error("expected held-tail source frames");
|
|
const lookup = createFrameLookupTable([video], result.extracted);
|
|
// The authored slot remains active through end=5, but lookup is rebased to
|
|
// one exact final frame instead of assuming the last second contains a
|
|
// timestamp or materializing the full source.
|
|
expect(video).toMatchObject({ start: 0, end: 5, mediaStart: 9.999999, loop: false });
|
|
expect(extracted.totalFrames).toBe(1);
|
|
expect(lookup.getFrame("negative-held-tail", 0)).toBe(
|
|
extracted.framePaths.get(extracted.totalFrames - 1),
|
|
);
|
|
}, 30_000);
|
|
|
|
it("detects the synthesized fixture as VFR", async () => {
|
|
const md = await extractVideoMetadata(VFR_FIXTURE);
|
|
expect(md.isVFR).toBe(true);
|
|
});
|
|
|
|
it.each([
|
|
["compiled EOF", 4, 1],
|
|
["compiled past EOF", 5, 2],
|
|
["ordinary explicit zero", 6, 0],
|
|
])(
|
|
"drops a known zero timeline window without extraction or error: %s",
|
|
async (label, start, mediaStart) => {
|
|
const src = await synthCfrClip(`zero-window-${label.replaceAll(" ", "-")}.mp4`, 1);
|
|
const videos = parseVideoElements(
|
|
`<video id="zero-window" src="${src}" data-start="${start}" data-duration="0" data-end="${start}" data-media-start="${mediaStart}" muted></video>`,
|
|
);
|
|
const outputDir = join(FIXTURE_DIR, `out-zero-window-${label.replaceAll(" ", "-")}`);
|
|
const result = await extractAllVideoFrames(videos, FIXTURE_DIR, { fps: 30, outputDir });
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.extracted).toEqual([]);
|
|
expect(result.totalFramesExtracted).toBe(0);
|
|
expect(videos).toEqual([]);
|
|
},
|
|
);
|
|
|
|
it("passes an exact 24000/1001 rate through VFR normalization", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-vfr-ntsc-boundary");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractVideoFramesRange(VFR_FIXTURE, "vfr-ntsc-boundary", 0, 0.125125, {
|
|
fps: { num: 24000, den: 1001 },
|
|
outputDir,
|
|
format: "jpg",
|
|
});
|
|
|
|
expect(result.metadata.isVFR).toBe(true);
|
|
expect(result.totalFrames).toBe(3);
|
|
}, 60_000);
|
|
|
|
// With B-frames, mp4 packet durations are decode-order gaps, so a frame can claim 0.5s while
|
|
// the next one starts 17ms later. Each slot must still show the frame on screen at its time:
|
|
// `-fps_mode cfr` on ffmpeg 8 held such a frame for 14 slots and cut the still before a gap short.
|
|
it("samples a B-frame VFR source by timestamp, not by packet duration", async () => {
|
|
const source = join(FIXTURE_DIR, "vfr_bframes.mp4");
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"testsrc2=s=320x180:d=4:rate=60",
|
|
"-vf",
|
|
"select='not(between(n\\,30\\,89))*not(between(n\\,150\\,179))'",
|
|
"-fps_mode",
|
|
"vfr",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"medium",
|
|
"-bf",
|
|
"3",
|
|
"-g",
|
|
"600",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
source,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const distinctFrames = (paths: Iterable<string>) =>
|
|
new Set([...paths].map((path) => readFileSync(path).toString("base64"))).size;
|
|
|
|
// 2.5s to 3s is a still; every 30fps slot after it lands on its own 60fps source frame.
|
|
const afterGap = await extractVideoFramesRange(source, "vfr-bframes-after-gap", 2.5, 1.5, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-vfr-bframes-after-gap"),
|
|
format: "png",
|
|
});
|
|
expect(afterGap.metadata.isVFR).toBe(true);
|
|
expect(afterGap.totalFrames).toBe(45);
|
|
const slots = [...afterGap.framePaths.entries()].sort(([a], [b]) => a - b);
|
|
expect(distinctFrames(slots.slice(0, 15).map(([, path]) => path))).toBe(1);
|
|
expect(distinctFrames(slots.slice(15).map(([, path]) => path))).toBe(30);
|
|
|
|
const endsOnStill = await extractVideoFramesRange(source, "vfr-bframes-still", 0, 3, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-vfr-bframes-still"),
|
|
format: "png",
|
|
});
|
|
expect(endsOnStill.totalFrames).toBe(90);
|
|
}, 60_000);
|
|
|
|
it("produces the expected frame count for a mid-file segment", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-mid-segment");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const video: VideoElement = {
|
|
id: "v1",
|
|
src: VFR_FIXTURE,
|
|
start: 0,
|
|
end: 4,
|
|
mediaStart: 3,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
|
|
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: 30,
|
|
outputDir,
|
|
});
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.extracted).toHaveLength(1);
|
|
const frames = readdirSync(join(outputDir, "v1")).filter((f) => f.endsWith(".jpg"));
|
|
// Pre-fix behavior produced ~90 frames (a 25% shortfall).
|
|
// ±3 tolerance: FFmpeg's one-pass VFR→CFR extraction yields slightly
|
|
// different frame counts across versions (timestamp rounding).
|
|
expect(frames.length).toBeGreaterThanOrEqual(117);
|
|
expect(frames.length).toBeLessThanOrEqual(123);
|
|
|
|
expect(result.phaseBreakdown).toBeDefined();
|
|
expect(result.phaseBreakdown.extractMs).toBeGreaterThan(0);
|
|
expect(result.phaseBreakdown.vfrPreflightCount).toBe(1);
|
|
expect(result.phaseBreakdown.vfrPreflightMs).toBeGreaterThanOrEqual(0);
|
|
}, 60_000);
|
|
|
|
// Shared fixture helpers for the cache tests below. All synthesize clean
|
|
// CFR SDR clips — keeps VFR preflight count at zero so cache keys are
|
|
// stable across runs within a test.
|
|
async function synthCfrClip(name: string, durationSeconds: number): Promise<string> {
|
|
const src = join(FIXTURE_DIR, name);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`testsrc2=s=320x180:d=${durationSeconds}:rate=30`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
src,
|
|
]);
|
|
if (!synth.success) {
|
|
throw new Error(`Fixture synthesis failed (${name}): ${synth.stderr.slice(-400)}`);
|
|
}
|
|
return src;
|
|
}
|
|
|
|
it("rejects a media start beyond source duration before invoking FFmpeg", async () => {
|
|
const src = await synthCfrClip("zero-output-src.mp4", 1);
|
|
const outputDir = join(FIXTURE_DIR, "out-zero-output");
|
|
await expect(
|
|
extractVideoFramesRange(src, "past-eof", 2, 1, { fps: 30, outputDir }),
|
|
).rejects.toMatchObject({
|
|
kind: "media_start_out_of_range",
|
|
retryable: false,
|
|
});
|
|
}, 60_000);
|
|
|
|
it("preserves legacy metadata rejection unless typed aggregation is explicitly enabled", async () => {
|
|
const src = join(FIXTURE_DIR, "invalid-probe.mp4");
|
|
writeFileSync(src, "not a media container");
|
|
const video = cfrClipElement("invalid-probe", src, 1);
|
|
|
|
await expect(
|
|
extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-invalid-probe-legacy"),
|
|
}),
|
|
).rejects.toThrow();
|
|
|
|
const collected = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-invalid-probe-typed"),
|
|
collectProbeFailures: true,
|
|
});
|
|
expect(collected.success).toBe(false);
|
|
expect(collected.extracted).toEqual([]);
|
|
expect(collected.errors).toEqual([
|
|
expect.objectContaining({
|
|
videoId: "invalid-probe",
|
|
kind: "invalid_media",
|
|
retryable: false,
|
|
}),
|
|
]);
|
|
}, 60_000);
|
|
|
|
async function synthHdrTaggedClip(name: string, durationSeconds: number): Promise<string> {
|
|
const src = join(FIXTURE_DIR, name);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`testsrc2=s=320x180:d=${durationSeconds}:rate=30`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
"-color_primaries",
|
|
"bt2020",
|
|
"-color_trc",
|
|
"smpte2084",
|
|
"-colorspace",
|
|
"bt2020nc",
|
|
// The -color_* flags above only tag the container/encoder context;
|
|
// whether they reach the H.264 VUI depends on the ffmpeg build (the
|
|
// pinned Windows CI build drops the transfer). Write the VUI directly
|
|
// so probing reports smpte2084 on every build (9/16/9 = bt2020/PQ/bt2020nc).
|
|
"-bsf:v",
|
|
"h264_metadata=colour_primaries=9:transfer_characteristics=16:matrix_coefficients=9",
|
|
src,
|
|
]);
|
|
if (!synth.success) {
|
|
throw new Error(`HDR fixture synthesis failed (${name}): ${synth.stderr.slice(-400)}`);
|
|
}
|
|
return src;
|
|
}
|
|
|
|
function cfrClipElement(
|
|
id: string,
|
|
src: string,
|
|
endSeconds: number,
|
|
mediaStart = 0,
|
|
): VideoElement {
|
|
return {
|
|
id,
|
|
src,
|
|
start: 0,
|
|
end: endSeconds,
|
|
mediaStart,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
}
|
|
|
|
async function extractWithCache(
|
|
video: VideoElement,
|
|
outName: string,
|
|
cacheDir: string,
|
|
fps = 30,
|
|
): Promise<ExtractionResult> {
|
|
const outputDir = join(FIXTURE_DIR, outName);
|
|
mkdirSync(outputDir, { recursive: true });
|
|
return extractAllVideoFrames([video], FIXTURE_DIR, { fps, outputDir }, undefined, {
|
|
extractCacheDir: cacheDir,
|
|
});
|
|
}
|
|
|
|
function cacheEntryNames(cacheDir: string): string[] {
|
|
return readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX));
|
|
}
|
|
|
|
function supersetDirNames(outputDir: string): string[] {
|
|
if (!existsSync(outputDir)) return [];
|
|
return readdirSync(outputDir).filter((name) => name.startsWith("__superset-"));
|
|
}
|
|
|
|
function extractedFor(result: ExtractionResult, videoId: string): ExtractedFrames {
|
|
const extracted = result.extracted.find((item) => item.videoId === videoId);
|
|
if (!extracted) throw new Error(`missing extraction result for ${videoId}`);
|
|
return extracted;
|
|
}
|
|
|
|
function framePath(result: ExtractionResult, videoId: string, frameIndex: number): string {
|
|
const extracted = extractedFor(result, videoId);
|
|
const frame = extracted?.framePaths.get(frameIndex);
|
|
if (!frame) throw new Error(`missing frame ${frameIndex} for ${videoId}`);
|
|
return frame;
|
|
}
|
|
|
|
it("reuses extracted frames on a warm cache hit", async () => {
|
|
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
|
|
const SRC = await synthCfrClip("cache-src.mp4", 2);
|
|
const video = cfrClipElement("cv1", SRC, 2);
|
|
|
|
const miss = await extractWithCache(video, "out-cache-miss", CACHE_DIR);
|
|
expect(miss.errors).toEqual([]);
|
|
expect(miss.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(miss.phaseBreakdown.cacheMisses).toBe(1);
|
|
const ffmpegRunsBeforeHit = vi.mocked(runFfmpeg).mock.calls.length;
|
|
|
|
const hit = await extractWithCache(video, "out-cache-hit", CACHE_DIR);
|
|
expect(hit.errors).toEqual([]);
|
|
expect(hit.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(hit.phaseBreakdown.cacheMisses).toBe(0);
|
|
expect(vi.mocked(runFfmpeg).mock.calls.length).toBe(ffmpegRunsBeforeHit);
|
|
expect(hit.extracted).toHaveLength(1);
|
|
expect(hit.extracted[0]!.totalFrames).toBe(miss.extracted[0]!.totalFrames);
|
|
|
|
rmSync(CACHE_DIR, { recursive: true, force: true });
|
|
}, 60_000);
|
|
|
|
it("does not reuse a decimal-rate VFR cache entry for the exact rational rate", async () => {
|
|
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-cache-ntsc-rate-test-"));
|
|
const decimalOutputDir = join(FIXTURE_DIR, "out-cache-vfr-ntsc-decimal");
|
|
const rationalOutputDir = join(FIXTURE_DIR, "out-cache-vfr-ntsc-rational");
|
|
mkdirSync(decimalOutputDir, { recursive: true });
|
|
mkdirSync(rationalOutputDir, { recursive: true });
|
|
try {
|
|
const decimal = await extractAllVideoFrames(
|
|
[cfrClipElement("vfr-ntsc-cache", VFR_FIXTURE, 0.125125)],
|
|
FIXTURE_DIR,
|
|
{ fps: 24000 / 1001, outputDir: decimalOutputDir },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(decimal.errors).toEqual([]);
|
|
expect(decimal.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(decimal.extracted[0]?.totalFrames).toBe(3);
|
|
// Model the warm v3 entry from before exact-rate extraction: the
|
|
// decimal path could persist one extra frame at this boundary. If the
|
|
// rational lookup collides, rehydration below will observe all four.
|
|
writeFileSync(
|
|
join(decimal.extracted[0]!.outputDir, "frame_00004.jpg"),
|
|
"stale-decimal-boundary-frame",
|
|
"utf-8",
|
|
);
|
|
|
|
const rational = await extractAllVideoFrames(
|
|
[cfrClipElement("vfr-ntsc-cache", VFR_FIXTURE, 0.125125)],
|
|
FIXTURE_DIR,
|
|
{ fps: { num: 24000, den: 1001 }, outputDir: rationalOutputDir },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(rational.errors).toEqual([]);
|
|
expect(rational.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(rational.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(rational.extracted[0]?.totalFrames).toBe(3);
|
|
expect(cacheEntryNames(cacheDir)).toHaveLength(2);
|
|
} finally {
|
|
rmSync(cacheDir, { recursive: true, force: true });
|
|
}
|
|
}, 60_000);
|
|
|
|
it("reuses one-cycle loop extraction across different authored starts", async () => {
|
|
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-loop-phase-cache-test-"));
|
|
const src = await synthCfrClip("cache-loop-phase-src.mp4", 3);
|
|
try {
|
|
const firstOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-first");
|
|
const secondOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-second");
|
|
mkdirSync(firstOutputDir, { recursive: true });
|
|
mkdirSync(secondOutputDir, { recursive: true });
|
|
|
|
const first = await extractAllVideoFrames(
|
|
[
|
|
{
|
|
...cfrClipElement("loop-cache-first", src, 60),
|
|
loop: true,
|
|
},
|
|
],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir: firstOutputDir, timelineEnd: 60 },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(first.errors).toEqual([]);
|
|
expect(first.phaseBreakdown.cacheMisses).toBe(1);
|
|
|
|
const second = await extractAllVideoFrames(
|
|
[
|
|
{
|
|
...cfrClipElement("loop-cache-second", src, 66),
|
|
start: -6,
|
|
loop: true,
|
|
},
|
|
],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir: secondOutputDir, timelineEnd: 60 },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(second.errors).toEqual([]);
|
|
expect(second.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(second.phaseBreakdown.cacheMisses).toBe(0);
|
|
expect(second.extracted[0]?.totalFrames).toBe(first.extracted[0]?.totalFrames);
|
|
} finally {
|
|
rmSync(cacheDir, { recursive: true, force: true });
|
|
}
|
|
}, 60_000);
|
|
|
|
it("updates the cache sentinel mtime on a hit", async () => {
|
|
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-touch-test-"));
|
|
const SRC = await synthCfrClip("cache-touch-src.mp4", 1);
|
|
const video = cfrClipElement("touch", SRC, 1);
|
|
|
|
const miss = await extractWithCache(video, "out-cache-touch-miss", CACHE_DIR);
|
|
expect(miss.errors).toEqual([]);
|
|
expect(miss.phaseBreakdown.cacheMisses).toBe(1);
|
|
|
|
const cacheEntryNames = readdirSync(CACHE_DIR).filter((name) => name.startsWith(SCHEMA_PREFIX));
|
|
expect(cacheEntryNames).toHaveLength(1);
|
|
const sentinel = join(CACHE_DIR, cacheEntryNames[0]!, COMPLETE_SENTINEL);
|
|
const old = new Date(Date.now() - 120_000);
|
|
utimesSync(sentinel, old, old);
|
|
const before = statSync(sentinel).mtimeMs;
|
|
|
|
const hit = await extractWithCache(video, "out-cache-touch-hit", CACHE_DIR);
|
|
|
|
expect(hit.errors).toEqual([]);
|
|
expect(hit.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(statSync(sentinel).mtimeMs).toBeGreaterThan(before);
|
|
|
|
rmSync(CACHE_DIR, { recursive: true, force: true });
|
|
}, 60_000);
|
|
|
|
it("skips cache GC on all-hit renders", async () => {
|
|
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-gc-skip-test-"));
|
|
const SRC = await synthCfrClip("cache-gc-skip-src.mp4", 1);
|
|
const video = cfrClipElement("gc-skip", SRC, 1);
|
|
|
|
const miss = await extractWithCache(video, "out-cache-gc-skip-miss", CACHE_DIR);
|
|
expect(miss.errors).toEqual([]);
|
|
expect(miss.phaseBreakdown.cacheMisses).toBe(1);
|
|
|
|
const agedPartial = join(CACHE_DIR, `${SCHEMA_PREFIX}aged.partial-1234-deadbeef`);
|
|
mkdirSync(agedPartial, { recursive: true });
|
|
writeFileSync(join(agedPartial, "frame_00001.jpg"), "stale", "utf-8");
|
|
const old = new Date(Date.now() - 2 * 60 * 60 * 1000);
|
|
utimesSync(agedPartial, old, old);
|
|
|
|
const hit = await extractWithCache(video, "out-cache-gc-skip-hit", CACHE_DIR);
|
|
expect(hit.errors).toEqual([]);
|
|
expect(hit.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(hit.phaseBreakdown.cacheMisses).toBe(0);
|
|
expect(existsSync(agedPartial)).toBe(true);
|
|
|
|
rmSync(CACHE_DIR, { recursive: true, force: true });
|
|
}, 60_000);
|
|
|
|
it("disables caching for this render when the cache dir is not writable", async () => {
|
|
const CACHE_FILE = join(FIXTURE_DIR, "cache-dir-is-a-file");
|
|
writeFileSync(CACHE_FILE, "not a directory", "utf-8");
|
|
const SRC = await synthCfrClip("cache-disabled-src.mp4", 1);
|
|
|
|
const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
|
|
try {
|
|
const result = await extractWithCache(
|
|
cfrClipElement("uncached", SRC, 1),
|
|
"out-cache-disabled",
|
|
CACHE_FILE,
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.extracted).toHaveLength(1);
|
|
expect(result.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(result.phaseBreakdown.cacheMisses).toBe(0);
|
|
expect(stderr).toHaveBeenCalledTimes(1);
|
|
expect(String(stderr.mock.calls[0]?.[0])).toContain("extraction cache dir");
|
|
expect(String(stderr.mock.calls[0]?.[0])).toContain("caching disabled for this render");
|
|
} finally {
|
|
stderr.mockRestore();
|
|
}
|
|
}, 60_000);
|
|
|
|
it("invalidates the cache when fps changes", async () => {
|
|
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-"));
|
|
const SRC = await synthCfrClip("cache-fps-src.mp4", 1);
|
|
const video = cfrClipElement("cv2", SRC, 1);
|
|
|
|
const first = await extractWithCache(video, "out-cache-fps-30", CACHE_DIR);
|
|
expect(first.phaseBreakdown.cacheMisses).toBe(1);
|
|
|
|
const second = await extractWithCache(video, "out-cache-fps-60", CACHE_DIR, 60);
|
|
expect(second.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(second.phaseBreakdown.cacheHits).toBe(0);
|
|
|
|
rmSync(CACHE_DIR, { recursive: true, force: true });
|
|
}, 60_000);
|
|
|
|
it("applies SDR→HDR conversion during extraction without normalized intermediates", async () => {
|
|
const SDR_LONG = await synthCfrClip("sdr-long.mp4", 10);
|
|
const HDR_SHORT = await synthHdrTaggedClip("hdr-short.mp4", 2);
|
|
const outputDir = join(FIXTURE_DIR, "out-hdr-segment");
|
|
const plainOutputDir = join(FIXTURE_DIR, "out-hdr-plain-sdr");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
mkdirSync(plainOutputDir, { recursive: true });
|
|
|
|
const videos: VideoElement[] = [
|
|
{ id: "sdr", src: SDR_LONG, start: 0, end: 2, mediaStart: 0, loop: false, hasAudio: false },
|
|
{
|
|
id: "hdr",
|
|
src: HDR_SHORT,
|
|
start: 2,
|
|
end: 4,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
},
|
|
];
|
|
|
|
const result = await extractAllVideoFrames(videos, FIXTURE_DIR, {
|
|
fps: 30,
|
|
outputDir,
|
|
});
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.phaseBreakdown.hdrPreflightCount).toBe(1);
|
|
expect(existsSync(join(outputDir, "_hdr_normalized"))).toBe(false);
|
|
|
|
const sdrFrames = result.extracted.find((item) => item.videoId === "sdr");
|
|
expect(sdrFrames?.totalFrames).toBe(60);
|
|
|
|
const plain = await extractVideoFramesRange(SDR_LONG, "plain-sdr", 0, 2, {
|
|
fps: 30,
|
|
outputDir: plainOutputDir,
|
|
format: "jpg",
|
|
});
|
|
expect(plain.totalFrames).toBe(60);
|
|
expect(
|
|
readFileSync(framePath(result, "sdr", 0)).equals(readFileSync(plain.framePaths.get(0)!)),
|
|
).toBe(false);
|
|
}, 60_000);
|
|
|
|
it("leaves SDR clips alone in a mixed timeline that tone-maps to SDR", async () => {
|
|
const SDR = await synthCfrClip("sdr-tonemap-peer.mp4", 1);
|
|
const HDR = await synthHdrTaggedClip("hdr-tonemap-peer.mp4", 1);
|
|
const outputDir = join(FIXTURE_DIR, "out-hdr-tonemap-mixed");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[cfrClipElement("sdr", SDR, 1), { ...cfrClipElement("hdr", HDR, 1), start: 1, end: 2 }],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir, toneMapHdrToSdr: true },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.phaseBreakdown.hdrPreflightCount).toBe(0);
|
|
}, 60_000);
|
|
|
|
it("keeps a finite SDR past-EOF slot in a mixed HDR timeline", async () => {
|
|
const SDR_SHORT = await synthCfrClip("sdr-past-eof.mp4", 1);
|
|
const HDR_SHORT = await synthHdrTaggedClip("hdr-past-eof-peer.mp4", 1);
|
|
const outputDir = join(FIXTURE_DIR, "out-hdr-past-eof");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[
|
|
cfrClipElement("sdr-past-eof", SDR_SHORT, 4, 5),
|
|
{ ...cfrClipElement("hdr-peer", HDR_SHORT, 1), start: 1, end: 2 },
|
|
],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir },
|
|
);
|
|
|
|
// The SDR slot must survive the mixed-HDR preflight and use the held-tail
|
|
// path. Reverting its guard to `mediaStart >= playableDuration` records an
|
|
// out-of-range error here and drops the slot before extraction.
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.phaseBreakdown.hdrPreflightCount).toBe(1);
|
|
expect(extractedFor(result, "sdr-past-eof").totalFrames).toBe(1);
|
|
expect(extractedFor(result, "hdr-peer").totalFrames).toBeGreaterThan(0);
|
|
}, 60_000);
|
|
|
|
it("keeps SDR→HDR cache entries distinct from plain SDR entries", async () => {
|
|
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-hdr-cache-test-"));
|
|
const SDR = await synthCfrClip("cache-hdr-sdr.mp4", 1);
|
|
const HDR = await synthHdrTaggedClip("cache-hdr-hdr.mp4", 1);
|
|
try {
|
|
const mixedOutputDir = join(FIXTURE_DIR, "out-cache-hdr-mixed");
|
|
mkdirSync(mixedOutputDir, { recursive: true });
|
|
const mixed = await extractAllVideoFrames(
|
|
[
|
|
cfrClipElement("sdr-transform", SDR, 1),
|
|
{ ...cfrClipElement("hdr-peer", HDR, 1), start: 1, end: 2 },
|
|
],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir: mixedOutputDir },
|
|
undefined,
|
|
{ extractCacheDir: CACHE_DIR },
|
|
);
|
|
expect(mixed.errors).toEqual([]);
|
|
expect(mixed.phaseBreakdown.hdrPreflightCount).toBe(1);
|
|
expect(mixed.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(mixed.phaseBreakdown.cacheMisses).toBe(2);
|
|
|
|
const plain = await extractWithCache(
|
|
cfrClipElement("sdr-plain", SDR, 1),
|
|
"out-cache-hdr-plain",
|
|
CACHE_DIR,
|
|
);
|
|
expect(plain.errors).toEqual([]);
|
|
expect(plain.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(plain.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(cacheEntryNames(CACHE_DIR)).toHaveLength(3);
|
|
|
|
// Cross-render poisoning regression: the plain-SDR render must not be
|
|
// served the BT.2020-converted frames the mixed render cached for the
|
|
// SAME source+trim. Compare actual frame bytes across the cache
|
|
// boundary, not just entry counts.
|
|
expect(
|
|
readFileSync(framePath(plain, "sdr-plain", 0)).equals(
|
|
readFileSync(framePath(mixed, "sdr-transform", 0)),
|
|
),
|
|
).toBe(false);
|
|
|
|
// And a repeat plain render must HIT the plain entry and serve
|
|
// byte-identical plain frames (proves the hit path keys correctly too).
|
|
const plainAgain = await extractWithCache(
|
|
cfrClipElement("sdr-plain-again", SDR, 1),
|
|
"out-cache-hdr-plain-again",
|
|
CACHE_DIR,
|
|
);
|
|
expect(plainAgain.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(
|
|
readFileSync(framePath(plainAgain, "sdr-plain-again", 0)).equals(
|
|
readFileSync(framePath(plain, "sdr-plain", 0)),
|
|
),
|
|
).toBe(true);
|
|
} finally {
|
|
rmSync(CACHE_DIR, { recursive: true, force: true });
|
|
}
|
|
}, 60_000);
|
|
|
|
it("clusters overlap components so a disjoint outlier does not break the group", async () => {
|
|
const SRC = await synthCfrClip("superset-cluster-src.mp4", 12);
|
|
const outputDir = join(FIXTURE_DIR, "out-superset-cluster");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
// Three overlapping trims [0..4], [2..6], [4..8] plus one disjoint trim
|
|
// [10..12] of the same source. Pre-clustering, the outlier failed the
|
|
// union<=sum check for the whole bucket and ALL FOUR fell back to direct
|
|
// extraction; the overlapping three must still share one superset.
|
|
const result = await extractAllVideoFrames(
|
|
[
|
|
cfrClipElement("cl-a", SRC, 4, 0),
|
|
cfrClipElement("cl-b", SRC, 4, 2),
|
|
cfrClipElement("cl-c", SRC, 4, 4),
|
|
cfrClipElement("cl-out", SRC, 2, 10),
|
|
],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
// Overlap region t=2..4 of the source: cl-a frame 60 and cl-b frame 0
|
|
// must be the SAME inode (shared superset extraction).
|
|
expect(statSync(framePath(result, "cl-a", 60)).ino).toBe(
|
|
statSync(framePath(result, "cl-b", 0)).ino,
|
|
);
|
|
expect(statSync(framePath(result, "cl-b", 60)).ino).toBe(
|
|
statSync(framePath(result, "cl-c", 0)).ino,
|
|
);
|
|
// The outlier extracted directly: its frames share no inode with the
|
|
// cluster (frame at source t=10 exists only in its own extraction).
|
|
expect(extractedFor(result, "cl-out").totalFrames).toBe(60);
|
|
expect(supersetDirNames(outputDir)).toEqual([]);
|
|
}, 60_000);
|
|
|
|
it("runs the GC staleness fallback sweep on all-hit renders with a stale marker", async () => {
|
|
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-gc-stale-test-"));
|
|
const SRC = await synthCfrClip("cache-gc-stale-src.mp4", 1);
|
|
const video = cfrClipElement("gc-stale", SRC, 1);
|
|
|
|
const miss = await extractWithCache(video, "out-cache-gc-stale-miss", CACHE_DIR);
|
|
expect(miss.phaseBreakdown.cacheMisses).toBe(1);
|
|
|
|
const agedPartial = join(CACHE_DIR, `${SCHEMA_PREFIX}aged.partial-1234-cafef00d`);
|
|
mkdirSync(agedPartial, { recursive: true });
|
|
writeFileSync(join(agedPartial, "frame_00001.jpg"), "stale", "utf-8");
|
|
const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000);
|
|
utimesSync(agedPartial, twoHoursAgo, twoHoursAgo);
|
|
|
|
// Age the sweep marker past the 24h staleness window: the next all-hit
|
|
// render must sweep anyway and clear the aged partial.
|
|
const twoDaysAgo = new Date(Date.now() - 48 * 60 * 60 * 1000);
|
|
utimesSync(join(CACHE_DIR, GC_MARKER), twoDaysAgo, twoDaysAgo);
|
|
|
|
const hit = await extractWithCache(video, "out-cache-gc-stale-hit", CACHE_DIR);
|
|
expect(hit.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(hit.phaseBreakdown.cacheMisses).toBe(0);
|
|
expect(existsSync(agedPartial)).toBe(false);
|
|
expect(hit.phaseBreakdown.cacheAgedPartialsCleared).toBe(1);
|
|
|
|
rmSync(CACHE_DIR, { recursive: true, force: true });
|
|
}, 60_000);
|
|
|
|
it("hardlinks overlapping aligned trims from one superset extraction", async () => {
|
|
const SRC = await synthCfrClip("superset-overlap-src.mp4", 10);
|
|
const outputDir = join(FIXTURE_DIR, "out-superset-overlap");
|
|
const directOutputDir = join(FIXTURE_DIR, "out-superset-direct");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
mkdirSync(directOutputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[cfrClipElement("trim-a", SRC, 4, 0), cfrClipElement("trim-b", SRC, 4, 2)],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(extractedFor(result, "trim-a").totalFrames).toBe(120);
|
|
expect(extractedFor(result, "trim-b").totalFrames).toBe(120);
|
|
expect(statSync(framePath(result, "trim-a", 60)).ino).toBe(
|
|
statSync(framePath(result, "trim-b", 0)).ino,
|
|
);
|
|
|
|
const direct = await extractVideoFramesRange(SRC, "direct-trim-b", 2, 4, {
|
|
fps: 30,
|
|
outputDir: directOutputDir,
|
|
format: "jpg",
|
|
});
|
|
expect(
|
|
readFileSync(framePath(result, "trim-b", 0)).equals(readFileSync(direct.framePaths.get(0)!)),
|
|
).toBe(true);
|
|
expect(supersetDirNames(outputDir)).toEqual([]);
|
|
}, 60_000);
|
|
|
|
it.each([
|
|
{ label: "decimal underflow half-frame", duration: 2.05, offset: 1, expectedFrames: 62 },
|
|
{
|
|
label: "non-half-integer boundary",
|
|
duration: 0.616666,
|
|
offset: 0.1,
|
|
expectedFrames: 18,
|
|
},
|
|
])(
|
|
"keeps direct and superset CFR extraction equal at a $label",
|
|
async ({ label, duration, offset, expectedFrames }) => {
|
|
const fixtureKey = label.replaceAll(" ", "-");
|
|
const src = await synthCfrClip(`superset-cfr-${fixtureKey}.mp4`, 4);
|
|
const groupedOutputDir = join(FIXTURE_DIR, `out-superset-cfr-${fixtureKey}`);
|
|
const directOutputDir = join(FIXTURE_DIR, `out-direct-cfr-${fixtureKey}`);
|
|
mkdirSync(groupedOutputDir, { recursive: true });
|
|
mkdirSync(directOutputDir, { recursive: true });
|
|
|
|
const grouped = await extractAllVideoFrames(
|
|
[
|
|
cfrClipElement(`${fixtureKey}-base`, src, duration, 0),
|
|
cfrClipElement(`${fixtureKey}-member`, src, duration, offset),
|
|
],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir: groupedOutputDir },
|
|
);
|
|
const direct = await extractVideoFramesRange(src, `${fixtureKey}-direct`, offset, duration, {
|
|
fps: 30,
|
|
outputDir: directOutputDir,
|
|
format: "jpg",
|
|
});
|
|
|
|
expect(grouped.errors).toEqual([]);
|
|
expect(direct.totalFrames).toBe(expectedFrames);
|
|
expect(extractedFor(grouped, `${fixtureKey}-base`).totalFrames).toBe(expectedFrames);
|
|
expect(extractedFor(grouped, `${fixtureKey}-member`).totalFrames).toBe(expectedFrames);
|
|
expect(statSync(framePath(grouped, `${fixtureKey}-base`, Math.round(offset * 30))).ino).toBe(
|
|
statSync(framePath(grouped, `${fixtureKey}-member`, 0)).ino,
|
|
);
|
|
for (let frame = 0; frame < expectedFrames; frame += 1) {
|
|
expect(
|
|
readFileSync(framePath(grouped, `${fixtureKey}-member`, frame)).equals(
|
|
readFileSync(direct.framePaths.get(frame)!),
|
|
),
|
|
).toBe(true);
|
|
}
|
|
expect(supersetDirNames(groupedOutputDir)).toEqual([]);
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("keeps direct and superset CFR extraction equal at 30000/1001", async () => {
|
|
const src = await synthCfrClip("superset-cfr-ntsc.mp4", 4);
|
|
const groupedOutputDir = join(FIXTURE_DIR, "out-superset-cfr-ntsc");
|
|
const directOutputDir = join(FIXTURE_DIR, "out-direct-cfr-ntsc");
|
|
const fps = { num: 30000, den: 1001 };
|
|
const duration = 0.25025;
|
|
mkdirSync(groupedOutputDir, { recursive: true });
|
|
mkdirSync(directOutputDir, { recursive: true });
|
|
|
|
const grouped = await extractAllVideoFrames(
|
|
[
|
|
cfrClipElement("ntsc-base", src, 0.5005, 0),
|
|
cfrClipElement("ntsc-member", src, duration, 0),
|
|
],
|
|
FIXTURE_DIR,
|
|
{ fps, outputDir: groupedOutputDir },
|
|
);
|
|
const direct = await extractVideoFramesRange(src, "ntsc-direct", 0, duration, {
|
|
fps,
|
|
outputDir: directOutputDir,
|
|
format: "jpg",
|
|
});
|
|
|
|
expect(grouped.errors).toEqual([]);
|
|
expect(direct.totalFrames).toBe(8);
|
|
expect(extractedFor(grouped, "ntsc-base").totalFrames).toBe(15);
|
|
expect(extractedFor(grouped, "ntsc-member").totalFrames).toBe(8);
|
|
expect(statSync(framePath(grouped, "ntsc-base", 0)).ino).toBe(
|
|
statSync(framePath(grouped, "ntsc-member", 0)).ino,
|
|
);
|
|
for (let frame = 0; frame < 8; frame += 1) {
|
|
expect(
|
|
readFileSync(framePath(grouped, "ntsc-member", frame)).equals(
|
|
readFileSync(direct.framePaths.get(frame)!),
|
|
),
|
|
).toBe(true);
|
|
}
|
|
expect(supersetDirNames(groupedOutputDir)).toEqual([]);
|
|
}, 60_000);
|
|
|
|
it("does not superset disjoint trims", async () => {
|
|
const SRC = await synthCfrClip("superset-disjoint-src.mp4", 10);
|
|
const outputDir = join(FIXTURE_DIR, "out-superset-disjoint");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[cfrClipElement("trim-a", SRC, 2, 0), cfrClipElement("trim-b", SRC, 2, 8)],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(statSync(framePath(result, "trim-a", 0)).ino).not.toBe(
|
|
statSync(framePath(result, "trim-b", 0)).ino,
|
|
);
|
|
expect(supersetDirNames(outputDir)).toEqual([]);
|
|
}, 60_000);
|
|
|
|
it("does not superset trims whose offsets are not frame-aligned", async () => {
|
|
const SRC = await synthCfrClip("superset-misaligned-src.mp4", 2);
|
|
const outputDir = join(FIXTURE_DIR, "out-superset-misaligned");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[cfrClipElement("trim-a", SRC, 1, 0), cfrClipElement("trim-b", SRC, 1, 0.017)],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(statSync(framePath(result, "trim-a", 0)).ino).not.toBe(
|
|
statSync(framePath(result, "trim-b", 0)).ino,
|
|
);
|
|
expect(supersetDirNames(outputDir)).toEqual([]);
|
|
}, 60_000);
|
|
|
|
it("keeps overlapping VFR trims direct because CFR resampling phase resets per seek", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-vfr-superset-short");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[
|
|
cfrClipElement("vfr-short-a", VFR_FIXTURE, 0.616666, 0),
|
|
cfrClipElement("vfr-short-b", VFR_FIXTURE, 0.616666, 0.1),
|
|
],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(extractedFor(result, "vfr-short-a").metadata.isVFR).toBe(true);
|
|
expect(extractedFor(result, "vfr-short-a").totalFrames).toBe(19);
|
|
expect(extractedFor(result, "vfr-short-b").totalFrames).toBe(19);
|
|
expect(statSync(framePath(result, "vfr-short-a", 3)).ino).not.toBe(
|
|
statSync(framePath(result, "vfr-short-b", 0)).ino,
|
|
);
|
|
expect(supersetDirNames(outputDir)).toEqual([]);
|
|
}, 60_000);
|
|
|
|
it("keeps batched and direct VFR extraction equal at a floating integral boundary", async () => {
|
|
const groupedOutputDir = join(FIXTURE_DIR, "out-vfr-superset-integral-boundary");
|
|
const directOutputDir = join(FIXTURE_DIR, "out-vfr-direct-integral-boundary");
|
|
mkdirSync(groupedOutputDir, { recursive: true });
|
|
mkdirSync(directOutputDir, { recursive: true });
|
|
|
|
const first: VideoElement = {
|
|
id: "vfr-integral-a",
|
|
src: VFR_FIXTURE,
|
|
start: 0.03,
|
|
end: 0.33,
|
|
mediaStart: 0.03,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
const second: VideoElement = {
|
|
...first,
|
|
id: "vfr-integral-b",
|
|
mediaStart: 0.13,
|
|
};
|
|
|
|
const direct = await extractAllVideoFrames([{ ...second }], FIXTURE_DIR, {
|
|
fps: 30,
|
|
outputDir: directOutputDir,
|
|
});
|
|
const grouped = await extractAllVideoFrames([{ ...first }, second], FIXTURE_DIR, {
|
|
fps: 30,
|
|
outputDir: groupedOutputDir,
|
|
});
|
|
|
|
expect(direct.errors).toEqual([]);
|
|
expect(grouped.errors).toEqual([]);
|
|
expect(extractedFor(direct, second.id).totalFrames).toBe(9);
|
|
expect(extractedFor(grouped, first.id).totalFrames).toBe(9);
|
|
expect(extractedFor(grouped, second.id).totalFrames).toBe(9);
|
|
for (let frame = 0; frame < 9; frame += 1) {
|
|
expect(
|
|
readFileSync(framePath(grouped, second.id, frame)).equals(
|
|
readFileSync(framePath(direct, second.id, frame)),
|
|
),
|
|
).toBe(true);
|
|
}
|
|
expect(statSync(framePath(grouped, first.id, 3)).ino).not.toBe(
|
|
statSync(framePath(grouped, second.id, 0)).ino,
|
|
);
|
|
expect(supersetDirNames(groupedOutputDir)).toEqual([]);
|
|
}, 60_000);
|
|
|
|
it("publishes overlapping superset slices to cache entries and hits them on the next render", async () => {
|
|
const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-superset-cache-test-"));
|
|
const SRC = await synthCfrClip("superset-cache-src.mp4", 10);
|
|
try {
|
|
const firstOutputDir = join(FIXTURE_DIR, "out-superset-cache-first");
|
|
const secondOutputDir = join(FIXTURE_DIR, "out-superset-cache-second");
|
|
mkdirSync(firstOutputDir, { recursive: true });
|
|
mkdirSync(secondOutputDir, { recursive: true });
|
|
const videos = [cfrClipElement("trim-a", SRC, 4, 0), cfrClipElement("trim-b", SRC, 4, 2)];
|
|
|
|
const first = await extractAllVideoFrames(
|
|
videos,
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir: firstOutputDir },
|
|
undefined,
|
|
{ extractCacheDir: CACHE_DIR },
|
|
);
|
|
expect(first.errors).toEqual([]);
|
|
expect(first.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(first.phaseBreakdown.cacheMisses).toBe(2);
|
|
expect(cacheEntryNames(CACHE_DIR)).toHaveLength(2);
|
|
expect(statSync(framePath(first, "trim-a", 60)).ino).toBe(
|
|
statSync(framePath(first, "trim-b", 0)).ino,
|
|
);
|
|
|
|
const second = await extractAllVideoFrames(
|
|
videos,
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir: secondOutputDir },
|
|
undefined,
|
|
{ extractCacheDir: CACHE_DIR },
|
|
);
|
|
expect(second.errors).toEqual([]);
|
|
expect(second.phaseBreakdown.cacheHits).toBe(2);
|
|
expect(second.phaseBreakdown.cacheMisses).toBe(0);
|
|
expect(supersetDirNames(secondOutputDir)).toEqual([]);
|
|
} finally {
|
|
rmSync(CACHE_DIR, { recursive: true, force: true });
|
|
}
|
|
}, 60_000);
|
|
|
|
it("clamps loop-past-EOF superset slices to available source frames", async () => {
|
|
const SRC = await synthCfrClip("superset-eof-src.mp4", 10);
|
|
const outputDir = join(FIXTURE_DIR, "out-superset-eof");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[cfrClipElement("covered", SRC, 4, 6), cfrClipElement("past-eof", SRC, 6, 8)],
|
|
FIXTURE_DIR,
|
|
{ fps: 30, outputDir },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(extractedFor(result, "covered").totalFrames).toBe(120);
|
|
expect(extractedFor(result, "past-eof").totalFrames).toBe(60);
|
|
expect(statSync(framePath(result, "covered", 60)).ino).toBe(
|
|
statSync(framePath(result, "past-eof", 0)).ino,
|
|
);
|
|
expect(supersetDirNames(outputDir)).toEqual([]);
|
|
}, 60_000);
|
|
|
|
// Asserts frame-count correctness for a full VFR file. One-pass CFR image
|
|
// extraction may repeat held source frames across timestamp gaps; the freeze
|
|
// regression is missing frames, which leaves late timeline lookups null.
|
|
it("produces the full frame count on the full VFR file", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-full");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const video: VideoElement = {
|
|
id: "vfull",
|
|
src: VFR_FIXTURE,
|
|
start: 0,
|
|
end: 10,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
|
|
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: 30,
|
|
outputDir,
|
|
});
|
|
expect(result.errors).toEqual([]);
|
|
|
|
const frameDir = join(outputDir, "vfull");
|
|
const frames = readdirSync(frameDir)
|
|
.filter((f) => f.endsWith(".jpg"))
|
|
.sort();
|
|
// ±3 tolerance: same FFmpeg one-pass VFR→CFR rounding variance as the
|
|
// mid-segment test.
|
|
expect(frames.length).toBeGreaterThanOrEqual(297);
|
|
expect(frames.length).toBeLessThanOrEqual(303);
|
|
}, 60_000);
|
|
});
|
|
|
|
// Release builds print "version 8.1.3" or "n8.1.3"; master builds print "N-<number>" and are newer.
|
|
const FFPROBE_VERSION = spawnSync("ffprobe", ["-version"]).stdout?.toString() ?? "";
|
|
const FFPROBE_RELEASE = FFPROBE_VERSION.match(/version n?(\d+)\.(\d+)/)
|
|
?.slice(1)
|
|
.map(Number);
|
|
const FFPROBE_READS_CONTAINER_TRANSFER =
|
|
/version N-/.test(FFPROBE_VERSION) ||
|
|
(FFPROBE_RELEASE !== undefined &&
|
|
(FFPROBE_RELEASE[0]! > 8 || (FFPROBE_RELEASE[0] === 8 && FFPROBE_RELEASE[1]! >= 1)));
|
|
|
|
describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
|
|
let fixtureDir = "";
|
|
|
|
beforeAll(() => {
|
|
fixtureDir = mkdtempSync(join(tmpdir(), "hf-forced-sdr-tonemap-test-"));
|
|
});
|
|
|
|
afterAll(() => {
|
|
rmSync(fixtureDir, { recursive: true, force: true });
|
|
}, 30_000);
|
|
|
|
async function synthesizeHlgClip(
|
|
path: string,
|
|
vui = "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9",
|
|
transfer = "arib-std-b67",
|
|
): Promise<void> {
|
|
const synthesized = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=0xe0b080:s=64x64:r=1:d=1",
|
|
"-vf",
|
|
`zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=${transfer}:m=bt2020nc:r=tv,format=yuv420p`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-color_primaries",
|
|
"bt2020",
|
|
"-color_trc",
|
|
transfer,
|
|
"-colorspace",
|
|
"bt2020nc",
|
|
"-bsf:v",
|
|
`h264_metadata=${vui}`,
|
|
path,
|
|
]);
|
|
if (!synthesized.success) {
|
|
throw new Error(`HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`);
|
|
}
|
|
}
|
|
|
|
it.each([
|
|
[
|
|
"HLG",
|
|
"matrix",
|
|
"arib-std-b67",
|
|
"colour_primaries=9:transfer_characteristics=18:matrix_coefficients=2",
|
|
],
|
|
[
|
|
"HLG",
|
|
"primaries",
|
|
"arib-std-b67",
|
|
"colour_primaries=2:transfer_characteristics=18:matrix_coefficients=9",
|
|
],
|
|
[
|
|
"HLG",
|
|
"matrix and primaries",
|
|
"arib-std-b67",
|
|
"colour_primaries=2:transfer_characteristics=18:matrix_coefficients=2",
|
|
],
|
|
[
|
|
"PQ",
|
|
"matrix and primaries",
|
|
"smpte2084",
|
|
"colour_primaries=2:transfer_characteristics=16:matrix_coefficients=2",
|
|
],
|
|
[
|
|
"HLG",
|
|
"transfer",
|
|
"arib-std-b67",
|
|
"colour_primaries=9:transfer_characteristics=2:matrix_coefficients=9",
|
|
],
|
|
])(
|
|
"tone-maps %s footage whose video stream has no %s tag like the fully tagged clip",
|
|
async (name, missing, transfer, vui) => {
|
|
const slug = `${name}-no-${missing}`.replace(/\W+/g, "-");
|
|
const tagged = join(fixtureDir, `${name}-fully-tagged.mp4`);
|
|
const untagged = join(fixtureDir, `${slug}.mp4`);
|
|
const vuiTransfer = transfer === "smpte2084" ? 16 : 18;
|
|
await synthesizeHlgClip(
|
|
tagged,
|
|
`colour_primaries=9:transfer_characteristics=${vuiTransfer}:matrix_coefficients=9`,
|
|
transfer,
|
|
);
|
|
await synthesizeHlgClip(untagged, vui, transfer);
|
|
const extract = (source: string, id: string) =>
|
|
extractVideoFramesRange(source, id, 0, 1, {
|
|
fps: 1,
|
|
outputDir: join(fixtureDir, `out-${id}`),
|
|
format: "png",
|
|
toneMapHdrToSdr: true,
|
|
});
|
|
|
|
if (missing === "transfer") {
|
|
// ffprobe before 8.1 reads the stream's unspecified transfer, so the clip is SDR there, as on main.
|
|
if (!FFPROBE_READS_CONTAINER_TRANSFER) {
|
|
await expect(extract(untagged, slug)).resolves.toBeDefined();
|
|
return;
|
|
}
|
|
expect((await extractVideoMetadata(untagged)).colorSpace?.colorTransfer).toBe(transfer);
|
|
}
|
|
const reference = await extract(tagged, `ref-${slug}`);
|
|
const result = await extract(untagged, slug);
|
|
|
|
expect(readFileSync(result.framePaths.get(0)!)).toEqual(
|
|
readFileSync(reference.framePaths.get(0)!),
|
|
);
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("probes each source's first frame once, however many cuts of it extract together", async () => {
|
|
const sources = ["a", "b"].map((name) => join(fixtureDir, `cuts-${name}.mp4`));
|
|
for (const source of sources) await synthesizeHlgClip(source);
|
|
const probes = () =>
|
|
vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;
|
|
const before = probes();
|
|
await Promise.all(
|
|
Array.from({ length: 12 }, (_, cut) =>
|
|
extractVideoFramesRange(sources[cut % 2]!, `cut-${cut}`, 0, 1, {
|
|
fps: 1,
|
|
outputDir: join(fixtureDir, "cuts"),
|
|
format: "png",
|
|
toneMapHdrToSdr: true,
|
|
}),
|
|
),
|
|
);
|
|
expect(probes() - before).toBe(2);
|
|
}, 60_000);
|
|
|
|
it("probes a source again after its first-frame probe failed", async () => {
|
|
const source = join(fixtureDir, "probe-retry.mp4");
|
|
await synthesizeHlgClip(source);
|
|
const probes = () =>
|
|
vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;
|
|
const { runFfmpeg: real } =
|
|
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
|
|
vi.mocked(runFfmpeg).mockImplementation((args, opts) =>
|
|
args.includes("showinfo")
|
|
? Promise.resolve({
|
|
success: false,
|
|
exitCode: 1,
|
|
stderr: "",
|
|
durationMs: 0,
|
|
terminationReason: "exit",
|
|
})
|
|
: real(args, opts),
|
|
);
|
|
const extract = (id: string) =>
|
|
extractVideoFramesRange(source, id, 0, 1, {
|
|
fps: 1,
|
|
outputDir: join(fixtureDir, "retry"),
|
|
format: "png",
|
|
toneMapHdrToSdr: true,
|
|
});
|
|
const before = probes();
|
|
try {
|
|
await extract("retry-failed");
|
|
} finally {
|
|
vi.mocked(runFfmpeg).mockImplementation(real);
|
|
}
|
|
await extract("retry-again");
|
|
expect(probes() - before).toBe(2);
|
|
}, 60_000);
|
|
|
|
// One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer.
|
|
it("tone-maps each frame of a mixed HLG and PQ stream with its own transfer", async () => {
|
|
const segment = async (name: string, transfer: number) => {
|
|
const path = join(fixtureDir, `mixed-${name}.h264`);
|
|
const result = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"testsrc2=s=160x90:r=25:d=0.2",
|
|
"-c:v",
|
|
"libx264",
|
|
// No B-frames: genpts stamps a raw stream in decode order, so reordered frames get wrong times.
|
|
"-bf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
"-bsf:v",
|
|
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
|
|
"-f",
|
|
"h264",
|
|
path,
|
|
]);
|
|
if (!result.success)
|
|
throw new Error(`segment synthesis failed: ${result.stderr.slice(-400)}`);
|
|
return readFileSync(path);
|
|
};
|
|
const mux = async (name: string, stream: Buffer) => {
|
|
const raw = join(fixtureDir, `${name}.h264`);
|
|
const path = join(fixtureDir, `${name}.mp4`);
|
|
writeFileSync(raw, stream);
|
|
const result = await runFfmpeg([
|
|
...["-y", "-v", "error", "-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw],
|
|
...["-c", "copy", path],
|
|
]);
|
|
if (!result.success) throw new Error(`mux failed: ${result.stderr.slice(-400)}`);
|
|
return path;
|
|
};
|
|
const hlg = await segment("hlg", 18);
|
|
const pq = await segment("pq", 16);
|
|
const mixed = await mux("mixed-hlg-pq", Buffer.concat([hlg, pq]));
|
|
const pqOnly = await mux("mixed-pq-only", pq);
|
|
const extract = (source: string, id: string, duration: number) =>
|
|
extractVideoFramesRange(source, id, 0, duration, {
|
|
fps: 25,
|
|
outputDir: join(fixtureDir, `out-${id}`),
|
|
format: "png",
|
|
toneMapHdrToSdr: true,
|
|
});
|
|
|
|
const mixedFrames = await extract(mixed, "mixed-hlg-pq", 0.4);
|
|
const pqFrames = await extract(pqOnly, "mixed-pq-only", 0.2);
|
|
|
|
expect(readFileSync(mixedFrames.framePaths.get(5)!)).toEqual(
|
|
readFileSync(pqFrames.framePaths.get(0)!),
|
|
);
|
|
}, 60_000);
|
|
|
|
// A stream-copy trim opens on edit-list pre-roll, which ffprobe counts as packets without frames.
|
|
it("keeps per-frame tags in a mixed HLG and PQ file trimmed with a stream copy", async () => {
|
|
const run = async (args: string[]) => {
|
|
const result = await runFfmpeg(["-y", "-v", "error", ...args]);
|
|
if (!result.success) throw new Error(`fixture failed: ${result.stderr.slice(-400)}`);
|
|
};
|
|
const segment = async (name: string, transfer: number) => {
|
|
const path = join(fixtureDir, `trim-${name}.h264`);
|
|
await run([
|
|
...["-f", "lavfi", "-i", "testsrc2=s=160x90:r=25:d=2"],
|
|
...["-c:v", "libx264", "-g", "50", "-bf", "0"],
|
|
...["-pix_fmt", "yuv420p", "-bsf:v"],
|
|
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
|
|
...["-f", "h264", path],
|
|
]);
|
|
return readFileSync(path);
|
|
};
|
|
const mux = async (name: string, stream: Buffer) => {
|
|
const raw = join(fixtureDir, `${name}.h264`);
|
|
const path = join(fixtureDir, `${name}.mp4`);
|
|
writeFileSync(raw, stream);
|
|
await run(["-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw, "-c", "copy", path]);
|
|
return path;
|
|
};
|
|
const pq = await segment("pq", 16);
|
|
const mixed = await mux("trim-hlg-then-pq", Buffer.concat([await segment("hlg", 18), pq]));
|
|
const pqOnly = await mux("trim-pq-only", pq);
|
|
const trimmed = join(fixtureDir, "trim-from-hlg.mp4");
|
|
await run(["-ss", "1", "-i", mixed, "-c", "copy", trimmed]);
|
|
const extract = (source: string, id: string, start: number) =>
|
|
extractVideoFramesRange(source, id, start, 0.04, {
|
|
fps: 25,
|
|
outputDir: join(fixtureDir, `out-${id}`),
|
|
format: "png",
|
|
toneMapHdrToSdr: true,
|
|
});
|
|
|
|
// Trimmed 1.6 s is the mixed file's 2.6 s: the PQ segment's 0.6 s.
|
|
const fromTrim = await extract(trimmed, "trim-from-hlg", 1.6);
|
|
const fromPq = await extract(pqOnly, "trim-pq-only", 0.6);
|
|
|
|
const shown = readFirstFramePixel(fromTrim.framePaths.get(0)!, 40, 40);
|
|
const expected = readFirstFramePixel(fromPq.framePaths.get(0)!, 40, 40);
|
|
expect(Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)))).toBeLessThanOrEqual(2);
|
|
}, 60_000);
|
|
|
|
it("matches Studio's HLG tone map and isolates transformed cache entries", async () => {
|
|
const source = join(fixtureDir, "hlg-warm.mp4");
|
|
await synthesizeHlgClip(source);
|
|
|
|
const reference = join(fixtureDir, "studio-reference.png");
|
|
const referenceResult = await runFfmpeg([
|
|
"-y",
|
|
"-ss",
|
|
"0",
|
|
"-i",
|
|
source,
|
|
"-t",
|
|
"1",
|
|
"-vf",
|
|
"fps=1,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv",
|
|
"-q:v",
|
|
"0",
|
|
"-compression_level",
|
|
"1",
|
|
reference,
|
|
]);
|
|
if (!referenceResult.success) {
|
|
throw new Error(`Studio reference extraction failed: ${referenceResult.stderr.slice(-400)}`);
|
|
}
|
|
|
|
const cacheDir = join(fixtureDir, "cache");
|
|
const video = (id: string): VideoElement => ({
|
|
id,
|
|
src: source,
|
|
start: 0,
|
|
end: 1,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
});
|
|
const extract = (id: string, toneMapHdrToSdr = false) =>
|
|
extractAllVideoFrames(
|
|
[video(id)],
|
|
fixtureDir,
|
|
{
|
|
fps: 1,
|
|
outputDir: join(fixtureDir, id),
|
|
format: "png",
|
|
toneMapHdrToSdr,
|
|
},
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
|
|
const plain = await extract("plain");
|
|
const toneMapped = await extract("tone-mapped", true);
|
|
const toneMappedAgain = await extract("tone-mapped-again", true);
|
|
|
|
expect(plain.errors).toEqual([]);
|
|
expect(toneMapped.errors).toEqual([]);
|
|
expect(toneMapped.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(toneMapped.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(toneMappedAgain.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX))).toHaveLength(2);
|
|
|
|
const frame = (result: ExtractionResult): Buffer => {
|
|
const path = result.extracted[0]?.framePaths.get(0);
|
|
if (!path) throw new Error("expected extracted frame");
|
|
return readFileSync(path);
|
|
};
|
|
// Same pixels as Studio's tone map, declared as sRGB so Chrome shows them unconverted.
|
|
const rgb = (path: string): Buffer =>
|
|
spawnSync("ffmpeg", ["-v", "error", "-i", path, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"])
|
|
.stdout;
|
|
const toneMappedPath = toneMapped.extracted[0]!.framePaths.get(0)!;
|
|
expect(rgb(toneMappedPath)).toEqual(rgb(reference));
|
|
expect(pngChunkTypes(toneMappedPath)).toContain("sRGB");
|
|
expect(pngChunkTypes(toneMappedPath)).not.toContain("cICP");
|
|
expect(frame(plain)).not.toEqual(frame(toneMapped));
|
|
|
|
const toneMappedJpg = await extractAllVideoFrames([video("tone-mapped-jpg")], fixtureDir, {
|
|
fps: 1,
|
|
outputDir: join(fixtureDir, "tone-mapped-jpg"),
|
|
format: "jpg",
|
|
toneMapHdrToSdr: true,
|
|
});
|
|
expect(toneMappedJpg.errors).toEqual([]);
|
|
const jpgPixels = rgb(toneMappedJpg.extracted[0]!.framePaths.get(0)!);
|
|
const referencePixels = rgb(reference);
|
|
const worst = Math.max(...[...jpgPixels].map((v, i) => Math.abs(v - referencePixels[i]!)));
|
|
expect(worst, "tone-mapped jpg against Studio's tone map").toBeLessThanOrEqual(3);
|
|
expect(frame(toneMappedAgain)).toEqual(frame(toneMapped));
|
|
|
|
// A macOS ffmpeg that has zscale uses the same tone map, not VideoToolbox.
|
|
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
|
|
Object.defineProperty(process, "platform", { value: "darwin" });
|
|
try {
|
|
const mac = await extractAllVideoFrames([video("tone-mapped-mac")], fixtureDir, {
|
|
fps: 1,
|
|
outputDir: join(fixtureDir, "tone-mapped-mac"),
|
|
format: "png",
|
|
toneMapHdrToSdr: true,
|
|
});
|
|
expect(mac.errors).toEqual([]);
|
|
expect(rgb(mac.extracted[0]!.framePaths.get(0)!)).toEqual(rgb(reference));
|
|
} finally {
|
|
Object.defineProperty(process, "platform", platform);
|
|
}
|
|
}, 60_000);
|
|
|
|
it("warns once and keeps VideoToolbox, under its own cache key, when a macOS ffmpeg has no zscale", async () => {
|
|
const source = join(fixtureDir, "hlg-no-zscale.mp4");
|
|
await synthesizeHlgClip(source);
|
|
const cacheDir = join(fixtureDir, "no-zscale-cache");
|
|
const clip = (id: string): VideoElement => ({
|
|
id,
|
|
src: source,
|
|
start: 0,
|
|
end: 1,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
});
|
|
const options = (id: string) => ({
|
|
fps: 1,
|
|
outputDir: join(fixtureDir, id),
|
|
format: "png" as const,
|
|
toneMapHdrToSdr: true,
|
|
});
|
|
const zscale = await extractAllVideoFrames(
|
|
[clip("zscale")],
|
|
fixtureDir,
|
|
options("zscale"),
|
|
undefined,
|
|
{
|
|
extractCacheDir: cacheDir,
|
|
},
|
|
);
|
|
expect(zscale.errors).toEqual([]);
|
|
|
|
const realPlatform = process.platform;
|
|
vi.resetModules();
|
|
vi.doMock("../utils/psnrFilterAvailability.js", () => ({
|
|
isFfmpegFilterAvailable: async () => false,
|
|
isPsnrFilterAvailable: async () => false,
|
|
}));
|
|
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
|
|
Object.defineProperty(process, "platform", { value: "darwin" });
|
|
const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
|
|
try {
|
|
const { extractAllVideoFrames: extractOnMac } = await import("./videoFrameExtractor.js");
|
|
for (const [i, id] of ["no-zscale-1", "no-zscale-2"].entries()) {
|
|
const result = await extractOnMac([clip(id)], fixtureDir, options(id), undefined, {
|
|
extractCacheDir: cacheDir,
|
|
});
|
|
// The zscale frames above must not be served for the VideoToolbox path.
|
|
if (i === 0) expect(result.phaseBreakdown.cacheHits).toBe(0);
|
|
// Off macOS the VideoToolbox decode itself fails, which proves it was attempted.
|
|
if (realPlatform !== "darwin")
|
|
expect(JSON.stringify(result.errors)).toMatch(/videotoolbox/i);
|
|
}
|
|
const zscaleWarnings = stderr.mock.calls.filter(([message]) =>
|
|
String(message).includes("no zscale filter"),
|
|
);
|
|
expect(zscaleWarnings).toHaveLength(1);
|
|
} finally {
|
|
stderr.mockRestore();
|
|
Object.defineProperty(process, "platform", platform);
|
|
vi.doUnmock("../utils/psnrFilterAvailability.js");
|
|
vi.resetModules();
|
|
}
|
|
}, 60_000);
|
|
|
|
// A second process would read the frames back without the HDR10 light-level metadata the tone map uses.
|
|
it("tone-maps a VFR HDR10 window in one process, as the reference tone map would", async () => {
|
|
const source = join(fixtureDir, "pq-vfr-4000nit.mp4");
|
|
const synthesized = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=0xf0e0c0:s=64x64:r=60:d=2",
|
|
"-vf",
|
|
"select='not(between(n\\,30\\,89))',zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=smpte2084:m=bt2020nc:r=tv:npl=1000,format=yuv420p10le",
|
|
"-fps_mode",
|
|
"vfr",
|
|
"-c:v",
|
|
"libx265",
|
|
"-x265-params",
|
|
"log-level=error:hdr10=1:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:max-cll=4000,400:master-display=G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(40000000,50)",
|
|
source,
|
|
]);
|
|
if (!synthesized.success)
|
|
throw new Error(`PQ fixture synthesis failed: ${synthesized.stderr.slice(-400)}`);
|
|
const referenceDir = join(fixtureDir, "pq-vfr-reference");
|
|
mkdirSync(referenceDir, { recursive: true });
|
|
const reference = await runFfmpeg([
|
|
"-v",
|
|
"error",
|
|
"-ss",
|
|
"0",
|
|
"-i",
|
|
source,
|
|
"-t",
|
|
"0.616666",
|
|
"-vf",
|
|
"zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv,setparams=color_primaries=bt709:color_trc=iec61966-2-1",
|
|
"-fps_mode",
|
|
"cfr",
|
|
"-r",
|
|
"30",
|
|
"-q:v",
|
|
"0",
|
|
"-compression_level",
|
|
"1",
|
|
join(referenceDir, "frame_%05d.png"),
|
|
]);
|
|
expect(reference.success, reference.stderr).toBe(true);
|
|
|
|
const result = await extractVideoFramesRange(source, "pq-vfr", 0, 0.616666, {
|
|
fps: 30,
|
|
outputDir: join(fixtureDir, "pq-vfr-out"),
|
|
format: "png",
|
|
toneMapHdrToSdr: true,
|
|
});
|
|
|
|
expect(result.metadata.isVFR).toBe(true);
|
|
const referenceFrames = readdirSync(referenceDir).sort();
|
|
// One slot per 30fps tick before 0.616666s; x265 packet durations cut the reference's still short.
|
|
expect(result.totalFrames).toBe(19);
|
|
expect(readFileSync(result.framePaths.get(0)!)).toEqual(
|
|
readFileSync(join(referenceDir, referenceFrames[0]!)),
|
|
);
|
|
}, 60_000);
|
|
});
|
|
|
|
describe("getFrameAtTime — IEEE 754 boundary precision", () => {
|
|
function makeExtracted(fps: number, totalFrames: number): ExtractedFrames {
|
|
const framePaths = new Map<number, string>();
|
|
for (let i = 0; i < totalFrames; i++) framePaths.set(i, `frame-${i}.jpg`);
|
|
return {
|
|
fps,
|
|
totalFrames,
|
|
framePaths,
|
|
metadata: {
|
|
durationSeconds: totalFrames / fps,
|
|
width: 1920,
|
|
height: 1080,
|
|
codec: "h264",
|
|
hasAudio: false,
|
|
fps,
|
|
},
|
|
} as ExtractedFrames;
|
|
}
|
|
|
|
it("indexes frames by integrated source time for a rate lane", () => {
|
|
const extracted = { ...makeExtracted(25, 351), videoId: "ramped" } as ExtractedFrames;
|
|
const rate = {
|
|
target: "rate",
|
|
points: [
|
|
{ t: 0, v: 1 },
|
|
{ t: 2, v: 3 },
|
|
],
|
|
};
|
|
const table = new FrameLookupTable();
|
|
table.addVideo(extracted, 0, 10, 0, false, rate);
|
|
// 2s * (3-1)/ln 3 = 3.6411 source seconds * 25 fps = frame 91
|
|
expect(table.getFrame("ramped", 2)).toBe("frame-91.jpg");
|
|
expect(table.getFrame("ramped", 0)).toBe("frame-0.jpg");
|
|
});
|
|
|
|
it("wraps a ramped loop in source space so the lane keeps running across cycles", () => {
|
|
const extracted = { ...makeExtracted(25, 100), videoId: "looped" } as ExtractedFrames;
|
|
const rate = {
|
|
target: "rate",
|
|
points: [
|
|
{ t: 0, v: 1 },
|
|
{ t: 4, v: 3 },
|
|
],
|
|
};
|
|
const table = new FrameLookupTable();
|
|
table.addVideo(extracted, 0, 10, 0, true, rate);
|
|
// source(3) = 4.6586 on a 4s source wraps to 0.6586 * 25 fps = frame 16
|
|
expect(table.getFrame("looped", 3)).toBe("frame-16.jpg");
|
|
});
|
|
|
|
it("does not produce duplicate frames when data-start is grid-aligned", () => {
|
|
const extracted = makeExtracted(25, 351);
|
|
const videoStart = 0;
|
|
const seen: string[] = [];
|
|
let duplicates = 0;
|
|
for (let i = 0; i < 351; i++) {
|
|
const globalTime = i / 25;
|
|
const frame = getFrameAtTime(extracted, globalTime, videoStart);
|
|
if (frame && seen.length > 0 && frame === seen[seen.length - 1]) duplicates++;
|
|
if (frame) seen.push(frame);
|
|
}
|
|
expect(duplicates).toBe(0);
|
|
});
|
|
|
|
it("returns monotonically increasing frame indices", () => {
|
|
const extracted = makeExtracted(25, 100);
|
|
let lastIndex = -1;
|
|
for (let i = 0; i < 100; i++) {
|
|
const globalTime = i / 25;
|
|
const frame = getFrameAtTime(extracted, globalTime, 0);
|
|
const idx = frame ? parseInt(frame.split("-")[1]!) : -1;
|
|
expect(idx).toBeGreaterThan(lastIndex);
|
|
lastIndex = idx;
|
|
}
|
|
});
|
|
|
|
it("handles the 0.28 * 25 boundary case (6.999999 vs 7)", () => {
|
|
const extracted = makeExtracted(25, 10);
|
|
const frame = getFrameAtTime(extracted, 0.28, 0);
|
|
expect(frame).toBe("frame-7.jpg");
|
|
});
|
|
|
|
it("picks the frame the preview shows when a clip starts a hair after a frame instant", () => {
|
|
// 449/30 written with six decimals starts the clip a third of a microsecond late. The preview's
|
|
// <video> still shows the take's second frame at output frame 450, so the render must too.
|
|
const extracted = makeExtracted(30, 100);
|
|
expect(getFrameAtTime(extracted, 450 / 30, 14.966667, false, 4.3)).toBe("frame-1.jpg");
|
|
expect(getFrameAtTime(extracted, 460 / 30, 14.966667, false, 4.3)).toBe("frame-11.jpg");
|
|
});
|
|
|
|
it("mediaStart does not offset frame index (extractor handles trim via -ss)", () => {
|
|
const extracted = makeExtracted(25, 100);
|
|
const frame = getFrameAtTime(extracted, 0, 0, false, 1.0);
|
|
expect(frame).toBe("frame-0.jpg");
|
|
});
|
|
|
|
it("returns null after source exhaustion without an authored slot boundary", () => {
|
|
const extracted = makeExtracted(25, 25);
|
|
expect(getFrameAtTime(extracted, 3, 0)).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe("parseVideoElements hidden flag", () => {
|
|
it("marks a video hidden on itself or through a hidden ancestor", () => {
|
|
const videos = parseVideoElements(
|
|
`<div data-hidden><video id="a" src="a.mp4" data-has-audio="true"></video></div>` +
|
|
`<video id="b" src="b.mp4" data-has-audio="true" data-hidden></video>` +
|
|
`<video id="c" src="c.mp4" data-has-audio="true"></video>`,
|
|
);
|
|
expect(videos.map((v) => [v.id, v.hidden])).toEqual([
|
|
["a", true],
|
|
["b", true],
|
|
["c", undefined],
|
|
]);
|
|
});
|
|
|
|
it("marks only an audible video hidden when its audio group is hidden", () => {
|
|
const videos = parseVideoElements(
|
|
`<hf-audio-group id="g" data-hidden></hf-audio-group>` +
|
|
`<video id="a" src="a.mp4" data-has-audio="true" data-audio-group="g"></video>` +
|
|
`<video id="b" src="b.mp4" muted data-audio-group="g"></video>`,
|
|
);
|
|
expect(videos.map((v) => [v.id, v.hidden])).toEqual([
|
|
["a", true],
|
|
["b", undefined],
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe.skipIf(!HAS_FFMPEG)("a clip stored on a non-square pixel grid", () => {
|
|
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-sar-"));
|
|
const SOURCE = join(FIXTURE_DIR, "anamorphic.mp4");
|
|
|
|
beforeAll(async () => {
|
|
const result = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=red:s=64x64:d=1:r=10",
|
|
"-vf",
|
|
"setsar=1/2",
|
|
"-c:v",
|
|
"libx264",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
SOURCE,
|
|
]);
|
|
if (!result.success) throw new Error(`fixture synthesis failed: ${result.stderr.slice(-400)}`);
|
|
}, 30_000);
|
|
|
|
afterAll(() => rmSync(FIXTURE_DIR, { recursive: true, force: true }), 30_000);
|
|
|
|
it("keeps the display shape of a rotated clip, whose autorotate swaps the pixel aspect", async () => {
|
|
const rotated = join(FIXTURE_DIR, "rotated.mp4");
|
|
// `-display_rotation` is an ffmpeg 6 input option; older builds take the same rotation as metadata.
|
|
const rotateWith = (args: string[]) =>
|
|
runFfmpeg(["-y", "-v", "error", ...args, "-c", "copy", rotated]);
|
|
let synth = await rotateWith(["-display_rotation:v:0", "90", "-i", SOURCE]);
|
|
if (!synth.success) synth = await rotateWith(["-i", SOURCE, "-metadata:s:v:0", "rotate=90"]);
|
|
expect(synth.success).toBe(true);
|
|
const outDir = join(FIXTURE_DIR, "out-rot");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const video: VideoElement = {
|
|
id: "r",
|
|
src: rotated,
|
|
start: 0,
|
|
end: 1,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: 10,
|
|
outputDir: outDir,
|
|
});
|
|
expect(result.errors).toEqual([]);
|
|
const probe = spawnSync("ffprobe", [
|
|
"-v",
|
|
"error",
|
|
"-show_entries",
|
|
"stream=width,height",
|
|
"-of",
|
|
"csv=p=0",
|
|
"--",
|
|
result.extracted[0]!.framePaths.get(0)!,
|
|
]);
|
|
expect(probe.stdout.toString().trim()).toBe("128,64");
|
|
});
|
|
|
|
it("extracts frames at the shape Chrome displays, not the stored grid", async () => {
|
|
const outDir = join(FIXTURE_DIR, "out");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const video: VideoElement = {
|
|
id: "v",
|
|
src: SOURCE,
|
|
start: 0,
|
|
end: 1,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: 10,
|
|
outputDir: outDir,
|
|
});
|
|
expect(result.errors).toEqual([]);
|
|
const probe = spawnSync("ffprobe", [
|
|
"-v",
|
|
"error",
|
|
"-show_entries",
|
|
"stream=width,height",
|
|
"-of",
|
|
"csv=p=0",
|
|
"--",
|
|
result.extracted[0]!.framePaths.get(0)!,
|
|
]);
|
|
expect(probe.stdout.toString().trim()).toBe("32,64");
|
|
});
|
|
});
|
|
|
|
// Each source frame N is a flat gray of 3N, so the frame a render shows reads straight off a pixel.
|
|
describe.skipIf(!HAS_FFMPEG)("a clip whose data-start is off the frame grid", () => {
|
|
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-offgrid-start-"));
|
|
const SOURCE = join(FIXTURE_DIR, "numbered.mp4");
|
|
const FPS = 30;
|
|
|
|
beforeAll(async () => {
|
|
const result = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=black:s=64x64:d=6:r=30",
|
|
"-vf",
|
|
"format=gray,geq=lum='N*3'",
|
|
"-c:v",
|
|
"libx264",
|
|
"-crf",
|
|
"0",
|
|
"-g",
|
|
"30",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
SOURCE,
|
|
]);
|
|
if (!result.success) throw new Error(`fixture synthesis failed: ${result.stderr.slice(-400)}`);
|
|
}, 30_000);
|
|
|
|
afterAll(() => rmSync(FIXTURE_DIR, { recursive: true, force: true }), 30_000);
|
|
|
|
async function sourceFramesShown(
|
|
start: number,
|
|
outputFrames: number[],
|
|
): Promise<(number | null)[]> {
|
|
const outDir = join(FIXTURE_DIR, `out-${start}`);
|
|
mkdirSync(outDir, { recursive: true });
|
|
const video: VideoElement = {
|
|
id: "v",
|
|
src: SOURCE,
|
|
start,
|
|
end: start + 1.9,
|
|
mediaStart: start,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: FPS,
|
|
outputDir: outDir,
|
|
});
|
|
expect(result.errors).toEqual([]);
|
|
const table = createFrameLookupTable([video], result.extracted, FPS);
|
|
return outputFrames.map((n) => {
|
|
const path = table.getFrame("v", n / FPS);
|
|
return path ? Math.round(readFirstFramePixel(path, 0, 0)[0] / 3) : null;
|
|
});
|
|
}
|
|
|
|
it("keeps a looped clip on the same source frame after every wrap", async () => {
|
|
const loopSource = join(FIXTURE_DIR, "loop.mp4");
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=black:s=64x64:d=2:r=30",
|
|
"-vf",
|
|
"format=gray,geq=lum='N*4'",
|
|
"-c:v",
|
|
"libx264",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
loopSource,
|
|
]);
|
|
expect(synth.success).toBe(true);
|
|
const outDir = join(FIXTURE_DIR, "out-loop");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const video: VideoElement = {
|
|
id: "v",
|
|
src: loopSource,
|
|
start: 0.35,
|
|
end: 6.35,
|
|
mediaStart: 1,
|
|
loop: true,
|
|
hasAudio: false,
|
|
};
|
|
const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
|
|
fps: FPS,
|
|
outputDir: outDir,
|
|
timelineEnd: 6,
|
|
});
|
|
expect(result.errors).toEqual([]);
|
|
const table = createFrameLookupTable([video], result.extracted, FPS);
|
|
const shown = [0, 1, 2, 3, 4].map((wrap) => {
|
|
const path = table.getFrame("v", (12 + wrap * FPS + 1) / FPS);
|
|
return path ? Math.round(readFirstFramePixel(path, 0, 0)[0] / 4) : null;
|
|
});
|
|
expect(new Set(shown).size).toBe(1);
|
|
}, 60_000);
|
|
|
|
it.each([1.95, 2.0001, 1.999999])(
|
|
"shows source frame N at output frame N for data-start=data-media-start=%s",
|
|
async (start) => {
|
|
const shown = await sourceFramesShown(start, [61, 62, 63, 64, 65]);
|
|
expect(shown).toEqual([61, 62, 63, 64, 65]);
|
|
},
|
|
60_000,
|
|
);
|
|
});
|