test(engine): split the frame extractor tests so each file stays under the flake gate budget (#7476)

* test(engine): split the frame extractor tests so each file stays under the flake gate budget

* test(engine): cut the comments the frame extractor test split carried over
This commit is contained in:
Miguel Ángel
2026-11-10 03:19:15 +00:00
committed by GitHub
parent b12365d67d
commit 1a758fd60f
5 changed files with 3058 additions and 2909 deletions
@@ -0,0 +1,759 @@
import { vi, afterAll, beforeAll, describe, expect, it } from "vitest";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
import { basename, join } from "node:path";
import { tmpdir } from "node:os";
import { spawnSync } from "node:child_process";
import {
extractAllVideoFrames,
extractVideoFramesRange,
type VideoElement,
} from "./videoFrameExtractor.js";
import { extractVideoMetadata } from "../utils/ffprobe.js";
import { runFfmpeg } from "../utils/runFfmpeg.js";
// Pass-through, so a test can tell whether extraction ran ffmpeg.
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
});
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
const FFMPEG_ENCODERS = HAS_FFMPEG
? spawnSync("ffmpeg", ["-hide_banner", "-encoders"]).stdout.toString()
: "";
// Homebrew's ffmpeg has libsvtav1 but not libaom-av1. Either one writes an untagged AV1 stream.
const AV1_ENCODER = ["libaom-av1", "libsvtav1"].find((name) =>
new RegExp(`\\s${name}\\s`).test(FFMPEG_ENCODERS),
);
type Rgb = [number, number, number];
const UI_FIXTURE_WIDTH = 240;
const UI_FIXTURE_HEIGHT = 160;
const RED_SAMPLE_PIXELS = [
[70, 72],
[118, 82],
[178, 92],
] as const;
function pngChunkTypes(path: string): string[] {
const bytes = readFileSync(path);
const types: string[] = [];
for (let offset = 8; offset + 8 <= bytes.length; offset += 12 + bytes.readUInt32BE(offset)) {
types.push(bytes.toString("latin1", offset + 4, offset + 8));
}
return types;
}
function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb {
const result = spawnSync(
"ffmpeg",
[
"-v",
"error",
"-i",
mediaPath,
"-frames:v",
"1",
"-f",
"rawvideo",
"-pix_fmt",
"rgb24",
"pipe:1",
],
{ maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 },
);
if (result.status !== 0) {
throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`);
}
const offset = (y * UI_FIXTURE_WIDTH + x) * 3;
return [
result.stdout[offset] ?? 0,
result.stdout[offset + 1] ?? 0,
result.stdout[offset + 2] ?? 0,
];
}
function maxChannelDelta(a: Rgb, b: Rgb): number {
return Math.max(Math.abs(a[0] - b[0]), Math.abs(a[1] - b[1]), Math.abs(a[2] - b[2]));
}
// Regression for saturated UI recordings: default JPEG extraction can shift
// high-chroma reds before browser capture. Forcing PNG should keep extracted
// source-video frames effectively identical to the decoded source pixels.
describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-format-"));
const UI_FIXTURE = join(FIXTURE_DIR, "ui-red.mp4");
beforeAll(async () => {
const result = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`color=c=0xffe7ee:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
"-vf",
"drawbox=x=44:y=58:w=152:h=44:color=0xdd382e@1:t=fill,drawbox=x=64:y=75:w=112:h=10:color=0xfff0f0@1:t=fill",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"0",
"-pix_fmt",
"yuv420p",
"-color_primaries",
"bt709",
"-color_trc",
"bt709",
"-colorspace",
"bt709",
UI_FIXTURE,
]);
if (!result.success) {
throw new Error(`UI color fixture synthesis failed: ${result.stderr.slice(-400)}`);
}
}, 30_000);
afterAll(() => {
if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
}, 30_000);
function fixtureVideo(): VideoElement {
return {
id: "ui",
src: UI_FIXTURE,
start: 0,
end: 1,
mediaStart: 0,
loop: false,
hasAudio: false,
};
}
it("extracts into a work directory whose name holds a printf code", async () => {
const outDir = join(FIXTURE_DIR, "x 12% Up");
mkdirSync(outDir, { recursive: true });
const result = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
fps: 1,
outputDir: outDir,
});
expect(result.errors).toEqual([]);
const frame = result.extracted[0]!.framePaths.get(0)!;
expect(frame.startsWith(outDir)).toBe(true);
expect(existsSync(frame)).toBe(true);
});
it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) jpg extraction", async () => {
const source = join(FIXTURE_DIR, "white-1080.mp4");
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"color=c=white:s=1080x16:d=1:r=1",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"0",
"-pix_fmt",
"yuv420p",
source,
]);
expect(synth.success).toBe(true);
const outDir = join(FIXTURE_DIR, "out-white-1080");
mkdirSync(outDir, { recursive: true });
const result = await extractAllVideoFrames(
[{ ...fixtureVideo(), id: "white", src: source }],
FIXTURE_DIR,
{ fps: 1, outputDir: outDir },
);
expect(result.errors).toEqual([]);
const frame = result.extracted[0]!.framePaths.get(0)!;
expect(frame.endsWith(".jpg")).toBe(true);
const lastColumns = (file: string) => [
...spawnSync("ffmpeg", [
"-v",
"error",
"-i",
file,
"-frames:v",
"1",
"-vf",
"format=gray,crop=8:1:iw-8:8",
"-f",
"rawvideo",
"-",
]).stdout,
];
const delivered = lastColumns(frame);
const expected = lastColumns(source);
expect(delivered).toHaveLength(8);
for (const [i, v] of delivered.entries())
expect(Math.abs(v - expected[i]!)).toBeLessThanOrEqual(5);
}, 60_000);
async function proresFrame(name: string, pixFmt: string, profile: string, source: string) {
const clip = join(FIXTURE_DIR, `${name}.mov`);
const synth = await runFfmpeg(
["-y", "-loglevel", "error", "-f", "lavfi", "-i", source].concat([
"-c:v",
"prores_ks",
"-profile:v",
profile,
"-pix_fmt",
pixFmt,
clip,
]),
);
expect(synth.success).toBe(true);
const outDir = join(FIXTURE_DIR, `out-${name}`);
mkdirSync(outDir, { recursive: true });
const result = await extractAllVideoFrames(
[{ ...fixtureVideo(), id: name, src: clip }],
FIXTURE_DIR,
{ fps: 1, outputDir: outDir },
);
expect(result.errors).toEqual([]);
return result.extracted[0]!.framePaths.get(0)!;
}
it("extracts ProRes without an alpha plane to jpg", async () => {
const frame = await proresFrame("prores422", "yuv422p10le", "2", "testsrc2=s=320x180:d=1:r=1");
expect(frame.endsWith(".jpg")).toBe(true);
}, 60_000);
it("keeps the alpha of ProRes 4444 exact in png frames", async () => {
const frame = await proresFrame(
"prores4444",
"yuva444p10le",
"4",
"color=c=0x00ff00@0.5:s=320x180:d=1:r=1,format=yuva444p10le",
);
expect(frame.endsWith(".png")).toBe(true);
const alpha = spawnSync("ffmpeg", [
"-v",
"error",
"-i",
frame,
"-vf",
"alphaextract,crop=1:1:10:10",
"-f",
"rawvideo",
"-",
]);
expect(Math.abs(alpha.stdout[0]! - 128)).toBeLessThanOrEqual(1);
}, 60_000);
it("keeps color-sensitive UI reds closer to source when extraction is forced to png", async () => {
const defaultOut = join(FIXTURE_DIR, "out-default");
const pngOut = join(FIXTURE_DIR, "out-png");
mkdirSync(defaultOut, { recursive: true });
mkdirSync(pngOut, { recursive: true });
const defaultResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
fps: 1,
outputDir: defaultOut,
});
const pngResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
fps: 1,
outputDir: pngOut,
format: "png",
});
expect(defaultResult.errors).toEqual([]);
expect(pngResult.errors).toEqual([]);
const defaultFrame = defaultResult.extracted[0]!.framePaths.get(0)!;
const pngFrame = pngResult.extracted[0]!.framePaths.get(0)!;
expect(defaultFrame.endsWith(".jpg")).toBe(true);
expect(pngFrame.endsWith(".png")).toBe(true);
let worstDefaultDelta = 0;
let worstPngDelta = 0;
for (const [x, y] of RED_SAMPLE_PIXELS) {
const sourcePixel = readFirstFramePixel(UI_FIXTURE, x, y);
worstDefaultDelta = Math.max(
worstDefaultDelta,
maxChannelDelta(sourcePixel, readFirstFramePixel(defaultFrame, x, y)),
);
worstPngDelta = Math.max(
worstPngDelta,
maxChannelDelta(sourcePixel, readFirstFramePixel(pngFrame, x, y)),
);
}
expect(worstPngDelta).toBeLessThanOrEqual(5);
expect(worstPngDelta).toBeLessThanOrEqual(worstDefaultDelta);
}, 60_000);
// Chrome runs with --force-color-profile=srgb and colour-manages any frame not tagged sRGB;
// the SDR encoder writes the canvas code values as they are, so frames must reach it unconverted.
it.each([
["BT.709", "bt709"],
["BT.601", "smpte170m"],
])(
"declares png frames of a %s source as sRGB so Chrome shows their code values",
async (name, tag) => {
const fixture = join(FIXTURE_DIR, `${tag}-shadow.mp4`);
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`color=c=0x101010:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
"-vf",
`setparams=color_primaries=${tag}:color_trc=${tag}:colorspace=${tag}:range=tv`,
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"0",
"-pix_fmt",
"yuv420p",
fixture,
]);
if (!synth.success) {
throw new Error(`${name} fixture synthesis failed: ${synth.stderr.slice(-400)}`);
}
expect((await extractVideoMetadata(fixture)).colorSpace?.colorTransfer).toBe(tag);
const outputDir = join(FIXTURE_DIR, `out-png-transfer-${tag}`);
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[{ ...fixtureVideo(), id: tag, src: fixture }],
FIXTURE_DIR,
{ fps: 1, outputDir, format: "png" },
);
expect(result.errors).toEqual([]);
const frame = result.extracted[0]!.framePaths.get(0)!;
// Chrome reads cICP before sRGB, and ffmpeg < 6.0 writes no cICP, so only an sRGB chunk
// without cICP means "these are sRGB code values" on every ffmpeg.
const chunks = pngChunkTypes(frame);
expect(chunks).toContain("sRGB");
expect(chunks).not.toContain("cICP");
expect(readFirstFramePixel(frame, 10, 10)).toEqual(readFirstFramePixel(fixture, 10, 10));
},
60_000,
);
it("extracts jpg frames of a BT.709 source as the BT.601 full range JPEG readers assume", async () => {
const fixture = join(FIXTURE_DIR, "bt709-patch.mp4");
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1,format=rgb24,drawbox=x=31:y=0:w=64:h=${UI_FIXTURE_HEIGHT}:c=0x0000FE:t=fill`,
"-vf",
"scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-crf",
"0",
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
const outputDir = join(FIXTURE_DIR, "out-jpg-matrix");
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[{ ...fixtureVideo(), id: "bt709-jpg", src: fixture }],
FIXTURE_DIR,
{ fps: 1, outputDir, format: "jpg" },
);
expect(result.errors).toEqual([]);
const frame = result.extracted[0]!.framePaths.get(0)!;
// x=10 is flat colour; x=32 sits one pixel inside the blue edge.
for (const x of [10, 32]) {
const source = readFirstFramePixel(fixture, x, 10);
const shown = readFirstFramePixel(frame, x, 10);
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
expect(worst, `x=${x}: source ${source} jpg ${shown}`).toBeLessThanOrEqual(3);
}
}, 60_000);
// Measured in Chrome 152: untagged H.264 plays as BT.601 at 1280x718 and BT.709 at 1280x720 or 406x720;
// untagged VP9 and AV1 play as BT.601 at every size.
it.each([
{ codec: "libx264", height: 718, matrix: "bt601", format: "png" },
{ codec: "libx264", height: 718, matrix: "bt601", format: "jpg" },
{ codec: "libx264", height: 720, matrix: "bt709", format: "png" },
{ codec: "libx264", height: 720, matrix: "bt709", format: "jpg" },
{ codec: "libvpx-vp9", height: 720, matrix: "bt601", format: "png" },
...(AV1_ENCODER
? [{ codec: AV1_ENCODER, height: 720, matrix: "bt601", format: "png" } as const]
: []),
] as const)(
"reads an untagged $height-line $codec source as $matrix like Chrome's own playback ($format)",
async ({ codec, height, matrix, format }) => {
const extension = codec === "libvpx-vp9" ? "webm" : "mp4";
const fixture = join(FIXTURE_DIR, `untagged-${codec}-${height}.${extension}`);
const synth = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
`color=c=0xC83C28:s=64x${height}:d=1:r=1`,
"-vf",
"scale=out_color_matrix=bt601:out_range=tv,format=yuv420p,setparams=colorspace=unknown:color_primaries=unknown:color_trc=unknown:range=tv",
"-c:v",
codec,
...(codec === "libx264" ? ["-qp", "0"] : ["-crf", "0", "-b:v", "0"]),
...(codec === "libaom-av1" ? ["-cpu-used", "8"] : []),
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
expect((await extractVideoMetadata(fixture)).colorSpace?.colorSpace ?? "unknown").toBe(
"unknown",
);
const result = await extractVideoFramesRange(
fixture,
`${basename(fixture)}-${format}`,
0,
1,
{
fps: 1,
outputDir: join(FIXTURE_DIR, "out-untagged"),
format,
},
);
const pixelAt = (file: string, decode: string): number[] => [
...spawnSync("ffmpeg", [
"-v",
"error",
"-i",
file,
"-vf",
`${decode}format=rgb24,crop=1:1:10:10`,
"-frames:v",
"1",
"-f",
"rawvideo",
"-",
]).stdout,
];
const expected = pixelAt(fixture, `scale=in_color_matrix=${matrix}:in_range=tv,`);
const shown = pixelAt(result.framePaths.get(0)!, "");
expect([expected.length, shown.length]).toEqual([3, 3]);
const worst = Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)));
expect(worst, `${matrix} decode ${expected}, ${format} ${shown}`).toBeLessThanOrEqual(
format === "png" ? 1 : 3,
);
},
60_000,
);
it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => {
// Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks.
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4");
const synth = await runFfmpeg([
...["-y", "-v", "error", "-f", "lavfi", "-i"],
"color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-ntsc-ms"),
format: "png",
});
const indexOf = (file: string): number => {
const row = spawnSync("ffmpeg", [
"-v",
"error",
"-i",
file,
"-vf",
"format=gray,crop=64:1:0:4",
"-f",
"rawvideo",
"-",
]).stdout;
return [...Array(8).keys()].reduce(
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
0,
);
};
const shown = [...result.framePaths.keys()]
.sort((a, b) => a - b)
.map((i) => indexOf(result.framePaths.get(i)!));
const steps = shown.slice(1).map((n, i) => n - shown[i]!);
expect(shown.length).toBeGreaterThanOrEqual(44);
expect(
steps.filter((step) => step > 1),
`source frames ${shown.join(",")}`,
).toEqual([]);
expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1);
}, 60_000);
it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => {
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4");
const synth = await runFfmpeg([
...["-y", "-v", "error", "-f", "lavfi", "-i"],
"color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
const indexOf = (file: string): number => {
const row = spawnSync("ffmpeg", [
"-v",
"error",
"-i",
file,
"-vf",
"format=gray,crop=64:1:0:4",
"-f",
"rawvideo",
"-",
]).stdout;
return [...Array(8).keys()].reduce(
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
0,
);
};
const options = {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-ntsc-seams"),
format: "png" as const,
};
const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options);
const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201];
const differing: string[] = [];
for (const firstFrame of seams) {
const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, {
...options,
frameRanges: [{ firstFrame, frames: 12 }],
});
for (const [slot, path] of segment.framePaths) {
const expected = indexOf(single.framePaths.get(slot)!);
const shown = indexOf(path);
if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`);
}
}
expect(differing).toEqual([]);
}, 120_000);
// ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour
// filters would cut the still that ends this window short.
it.each([
{
name: "full range BT.709",
format: "png",
codec: "libx264",
pixFmt: "yuvj420p",
probed: { colorTransfer: "bt709" },
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
},
{
name: "full range BT.709",
format: "jpg",
codec: "libx264",
pixFmt: "yuvj420p",
probed: { colorTransfer: "bt709" },
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
},
{
name: "BT.470BG gamma 2.8",
format: "png",
codec: "libx264",
pixFmt: "yuv420p",
probed: { colorTransfer: "bt470bg" },
tags: "color_primaries=bt470bg:color_trc=bt470bg:colorspace=bt470bg",
},
{
name: "RGB",
format: "png",
codec: "libx264rgb",
pixFmt: "rgb24",
probed: { colorSpace: "gbr" },
tags: "",
},
] as const)(
"keeps every frame and the colours of a $name VFR window that ends on a still ($format)",
async ({ name, format, codec, pixFmt, probed, tags }) => {
const fixture = join(FIXTURE_DIR, `vfr-still-${name.replace(/\W+/g, "-")}-${format}.mp4`);
const synth = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=2:r=60`,
"-vf",
`select='not(between(n\\,30\\,89))'${tags ? `,setparams=${tags}` : ""}`,
"-fps_mode",
"vfr",
"-c:v",
codec,
"-preset",
"ultrafast",
"-pix_fmt",
pixFmt,
fixture,
]);
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
const window = 0.616666;
const unfilteredDir = join(FIXTURE_DIR, `out-${basename(fixture)}-unfiltered`);
mkdirSync(unfilteredDir, { recursive: true });
const unfiltered = await runFfmpeg([
"-v",
"error",
"-ss",
"0",
"-i",
fixture,
"-t",
String(window),
"-fps_mode",
"cfr",
"-r",
"30",
join(unfilteredDir, "f_%05d.png"),
]);
expect(unfiltered.success).toBe(true);
const result = await extractVideoFramesRange(fixture, basename(fixture), 0, window, {
fps: 30,
outputDir: join(FIXTURE_DIR, "out-vfr-still"),
format,
});
expect(result.metadata.isVFR).toBe(true);
expect(result.metadata.colorSpace).toMatchObject(probed);
expect(result.totalFrames).toBe(readdirSync(unfilteredDir).length);
const last = result.framePaths.get(result.totalFrames - 1)!;
const source = readFirstFramePixel(fixture, 10, 10);
const shown = readFirstFramePixel(last, 10, 10);
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
expect(worst, `source ${source} ${format} ${shown}`).toBeLessThanOrEqual(
format === "png" ? 0 : 3,
);
if (format === "png") expect(pngChunkTypes(last)).toContain("sRGB");
},
60_000,
);
it("leaves SDR-to-HDR frames in the HDR colours they were converted to", async () => {
const outputDir = join(FIXTURE_DIR, "out-sdr-to-hdr-gate");
mkdirSync(outputDir, { recursive: true });
const result = await extractVideoFramesRange(UI_FIXTURE, "sdr-to-hdr-gate", 0, 1, {
fps: 1,
outputDir,
format: "png",
sdrToHdrTransfer: "pq",
});
expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB");
}, 60_000);
it("keeps jpg and png extraction caches separate", async () => {
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-"));
try {
const defaultOut = join(FIXTURE_DIR, "cache-default");
const pngOut = join(FIXTURE_DIR, "cache-png");
const pngHitOut = join(FIXTURE_DIR, "cache-png-hit");
mkdirSync(defaultOut, { recursive: true });
mkdirSync(pngOut, { recursive: true });
mkdirSync(pngHitOut, { recursive: true });
const defaultResult = await extractAllVideoFrames(
[fixtureVideo()],
FIXTURE_DIR,
{ fps: 1, outputDir: defaultOut },
undefined,
{ extractCacheDir: cacheDir },
);
expect(defaultResult.errors).toEqual([]);
expect(defaultResult.phaseBreakdown.cacheHits).toBe(0);
expect(defaultResult.phaseBreakdown.cacheMisses).toBe(1);
expect(defaultResult.extracted[0]!.framePaths.get(0)!.endsWith(".jpg")).toBe(true);
const pngMiss = await extractAllVideoFrames(
[fixtureVideo()],
FIXTURE_DIR,
{ fps: 1, outputDir: pngOut, format: "png" },
undefined,
{ extractCacheDir: cacheDir },
);
expect(pngMiss.errors).toEqual([]);
expect(pngMiss.phaseBreakdown.cacheHits).toBe(0);
expect(pngMiss.phaseBreakdown.cacheMisses).toBe(1);
expect(pngMiss.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
const pngHit = await extractAllVideoFrames(
[fixtureVideo()],
FIXTURE_DIR,
{ fps: 1, outputDir: pngHitOut, format: "png" },
undefined,
{ extractCacheDir: cacheDir },
);
expect(pngHit.errors).toEqual([]);
expect(pngHit.phaseBreakdown.cacheHits).toBe(1);
expect(pngHit.phaseBreakdown.cacheMisses).toBe(0);
expect(pngHit.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
} finally {
rmSync(cacheDir, { recursive: true, force: true });
}
}, 60_000);
it("dedupes identical extractions within one render", async () => {
const outputDir = join(FIXTURE_DIR, "out-dedupe");
mkdirSync(outputDir, { recursive: true });
const videoA: VideoElement = { ...fixtureVideo(), id: "dupe-a" };
const videoB: VideoElement = { ...fixtureVideo(), id: "dupe-b" };
const result = await extractAllVideoFrames([videoA, videoB], FIXTURE_DIR, {
fps: 1,
outputDir,
});
expect(result.errors).toEqual([]);
expect(result.extracted).toHaveLength(2);
const first = result.extracted[0]!;
const second = result.extracted[1]!;
expect(first.videoId).toBe("dupe-a");
expect(second.videoId).toBe("dupe-b");
expect(second.outputDir).toBe(first.outputDir);
expect(Array.from(second.framePaths.entries())).toEqual(Array.from(first.framePaths.entries()));
const frameDirs = readdirSync(outputDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name);
expect(frameDirs).toEqual(["dupe-a"]);
expect(readdirSync(first.outputDir).filter((f) => f.endsWith(".jpg"))).toHaveLength(
first.totalFrames,
);
}, 60_000);
});
@@ -0,0 +1,675 @@
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { spawnSync } from "node:child_process";
import {
extractAllVideoFrames,
extractVideoFramesRange,
type VideoElement,
type ExtractionResult,
} from "./videoFrameExtractor.js";
import { extractVideoMetadata } from "../utils/ffprobe.js";
import { runFfmpeg } from "../utils/runFfmpeg.js";
import { SCHEMA_PREFIX } from "./extractionCache.js";
// Pass-through, so a test can tell whether extraction ran ffmpeg.
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
});
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
const HAS_ZSCALE =
HAS_FFMPEG &&
/\szscale\s/.test(spawnSync("ffmpeg", ["-hide_banner", "-filters"]).stdout.toString());
type Rgb = [number, number, number];
const UI_FIXTURE_WIDTH = 240;
const UI_FIXTURE_HEIGHT = 160;
function pngChunkTypes(path: string): string[] {
const bytes = readFileSync(path);
const types: string[] = [];
for (let offset = 8; offset + 8 <= bytes.length; offset += 12 + bytes.readUInt32BE(offset)) {
types.push(bytes.toString("latin1", offset + 4, offset + 8));
}
return types;
}
function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb {
const result = spawnSync(
"ffmpeg",
[
"-v",
"error",
"-i",
mediaPath,
"-frames:v",
"1",
"-f",
"rawvideo",
"-pix_fmt",
"rgb24",
"pipe:1",
],
{ maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 },
);
if (result.status !== 0) {
throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`);
}
const offset = (y * UI_FIXTURE_WIDTH + x) * 3;
return [
result.stdout[offset] ?? 0,
result.stdout[offset + 1] ?? 0,
result.stdout[offset + 2] ?? 0,
];
}
// Release builds print "version 8.1.3" or "n8.1.3"; master builds print "N-<number>" and are newer.
const FFPROBE_VERSION = spawnSync("ffprobe", ["-version"]).stdout?.toString() ?? "";
const FFPROBE_RELEASE = FFPROBE_VERSION.match(/version n?(\d+)\.(\d+)/)
?.slice(1)
.map(Number);
const FFPROBE_READS_CONTAINER_TRANSFER =
/version N-/.test(FFPROBE_VERSION) ||
(FFPROBE_RELEASE !== undefined &&
(FFPROBE_RELEASE[0]! > 8 || (FFPROBE_RELEASE[0] === 8 && FFPROBE_RELEASE[1]! >= 1)));
/** The colour probes run so far: first-frame reads, and SPS transfer reads of an HLG source. */
function colourProbeRuns() {
const runs = vi
.mocked(runFfmpeg)
.mock.calls.map(([args]) => args)
.filter((args) => args.some((arg) => /showinfo|trace_headers/.test(arg)));
return {
firstFrame: runs.filter((args) => args.includes("-frames:v")).length,
keyframes: runs.filter((args) => args.includes("nokey")).length,
};
}
function colourProbesSince(before: ReturnType<typeof colourProbeRuns>) {
const now = colourProbeRuns();
return {
firstFrame: now.firstFrame - before.firstFrame,
keyframes: now.keyframes - before.keyframes,
};
}
describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
let fixtureDir = "";
beforeAll(() => {
fixtureDir = mkdtempSync(join(tmpdir(), "hf-forced-sdr-tonemap-test-"));
});
afterAll(() => {
rmSync(fixtureDir, { recursive: true, force: true });
}, 30_000);
async function synthesizeHlgClip(
path: string,
vui = "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9",
transfer = "arib-std-b67",
): Promise<void> {
const synthesized = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"color=c=0xe0b080:s=64x64:r=1:d=1",
"-vf",
`zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=${transfer}:m=bt2020nc:r=tv,format=yuv420p`,
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-color_primaries",
"bt2020",
"-color_trc",
transfer,
"-colorspace",
"bt2020nc",
"-bsf:v",
`h264_metadata=${vui}`,
path,
]);
if (!synthesized.success) {
throw new Error(`HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`);
}
}
it.each([
[
"HLG",
"matrix",
"arib-std-b67",
"colour_primaries=9:transfer_characteristics=18:matrix_coefficients=2",
],
[
"HLG",
"primaries",
"arib-std-b67",
"colour_primaries=2:transfer_characteristics=18:matrix_coefficients=9",
],
[
"HLG",
"matrix and primaries",
"arib-std-b67",
"colour_primaries=2:transfer_characteristics=18:matrix_coefficients=2",
],
[
"PQ",
"matrix and primaries",
"smpte2084",
"colour_primaries=2:transfer_characteristics=16:matrix_coefficients=2",
],
[
"HLG",
"transfer",
"arib-std-b67",
"colour_primaries=9:transfer_characteristics=2:matrix_coefficients=9",
],
])(
"tone-maps %s footage whose video stream has no %s tag like the fully tagged clip",
async (name, missing, transfer, vui) => {
const slug = `${name}-no-${missing}`.replace(/\W+/g, "-");
const tagged = join(fixtureDir, `${name}-fully-tagged.mp4`);
const untagged = join(fixtureDir, `${slug}.mp4`);
const vuiTransfer = transfer === "smpte2084" ? 16 : 18;
await synthesizeHlgClip(
tagged,
`colour_primaries=9:transfer_characteristics=${vuiTransfer}:matrix_coefficients=9`,
transfer,
);
await synthesizeHlgClip(untagged, vui, transfer);
const extract = (source: string, id: string) =>
extractVideoFramesRange(source, id, 0, 1, {
fps: 1,
outputDir: join(fixtureDir, `out-${id}`),
format: "png",
toneMapHdrToSdr: true,
});
if (missing === "transfer") {
// ffprobe before 8.1 reads the stream's unspecified transfer, so the clip is SDR there, as on main.
if (!FFPROBE_READS_CONTAINER_TRANSFER) {
await expect(extract(untagged, slug)).resolves.toBeDefined();
return;
}
expect((await extractVideoMetadata(untagged)).colorSpace?.colorTransfer).toBe(transfer);
}
const reference = await extract(tagged, `ref-${slug}`);
const result = await extract(untagged, slug);
expect(readFileSync(result.framePaths.get(0)!)).toEqual(
readFileSync(reference.framePaths.get(0)!),
);
},
60_000,
);
it("probes each source's first frame once, however many cuts of it extract together", async () => {
const sources = ["a", "b"].map((name) => join(fixtureDir, `cuts-${name}.mp4`));
for (const source of sources) await synthesizeHlgClip(source);
const probes = colourProbeRuns;
const before = probes();
await Promise.all(
Array.from({ length: 12 }, (_, cut) =>
extractVideoFramesRange(sources[cut % 2]!, `cut-${cut}`, 0, 1, {
fps: 1,
outputDir: join(fixtureDir, "cuts"),
format: "png",
toneMapHdrToSdr: true,
}),
),
);
expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 });
}, 60_000);
it("probes a source again after its first-frame probe failed", async () => {
const source = join(fixtureDir, "probe-retry.mp4");
await synthesizeHlgClip(source);
const probes = colourProbeRuns;
const { runFfmpeg: real } =
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
vi.mocked(runFfmpeg).mockImplementation((args, opts) =>
args.includes("showinfo")
? Promise.resolve({
success: false,
exitCode: 1,
stderr: "",
durationMs: 0,
terminationReason: "exit",
})
: real(args, opts),
);
const extract = (id: string) =>
extractVideoFramesRange(source, id, 0, 1, {
fps: 1,
outputDir: join(fixtureDir, "retry"),
format: "png",
toneMapHdrToSdr: true,
});
const before = probes();
try {
await extract("retry-failed");
} finally {
vi.mocked(runFfmpeg).mockImplementation(real);
}
await extract("retry-again");
expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 1 });
}, 60_000);
it("tone-maps an HLG source at HLG's white when a large header SEI follows its SPS", async () => {
const source = join(fixtureDir, "hlg-x265-header.mp4");
// x265 stores its settings as an SEI after the SPS in the file header; traced, it runs past 64 KB.
// No B-frames: at 1 fps their reorder gap is a whole second, which Windows ffmpeg's mp4 muxer can refuse.
const synthesized = await runFfmpeg([
...["-y", "-hide_banner", "-loglevel", "error"],
...["-f", "lavfi", "-i", "color=c=0xe0b080:s=64x64:r=1:d=1"],
"-vf",
"zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=arib-std-b67:m=bt2020nc:r=tv,format=yuv420p10le",
...[
"-c:v",
"libx265",
"-x265-params",
"colorprim=bt2020:transfer=arib-std-b67:colormatrix=bt2020nc:log-level=error:bframes=0",
],
...["-color_primaries", "bt2020", "-color_trc", "arib-std-b67", "-colorspace", "bt2020nc"],
source,
]);
if (!synthesized.success) {
throw new Error(`x265 HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`);
}
const before = vi.mocked(runFfmpeg).mock.calls.length;
await extractVideoFramesRange(source, "hlg-x265-header", 0, 1, {
fps: 1,
outputDir: join(fixtureDir, "x265-header"),
format: "png",
toneMapHdrToSdr: true,
});
const toneMaps = vi
.mocked(runFfmpeg)
.mock.calls.slice(before)
.flatMap(([args]) => args.filter((arg) => arg.includes("tonemap=")));
expect(toneMaps).toHaveLength(1);
expect(toneMaps[0]).toContain("npl=203");
}, 60_000);
it("reads an HLG source's keyframes again after that read failed", async () => {
const source = join(fixtureDir, "keyframe-retry.mp4");
await synthesizeHlgClip(source);
const { runFfmpeg: real } =
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
vi.mocked(runFfmpeg).mockImplementation((args, opts) =>
args.includes("nokey")
? Promise.resolve({
success: false,
exitCode: 1,
stderr: "",
durationMs: 0,
terminationReason: "exit",
})
: real(args, opts),
);
const extract = (id: string) =>
extractVideoFramesRange(source, id, 0, 1, {
fps: 1,
outputDir: join(fixtureDir, "keyframe-retry"),
format: "png",
toneMapHdrToSdr: true,
});
const before = colourProbeRuns();
try {
await extract("keyframes-failed");
} finally {
vi.mocked(runFfmpeg).mockImplementation(real);
}
await extract("keyframes-again");
expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 });
}, 60_000);
// One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer.
it("tone-maps each frame of a mixed HLG and PQ stream with its own transfer", async () => {
const segment = async (name: string, transfer: number) => {
const path = join(fixtureDir, `mixed-${name}.h264`);
const result = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
"testsrc2=s=160x90:r=25:d=0.2",
"-c:v",
"libx264",
// No B-frames: genpts stamps a raw stream in decode order, so reordered frames get wrong times.
"-bf",
"0",
"-pix_fmt",
"yuv420p",
"-bsf:v",
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
"-f",
"h264",
path,
]);
if (!result.success)
throw new Error(`segment synthesis failed: ${result.stderr.slice(-400)}`);
return readFileSync(path);
};
const mux = async (name: string, stream: Buffer) => {
const raw = join(fixtureDir, `${name}.h264`);
const path = join(fixtureDir, `${name}.mp4`);
writeFileSync(raw, stream);
const result = await runFfmpeg([
...["-y", "-v", "error", "-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw],
...["-c", "copy", path],
]);
if (!result.success) throw new Error(`mux failed: ${result.stderr.slice(-400)}`);
return path;
};
const hlg = await segment("hlg", 18);
const pq = await segment("pq", 16);
const mixed = await mux("mixed-hlg-pq", Buffer.concat([hlg, pq]));
const pqOnly = await mux("mixed-pq-only", pq);
const extract = (source: string, id: string, duration: number) =>
extractVideoFramesRange(source, id, 0, duration, {
fps: 25,
outputDir: join(fixtureDir, `out-${id}`),
format: "png",
toneMapHdrToSdr: true,
});
const mixedFrames = await extract(mixed, "mixed-hlg-pq", 0.4);
const pqFrames = await extract(pqOnly, "mixed-pq-only", 0.2);
expect(readFileSync(mixedFrames.framePaths.get(5)!)).toEqual(
readFileSync(pqFrames.framePaths.get(0)!),
);
}, 60_000);
// A stream-copy trim opens on edit-list pre-roll, which ffprobe counts as packets without frames.
it("keeps per-frame tags in a mixed HLG and PQ file trimmed with a stream copy", async () => {
const run = async (args: string[]) => {
const result = await runFfmpeg(["-y", "-v", "error", ...args]);
if (!result.success) throw new Error(`fixture failed: ${result.stderr.slice(-400)}`);
};
const segment = async (name: string, transfer: number) => {
const path = join(fixtureDir, `trim-${name}.h264`);
await run([
...["-f", "lavfi", "-i", "testsrc2=s=160x90:r=25:d=2"],
...["-c:v", "libx264", "-g", "50", "-bf", "0"],
...["-pix_fmt", "yuv420p", "-bsf:v"],
`h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
...["-f", "h264", path],
]);
return readFileSync(path);
};
const mux = async (name: string, stream: Buffer) => {
const raw = join(fixtureDir, `${name}.h264`);
const path = join(fixtureDir, `${name}.mp4`);
writeFileSync(raw, stream);
await run(["-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw, "-c", "copy", path]);
return path;
};
const pq = await segment("pq", 16);
const mixed = await mux("trim-hlg-then-pq", Buffer.concat([await segment("hlg", 18), pq]));
const pqOnly = await mux("trim-pq-only", pq);
const trimmed = join(fixtureDir, "trim-from-hlg.mp4");
await run(["-ss", "1", "-i", mixed, "-c", "copy", trimmed]);
const extract = (source: string, id: string, start: number) =>
extractVideoFramesRange(source, id, start, 0.04, {
fps: 25,
outputDir: join(fixtureDir, `out-${id}`),
format: "png",
toneMapHdrToSdr: true,
});
// Trimmed 1.6 s is the mixed file's 2.6 s: the PQ segment's 0.6 s.
const fromTrim = await extract(trimmed, "trim-from-hlg", 1.6);
const fromPq = await extract(pqOnly, "trim-pq-only", 0.6);
const shown = readFirstFramePixel(fromTrim.framePaths.get(0)!, 40, 40);
const expected = readFirstFramePixel(fromPq.framePaths.get(0)!, 40, 40);
expect(Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)))).toBeLessThanOrEqual(2);
}, 60_000);
it("matches Studio's HLG tone map and isolates transformed cache entries", async () => {
const source = join(fixtureDir, "hlg-warm.mp4");
await synthesizeHlgClip(source);
const reference = join(fixtureDir, "studio-reference.png");
const referenceResult = await runFfmpeg([
"-y",
"-ss",
"0",
"-i",
source,
"-t",
"1",
"-vf",
"fps=1,zscale=t=linear:npl=203,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv",
"-q:v",
"0",
"-compression_level",
"1",
reference,
]);
if (!referenceResult.success) {
throw new Error(`Studio reference extraction failed: ${referenceResult.stderr.slice(-400)}`);
}
const cacheDir = join(fixtureDir, "cache");
const video = (id: string): VideoElement => ({
id,
src: source,
start: 0,
end: 1,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const extract = (id: string, toneMapHdrToSdr = false) =>
extractAllVideoFrames(
[video(id)],
fixtureDir,
{
fps: 1,
outputDir: join(fixtureDir, id),
format: "png",
toneMapHdrToSdr,
},
undefined,
{ extractCacheDir: cacheDir },
);
const plain = await extract("plain");
const toneMapped = await extract("tone-mapped", true);
const toneMappedAgain = await extract("tone-mapped-again", true);
expect(plain.errors).toEqual([]);
expect(toneMapped.errors).toEqual([]);
expect(toneMapped.phaseBreakdown.cacheHits).toBe(0);
expect(toneMapped.phaseBreakdown.cacheMisses).toBe(1);
expect(toneMappedAgain.phaseBreakdown.cacheHits).toBe(1);
expect(readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX))).toHaveLength(2);
const frame = (result: ExtractionResult): Buffer => {
const path = result.extracted[0]?.framePaths.get(0);
if (!path) throw new Error("expected extracted frame");
return readFileSync(path);
};
// Same pixels as Studio's tone map, declared as sRGB so Chrome shows them unconverted.
const rgb = (path: string): Buffer =>
spawnSync("ffmpeg", ["-v", "error", "-i", path, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"])
.stdout;
const toneMappedPath = toneMapped.extracted[0]!.framePaths.get(0)!;
expect(rgb(toneMappedPath)).toEqual(rgb(reference));
expect(pngChunkTypes(toneMappedPath)).toContain("sRGB");
expect(pngChunkTypes(toneMappedPath)).not.toContain("cICP");
expect(frame(plain)).not.toEqual(frame(toneMapped));
const toneMappedJpg = await extractAllVideoFrames([video("tone-mapped-jpg")], fixtureDir, {
fps: 1,
outputDir: join(fixtureDir, "tone-mapped-jpg"),
format: "jpg",
toneMapHdrToSdr: true,
});
expect(toneMappedJpg.errors).toEqual([]);
const jpgPixels = rgb(toneMappedJpg.extracted[0]!.framePaths.get(0)!);
const referencePixels = rgb(reference);
const worst = Math.max(...[...jpgPixels].map((v, i) => Math.abs(v - referencePixels[i]!)));
expect(worst, "tone-mapped jpg against Studio's tone map").toBeLessThanOrEqual(3);
expect(frame(toneMappedAgain)).toEqual(frame(toneMapped));
// A macOS ffmpeg that has zscale uses the same tone map, not VideoToolbox.
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
Object.defineProperty(process, "platform", { value: "darwin" });
try {
const mac = await extractAllVideoFrames([video("tone-mapped-mac")], fixtureDir, {
fps: 1,
outputDir: join(fixtureDir, "tone-mapped-mac"),
format: "png",
toneMapHdrToSdr: true,
});
expect(mac.errors).toEqual([]);
expect(rgb(mac.extracted[0]!.framePaths.get(0)!)).toEqual(rgb(reference));
} finally {
Object.defineProperty(process, "platform", platform);
}
}, 60_000);
it("warns once and keeps VideoToolbox, under its own cache key, when a macOS ffmpeg has no zscale", async () => {
const source = join(fixtureDir, "hlg-no-zscale.mp4");
await synthesizeHlgClip(source);
const cacheDir = join(fixtureDir, "no-zscale-cache");
const clip = (id: string): VideoElement => ({
id,
src: source,
start: 0,
end: 1,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const options = (id: string) => ({
fps: 1,
outputDir: join(fixtureDir, id),
format: "png" as const,
toneMapHdrToSdr: true,
});
const zscale = await extractAllVideoFrames(
[clip("zscale")],
fixtureDir,
options("zscale"),
undefined,
{
extractCacheDir: cacheDir,
},
);
expect(zscale.errors).toEqual([]);
const realPlatform = process.platform;
vi.resetModules();
vi.doMock("../utils/psnrFilterAvailability.js", () => ({
isFfmpegFilterAvailable: async () => false,
isPsnrFilterAvailable: async () => false,
}));
const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
Object.defineProperty(process, "platform", { value: "darwin" });
const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
try {
const { extractAllVideoFrames: extractOnMac } = await import("./videoFrameExtractor.js");
for (const [i, id] of ["no-zscale-1", "no-zscale-2"].entries()) {
const result = await extractOnMac([clip(id)], fixtureDir, options(id), undefined, {
extractCacheDir: cacheDir,
});
// The zscale frames above must not be served for the VideoToolbox path.
if (i === 0) expect(result.phaseBreakdown.cacheHits).toBe(0);
// Off macOS the VideoToolbox decode itself fails, which proves it was attempted.
if (realPlatform !== "darwin")
expect(JSON.stringify(result.errors)).toMatch(/videotoolbox/i);
}
const zscaleWarnings = stderr.mock.calls.filter(([message]) =>
String(message).includes("no zscale filter"),
);
expect(zscaleWarnings).toHaveLength(1);
} finally {
stderr.mockRestore();
Object.defineProperty(process, "platform", platform);
vi.doUnmock("../utils/psnrFilterAvailability.js");
vi.resetModules();
}
}, 60_000);
// A second process would read the frames back without the HDR10 light-level metadata the tone map uses.
it("tone-maps a VFR HDR10 window in one process, as the reference tone map would", async () => {
const source = join(fixtureDir, "pq-vfr-4000nit.mp4");
const synthesized = await runFfmpeg([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
"color=c=0xf0e0c0:s=64x64:r=60:d=2",
"-vf",
"select='not(between(n\\,30\\,89))',zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=smpte2084:m=bt2020nc:r=tv:npl=1000,format=yuv420p10le",
"-fps_mode",
"vfr",
"-c:v",
"libx265",
"-x265-params",
"log-level=error:hdr10=1:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:max-cll=4000,400:master-display=G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(40000000,50)",
source,
]);
if (!synthesized.success)
throw new Error(`PQ fixture synthesis failed: ${synthesized.stderr.slice(-400)}`);
const referenceDir = join(fixtureDir, "pq-vfr-reference");
mkdirSync(referenceDir, { recursive: true });
const reference = await runFfmpeg([
"-v",
"error",
"-ss",
"0",
"-i",
source,
"-t",
"0.616666",
"-vf",
"zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv,setparams=color_primaries=bt709:color_trc=iec61966-2-1",
"-fps_mode",
"cfr",
"-r",
"30",
"-q:v",
"0",
"-compression_level",
"1",
join(referenceDir, "frame_%05d.png"),
]);
expect(reference.success, reference.stderr).toBe(true);
const result = await extractVideoFramesRange(source, "pq-vfr", 0, 0.616666, {
fps: 30,
outputDir: join(fixtureDir, "pq-vfr-out"),
format: "png",
toneMapHdrToSdr: true,
});
expect(result.metadata.isVFR).toBe(true);
const referenceFrames = readdirSync(referenceDir).sort();
// One slot per 30fps tick before 0.616666s; x265 packet durations cut the reference's still short.
expect(result.totalFrames).toBe(19);
expect(readFileSync(result.framePaths.get(0)!)).toEqual(
readFileSync(join(referenceDir, referenceFrames[0]!)),
);
}, 60_000);
});
@@ -0,0 +1,386 @@
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { spawnSync } from "node:child_process";
import {
extractAllVideoFrames,
extractVideoFramesRange,
type VideoElement,
type ExtractedFrames,
} from "./videoFrameExtractor.js";
import { ffmpegStatsReader, runFfmpeg, type RunFfmpegOptions } from "../utils/runFfmpeg.js";
// Pass-through, so a test can tell whether extraction ran ffmpeg.
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
});
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
// Each output slot shows the frame on screen at its time, as the preview does.
describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-sampling-"));
const WIDTH = 32;
const HEIGHT = 16;
// Matroska stores whole-millisecond timestamps (a 30 fps frame at 66.67 ms reads 67 ms);
// a 1/30 timescale stores each frame exactly on a coarse tick.
const SOURCES = {
"mp4-60": { fps: 60, file: "index-60fps.mp4", muxer: [] },
"mkv-30": { fps: 30, file: "index-30fps.mkv", muxer: [] },
"mp4-30-timescale-30": {
fps: 30,
file: "index-30fps-ts30.mp4",
muxer: ["-video_track_timescale", "30"],
},
} as const;
type SourceName = keyof typeof SOURCES;
const sourcePath = (name: SourceName) => join(FIXTURE_DIR, SOURCES[name].file);
beforeAll(async () => {
for (const name of Object.keys(SOURCES) as SourceName[]) {
// Frame k carries luma 16 + 2k, so each extracted frame names its source index.
const result = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`nullsrc=s=${WIDTH}x${HEIGHT}:r=${SOURCES[name].fps}:d=1.5,geq=lum='16+2*N':cb=128:cr=128`,
"-c:v",
"libx264",
"-qp",
"0",
"-pix_fmt",
"yuv420p",
...SOURCES[name].muxer,
sourcePath(name),
]);
if (!result.success) throw new Error(`index fixture synthesis failed: ${result.stderr}`);
}
}, 30_000);
afterAll(() => {
rmSync(FIXTURE_DIR, { recursive: true, force: true });
}, 30_000);
function sourceIndexes(extracted: ExtractedFrames): number[] {
const decoded = spawnSync("ffmpeg", [
"-v",
"error",
"-i",
join(extracted.outputDir, extracted.framePattern),
"-f",
"rawvideo",
"-pix_fmt",
"gray",
"pipe:1",
]);
if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
const indexes: number[] = [];
for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) {
indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2));
}
return indexes;
}
it.each([
{ source: "mp4-60", fps: 24, startTime: 0, duration: 0.5 },
{ source: "mp4-60", fps: 10, startTime: 0.37, duration: 0.6 },
{ source: "mkv-30", fps: 30, startTime: 0, duration: 0.5 },
{ source: "mp4-30-timescale-30", fps: 60, startTime: 0, duration: 0.5 },
] as const)(
"$source: samples the frame on screen at each $fps fps slot from $startTime s",
async (c) => {
const extracted = await extractVideoFramesRange(
sourcePath(c.source),
`${c.source}-${c.fps}`,
c.startTime,
c.duration,
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "png" },
);
const onScreen = Array.from({ length: Math.round(c.duration * c.fps) }, (_, i) =>
Math.floor((c.startTime + i / c.fps) * SOURCES[c.source].fps + 1e-9),
);
expect(sourceIndexes(extracted)).toEqual(onScreen);
},
30_000,
);
// #5260: one ffmpeg process over a 26-minute clip hit ffmpegProcessTimeout.
it.each([
// 260 frames: one 240-frame segment, then 20 more.
{ startTime: 0.5, duration: 130, frames: 260 },
// 200.3 - 80.3 is 120.00000000000001 s: exactly one segment, not a second near-empty one.
{ startTime: 80.3, duration: 200.3 - 80.3, frames: 240 },
])(
"extracts $duration s from $startTime s at 2 fps across segments without a gap or repeat",
async (c) => {
const source = join(FIXTURE_DIR, "index-2fps-long.mp4");
if (!existsSync(source)) {
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`nullsrc=s=${WIDTH}x${HEIGHT}:r=2:d=205,geq=lum='16+2*mod(N\\,100)':cb=128:cr=128`,
"-c:v",
"libx264",
"-qp",
"0",
"-pix_fmt",
"yuv420p",
source,
]);
if (!synth.success) throw new Error(`long fixture synthesis failed: ${synth.stderr}`);
}
const extracted = await extractVideoFramesRange(
source,
`long-2fps-${c.startTime}`,
c.startTime,
c.duration,
{ fps: 2, outputDir: FIXTURE_DIR, format: "png" },
);
const onScreen = Array.from(
{ length: c.frames },
(_, i) => Math.floor((c.startTime + i / 2) * 2 + 1e-9) % 100,
);
expect(extracted.totalFrames).toBe(c.frames);
expect(sourceIndexes(extracted)).toEqual(onScreen);
},
60_000,
);
it("stops extraction only on a stall, and its ffmpeg reports each frame it writes", async () => {
const source = join(FIXTURE_DIR, "paced-2s.mp4");
const synth = await runFfmpeg([
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],
...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=2`, "-pix_fmt", "yuv420p", source],
]);
if (!synth.success) throw new Error(`paced fixture synthesis failed: ${synth.stderr}`);
const { runFfmpeg: real } =
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
let requested: RunFfmpegOptions | undefined;
const reported: number[] = [];
const readStats = ffmpegStatsReader(({ frames }) => {
if (frames) reported.push(frames);
});
// Real-time pacing spreads the stats over the run. The stall clock itself is runFfmpeg's to
// test; here it is lifted so a starved runner cannot race it.
vi.mocked(runFfmpeg).mockImplementation((args, opts) => {
requested = opts;
return real(["-stats_period", "0.1", "-re", ...args], {
...opts,
timeout: 60_000,
onStderr: readStats,
});
});
try {
const extracted = await extractVideoFramesRange(
source,
"paced-stall-only",
0,
2,
{ fps: 10, outputDir: FIXTURE_DIR, format: "png" },
undefined,
{ ffmpegProcessTimeout: 2000 },
);
expect(extracted.totalFrames).toBe(20);
expect(requested).toMatchObject({ timeout: 2000 });
expect(new Set(reported).size).toBeGreaterThan(1);
} finally {
vi.mocked(runFfmpeg).mockImplementation(real);
}
}, 30_000);
it.each([
{ name: "one keyframe", keyint: "infinite", gapSeconds: 8 },
{ name: "a keyframe every second", keyint: "30", gapSeconds: 0 },
])(
"gives the stall clock the time to decode up to the seek point ($name)",
async ({ keyint, gapSeconds }) => {
const source = join(FIXTURE_DIR, `gop-${keyint}.mp4`);
const synth = await runFfmpeg([
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],
...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=10`, "-pix_fmt", "yuv420p"],
...["-x264-params", `keyint=${keyint}:scenecut=0`, source],
]);
if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`);
const { runFfmpeg: real } =
await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
let requested: RunFfmpegOptions | undefined;
vi.mocked(runFfmpeg).mockImplementation((args, opts) => {
requested = opts;
return real(args, opts);
});
try {
await extractVideoFramesRange(
source,
`stall-seek-${keyint}`,
8,
1,
{ fps: 10, outputDir: FIXTURE_DIR, format: "png" },
undefined,
{ ffmpegProcessTimeout: 2000 },
);
expect(requested?.timeout).toBe(2000 + gapSeconds * 1000);
} finally {
vi.mocked(runFfmpeg).mockImplementation(real);
}
},
30_000,
);
it("stops a run that writes no new frame for the stall timeout", async () => {
// stdin stays open and empty, so ffmpeg waits for its first frame forever.
const stalled = await runFfmpeg(
["-f", "rawvideo", "-pix_fmt", "gray", "-s", "2x2", "-i", "pipe:0", "-f", "null", "-"],
{ timeout: 500 },
);
expect(stalled.terminationReason).toBe("inactivity");
}, 30_000);
// A distributed chunk extracts only the frames it renders; they must be the full extraction's.
describe("partial extraction", () => {
const gopSource = join(FIXTURE_DIR, "gop-30fps.mp4");
beforeAll(async () => {
// Real motion and long GOPs, so range seeks land mid-GOP.
const synth = await runFfmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"testsrc2=s=64x36:r=30:d=20",
"-c:v",
"libx264",
"-g",
"250",
"-pix_fmt",
"yuv420p",
gopSource,
]);
if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`);
}, 30_000);
function frameBytes(dir: string): Map<string, Buffer> {
return new Map(readdirSync(dir).map((name) => [name, readFileSync(join(dir, name))]));
}
it.each([
{ fps: 24, startTime: 1.3, duration: 17 },
{ fps: 60, startTime: 0, duration: 20 },
])(
"frameRanges at $fps fps write the same files as a full extraction",
async (c) => {
const full = await extractVideoFramesRange(
gopSource,
`full-${c.fps}`,
c.startTime,
c.duration,
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg" },
);
const last = full.totalFrames;
const ranges = [
{ firstFrame: 0, frames: 5 },
{ firstFrame: 97, frames: 40 },
{ firstFrame: last - 3, frames: 3 },
];
const partial = await extractVideoFramesRange(
gopSource,
`partial-${c.fps}`,
c.startTime,
c.duration,
{ fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg", frameRanges: ranges },
);
const fullFrames = frameBytes(full.outputDir);
const partialFrames = frameBytes(partial.outputDir);
expect(partialFrames.size).toBe(48);
for (const [name, bytes] of partialFrames) {
expect(fullFrames.get(name)?.equals(bytes), name).toBe(true);
}
},
60_000,
);
it.each([
{ fps: 24, mediaStart: 3.3 },
{ fps: 30, mediaStart: 0 },
{ fps: 60, mediaStart: 7.77 },
])(
"a deferred window to the source end at $fps fps reports the full frame count",
async (c) => {
const video = (): VideoElement => ({
id: `defer-${c.fps}`,
src: gopSource,
start: 0,
end: Infinity,
mediaStart: c.mediaStart,
loop: false,
hasAudio: false,
});
const full = await extractAllVideoFrames([video()], FIXTURE_DIR, {
fps: c.fps,
outputDir: join(FIXTURE_DIR, `defer-full-${c.fps}`),
});
const deferred = await extractAllVideoFrames([video()], FIXTURE_DIR, {
fps: c.fps,
outputDir: join(FIXTURE_DIR, `defer-${c.fps}`),
deferRangeExtraction: true,
});
expect(deferred.errors).toEqual([]);
const result = deferred.extracted[0]!;
expect(result.totalFrames).toBe(full.extracted[0]!.totalFrames);
expect(result.framePaths.size).toBe(0);
expect(readdirSync(result.outputDir)).toEqual([]);
expect(result.deferredRange).toEqual({
startTime: c.mediaStart,
durationSeconds: expect.any(Number),
format: "jpg",
});
},
60_000,
);
it("halves frames while they still cover the box a video is shown in", async () => {
const firstFrameSize = async (
videoId: string,
shownBox: { width: number; height: number },
) => {
const result = await extractVideoFramesRange(gopSource, videoId, 0, 1, {
fps: 30,
outputDir: FIXTURE_DIR,
format: "jpg",
frameRanges: [{ firstFrame: 0, frames: 1 }],
shownBox,
});
const [frame] = result.framePaths.values();
return spawnSync("ffprobe", [
"-v",
"error",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0",
"--",
frame!,
])
.stdout.toString()
.trim();
};
// 64x36 source: halved as often as the box allows (even sides round up); never below the box.
expect(await firstFrameSize("shown-quarter", { width: 16, height: 9 })).toBe("16,10");
expect(await firstFrameSize("shown-third", { width: 20, height: 12 })).toBe("32,18");
expect(await firstFrameSize("shown-near-full", { width: 40, height: 20 })).toBe("64,36");
}, 30_000);
});
});
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff