mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
834 lines
28 KiB
TypeScript
834 lines
28 KiB
TypeScript
import { vi, afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
import { basename, dirname, join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { spawnSync } from "node:child_process";
|
|
import {
|
|
extractAllVideoFrames,
|
|
extractVideoFramesRange,
|
|
type VideoElement,
|
|
} from "./videoFrameExtractor.js";
|
|
import { extractVideoMetadata } from "../utils/ffprobe.js";
|
|
import { runFfmpeg } from "../utils/runFfmpeg.js";
|
|
|
|
// Pass-through, so a test can tell whether extraction ran ffmpeg.
|
|
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
|
|
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
|
|
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
|
|
});
|
|
|
|
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
|
|
const FFMPEG_ENCODERS = HAS_FFMPEG
|
|
? spawnSync("ffmpeg", ["-hide_banner", "-encoders"]).stdout.toString()
|
|
: "";
|
|
// Homebrew's ffmpeg has libsvtav1 but not libaom-av1. Either one writes an untagged AV1 stream.
|
|
const AV1_ENCODER = ["libaom-av1", "libsvtav1"].find((name) =>
|
|
new RegExp(`\\s${name}\\s`).test(FFMPEG_ENCODERS),
|
|
);
|
|
|
|
type Rgb = [number, number, number];
|
|
|
|
const UI_FIXTURE_WIDTH = 240;
|
|
const UI_FIXTURE_HEIGHT = 160;
|
|
const RED_SAMPLE_PIXELS = [
|
|
[70, 72],
|
|
[118, 82],
|
|
[178, 92],
|
|
] as const;
|
|
|
|
function pngChunkTypes(path: string): string[] {
|
|
const bytes = readFileSync(path);
|
|
const types: string[] = [];
|
|
for (let offset = 8; offset + 8 <= bytes.length; offset += 12 + bytes.readUInt32BE(offset)) {
|
|
types.push(bytes.toString("latin1", offset + 4, offset + 8));
|
|
}
|
|
return types;
|
|
}
|
|
|
|
function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb {
|
|
const result = spawnSync(
|
|
"ffmpeg",
|
|
[
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
mediaPath,
|
|
"-frames:v",
|
|
"1",
|
|
"-f",
|
|
"rawvideo",
|
|
"-pix_fmt",
|
|
"rgb24",
|
|
"pipe:1",
|
|
],
|
|
{ maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 },
|
|
);
|
|
if (result.status !== 0) {
|
|
throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`);
|
|
}
|
|
|
|
const offset = (y * UI_FIXTURE_WIDTH + x) * 3;
|
|
return [
|
|
result.stdout[offset] ?? 0,
|
|
result.stdout[offset + 1] ?? 0,
|
|
result.stdout[offset + 2] ?? 0,
|
|
];
|
|
}
|
|
|
|
function maxChannelDelta(a: Rgb, b: Rgb): number {
|
|
return Math.max(Math.abs(a[0] - b[0]), Math.abs(a[1] - b[1]), Math.abs(a[2] - b[2]));
|
|
}
|
|
|
|
// Regression for saturated UI recordings: default JPEG extraction can shift
|
|
// high-chroma reds before browser capture. Forcing PNG should keep extracted
|
|
// source-video frames effectively identical to the decoded source pixels.
|
|
describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
|
|
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-format-"));
|
|
const UI_FIXTURE = join(FIXTURE_DIR, "ui-red.mp4");
|
|
|
|
beforeAll(async () => {
|
|
const result = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xffe7ee:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
|
|
"-vf",
|
|
"drawbox=x=44:y=58:w=152:h=44:color=0xdd382e@1:t=fill,drawbox=x=64:y=75:w=112:h=10:color=0xfff0f0@1:t=fill",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
"-color_primaries",
|
|
"bt709",
|
|
"-color_trc",
|
|
"bt709",
|
|
"-colorspace",
|
|
"bt709",
|
|
UI_FIXTURE,
|
|
]);
|
|
if (!result.success) {
|
|
throw new Error(`UI color fixture synthesis failed: ${result.stderr.slice(-400)}`);
|
|
}
|
|
}, 30_000);
|
|
|
|
afterAll(() => {
|
|
if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
|
|
}, 30_000);
|
|
|
|
function fixtureVideo(): VideoElement {
|
|
return {
|
|
id: "ui",
|
|
src: UI_FIXTURE,
|
|
start: 0,
|
|
end: 1,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
}
|
|
|
|
it("extracts into a work directory whose name holds a printf code", async () => {
|
|
const outDir = join(FIXTURE_DIR, "x 12% Up");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir: outDir,
|
|
});
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
expect(frame.startsWith(outDir)).toBe(true);
|
|
expect(existsSync(frame)).toBe(true);
|
|
});
|
|
|
|
it("keeps the frames of a video whose id has parent segments inside the output directory", async () => {
|
|
const outDir = join(FIXTURE_DIR, "id-path", "work", "frames");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: "../../escaped" }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: outDir },
|
|
);
|
|
expect(result.errors).toEqual([]);
|
|
const extracted = result.extracted[0]!;
|
|
expect(dirname(extracted.outputDir)).toBe(outDir);
|
|
expect(dirname(extracted.framePaths.get(0)!)).toBe(extracted.outputDir);
|
|
expect(existsSync(join(FIXTURE_DIR, "id-path", "escaped"))).toBe(false);
|
|
});
|
|
|
|
it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) jpg extraction", async () => {
|
|
const source = join(FIXTURE_DIR, "white-1080.mp4");
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=white:s=1080x16:d=1:r=1",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
source,
|
|
]);
|
|
expect(synth.success).toBe(true);
|
|
const outDir = join(FIXTURE_DIR, "out-white-1080");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: "white", src: source }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: outDir },
|
|
);
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
expect(frame.endsWith(".jpg")).toBe(true);
|
|
const lastColumns = (file: string) => [
|
|
...spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-frames:v",
|
|
"1",
|
|
"-vf",
|
|
"format=gray,crop=8:1:iw-8:8",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout,
|
|
];
|
|
const delivered = lastColumns(frame);
|
|
const expected = lastColumns(source);
|
|
expect(delivered).toHaveLength(8);
|
|
for (const [i, v] of delivered.entries())
|
|
expect(Math.abs(v - expected[i]!)).toBeLessThanOrEqual(5);
|
|
}, 60_000);
|
|
|
|
async function proresFrame(name: string, pixFmt: string, profile: string, source: string) {
|
|
const clip = join(FIXTURE_DIR, `${name}.mov`);
|
|
const synth = await runFfmpeg(
|
|
["-y", "-loglevel", "error", "-f", "lavfi", "-i", source].concat([
|
|
"-c:v",
|
|
"prores_ks",
|
|
"-profile:v",
|
|
profile,
|
|
"-pix_fmt",
|
|
pixFmt,
|
|
clip,
|
|
]),
|
|
);
|
|
expect(synth.success).toBe(true);
|
|
const outDir = join(FIXTURE_DIR, `out-${name}`);
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: name, src: clip }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: outDir },
|
|
);
|
|
expect(result.errors).toEqual([]);
|
|
return result.extracted[0]!.framePaths.get(0)!;
|
|
}
|
|
|
|
it("extracts ProRes without an alpha plane to jpg", async () => {
|
|
const frame = await proresFrame("prores422", "yuv422p10le", "2", "testsrc2=s=320x180:d=1:r=1");
|
|
expect(frame.endsWith(".jpg")).toBe(true);
|
|
}, 60_000);
|
|
|
|
it("keeps the alpha of ProRes 4444 exact in png frames", async () => {
|
|
const frame = await proresFrame(
|
|
"prores4444",
|
|
"yuva444p10le",
|
|
"4",
|
|
"color=c=0x00ff00@0.5:s=320x180:d=1:r=1,format=yuva444p10le",
|
|
);
|
|
expect(frame.endsWith(".png")).toBe(true);
|
|
const alpha = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
frame,
|
|
"-vf",
|
|
"alphaextract,crop=1:1:10:10",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]);
|
|
expect(Math.abs(alpha.stdout[0]! - 128)).toBeLessThanOrEqual(1);
|
|
}, 60_000);
|
|
|
|
it("keeps color-sensitive UI reds closer to source when extraction is forced to png", async () => {
|
|
const defaultOut = join(FIXTURE_DIR, "out-default");
|
|
const pngOut = join(FIXTURE_DIR, "out-png");
|
|
mkdirSync(defaultOut, { recursive: true });
|
|
mkdirSync(pngOut, { recursive: true });
|
|
|
|
const defaultResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir: defaultOut,
|
|
});
|
|
const pngResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir: pngOut,
|
|
format: "png",
|
|
});
|
|
|
|
expect(defaultResult.errors).toEqual([]);
|
|
expect(pngResult.errors).toEqual([]);
|
|
const defaultFrame = defaultResult.extracted[0]!.framePaths.get(0)!;
|
|
const pngFrame = pngResult.extracted[0]!.framePaths.get(0)!;
|
|
expect(defaultFrame.endsWith(".jpg")).toBe(true);
|
|
expect(pngFrame.endsWith(".png")).toBe(true);
|
|
|
|
let worstDefaultDelta = 0;
|
|
let worstPngDelta = 0;
|
|
for (const [x, y] of RED_SAMPLE_PIXELS) {
|
|
const sourcePixel = readFirstFramePixel(UI_FIXTURE, x, y);
|
|
worstDefaultDelta = Math.max(
|
|
worstDefaultDelta,
|
|
maxChannelDelta(sourcePixel, readFirstFramePixel(defaultFrame, x, y)),
|
|
);
|
|
worstPngDelta = Math.max(
|
|
worstPngDelta,
|
|
maxChannelDelta(sourcePixel, readFirstFramePixel(pngFrame, x, y)),
|
|
);
|
|
}
|
|
|
|
expect(worstPngDelta).toBeLessThanOrEqual(5);
|
|
expect(worstPngDelta).toBeLessThanOrEqual(worstDefaultDelta);
|
|
}, 60_000);
|
|
|
|
// Chrome runs with --force-color-profile=srgb and colour-manages any frame not tagged sRGB;
|
|
// the SDR encoder writes the canvas code values as they are, so frames must reach it unconverted.
|
|
it.each([
|
|
["BT.709", "bt709"],
|
|
["BT.601", "smpte170m"],
|
|
])(
|
|
"declares png frames of a %s source as sRGB so Chrome shows their code values",
|
|
async (name, tag) => {
|
|
const fixture = join(FIXTURE_DIR, `${tag}-shadow.mp4`);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0x101010:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
|
|
"-vf",
|
|
`setparams=color_primaries=${tag}:color_trc=${tag}:colorspace=${tag}:range=tv`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
fixture,
|
|
]);
|
|
if (!synth.success) {
|
|
throw new Error(`${name} fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
}
|
|
expect((await extractVideoMetadata(fixture)).colorSpace?.colorTransfer).toBe(tag);
|
|
const outputDir = join(FIXTURE_DIR, `out-png-transfer-${tag}`);
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: tag, src: fixture }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir, format: "png" },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
// Chrome reads cICP before sRGB, and ffmpeg < 6.0 writes no cICP, so only an sRGB chunk
|
|
// without cICP means "these are sRGB code values" on every ffmpeg.
|
|
const chunks = pngChunkTypes(frame);
|
|
expect(chunks).toContain("sRGB");
|
|
expect(chunks).not.toContain("cICP");
|
|
expect(readFirstFramePixel(frame, 10, 10)).toEqual(readFirstFramePixel(fixture, 10, 10));
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("extracts jpg frames of a BT.709 source as the BT.601 full range JPEG readers assume", async () => {
|
|
const fixture = join(FIXTURE_DIR, "bt709-patch.mp4");
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1,format=rgb24,drawbox=x=31:y=0:w=64:h=${UI_FIXTURE_HEIGHT}:c=0x0000FE:t=fill`,
|
|
"-vf",
|
|
"scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const outputDir = join(FIXTURE_DIR, "out-jpg-matrix");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: "bt709-jpg", src: fixture }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir, format: "jpg" },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
// x=10 is flat colour; x=32 sits one pixel inside the blue edge.
|
|
for (const x of [10, 32]) {
|
|
const source = readFirstFramePixel(fixture, x, 10);
|
|
const shown = readFirstFramePixel(frame, x, 10);
|
|
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
|
|
expect(worst, `x=${x}: source ${source} jpg ${shown}`).toBeLessThanOrEqual(3);
|
|
}
|
|
}, 60_000);
|
|
|
|
// Measured in Chrome 152: untagged H.264 plays as BT.601 at 1280x718 and BT.709 at 1280x720 or 406x720;
|
|
// untagged VP9 and AV1 play as BT.601 at every size.
|
|
it.each([
|
|
{ codec: "libx264", height: 718, matrix: "bt601", format: "png" },
|
|
{ codec: "libx264", height: 718, matrix: "bt601", format: "jpg" },
|
|
{ codec: "libx264", height: 720, matrix: "bt709", format: "png" },
|
|
{ codec: "libx264", height: 720, matrix: "bt709", format: "jpg" },
|
|
{ codec: "libvpx-vp9", height: 720, matrix: "bt601", format: "png" },
|
|
...(AV1_ENCODER
|
|
? [{ codec: AV1_ENCODER, height: 720, matrix: "bt601", format: "png" } as const]
|
|
: []),
|
|
] as const)(
|
|
"reads an untagged $height-line $codec source as $matrix like Chrome's own playback ($format)",
|
|
async ({ codec, height, matrix, format }) => {
|
|
const extension = codec === "libvpx-vp9" ? "webm" : "mp4";
|
|
const fixture = join(FIXTURE_DIR, `untagged-${codec}-${height}.${extension}`);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xC83C28:s=64x${height}:d=1:r=1`,
|
|
"-vf",
|
|
"scale=out_color_matrix=bt601:out_range=tv,format=yuv420p,setparams=colorspace=unknown:color_primaries=unknown:color_trc=unknown:range=tv",
|
|
"-c:v",
|
|
codec,
|
|
...(codec === "libx264" ? ["-qp", "0"] : ["-crf", "0", "-b:v", "0"]),
|
|
...(codec === "libaom-av1" ? ["-cpu-used", "8"] : []),
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
expect((await extractVideoMetadata(fixture)).colorSpace?.colorSpace ?? "unknown").toBe(
|
|
"unknown",
|
|
);
|
|
|
|
const result = await extractVideoFramesRange(
|
|
fixture,
|
|
`${basename(fixture)}-${format}`,
|
|
0,
|
|
1,
|
|
{
|
|
fps: 1,
|
|
outputDir: join(FIXTURE_DIR, "out-untagged"),
|
|
format,
|
|
},
|
|
);
|
|
|
|
const pixelAt = (file: string, decode: string): number[] => [
|
|
...spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-vf",
|
|
`${decode}format=rgb24,crop=1:1:10:10`,
|
|
"-frames:v",
|
|
"1",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout,
|
|
];
|
|
const expected = pixelAt(fixture, `scale=in_color_matrix=${matrix}:in_range=tv,`);
|
|
const shown = pixelAt(result.framePaths.get(0)!, "");
|
|
expect([expected.length, shown.length]).toEqual([3, 3]);
|
|
const worst = Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)));
|
|
expect(worst, `${matrix} decode ${expected}, ${format} ${shown}`).toBeLessThanOrEqual(
|
|
format === "png" ? 1 : 3,
|
|
);
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => {
|
|
// Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks.
|
|
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4");
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-v", "error", "-f", "lavfi", "-i"],
|
|
"color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
|
|
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
|
|
const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-ntsc-ms"),
|
|
format: "png",
|
|
});
|
|
const indexOf = (file: string): number => {
|
|
const row = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-vf",
|
|
"format=gray,crop=64:1:0:4",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout;
|
|
return [...Array(8).keys()].reduce(
|
|
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
|
|
0,
|
|
);
|
|
};
|
|
const shown = [...result.framePaths.keys()]
|
|
.sort((a, b) => a - b)
|
|
.map((i) => indexOf(result.framePaths.get(i)!));
|
|
const steps = shown.slice(1).map((n, i) => n - shown[i]!);
|
|
|
|
expect(shown.length).toBeGreaterThanOrEqual(44);
|
|
expect(
|
|
steps.filter((step) => step > 1),
|
|
`source frames ${shown.join(",")}`,
|
|
).toEqual([]);
|
|
expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1);
|
|
}, 60_000);
|
|
|
|
it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => {
|
|
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4");
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-v", "error", "-f", "lavfi", "-i"],
|
|
"color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
|
|
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const indexOf = (file: string): number => {
|
|
const row = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-vf",
|
|
"format=gray,crop=64:1:0:4",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout;
|
|
return [...Array(8).keys()].reduce(
|
|
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
|
|
0,
|
|
);
|
|
};
|
|
const options = {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-ntsc-seams"),
|
|
format: "png" as const,
|
|
};
|
|
const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options);
|
|
|
|
const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201];
|
|
const differing: string[] = [];
|
|
for (const firstFrame of seams) {
|
|
const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, {
|
|
...options,
|
|
frameRanges: [{ firstFrame, frames: 12 }],
|
|
});
|
|
for (const [slot, path] of segment.framePaths) {
|
|
const expected = indexOf(single.framePaths.get(slot)!);
|
|
const shown = indexOf(path);
|
|
if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`);
|
|
}
|
|
}
|
|
expect(differing).toEqual([]);
|
|
}, 120_000);
|
|
|
|
// ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour
|
|
// filters would cut the still that ends this window short.
|
|
it.each([
|
|
{
|
|
name: "full range BT.709",
|
|
format: "png",
|
|
codec: "libx264",
|
|
pixFmt: "yuvj420p",
|
|
probed: { colorTransfer: "bt709" },
|
|
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
|
|
},
|
|
{
|
|
name: "full range BT.709",
|
|
format: "jpg",
|
|
codec: "libx264",
|
|
pixFmt: "yuvj420p",
|
|
probed: { colorTransfer: "bt709" },
|
|
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
|
|
},
|
|
{
|
|
name: "BT.470BG gamma 2.8",
|
|
format: "png",
|
|
codec: "libx264",
|
|
pixFmt: "yuv420p",
|
|
probed: { colorTransfer: "bt470bg" },
|
|
tags: "color_primaries=bt470bg:color_trc=bt470bg:colorspace=bt470bg",
|
|
},
|
|
{
|
|
name: "RGB",
|
|
format: "png",
|
|
codec: "libx264rgb",
|
|
pixFmt: "rgb24",
|
|
probed: { colorSpace: "gbr" },
|
|
tags: "",
|
|
},
|
|
] as const)(
|
|
"keeps every frame and the colours of a $name VFR window that ends on a still ($format)",
|
|
async ({ name, format, codec, pixFmt, probed, tags }) => {
|
|
const fixture = join(FIXTURE_DIR, `vfr-still-${name.replace(/\W+/g, "-")}-${format}.mp4`);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=2:r=60`,
|
|
"-vf",
|
|
`select='not(between(n\\,30\\,89))'${tags ? `,setparams=${tags}` : ""}`,
|
|
"-fps_mode",
|
|
"vfr",
|
|
"-c:v",
|
|
codec,
|
|
"-preset",
|
|
"ultrafast",
|
|
"-pix_fmt",
|
|
pixFmt,
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const window = 0.616666;
|
|
const unfilteredDir = join(FIXTURE_DIR, `out-${basename(fixture)}-unfiltered`);
|
|
mkdirSync(unfilteredDir, { recursive: true });
|
|
const unfiltered = await runFfmpeg([
|
|
"-v",
|
|
"error",
|
|
"-ss",
|
|
"0",
|
|
"-i",
|
|
fixture,
|
|
"-t",
|
|
String(window),
|
|
"-fps_mode",
|
|
"cfr",
|
|
"-r",
|
|
"30",
|
|
join(unfilteredDir, "f_%05d.png"),
|
|
]);
|
|
expect(unfiltered.success).toBe(true);
|
|
|
|
const result = await extractVideoFramesRange(fixture, basename(fixture), 0, window, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-vfr-still"),
|
|
format,
|
|
});
|
|
|
|
expect(result.metadata.isVFR).toBe(true);
|
|
expect(result.metadata.colorSpace).toMatchObject(probed);
|
|
expect(result.totalFrames).toBe(readdirSync(unfilteredDir).length);
|
|
const last = result.framePaths.get(result.totalFrames - 1)!;
|
|
const source = readFirstFramePixel(fixture, 10, 10);
|
|
const shown = readFirstFramePixel(last, 10, 10);
|
|
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
|
|
expect(worst, `source ${source} ${format} ${shown}`).toBeLessThanOrEqual(
|
|
format === "png" ? 0 : 3,
|
|
);
|
|
if (format === "png") expect(pngChunkTypes(last)).toContain("sRGB");
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("leaves SDR-to-HDR frames in the HDR colours they were converted to", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-sdr-to-hdr-gate");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractVideoFramesRange(UI_FIXTURE, "sdr-to-hdr-gate", 0, 1, {
|
|
fps: 1,
|
|
outputDir,
|
|
format: "png",
|
|
sdrToHdrTransfer: "pq",
|
|
});
|
|
|
|
expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB");
|
|
}, 60_000);
|
|
|
|
it("extracts each video at its own shown box, and caches halved frames apart from full ones", async () => {
|
|
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-shown-cache-"));
|
|
const size = (path: string) =>
|
|
spawnSync("ffprobe", [
|
|
"-v",
|
|
"error",
|
|
"-show_entries",
|
|
"stream=width,height",
|
|
"-of",
|
|
"csv=p=0",
|
|
"--",
|
|
path,
|
|
])
|
|
.stdout.toString()
|
|
.trim();
|
|
const run = async (
|
|
name: string,
|
|
shownBoxes?: Map<string, { width: number; height: number }>,
|
|
) => {
|
|
const outputDir = join(FIXTURE_DIR, name);
|
|
mkdirSync(outputDir, { recursive: true });
|
|
const result = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir, shownBoxes },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(result.errors).toEqual([]);
|
|
return {
|
|
size: size(result.extracted[0]!.framePaths.get(0)!),
|
|
hits: result.phaseBreakdown.cacheHits,
|
|
};
|
|
};
|
|
try {
|
|
const half = new Map([
|
|
["ui", { width: UI_FIXTURE_WIDTH / 2, height: UI_FIXTURE_HEIGHT / 2 }],
|
|
]);
|
|
expect(await run("shown-full")).toEqual({
|
|
size: `${UI_FIXTURE_WIDTH},${UI_FIXTURE_HEIGHT}`,
|
|
hits: 0,
|
|
});
|
|
expect(await run("shown-half", half)).toEqual({
|
|
size: `${UI_FIXTURE_WIDTH / 2},${UI_FIXTURE_HEIGHT / 2}`,
|
|
hits: 0,
|
|
});
|
|
expect(await run("shown-half-again", half)).toEqual({
|
|
size: `${UI_FIXTURE_WIDTH / 2},${UI_FIXTURE_HEIGHT / 2}`,
|
|
hits: 1,
|
|
});
|
|
expect(await run("shown-full-again")).toEqual({
|
|
size: `${UI_FIXTURE_WIDTH},${UI_FIXTURE_HEIGHT}`,
|
|
hits: 1,
|
|
});
|
|
} finally {
|
|
rmSync(cacheDir, { recursive: true, force: true });
|
|
}
|
|
}, 60_000);
|
|
|
|
it("keeps jpg and png extraction caches separate", async () => {
|
|
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-"));
|
|
try {
|
|
const defaultOut = join(FIXTURE_DIR, "cache-default");
|
|
const pngOut = join(FIXTURE_DIR, "cache-png");
|
|
const pngHitOut = join(FIXTURE_DIR, "cache-png-hit");
|
|
mkdirSync(defaultOut, { recursive: true });
|
|
mkdirSync(pngOut, { recursive: true });
|
|
mkdirSync(pngHitOut, { recursive: true });
|
|
|
|
const defaultResult = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: defaultOut },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(defaultResult.errors).toEqual([]);
|
|
expect(defaultResult.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(defaultResult.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(defaultResult.extracted[0]!.framePaths.get(0)!.endsWith(".jpg")).toBe(true);
|
|
|
|
const pngMiss = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: pngOut, format: "png" },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(pngMiss.errors).toEqual([]);
|
|
expect(pngMiss.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(pngMiss.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(pngMiss.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
|
|
|
|
const pngHit = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: pngHitOut, format: "png" },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(pngHit.errors).toEqual([]);
|
|
expect(pngHit.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(pngHit.phaseBreakdown.cacheMisses).toBe(0);
|
|
expect(pngHit.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
|
|
} finally {
|
|
rmSync(cacheDir, { recursive: true, force: true });
|
|
}
|
|
}, 60_000);
|
|
|
|
it("dedupes identical extractions within one render", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-dedupe");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const videoA: VideoElement = { ...fixtureVideo(), id: "dupe-a" };
|
|
const videoB: VideoElement = { ...fixtureVideo(), id: "dupe-b" };
|
|
|
|
const result = await extractAllVideoFrames([videoA, videoB], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir,
|
|
});
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.extracted).toHaveLength(2);
|
|
const first = result.extracted[0]!;
|
|
const second = result.extracted[1]!;
|
|
expect(first.videoId).toBe("dupe-a");
|
|
expect(second.videoId).toBe("dupe-b");
|
|
expect(second.outputDir).toBe(first.outputDir);
|
|
expect(Array.from(second.framePaths.entries())).toEqual(Array.from(first.framePaths.entries()));
|
|
|
|
const frameDirs = readdirSync(outputDir, { withFileTypes: true })
|
|
.filter((entry) => entry.isDirectory())
|
|
.map((entry) => entry.name);
|
|
expect(frameDirs).toEqual(["dupe-a"]);
|
|
expect(readdirSync(first.outputDir).filter((f) => f.endsWith(".jpg"))).toHaveLength(
|
|
first.totalFrames,
|
|
);
|
|
}, 60_000);
|
|
});
|