mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:00:07 +00:00
* fix(producer): build the render job before taking a render slot A request whose options fail the strict render-request validator now gets a 400 on both the blocking and streaming routes, and its inline project directory is removed. Validate workers up front like the other overrides. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(producer): write compiled sub-compositions only inside the compiled directory Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(engine): keep media work files inside their directory for any element id Map a media element id to one plain path segment before it names a frame directory or an intermediate audio file. Ids that already are one plain segment map to themselves, so existing paths stay the same; the producer and the distributed plan derive the same segment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(engine): listen on loopback by default in the engine file server The file server feeds the co-located capture browser, which loads it from localhost. A caller that wants another interface passes hostname. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(producer): refuse cross-origin browser posts to the standalone render server A web page on another origin can no longer start a render or lint through the server's POST routes. Requests without an Origin header (CLI tools, server-to-server callers) and same-origin pages are unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore(engine): trim comments on the media path segment and file server host Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(producer): trust the browser's Sec-Fetch-Site verdict for render server posts A same-origin page behind a reverse proxy that rewrites the Host header sent an Origin that no longer matched Host, so the cross-origin guard refused it. The guard now uses Sec-Fetch-Site when the browser sends it and falls back to the Origin and Host comparison only for browsers that do not. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(engine): keep generated media path names apart from plain ids and within filename limits A plain media id that already ends like a generated name now takes the hashed branch too, so it can no longer share frame folders or audio work files with the unsafe id that produced that name. The readable part of a generated name is capped at 64 UTF-8 bytes before the digest, so long or multibyte ids that contain separators stay within the 255-byte filename limit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
775 lines
26 KiB
TypeScript
775 lines
26 KiB
TypeScript
import { vi, afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
import { basename, dirname, join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { spawnSync } from "node:child_process";
|
|
import {
|
|
extractAllVideoFrames,
|
|
extractVideoFramesRange,
|
|
type VideoElement,
|
|
} from "./videoFrameExtractor.js";
|
|
import { extractVideoMetadata } from "../utils/ffprobe.js";
|
|
import { runFfmpeg } from "../utils/runFfmpeg.js";
|
|
|
|
// Pass-through, so a test can tell whether extraction ran ffmpeg.
|
|
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
|
|
const real = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
|
|
return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) };
|
|
});
|
|
|
|
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
|
|
const FFMPEG_ENCODERS = HAS_FFMPEG
|
|
? spawnSync("ffmpeg", ["-hide_banner", "-encoders"]).stdout.toString()
|
|
: "";
|
|
// Homebrew's ffmpeg has libsvtav1 but not libaom-av1. Either one writes an untagged AV1 stream.
|
|
const AV1_ENCODER = ["libaom-av1", "libsvtav1"].find((name) =>
|
|
new RegExp(`\\s${name}\\s`).test(FFMPEG_ENCODERS),
|
|
);
|
|
|
|
type Rgb = [number, number, number];
|
|
|
|
const UI_FIXTURE_WIDTH = 240;
|
|
const UI_FIXTURE_HEIGHT = 160;
|
|
const RED_SAMPLE_PIXELS = [
|
|
[70, 72],
|
|
[118, 82],
|
|
[178, 92],
|
|
] as const;
|
|
|
|
function pngChunkTypes(path: string): string[] {
|
|
const bytes = readFileSync(path);
|
|
const types: string[] = [];
|
|
for (let offset = 8; offset + 8 <= bytes.length; offset += 12 + bytes.readUInt32BE(offset)) {
|
|
types.push(bytes.toString("latin1", offset + 4, offset + 8));
|
|
}
|
|
return types;
|
|
}
|
|
|
|
function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb {
|
|
const result = spawnSync(
|
|
"ffmpeg",
|
|
[
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
mediaPath,
|
|
"-frames:v",
|
|
"1",
|
|
"-f",
|
|
"rawvideo",
|
|
"-pix_fmt",
|
|
"rgb24",
|
|
"pipe:1",
|
|
],
|
|
{ maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 },
|
|
);
|
|
if (result.status !== 0) {
|
|
throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`);
|
|
}
|
|
|
|
const offset = (y * UI_FIXTURE_WIDTH + x) * 3;
|
|
return [
|
|
result.stdout[offset] ?? 0,
|
|
result.stdout[offset + 1] ?? 0,
|
|
result.stdout[offset + 2] ?? 0,
|
|
];
|
|
}
|
|
|
|
function maxChannelDelta(a: Rgb, b: Rgb): number {
|
|
return Math.max(Math.abs(a[0] - b[0]), Math.abs(a[1] - b[1]), Math.abs(a[2] - b[2]));
|
|
}
|
|
|
|
// Regression for saturated UI recordings: default JPEG extraction can shift
|
|
// high-chroma reds before browser capture. Forcing PNG should keep extracted
|
|
// source-video frames effectively identical to the decoded source pixels.
|
|
describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
|
|
const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-format-"));
|
|
const UI_FIXTURE = join(FIXTURE_DIR, "ui-red.mp4");
|
|
|
|
beforeAll(async () => {
|
|
const result = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xffe7ee:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
|
|
"-vf",
|
|
"drawbox=x=44:y=58:w=152:h=44:color=0xdd382e@1:t=fill,drawbox=x=64:y=75:w=112:h=10:color=0xfff0f0@1:t=fill",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
"-color_primaries",
|
|
"bt709",
|
|
"-color_trc",
|
|
"bt709",
|
|
"-colorspace",
|
|
"bt709",
|
|
UI_FIXTURE,
|
|
]);
|
|
if (!result.success) {
|
|
throw new Error(`UI color fixture synthesis failed: ${result.stderr.slice(-400)}`);
|
|
}
|
|
}, 30_000);
|
|
|
|
afterAll(() => {
|
|
if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
|
|
}, 30_000);
|
|
|
|
function fixtureVideo(): VideoElement {
|
|
return {
|
|
id: "ui",
|
|
src: UI_FIXTURE,
|
|
start: 0,
|
|
end: 1,
|
|
mediaStart: 0,
|
|
loop: false,
|
|
hasAudio: false,
|
|
};
|
|
}
|
|
|
|
it("extracts into a work directory whose name holds a printf code", async () => {
|
|
const outDir = join(FIXTURE_DIR, "x 12% Up");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir: outDir,
|
|
});
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
expect(frame.startsWith(outDir)).toBe(true);
|
|
expect(existsSync(frame)).toBe(true);
|
|
});
|
|
|
|
it("keeps the frames of a video whose id has parent segments inside the output directory", async () => {
|
|
const outDir = join(FIXTURE_DIR, "id-path", "work", "frames");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: "../../escaped" }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: outDir },
|
|
);
|
|
expect(result.errors).toEqual([]);
|
|
const extracted = result.extracted[0]!;
|
|
expect(dirname(extracted.outputDir)).toBe(outDir);
|
|
expect(dirname(extracted.framePaths.get(0)!)).toBe(extracted.outputDir);
|
|
expect(existsSync(join(FIXTURE_DIR, "id-path", "escaped"))).toBe(false);
|
|
});
|
|
|
|
it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) jpg extraction", async () => {
|
|
const source = join(FIXTURE_DIR, "white-1080.mp4");
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=white:s=1080x16:d=1:r=1",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
source,
|
|
]);
|
|
expect(synth.success).toBe(true);
|
|
const outDir = join(FIXTURE_DIR, "out-white-1080");
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: "white", src: source }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: outDir },
|
|
);
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
expect(frame.endsWith(".jpg")).toBe(true);
|
|
const lastColumns = (file: string) => [
|
|
...spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-frames:v",
|
|
"1",
|
|
"-vf",
|
|
"format=gray,crop=8:1:iw-8:8",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout,
|
|
];
|
|
const delivered = lastColumns(frame);
|
|
const expected = lastColumns(source);
|
|
expect(delivered).toHaveLength(8);
|
|
for (const [i, v] of delivered.entries())
|
|
expect(Math.abs(v - expected[i]!)).toBeLessThanOrEqual(5);
|
|
}, 60_000);
|
|
|
|
async function proresFrame(name: string, pixFmt: string, profile: string, source: string) {
|
|
const clip = join(FIXTURE_DIR, `${name}.mov`);
|
|
const synth = await runFfmpeg(
|
|
["-y", "-loglevel", "error", "-f", "lavfi", "-i", source].concat([
|
|
"-c:v",
|
|
"prores_ks",
|
|
"-profile:v",
|
|
profile,
|
|
"-pix_fmt",
|
|
pixFmt,
|
|
clip,
|
|
]),
|
|
);
|
|
expect(synth.success).toBe(true);
|
|
const outDir = join(FIXTURE_DIR, `out-${name}`);
|
|
mkdirSync(outDir, { recursive: true });
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: name, src: clip }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: outDir },
|
|
);
|
|
expect(result.errors).toEqual([]);
|
|
return result.extracted[0]!.framePaths.get(0)!;
|
|
}
|
|
|
|
it("extracts ProRes without an alpha plane to jpg", async () => {
|
|
const frame = await proresFrame("prores422", "yuv422p10le", "2", "testsrc2=s=320x180:d=1:r=1");
|
|
expect(frame.endsWith(".jpg")).toBe(true);
|
|
}, 60_000);
|
|
|
|
it("keeps the alpha of ProRes 4444 exact in png frames", async () => {
|
|
const frame = await proresFrame(
|
|
"prores4444",
|
|
"yuva444p10le",
|
|
"4",
|
|
"color=c=0x00ff00@0.5:s=320x180:d=1:r=1,format=yuva444p10le",
|
|
);
|
|
expect(frame.endsWith(".png")).toBe(true);
|
|
const alpha = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
frame,
|
|
"-vf",
|
|
"alphaextract,crop=1:1:10:10",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]);
|
|
expect(Math.abs(alpha.stdout[0]! - 128)).toBeLessThanOrEqual(1);
|
|
}, 60_000);
|
|
|
|
it("keeps color-sensitive UI reds closer to source when extraction is forced to png", async () => {
|
|
const defaultOut = join(FIXTURE_DIR, "out-default");
|
|
const pngOut = join(FIXTURE_DIR, "out-png");
|
|
mkdirSync(defaultOut, { recursive: true });
|
|
mkdirSync(pngOut, { recursive: true });
|
|
|
|
const defaultResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir: defaultOut,
|
|
});
|
|
const pngResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir: pngOut,
|
|
format: "png",
|
|
});
|
|
|
|
expect(defaultResult.errors).toEqual([]);
|
|
expect(pngResult.errors).toEqual([]);
|
|
const defaultFrame = defaultResult.extracted[0]!.framePaths.get(0)!;
|
|
const pngFrame = pngResult.extracted[0]!.framePaths.get(0)!;
|
|
expect(defaultFrame.endsWith(".jpg")).toBe(true);
|
|
expect(pngFrame.endsWith(".png")).toBe(true);
|
|
|
|
let worstDefaultDelta = 0;
|
|
let worstPngDelta = 0;
|
|
for (const [x, y] of RED_SAMPLE_PIXELS) {
|
|
const sourcePixel = readFirstFramePixel(UI_FIXTURE, x, y);
|
|
worstDefaultDelta = Math.max(
|
|
worstDefaultDelta,
|
|
maxChannelDelta(sourcePixel, readFirstFramePixel(defaultFrame, x, y)),
|
|
);
|
|
worstPngDelta = Math.max(
|
|
worstPngDelta,
|
|
maxChannelDelta(sourcePixel, readFirstFramePixel(pngFrame, x, y)),
|
|
);
|
|
}
|
|
|
|
expect(worstPngDelta).toBeLessThanOrEqual(5);
|
|
expect(worstPngDelta).toBeLessThanOrEqual(worstDefaultDelta);
|
|
}, 60_000);
|
|
|
|
// Chrome runs with --force-color-profile=srgb and colour-manages any frame not tagged sRGB;
|
|
// the SDR encoder writes the canvas code values as they are, so frames must reach it unconverted.
|
|
it.each([
|
|
["BT.709", "bt709"],
|
|
["BT.601", "smpte170m"],
|
|
])(
|
|
"declares png frames of a %s source as sRGB so Chrome shows their code values",
|
|
async (name, tag) => {
|
|
const fixture = join(FIXTURE_DIR, `${tag}-shadow.mp4`);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0x101010:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`,
|
|
"-vf",
|
|
`setparams=color_primaries=${tag}:color_trc=${tag}:colorspace=${tag}:range=tv`,
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
fixture,
|
|
]);
|
|
if (!synth.success) {
|
|
throw new Error(`${name} fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
}
|
|
expect((await extractVideoMetadata(fixture)).colorSpace?.colorTransfer).toBe(tag);
|
|
const outputDir = join(FIXTURE_DIR, `out-png-transfer-${tag}`);
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: tag, src: fixture }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir, format: "png" },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
// Chrome reads cICP before sRGB, and ffmpeg < 6.0 writes no cICP, so only an sRGB chunk
|
|
// without cICP means "these are sRGB code values" on every ffmpeg.
|
|
const chunks = pngChunkTypes(frame);
|
|
expect(chunks).toContain("sRGB");
|
|
expect(chunks).not.toContain("cICP");
|
|
expect(readFirstFramePixel(frame, 10, 10)).toEqual(readFirstFramePixel(fixture, 10, 10));
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("extracts jpg frames of a BT.709 source as the BT.601 full range JPEG readers assume", async () => {
|
|
const fixture = join(FIXTURE_DIR, "bt709-patch.mp4");
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1,format=rgb24,drawbox=x=31:y=0:w=64:h=${UI_FIXTURE_HEIGHT}:c=0x0000FE:t=fill`,
|
|
"-vf",
|
|
"scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"0",
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const outputDir = join(FIXTURE_DIR, "out-jpg-matrix");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractAllVideoFrames(
|
|
[{ ...fixtureVideo(), id: "bt709-jpg", src: fixture }],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir, format: "jpg" },
|
|
);
|
|
|
|
expect(result.errors).toEqual([]);
|
|
const frame = result.extracted[0]!.framePaths.get(0)!;
|
|
// x=10 is flat colour; x=32 sits one pixel inside the blue edge.
|
|
for (const x of [10, 32]) {
|
|
const source = readFirstFramePixel(fixture, x, 10);
|
|
const shown = readFirstFramePixel(frame, x, 10);
|
|
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
|
|
expect(worst, `x=${x}: source ${source} jpg ${shown}`).toBeLessThanOrEqual(3);
|
|
}
|
|
}, 60_000);
|
|
|
|
// Measured in Chrome 152: untagged H.264 plays as BT.601 at 1280x718 and BT.709 at 1280x720 or 406x720;
|
|
// untagged VP9 and AV1 play as BT.601 at every size.
|
|
it.each([
|
|
{ codec: "libx264", height: 718, matrix: "bt601", format: "png" },
|
|
{ codec: "libx264", height: 718, matrix: "bt601", format: "jpg" },
|
|
{ codec: "libx264", height: 720, matrix: "bt709", format: "png" },
|
|
{ codec: "libx264", height: 720, matrix: "bt709", format: "jpg" },
|
|
{ codec: "libvpx-vp9", height: 720, matrix: "bt601", format: "png" },
|
|
...(AV1_ENCODER
|
|
? [{ codec: AV1_ENCODER, height: 720, matrix: "bt601", format: "png" } as const]
|
|
: []),
|
|
] as const)(
|
|
"reads an untagged $height-line $codec source as $matrix like Chrome's own playback ($format)",
|
|
async ({ codec, height, matrix, format }) => {
|
|
const extension = codec === "libvpx-vp9" ? "webm" : "mp4";
|
|
const fixture = join(FIXTURE_DIR, `untagged-${codec}-${height}.${extension}`);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xC83C28:s=64x${height}:d=1:r=1`,
|
|
"-vf",
|
|
"scale=out_color_matrix=bt601:out_range=tv,format=yuv420p,setparams=colorspace=unknown:color_primaries=unknown:color_trc=unknown:range=tv",
|
|
"-c:v",
|
|
codec,
|
|
...(codec === "libx264" ? ["-qp", "0"] : ["-crf", "0", "-b:v", "0"]),
|
|
...(codec === "libaom-av1" ? ["-cpu-used", "8"] : []),
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
expect((await extractVideoMetadata(fixture)).colorSpace?.colorSpace ?? "unknown").toBe(
|
|
"unknown",
|
|
);
|
|
|
|
const result = await extractVideoFramesRange(
|
|
fixture,
|
|
`${basename(fixture)}-${format}`,
|
|
0,
|
|
1,
|
|
{
|
|
fps: 1,
|
|
outputDir: join(FIXTURE_DIR, "out-untagged"),
|
|
format,
|
|
},
|
|
);
|
|
|
|
const pixelAt = (file: string, decode: string): number[] => [
|
|
...spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-vf",
|
|
`${decode}format=rgb24,crop=1:1:10:10`,
|
|
"-frames:v",
|
|
"1",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout,
|
|
];
|
|
const expected = pixelAt(fixture, `scale=in_color_matrix=${matrix}:in_range=tv,`);
|
|
const shown = pixelAt(result.framePaths.get(0)!, "");
|
|
expect([expected.length, shown.length]).toEqual([3, 3]);
|
|
const worst = Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)));
|
|
expect(worst, `${matrix} decode ${expected}, ${format} ${shown}`).toBeLessThanOrEqual(
|
|
format === "png" ? 1 : 3,
|
|
);
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => {
|
|
// Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks.
|
|
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4");
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-v", "error", "-f", "lavfi", "-i"],
|
|
"color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
|
|
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
|
|
const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-ntsc-ms"),
|
|
format: "png",
|
|
});
|
|
const indexOf = (file: string): number => {
|
|
const row = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-vf",
|
|
"format=gray,crop=64:1:0:4",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout;
|
|
return [...Array(8).keys()].reduce(
|
|
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
|
|
0,
|
|
);
|
|
};
|
|
const shown = [...result.framePaths.keys()]
|
|
.sort((a, b) => a - b)
|
|
.map((i) => indexOf(result.framePaths.get(i)!));
|
|
const steps = shown.slice(1).map((n, i) => n - shown[i]!);
|
|
|
|
expect(shown.length).toBeGreaterThanOrEqual(44);
|
|
expect(
|
|
steps.filter((step) => step > 1),
|
|
`source frames ${shown.join(",")}`,
|
|
).toEqual([]);
|
|
expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1);
|
|
}, 60_000);
|
|
|
|
it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => {
|
|
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4");
|
|
const synth = await runFfmpeg([
|
|
...["-y", "-v", "error", "-f", "lavfi", "-i"],
|
|
"color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
|
|
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const indexOf = (file: string): number => {
|
|
const row = spawnSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
file,
|
|
"-vf",
|
|
"format=gray,crop=64:1:0:4",
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
]).stdout;
|
|
return [...Array(8).keys()].reduce(
|
|
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
|
|
0,
|
|
);
|
|
};
|
|
const options = {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-ntsc-seams"),
|
|
format: "png" as const,
|
|
};
|
|
const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options);
|
|
|
|
const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201];
|
|
const differing: string[] = [];
|
|
for (const firstFrame of seams) {
|
|
const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, {
|
|
...options,
|
|
frameRanges: [{ firstFrame, frames: 12 }],
|
|
});
|
|
for (const [slot, path] of segment.framePaths) {
|
|
const expected = indexOf(single.framePaths.get(slot)!);
|
|
const shown = indexOf(path);
|
|
if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`);
|
|
}
|
|
}
|
|
expect(differing).toEqual([]);
|
|
}, 120_000);
|
|
|
|
// ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour
|
|
// filters would cut the still that ends this window short.
|
|
it.each([
|
|
{
|
|
name: "full range BT.709",
|
|
format: "png",
|
|
codec: "libx264",
|
|
pixFmt: "yuvj420p",
|
|
probed: { colorTransfer: "bt709" },
|
|
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
|
|
},
|
|
{
|
|
name: "full range BT.709",
|
|
format: "jpg",
|
|
codec: "libx264",
|
|
pixFmt: "yuvj420p",
|
|
probed: { colorTransfer: "bt709" },
|
|
tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709",
|
|
},
|
|
{
|
|
name: "BT.470BG gamma 2.8",
|
|
format: "png",
|
|
codec: "libx264",
|
|
pixFmt: "yuv420p",
|
|
probed: { colorTransfer: "bt470bg" },
|
|
tags: "color_primaries=bt470bg:color_trc=bt470bg:colorspace=bt470bg",
|
|
},
|
|
{
|
|
name: "RGB",
|
|
format: "png",
|
|
codec: "libx264rgb",
|
|
pixFmt: "rgb24",
|
|
probed: { colorSpace: "gbr" },
|
|
tags: "",
|
|
},
|
|
] as const)(
|
|
"keeps every frame and the colours of a $name VFR window that ends on a still ($format)",
|
|
async ({ name, format, codec, pixFmt, probed, tags }) => {
|
|
const fixture = join(FIXTURE_DIR, `vfr-still-${name.replace(/\W+/g, "-")}-${format}.mp4`);
|
|
const synth = await runFfmpeg([
|
|
"-y",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=2:r=60`,
|
|
"-vf",
|
|
`select='not(between(n\\,30\\,89))'${tags ? `,setparams=${tags}` : ""}`,
|
|
"-fps_mode",
|
|
"vfr",
|
|
"-c:v",
|
|
codec,
|
|
"-preset",
|
|
"ultrafast",
|
|
"-pix_fmt",
|
|
pixFmt,
|
|
fixture,
|
|
]);
|
|
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
|
|
const window = 0.616666;
|
|
const unfilteredDir = join(FIXTURE_DIR, `out-${basename(fixture)}-unfiltered`);
|
|
mkdirSync(unfilteredDir, { recursive: true });
|
|
const unfiltered = await runFfmpeg([
|
|
"-v",
|
|
"error",
|
|
"-ss",
|
|
"0",
|
|
"-i",
|
|
fixture,
|
|
"-t",
|
|
String(window),
|
|
"-fps_mode",
|
|
"cfr",
|
|
"-r",
|
|
"30",
|
|
join(unfilteredDir, "f_%05d.png"),
|
|
]);
|
|
expect(unfiltered.success).toBe(true);
|
|
|
|
const result = await extractVideoFramesRange(fixture, basename(fixture), 0, window, {
|
|
fps: 30,
|
|
outputDir: join(FIXTURE_DIR, "out-vfr-still"),
|
|
format,
|
|
});
|
|
|
|
expect(result.metadata.isVFR).toBe(true);
|
|
expect(result.metadata.colorSpace).toMatchObject(probed);
|
|
expect(result.totalFrames).toBe(readdirSync(unfilteredDir).length);
|
|
const last = result.framePaths.get(result.totalFrames - 1)!;
|
|
const source = readFirstFramePixel(fixture, 10, 10);
|
|
const shown = readFirstFramePixel(last, 10, 10);
|
|
const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!)));
|
|
expect(worst, `source ${source} ${format} ${shown}`).toBeLessThanOrEqual(
|
|
format === "png" ? 0 : 3,
|
|
);
|
|
if (format === "png") expect(pngChunkTypes(last)).toContain("sRGB");
|
|
},
|
|
60_000,
|
|
);
|
|
|
|
it("leaves SDR-to-HDR frames in the HDR colours they were converted to", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-sdr-to-hdr-gate");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const result = await extractVideoFramesRange(UI_FIXTURE, "sdr-to-hdr-gate", 0, 1, {
|
|
fps: 1,
|
|
outputDir,
|
|
format: "png",
|
|
sdrToHdrTransfer: "pq",
|
|
});
|
|
|
|
expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB");
|
|
}, 60_000);
|
|
|
|
it("keeps jpg and png extraction caches separate", async () => {
|
|
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-"));
|
|
try {
|
|
const defaultOut = join(FIXTURE_DIR, "cache-default");
|
|
const pngOut = join(FIXTURE_DIR, "cache-png");
|
|
const pngHitOut = join(FIXTURE_DIR, "cache-png-hit");
|
|
mkdirSync(defaultOut, { recursive: true });
|
|
mkdirSync(pngOut, { recursive: true });
|
|
mkdirSync(pngHitOut, { recursive: true });
|
|
|
|
const defaultResult = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: defaultOut },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(defaultResult.errors).toEqual([]);
|
|
expect(defaultResult.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(defaultResult.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(defaultResult.extracted[0]!.framePaths.get(0)!.endsWith(".jpg")).toBe(true);
|
|
|
|
const pngMiss = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: pngOut, format: "png" },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(pngMiss.errors).toEqual([]);
|
|
expect(pngMiss.phaseBreakdown.cacheHits).toBe(0);
|
|
expect(pngMiss.phaseBreakdown.cacheMisses).toBe(1);
|
|
expect(pngMiss.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
|
|
|
|
const pngHit = await extractAllVideoFrames(
|
|
[fixtureVideo()],
|
|
FIXTURE_DIR,
|
|
{ fps: 1, outputDir: pngHitOut, format: "png" },
|
|
undefined,
|
|
{ extractCacheDir: cacheDir },
|
|
);
|
|
expect(pngHit.errors).toEqual([]);
|
|
expect(pngHit.phaseBreakdown.cacheHits).toBe(1);
|
|
expect(pngHit.phaseBreakdown.cacheMisses).toBe(0);
|
|
expect(pngHit.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true);
|
|
} finally {
|
|
rmSync(cacheDir, { recursive: true, force: true });
|
|
}
|
|
}, 60_000);
|
|
|
|
it("dedupes identical extractions within one render", async () => {
|
|
const outputDir = join(FIXTURE_DIR, "out-dedupe");
|
|
mkdirSync(outputDir, { recursive: true });
|
|
|
|
const videoA: VideoElement = { ...fixtureVideo(), id: "dupe-a" };
|
|
const videoB: VideoElement = { ...fixtureVideo(), id: "dupe-b" };
|
|
|
|
const result = await extractAllVideoFrames([videoA, videoB], FIXTURE_DIR, {
|
|
fps: 1,
|
|
outputDir,
|
|
});
|
|
|
|
expect(result.errors).toEqual([]);
|
|
expect(result.extracted).toHaveLength(2);
|
|
const first = result.extracted[0]!;
|
|
const second = result.extracted[1]!;
|
|
expect(first.videoId).toBe("dupe-a");
|
|
expect(second.videoId).toBe("dupe-b");
|
|
expect(second.outputDir).toBe(first.outputDir);
|
|
expect(Array.from(second.framePaths.entries())).toEqual(Array.from(first.framePaths.entries()));
|
|
|
|
const frameDirs = readdirSync(outputDir, { withFileTypes: true })
|
|
.filter((entry) => entry.isDirectory())
|
|
.map((entry) => entry.name);
|
|
expect(frameDirs).toEqual(["dupe-a"]);
|
|
expect(readdirSync(first.outputDir).filter((f) => f.endsWith(".jpg"))).toHaveLength(
|
|
first.totalFrames,
|
|
);
|
|
}, 60_000);
|
|
});
|