test(cli): split diff's video tests into three files under CI's per-file limit

The file took ~131 s on CI against the flake gate's 120 s, and its heaviest test (four
comparisons) hit the 20 s per-test limit. The video tests now live in three files sharing
diff.test-helpers.ts, and that test is two tests of two comparisons each.

analyze reads each part once and checks the bytes it read, instead of a stat then a read
(CodeQL js/file-system-race).
This commit is contained in:
Miao Yang
2026-10-30 15:22:41 +08:00
parent 0909ef6d44
commit 442ef728a3
5 changed files with 416 additions and 391 deletions
+4 -10
View File
@@ -1,15 +1,7 @@
// HeyGen's video model watches a video and drafts its timeline as JSON. Its route samples at most 2 fps and reads
// one clip of at most 90 s at that rate, so the video goes in parts of up to 30 s split at cuts, each slowed until
// 2 fps there is about 6 fps of the original, and every time in the answer is mapped back to the film.
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, dirname, extname, join, resolve } from "node:path";
import { defineCommand } from "../utils/defineCommand.js";
@@ -75,7 +67,9 @@ export function underInlineCap(
for (const crf of [23, 30, 36]) {
const out = join(dir, `${name}-${crf}.mp4`);
runFfmpeg(args(crf, out));
if (statSync(out).size <= INLINE_BYTES) return readFileSync(out);
// Read once and measure the bytes read: a stat then a read could see two different files.
const bytes = readFileSync(out);
if (bytes.length <= INLINE_BYTES) return bytes;
}
throw new Fix(
`a ${round(seconds, 1)} s part stays over HeyGen's 16 MB inline cap`,
@@ -0,0 +1,255 @@
import { execFileSync } from "node:child_process";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { diffVideos, soundOffset } from "./diff.js";
import { clipFolder, ffmpeg, noTools, REF_ARGS } from "./diff.test-helpers.js";
import { probeVideo } from "./media.js";
import { parseSegments } from "./segments.js";
const { dir, clip } = clipFolder("hf-diff-person-test-");
describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
const ref = noTools ? "" : clip("ref.mp4", REF_ARGS);
it("fails the original's whole frame played where copy is meant to be rebuilt", async () => {
const ledger = join(dir, "manifest.jsonl");
writeFileSync(
ledger,
JSON.stringify({
type: "video",
path: "assets/all.mp4",
provenance: { provider: "original", window: [0, 3], roi: [0, 0, 320, 180] },
}) + "\n",
);
const titled = '## S01 · 0–3 s · Title\nPixels: rebuilt\nCopy: "TITLE"\n';
const copy = await diffVideos({
reference: ref,
yours: ref,
segments: parseSegments(titled),
ledger,
outDir: join(dir, "copy"),
});
expect(copy.whole_frame_reuse).toMatchObject({ spans: [[0, 3]], drawn: [[0, 3]], pass: false });
expect(copy.segments?.[0]?.pass).toBe(false);
expect(copy.pass).toBe(false);
// Footage reused with its burned-in title is a copy too (Miao, 2026-10-23: no whole-frame cut with drawn things).
const burned = await diffVideos({
reference: ref,
yours: ref,
segments: parseSegments(
'## S01 · 0–3 s · Title over footage\nPixels: reused\nCopy: "TITLE"\n',
),
ledger,
outDir: join(dir, "burned"),
});
expect(burned.whole_frame_reuse).toMatchObject({ share: 1, drawn: [[0, 3]], pass: false });
// Footage with nothing drawn on it may be cut whole.
const footage = await diffVideos({
reference: ref,
yours: ref,
segments: parseSegments("## S01 · 0–3 s · Footage\nPixels: reused\n"),
ledger,
outDir: join(dir, "footage"),
});
expect(footage.whole_frame_reuse).toMatchObject({ share: 1, drawn: [], pass: true });
expect(footage.pass).toBe(true);
// A cut that only fed a generation (a motion guide) is not played: yours differs from it.
const flipped = clip("guided.mp4", [...["-i", ref, "-vf"], "hflip"]);
const guided = await diffVideos({
reference: ref,
yours: flipped,
segments: parseSegments(titled),
ledger,
outDir: join(dir, "guided"),
});
expect(guided.whole_frame_reuse).toMatchObject({ spans: [], pass: true });
// A generated shot that starts from the original's first frame matches it for a few frames only: not reuse.
const started = clip("started.mp4", [
...["-i", ref, "-filter_complex"],
"[0]split[a][b];[b]hflip[f];[a][f]overlay=enable='gte(t\\,0.2)'",
]);
const start = await diffVideos({
reference: ref,
yours: started,
segments: parseSegments(titled),
ledger,
outDir: join(dir, "started"),
});
expect(start.whole_frame_reuse).toMatchObject({ spans: [], pass: true });
});
it("finds a whole-frame still of the original shown in yours", async () => {
const ledger = join(dir, "stills.jsonl");
writeFileSync(
ledger,
[
{ type: "image", provenance: { provider: "original", at: 1, roi: [0, 0, 320, 180] } },
{ type: "image", provenance: { provider: "original", at: 2.5 } },
]
.map((e) => JSON.stringify(e))
.join("\n"),
);
// Yours holds the original's frame at 1 s for the whole clip.
const held = clip("held.mp4", [
...["-i", ref, "-vf"],
"select=eq(n\\,30),loop=loop=89:size=1,setpts=N/30/TB",
]);
const r = await diffVideos({
reference: ref,
yours: held,
segments: parseSegments('## S01 · 0–3 s · Collage\nPixels: rebuilt\nCopy: "TITLE"\n'),
ledger,
outDir: join(dir, "held"),
});
expect(r.whole_frame_reuse).toMatchObject({ stills: [1], spans: [[0, 3]], pass: false });
});
it("reads sound that starts late as late, and the picture's length as the duration", async () => {
// Clicks at uneven times, with the sound running 0.4 s past the picture.
const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3.4";
const voiced = (name: string, delay: number) =>
clip(name, [
...["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"],
...["-itsoffset", String(delay), "-f", "lavfi", "-i", clicks],
...["-map", "0:v", "-map", "1:a", "-c:a", "aac"],
]);
const on = voiced("voiced.mp4", 0);
const same = await diffVideos({ reference: on, yours: on, outDir: join(dir, "voiced") });
expect(same.duration).toMatchObject({ reference: 3, pass: true });
expect(same.sound).toMatchObject({ offset_ms: 0, pass: true });
const late = await diffVideos({
reference: on,
yours: voiced("late.mp4", 0.2),
outDir: join(dir, "late"),
});
expect(late.sound).toMatchObject({ offset_ms: 200, pass: false });
});
it("places the sound by its decoded timestamps, not the start time the probe reports", () => {
// ffmpeg 5.1 drops an HE-AAC edit list's skip twice: the first sample it keeps is timestamped 0.1 s in while
// the probe still says the track starts at 0. Here a sound timestamped 0.25 s in, probed as starting at 0,
// against the same sound with 0.25 s of silence baked into its samples: in sync.
const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3";
const picture = ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"];
const stamped = clip("stamped.mp4", [
...picture,
...["-itsoffset", "0.25", "-f", "lavfi", "-i", clicks],
...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"],
]);
const baked = clip("baked.mp4", [
...picture,
...["-f", "lavfi", "-i", clicks, "-af", "adelay=250:all=1"],
...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"],
]);
const probedAtZero = { ...probeVideo(stamped), audioStart: 0 };
expect(soundOffset(stamped, baked, probedAtZero, probeVideo(baked))).toMatchObject({
offset_ms: 0,
pass: true,
});
});
it("passes a re-shot person whose motion is off when the components are right", async () => {
// The bottom half (the person) frozen on its first frame: no motion at all. Pixels above match.
const frozen = clip("frozen.mp4", [
...["-i", ref, "-filter_complex"],
"[0]split[a][b];[b]crop=320:90:0:90,trim=end_frame=1,tpad=stop=89:stop_mode=clone[f];[a][f]overlay=0:90",
]);
const r = await diffVideos({
reference: ref,
yours: frozen,
segments: parseSegments(
"## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\nRegenerated: 0, 0.5, 1, 0.5\n",
),
outDir: join(dir, "frozen"),
});
const [s] = r.segments!;
expect(s!.person?.energy?.mean_ratio).toBeLessThan(0.5);
expect(s!.person?.gates).toEqual({ framing: null, text: null, sync: null });
expect(s!.pass).toBe(true);
expect(r.components).toMatchObject({ passed: 1, total: 1 });
expect(r.person?.energy).toBeLessThan(0.5);
expect(r.pass).toBe(true);
});
it("fails a re-shot person whose generated clip carries the original's text", async () => {
// The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half.
const strokes = (x0: number) =>
[0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(",");
const label = strokes(120);
const soft = (delay: number) =>
`[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`;
const base = clip("carried-ref.mp4", [
...["-i", ref, "-filter_complex"],
`${soft(0)},${label}`,
]);
const project = join(dir, "carried");
mkdirSync(join(project, "assets"), { recursive: true });
// The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the
// label over each; over the first, 8 px off, the letters show doubled.
clip("carried/assets/dirty.mp4", [...["-i", ref, "-filter_complex"], `${soft(5)},${label}`]);
clip("carried/assets/clean.mp4", [...["-i", ref, "-filter_complex"], soft(5)]);
execFileSync(ffmpeg!, [
...["-hide_banner", "-loglevel", "error", "-y", "-i", ref, "-filter_complex"],
`${soft(0)},trim=end_frame=1`,
...["-frames:v", "1", join(project, "assets", "still.png")],
]);
const ledger = (path: string) => {
const file = join(project, `${path}.jsonl`);
writeFileSync(
file,
JSON.stringify({
type: "video",
path: `assets/${path}.mp4`,
provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] },
}) + "\n",
);
return file;
};
const md =
"## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n";
const judge = (path: string, x0: number) =>
diffVideos({
reference: base,
yours: clip(`carried-yours-${path}.mp4`, [
...["-i", join(project, "assets", `${path}.mp4`), "-vf"],
strokes(x0),
]),
segments: parseSegments(md),
ledger: ledger(path),
project,
outDir: join(dir, `carried-${path}`),
});
const dirty = await judge("dirty", 128);
expect(dirty.segments?.[0]?.person?.text?.[0]).toMatchObject({
drawn: "S01.D1",
carried: true,
});
expect(dirty.segments?.[0]?.pass).toBe(false);
expect(dirty.person).toMatchObject({ gates: { text: false }, failed: { text: ["S01"] } });
expect(dirty.pass).toBe(false);
const clean = await judge("clean", 120);
expect(clean.segments?.[0]?.drawn?.[0]?.pass).toBe(true);
expect(clean.segments?.[0]?.person?.gates.text).toBe(true);
expect(clean.segments?.[0]?.pass).toBe(true);
expect(clean.person?.shots_at_cuts).toMatchObject({
at_cuts: 1,
total: 1,
cuts_without_shot: [],
});
});
it("judges a regenerated segment by motion and look, not pixels", async () => {
const shifted = clip("shifted.mp4", [...["-i", ref, "-vf"], "hflip"]);
const segments = parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n");
const r = await diffVideos({
reference: ref,
yours: shifted,
segments,
outDir: join(dir, "regen"),
});
expect(r.ssim.mean).toBeLessThan(0.9);
expect(r.segments?.[0]?.energy?.mean_ratio).toBeCloseTo(1, 1);
expect(r.segments?.[0]?.pass).toBe(true);
});
});
@@ -0,0 +1,121 @@
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { diffVideos } from "./diff.js";
import { clipFolder, noTools, REF_ARGS } from "./diff.test-helpers.js";
import { parseSegments } from "./segments.js";
const { dir, clip } = clipFolder("hf-diff-seg-test-");
describe.skipIf(noTools)("diffVideos with segments", () => {
const ref = noTools ? "" : clip("ref.mp4", REF_ARGS);
it("judges a rebuilt segment's regenerated region by motion and look, the rest by SSIM", async () => {
// The bottom half mirrored, as a re-shot presenter differs in pixels but moves the same.
const mixed = clip("mixed.mp4", [
...["-i", ref, "-filter_complex"],
"[0]split[a][b];[b]crop=320:90:0:90,hflip[f];[a][f]overlay=0:90",
]);
const md = "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\n";
const plain = await diffVideos({
reference: ref,
yours: mixed,
segments: parseSegments(md),
outDir: join(dir, "mixed-plain"),
});
expect(plain.segments?.[0]?.pass).toBe(false);
const r = await diffVideos({
reference: ref,
yours: mixed,
segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`),
outDir: join(dir, "mixed"),
});
const [s] = r.segments!;
expect(s!.ssim!.mean).toBeGreaterThan(0.99);
expect(s!.regenerated?.roi).toEqual([0, 90, 320, 90]);
expect(s!.person?.energy?.mean_ratio).toBeCloseTo(1, 1);
expect(s!.pass).toBe(true);
// A wrong box outside the region still fails the segment.
const boxed = clip("mixed-box.mp4", [
...["-i", mixed, "-vf"],
"drawbox=x=20:y=10:w=60:h=40:color=red:t=fill",
]);
const bad = await diffVideos({
reference: ref,
yours: boxed,
segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`),
outDir: join(dir, "mixed-box"),
});
expect(bad.segments?.[0]?.pass).toBe(false);
});
// A label (three white strokes) over a soft bottom half, a regenerated person (here 5 frames late); yours
// draws it 24 px off. Two tests of two comparisons each, so neither nears the per-test limit on a busy runner.
const strokes = (x0: number) =>
[0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(",");
const soft = (delay: number) =>
`[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`;
const labelled = (name: string, delay: number, x0: number) =>
noTools ? "" : clip(name, [...["-i", ref, "-filter_complex"], `${soft(delay)},${strokes(x0)}`]);
const base = labelled("label-ref.mp4", 0, 120);
const right = labelled("label-right.mp4", 5, 120);
const off = labelled("label-off.mp4", 5, 144);
const md =
"## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\n";
const judge = (yours: string, text: string, name: string) =>
diffVideos({ reference: base, yours, segments: parseSegments(text), outDir: join(dir, name) });
it("hides a drawn element in a regenerated region without a Drawn line, and flags a box over the whole frame", async () => {
expect((await judge(off, md, "drawn-hidden")).segments?.[0]?.pass).toBe(true);
const whole = await judge(right, md.replace("0, 90, 320, 90", "0, 0, 320, 180"), "drawn-whole");
expect(whole.warnings.some((w) => w.includes("covers the whole frame"))).toBe(true);
});
it("judges each drawn element inside a regenerated region by itself, not by what is behind it", async () => {
const drawn = `${md}Drawn: 118, 108, 42, 28\n`;
const good = (await judge(right, drawn, "drawn-right")).segments![0]!;
expect(good.pass).toBe(true);
expect(good.drawn?.[0]).toMatchObject({ id: "S01.D1", box: [118, 108, 42, 28], pass: true });
expect(good.scored).toContain("1 drawn elements, each scored by itself");
const bad = await judge(off, drawn, "drawn-off");
expect(bad.segments?.[0]?.pass).toBe(false);
expect(bad.segments?.[0]?.drawn?.[0]?.pass).toBe(false);
});
it("scores a drawn element by itself without segments, and its ink says when yours is thinner", async () => {
// The same three-stroke label over two different flat backdrops; yours draws it at half the stroke width.
const strokes = (w: number) =>
[0, 14, 28].map((x) => `drawbox=x=${120 + x}:y=110:w=${w}:h=24:color=white:t=fill`).join(",");
const backdrop = (colour: string) => [
"-f",
"lavfi",
"-i",
`color=c=${colour}:s=320x180:d=3:r=30`,
];
const base = clip("solo-ref.mp4", [...backdrop("0x305070"), "-vf", strokes(6)]);
const right = clip("solo-right.mp4", [...backdrop("0x604030"), "-vf", strokes(6)]);
const thin = clip("solo-thin.mp4", [...backdrop("0x604030"), "-vf", strokes(3)]);
const box: [number, number, number, number] = [116, 106, 44, 32];
const good = await diffVideos({
reference: base,
yours: right,
at: [0.5, 1, 1.5, 2],
drawn: [{ box }, { box, window: [1, 3] }],
outDir: join(dir, "solo-right"),
});
expect(good.segments).toBeUndefined();
expect(good.drawn).toHaveLength(2);
expect(good.drawn?.[0]).toMatchObject({ id: "D1", box, window: [0, 3], frames: 4, pass: true });
expect(good.drawn?.[1]).toMatchObject({ id: "D2", window: [1, 3], frames: 3, pass: true });
const bad = await diffVideos({
reference: base,
yours: thin,
at: [0.5, 1, 1.5, 2],
drawn: [{ box }],
outDir: join(dir, "solo-thin"),
});
expect(good.drawn?.[0]).toMatchObject({ score: 1, offset: [0, 0], ink: 1 });
// Half-weight letters in the right place: the score barely drops, the ink says it.
expect(bad.drawn?.[0]?.score).toBeLessThan(0.9);
expect(bad.drawn?.[0]?.ink).toBe(0.5);
});
});
@@ -0,0 +1,31 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
export const ffmpeg = findFFmpeg();
export const noTools = !ffmpeg || !findFFprobe();
/** testsrc2, 3 s at 30 fps: the original most of diff's tests compare against. */
export const REF_ARGS = ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"];
/** A temp folder for one test file's clips, removed after its tests, and a maker of lossless synthetic clips in it,
* so SSIM differences come from the picture, not the encoder. */
export function clipFolder(prefix: string): {
dir: string;
clip: (name: string, args: string[]) => string;
} {
const dir = mkdtempSync(join(tmpdir(), prefix));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
const clip = (name: string, args: string[]): string => {
const out = join(dir, name);
execFileSync(ffmpeg!, [
...["-hide_banner", "-loglevel", "error", "-y", ...args],
...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "0", out],
]);
return out;
};
return { dir, clip };
}
+5 -381
View File
@@ -1,9 +1,6 @@
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { existsSync } from "node:fs";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { describe, expect, it } from "vitest";
import {
cellsOutside,
diffVideos,
@@ -11,27 +8,13 @@ import {
pickedFrames,
regionRoi,
selectFilter,
soundOffset,
wholeFrameCuts,
} from "./diff.js";
import { probeVideo } from "./media.js";
import { clipFolder, noTools, REF_ARGS } from "./diff.test-helpers.js";
import { carriedFrame, shotFraming, syncPass } from "./person.js";
import { parseSegments } from "./segments.js";
const ffmpeg = findFFmpeg();
const noTools = !ffmpeg || !findFFprobe();
const dir = mkdtempSync(join(tmpdir(), "hf-diff-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
/** A lossless synthetic clip, so SSIM differences come from the picture, not the encoder. */
function clip(name: string, args: string[]): string {
const out = join(dir, name);
execFileSync(ffmpeg!, [
...["-hide_banner", "-loglevel", "error", "-y", ...args],
...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "0", out],
]);
return out;
}
const { dir, clip } = clipFolder("hf-diff-test-");
describe("parseSegments", () => {
it("reads headings and where each window's pixels come from", () => {
@@ -207,7 +190,7 @@ describe("frame picks", () => {
});
describe.skipIf(noTools)("diffVideos", () => {
const ref = noTools ? "" : clip("ref.mp4", ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"]);
const ref = noTools ? "" : clip("ref.mp4", REF_ARGS);
it("passes a video against itself", async () => {
const r = await diffVideos({ reference: ref, yours: ref, outDir: join(dir, "same") });
@@ -353,363 +336,4 @@ describe.skipIf(noTools)("diffVideos", () => {
});
expect(late.cuts).toMatchObject({ missing: [1], extra: [1.2], pass: false });
});
it("judges a rebuilt segment's regenerated region by motion and look, the rest by SSIM", async () => {
// The bottom half mirrored, as a re-shot presenter differs in pixels but moves the same.
const mixed = clip("mixed.mp4", [
...["-i", ref, "-filter_complex"],
"[0]split[a][b];[b]crop=320:90:0:90,hflip[f];[a][f]overlay=0:90",
]);
const md = "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\n";
const plain = await diffVideos({
reference: ref,
yours: mixed,
segments: parseSegments(md),
outDir: join(dir, "mixed-plain"),
});
expect(plain.segments?.[0]?.pass).toBe(false);
const r = await diffVideos({
reference: ref,
yours: mixed,
segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`),
outDir: join(dir, "mixed"),
});
const [s] = r.segments!;
expect(s!.ssim!.mean).toBeGreaterThan(0.99);
expect(s!.regenerated?.roi).toEqual([0, 90, 320, 90]);
expect(s!.person?.energy?.mean_ratio).toBeCloseTo(1, 1);
expect(s!.pass).toBe(true);
// A wrong box outside the region still fails the segment.
const boxed = clip("mixed-box.mp4", [
...["-i", mixed, "-vf"],
"drawbox=x=20:y=10:w=60:h=40:color=red:t=fill",
]);
const bad = await diffVideos({
reference: ref,
yours: boxed,
segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`),
outDir: join(dir, "mixed-box"),
});
expect(bad.segments?.[0]?.pass).toBe(false);
});
it("judges each drawn element inside a regenerated region by itself, not by what is behind it", async () => {
// A label (three white strokes) over a soft bottom half, a regenerated person (here 5 frames late); yours
// draws it 24 px off.
const strokes = (x0: number) =>
[0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(",");
const label = strokes(120);
const soft = (delay: number) =>
`[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`;
const base = clip("label-ref.mp4", [...["-i", ref, "-filter_complex"], `${soft(0)},${label}`]);
const right = clip("label-right.mp4", [
...["-i", ref, "-filter_complex"],
`${soft(5)},${label}`,
]);
const off = clip("label-off.mp4", [
...["-i", ref, "-filter_complex"],
`${soft(5)},${strokes(144)}`,
]);
const md =
"## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\n";
const judge = (yours: string, text: string, name: string) =>
diffVideos({
reference: base,
yours,
segments: parseSegments(text),
outDir: join(dir, name),
});
// Without a Drawn line the label hides in the person's box.
expect((await judge(off, md, "drawn-hidden")).segments?.[0]?.pass).toBe(true);
const drawn = `${md}Drawn: 118, 108, 42, 28\n`;
const good = (await judge(right, drawn, "drawn-right")).segments![0]!;
expect(good.pass).toBe(true);
expect(good.drawn?.[0]).toMatchObject({ id: "S01.D1", box: [118, 108, 42, 28], pass: true });
expect(good.scored).toContain("1 drawn elements, each scored by itself");
const bad = await judge(off, drawn, "drawn-off");
expect(bad.segments?.[0]?.pass).toBe(false);
expect(bad.segments?.[0]?.drawn?.[0]?.pass).toBe(false);
// A box over the whole frame is flagged.
const whole = await judge(right, md.replace("0, 90, 320, 90", "0, 0, 320, 180"), "drawn-whole");
expect(whole.warnings.some((w) => w.includes("covers the whole frame"))).toBe(true);
});
it("scores a drawn element by itself without segments, and its ink says when yours is thinner", async () => {
// The same three-stroke label over two different flat backdrops; yours draws it at half the stroke width.
const strokes = (w: number) =>
[0, 14, 28].map((x) => `drawbox=x=${120 + x}:y=110:w=${w}:h=24:color=white:t=fill`).join(",");
const backdrop = (colour: string) => [
"-f",
"lavfi",
"-i",
`color=c=${colour}:s=320x180:d=3:r=30`,
];
const base = clip("solo-ref.mp4", [...backdrop("0x305070"), "-vf", strokes(6)]);
const right = clip("solo-right.mp4", [...backdrop("0x604030"), "-vf", strokes(6)]);
const thin = clip("solo-thin.mp4", [...backdrop("0x604030"), "-vf", strokes(3)]);
const box: [number, number, number, number] = [116, 106, 44, 32];
const good = await diffVideos({
reference: base,
yours: right,
at: [0.5, 1, 1.5, 2],
drawn: [{ box }, { box, window: [1, 3] }],
outDir: join(dir, "solo-right"),
});
expect(good.segments).toBeUndefined();
expect(good.drawn).toHaveLength(2);
expect(good.drawn?.[0]).toMatchObject({ id: "D1", box, window: [0, 3], frames: 4, pass: true });
expect(good.drawn?.[1]).toMatchObject({ id: "D2", window: [1, 3], frames: 3, pass: true });
const bad = await diffVideos({
reference: base,
yours: thin,
at: [0.5, 1, 1.5, 2],
drawn: [{ box }],
outDir: join(dir, "solo-thin"),
});
expect(good.drawn?.[0]).toMatchObject({ score: 1, offset: [0, 0], ink: 1 });
// Half-weight letters in the right place: the score barely drops, the ink says it.
expect(bad.drawn?.[0]?.score).toBeLessThan(0.9);
expect(bad.drawn?.[0]?.ink).toBe(0.5);
});
it("fails the original's whole frame played where copy is meant to be rebuilt", async () => {
const ledger = join(dir, "manifest.jsonl");
writeFileSync(
ledger,
JSON.stringify({
type: "video",
path: "assets/all.mp4",
provenance: { provider: "original", window: [0, 3], roi: [0, 0, 320, 180] },
}) + "\n",
);
const titled = '## S01 · 0–3 s · Title\nPixels: rebuilt\nCopy: "TITLE"\n';
const copy = await diffVideos({
reference: ref,
yours: ref,
segments: parseSegments(titled),
ledger,
outDir: join(dir, "copy"),
});
expect(copy.whole_frame_reuse).toMatchObject({ spans: [[0, 3]], drawn: [[0, 3]], pass: false });
expect(copy.segments?.[0]?.pass).toBe(false);
expect(copy.pass).toBe(false);
// Footage reused with its burned-in title is a copy too (Miao, 2026-10-23: no whole-frame cut with drawn things).
const burned = await diffVideos({
reference: ref,
yours: ref,
segments: parseSegments(
'## S01 · 0–3 s · Title over footage\nPixels: reused\nCopy: "TITLE"\n',
),
ledger,
outDir: join(dir, "burned"),
});
expect(burned.whole_frame_reuse).toMatchObject({ share: 1, drawn: [[0, 3]], pass: false });
// Footage with nothing drawn on it may be cut whole.
const footage = await diffVideos({
reference: ref,
yours: ref,
segments: parseSegments("## S01 · 0–3 s · Footage\nPixels: reused\n"),
ledger,
outDir: join(dir, "footage"),
});
expect(footage.whole_frame_reuse).toMatchObject({ share: 1, drawn: [], pass: true });
expect(footage.pass).toBe(true);
// A cut that only fed a generation (a motion guide) is not played: yours differs from it.
const flipped = clip("guided.mp4", [...["-i", ref, "-vf"], "hflip"]);
const guided = await diffVideos({
reference: ref,
yours: flipped,
segments: parseSegments(titled),
ledger,
outDir: join(dir, "guided"),
});
expect(guided.whole_frame_reuse).toMatchObject({ spans: [], pass: true });
// A generated shot that starts from the original's first frame matches it for a few frames only: not reuse.
const started = clip("started.mp4", [
...["-i", ref, "-filter_complex"],
"[0]split[a][b];[b]hflip[f];[a][f]overlay=enable='gte(t\\,0.2)'",
]);
const start = await diffVideos({
reference: ref,
yours: started,
segments: parseSegments(titled),
ledger,
outDir: join(dir, "started"),
});
expect(start.whole_frame_reuse).toMatchObject({ spans: [], pass: true });
});
it("finds a whole-frame still of the original shown in yours", async () => {
const ledger = join(dir, "stills.jsonl");
writeFileSync(
ledger,
[
{ type: "image", provenance: { provider: "original", at: 1, roi: [0, 0, 320, 180] } },
{ type: "image", provenance: { provider: "original", at: 2.5 } },
]
.map((e) => JSON.stringify(e))
.join("\n"),
);
// Yours holds the original's frame at 1 s for the whole clip.
const held = clip("held.mp4", [
...["-i", ref, "-vf"],
"select=eq(n\\,30),loop=loop=89:size=1,setpts=N/30/TB",
]);
const r = await diffVideos({
reference: ref,
yours: held,
segments: parseSegments('## S01 · 0–3 s · Collage\nPixels: rebuilt\nCopy: "TITLE"\n'),
ledger,
outDir: join(dir, "held"),
});
expect(r.whole_frame_reuse).toMatchObject({ stills: [1], spans: [[0, 3]], pass: false });
});
it("reads sound that starts late as late, and the picture's length as the duration", async () => {
// Clicks at uneven times, with the sound running 0.4 s past the picture.
const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3.4";
const voiced = (name: string, delay: number) =>
clip(name, [
...["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"],
...["-itsoffset", String(delay), "-f", "lavfi", "-i", clicks],
...["-map", "0:v", "-map", "1:a", "-c:a", "aac"],
]);
const on = voiced("voiced.mp4", 0);
const same = await diffVideos({ reference: on, yours: on, outDir: join(dir, "voiced") });
expect(same.duration).toMatchObject({ reference: 3, pass: true });
expect(same.sound).toMatchObject({ offset_ms: 0, pass: true });
const late = await diffVideos({
reference: on,
yours: voiced("late.mp4", 0.2),
outDir: join(dir, "late"),
});
expect(late.sound).toMatchObject({ offset_ms: 200, pass: false });
});
it("places the sound by its decoded timestamps, not the start time the probe reports", () => {
// ffmpeg 5.1 drops an HE-AAC edit list's skip twice: the first sample it keeps is timestamped 0.1 s in while
// the probe still says the track starts at 0. Here a sound timestamped 0.25 s in, probed as starting at 0,
// against the same sound with 0.25 s of silence baked into its samples: in sync.
const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3";
const picture = ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"];
const stamped = clip("stamped.mp4", [
...picture,
...["-itsoffset", "0.25", "-f", "lavfi", "-i", clicks],
...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"],
]);
const baked = clip("baked.mp4", [
...picture,
...["-f", "lavfi", "-i", clicks, "-af", "adelay=250:all=1"],
...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"],
]);
const probedAtZero = { ...probeVideo(stamped), audioStart: 0 };
expect(soundOffset(stamped, baked, probedAtZero, probeVideo(baked))).toMatchObject({
offset_ms: 0,
pass: true,
});
});
it("passes a re-shot person whose motion is off when the components are right", async () => {
// The bottom half (the person) frozen on its first frame: no motion at all. Pixels above match.
const frozen = clip("frozen.mp4", [
...["-i", ref, "-filter_complex"],
"[0]split[a][b];[b]crop=320:90:0:90,trim=end_frame=1,tpad=stop=89:stop_mode=clone[f];[a][f]overlay=0:90",
]);
const r = await diffVideos({
reference: ref,
yours: frozen,
segments: parseSegments(
"## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\nRegenerated: 0, 0.5, 1, 0.5\n",
),
outDir: join(dir, "frozen"),
});
const [s] = r.segments!;
expect(s!.person?.energy?.mean_ratio).toBeLessThan(0.5);
expect(s!.person?.gates).toEqual({ framing: null, text: null, sync: null });
expect(s!.pass).toBe(true);
expect(r.components).toMatchObject({ passed: 1, total: 1 });
expect(r.person?.energy).toBeLessThan(0.5);
expect(r.pass).toBe(true);
});
it("fails a re-shot person whose generated clip carries the original's text", async () => {
// The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half.
const strokes = (x0: number) =>
[0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(",");
const label = strokes(120);
const soft = (delay: number) =>
`[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`;
const base = clip("carried-ref.mp4", [
...["-i", ref, "-filter_complex"],
`${soft(0)},${label}`,
]);
const project = join(dir, "carried");
mkdirSync(join(project, "assets"), { recursive: true });
// The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the
// label over each; over the first, 8 px off, the letters show doubled.
clip("carried/assets/dirty.mp4", [...["-i", ref, "-filter_complex"], `${soft(5)},${label}`]);
clip("carried/assets/clean.mp4", [...["-i", ref, "-filter_complex"], soft(5)]);
execFileSync(ffmpeg!, [
...["-hide_banner", "-loglevel", "error", "-y", "-i", ref, "-filter_complex"],
`${soft(0)},trim=end_frame=1`,
...["-frames:v", "1", join(project, "assets", "still.png")],
]);
const ledger = (path: string) => {
const file = join(project, `${path}.jsonl`);
writeFileSync(
file,
JSON.stringify({
type: "video",
path: `assets/${path}.mp4`,
provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] },
}) + "\n",
);
return file;
};
const md =
"## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n";
const judge = (path: string, x0: number) =>
diffVideos({
reference: base,
yours: clip(`carried-yours-${path}.mp4`, [
...["-i", join(project, "assets", `${path}.mp4`), "-vf"],
strokes(x0),
]),
segments: parseSegments(md),
ledger: ledger(path),
project,
outDir: join(dir, `carried-${path}`),
});
const dirty = await judge("dirty", 128);
expect(dirty.segments?.[0]?.person?.text?.[0]).toMatchObject({
drawn: "S01.D1",
carried: true,
});
expect(dirty.segments?.[0]?.pass).toBe(false);
expect(dirty.person).toMatchObject({ gates: { text: false }, failed: { text: ["S01"] } });
expect(dirty.pass).toBe(false);
const clean = await judge("clean", 120);
expect(clean.segments?.[0]?.drawn?.[0]?.pass).toBe(true);
expect(clean.segments?.[0]?.person?.gates.text).toBe(true);
expect(clean.segments?.[0]?.pass).toBe(true);
expect(clean.person?.shots_at_cuts).toMatchObject({
at_cuts: 1,
total: 1,
cuts_without_shot: [],
});
});
it("judges a regenerated segment by motion and look, not pixels", async () => {
const shifted = clip("shifted.mp4", [...["-i", ref, "-vf"], "hflip"]);
const segments = parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n");
const r = await diffVideos({
reference: ref,
yours: shifted,
segments,
outDir: join(dir, "regen"),
});
expect(r.ssim.mean).toBeLessThan(0.9);
expect(r.segments?.[0]?.energy?.mean_ratio).toBeCloseTo(1, 1);
expect(r.segments?.[0]?.pass).toBe(true);
});
});