diff --git a/packages/cli/src/measure/diff-people.test.ts b/packages/cli/src/measure/diff-people.test.ts index 0fb0933c7f..2f0ad4d2b5 100644 --- a/packages/cli/src/measure/diff-people.test.ts +++ b/packages/cli/src/measure/diff-people.test.ts @@ -2,14 +2,22 @@ import { execFileSync } from "node:child_process"; import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { diffVideos, soundOffset } from "./diff.js"; -import { clipFolder, ffmpeg, noTools, REF_ARGS } from "./diff.test-helpers.js"; +import { soundOffset } from "./diff.js"; +import { + clipFolder, + diffVerdicts, + ffmpeg, + loadOpenCvFirst, + noTools, + REF_ARGS, +} from "./diff.test-helpers.js"; import { probeVideo } from "./media.js"; import { parseSegments } from "./segments.js"; const { dir, clip } = clipFolder("hf-diff-person-test-"); describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { + loadOpenCvFirst(); const ref = noTools ? "" : clip("ref.mp4", REF_ARGS); it("fails the original's whole frame played where copy is meant to be rebuilt", async () => { @@ -23,7 +31,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { }) + "\n", ); const titled = '## S01 · 0–3 s · Title\nPixels: rebuilt\nCopy: "TITLE"\n'; - const copy = await diffVideos({ + const copy = await diffVerdicts({ reference: ref, yours: ref, segments: parseSegments(titled), @@ -34,7 +42,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { expect(copy.segments?.[0]?.pass).toBe(false); expect(copy.pass).toBe(false); // Footage reused with its burned-in title is a copy too (Miao, 2026-10-23: no whole-frame cut with drawn things). - const burned = await diffVideos({ + const burned = await diffVerdicts({ reference: ref, yours: ref, segments: parseSegments( @@ -45,7 +53,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { }); expect(burned.whole_frame_reuse).toMatchObject({ share: 1, drawn: [[0, 3]], pass: false }); // Footage with nothing drawn on it may be cut whole. - const footage = await diffVideos({ + const footage = await diffVerdicts({ reference: ref, yours: ref, segments: parseSegments("## S01 · 0–3 s · Footage\nPixels: reused\n"), @@ -56,7 +64,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { expect(footage.pass).toBe(true); // A cut that only fed a generation (a motion guide) is not played: yours differs from it. const flipped = clip("guided.mp4", [...["-i", ref, "-vf"], "hflip"]); - const guided = await diffVideos({ + const guided = await diffVerdicts({ reference: ref, yours: flipped, segments: parseSegments(titled), @@ -69,7 +77,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { ...["-i", ref, "-filter_complex"], "[0]split[a][b];[b]hflip[f];[a][f]overlay=enable='gte(t\\,0.2)'", ]); - const start = await diffVideos({ + const start = await diffVerdicts({ reference: ref, yours: started, segments: parseSegments(titled), @@ -95,7 +103,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { ...["-i", ref, "-vf"], "select=eq(n\\,30),loop=loop=89:size=1,setpts=N/30/TB", ]); - const r = await diffVideos({ + const r = await diffVerdicts({ reference: ref, yours: held, segments: parseSegments('## S01 · 0–3 s · Collage\nPixels: rebuilt\nCopy: "TITLE"\n'), @@ -115,10 +123,10 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { ...["-map", "0:v", "-map", "1:a", "-c:a", "aac"], ]); const on = voiced("voiced.mp4", 0); - const same = await diffVideos({ reference: on, yours: on, outDir: join(dir, "voiced") }); + const same = await diffVerdicts({ reference: on, yours: on, outDir: join(dir, "voiced") }); expect(same.duration).toMatchObject({ reference: 3, pass: true }); expect(same.sound).toMatchObject({ offset_ms: 0, pass: true }); - const late = await diffVideos({ + const late = await diffVerdicts({ reference: on, yours: voiced("late.mp4", 0.2), outDir: join(dir, "late"), @@ -155,7 +163,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { ...["-i", ref, "-filter_complex"], "[0]split[a][b];[b]crop=320:90:0:90,trim=end_frame=1,tpad=stop=89:stop_mode=clone[f];[a][f]overlay=0:90", ]); - const r = await diffVideos({ + const r = await diffVerdicts({ reference: ref, yours: frozen, segments: parseSegments( @@ -172,55 +180,58 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { expect(r.pass).toBe(true); }); - it("fails a re-shot person whose generated clip carries the original's text", async () => { - // The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half. - const strokes = (x0: number) => - [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(","); - const label = strokes(120); - const soft = (delay: number) => - `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`; + // The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half. + const strokes = (x0: number) => + [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(","); + const label = strokes(120); + const soft = (delay: number) => + `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`; + const carried = join(dir, "carried"); + // The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the + // label over each; over the first, 8 px off, the letters show doubled. + const carriedRef = (): string => { const base = clip("carried-ref.mp4", [ ...["-i", ref, "-filter_complex"], `${soft(0)},${label}`, ]); - const project = join(dir, "carried"); - mkdirSync(join(project, "assets"), { recursive: true }); - // The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the - // label over each; over the first, 8 px off, the letters show doubled. + mkdirSync(join(carried, "assets"), { recursive: true }); clip("carried/assets/dirty.mp4", [...["-i", ref, "-filter_complex"], `${soft(5)},${label}`]); clip("carried/assets/clean.mp4", [...["-i", ref, "-filter_complex"], soft(5)]); execFileSync(ffmpeg!, [ ...["-hide_banner", "-loglevel", "error", "-y", "-i", ref, "-filter_complex"], `${soft(0)},trim=end_frame=1`, - ...["-frames:v", "1", join(project, "assets", "still.png")], + ...["-frames:v", "1", join(carried, "assets", "still.png")], ]); - const ledger = (path: string) => { - const file = join(project, `${path}.jsonl`); - writeFileSync( - file, - JSON.stringify({ - type: "video", - path: `assets/${path}.mp4`, - provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] }, - }) + "\n", - ); - return file; - }; - const md = - "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n"; - const judge = (path: string, x0: number) => - diffVideos({ - reference: base, - yours: clip(`carried-yours-${path}.mp4`, [ - ...["-i", join(project, "assets", `${path}.mp4`), "-vf"], - strokes(x0), - ]), - segments: parseSegments(md), - ledger: ledger(path), - project, - outDir: join(dir, `carried-${path}`), - }); - const dirty = await judge("dirty", 128); + return base; + }; + const carriedBase = noTools ? "" : carriedRef(); + const judgeCarried = (path: string, x0: number) => { + const ledger = join(carried, `${path}.jsonl`); + writeFileSync( + ledger, + JSON.stringify({ + type: "video", + path: `assets/${path}.mp4`, + provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] }, + }) + "\n", + ); + return diffVerdicts({ + reference: carriedBase, + yours: clip(`carried-yours-${path}.mp4`, [ + ...["-i", join(carried, "assets", `${path}.mp4`), "-vf"], + strokes(x0), + ]), + segments: parseSegments( + "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n", + ), + ledger, + project: carried, + outDir: join(dir, `carried-${path}`), + }); + }; + + it("fails a re-shot person whose generated clip carries the original's text", async () => { + const dirty = await judgeCarried("dirty", 128); expect(dirty.segments?.[0]?.person?.text?.[0]).toMatchObject({ drawn: "S01.D1", carried: true, @@ -228,7 +239,10 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { expect(dirty.segments?.[0]?.pass).toBe(false); expect(dirty.person).toMatchObject({ gates: { text: false }, failed: { text: ["S01"] } }); expect(dirty.pass).toBe(false); - const clean = await judge("clean", 120); + }); + + it("passes a re-shot person whose generated clip came from a masked guide", async () => { + const clean = await judgeCarried("clean", 120); expect(clean.segments?.[0]?.drawn?.[0]?.pass).toBe(true); expect(clean.segments?.[0]?.person?.gates.text).toBe(true); expect(clean.segments?.[0]?.pass).toBe(true); @@ -242,7 +256,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { it("judges a regenerated segment by motion and look, not pixels", async () => { const shifted = clip("shifted.mp4", [...["-i", ref, "-vf"], "hflip"]); const segments = parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n"); - const r = await diffVideos({ + const r = await diffVerdicts({ reference: ref, yours: shifted, segments, diff --git a/packages/cli/src/measure/diff-segments.test.ts b/packages/cli/src/measure/diff-segments.test.ts index b2f1861d28..db28dc44d1 100644 --- a/packages/cli/src/measure/diff-segments.test.ts +++ b/packages/cli/src/measure/diff-segments.test.ts @@ -1,12 +1,18 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { diffVideos } from "./diff.js"; -import { clipFolder, noTools, REF_ARGS } from "./diff.test-helpers.js"; +import { + clipFolder, + diffVerdicts, + loadOpenCvFirst, + noTools, + REF_ARGS, +} from "./diff.test-helpers.js"; import { parseSegments } from "./segments.js"; const { dir, clip } = clipFolder("hf-diff-seg-test-"); describe.skipIf(noTools)("diffVideos with segments", () => { + loadOpenCvFirst(); const ref = noTools ? "" : clip("ref.mp4", REF_ARGS); it("judges a rebuilt segment's regenerated region by motion and look, the rest by SSIM", async () => { @@ -16,14 +22,14 @@ describe.skipIf(noTools)("diffVideos with segments", () => { "[0]split[a][b];[b]crop=320:90:0:90,hflip[f];[a][f]overlay=0:90", ]); const md = "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\n"; - const plain = await diffVideos({ + const plain = await diffVerdicts({ reference: ref, yours: mixed, segments: parseSegments(md), outDir: join(dir, "mixed-plain"), }); expect(plain.segments?.[0]?.pass).toBe(false); - const r = await diffVideos({ + const r = await diffVerdicts({ reference: ref, yours: mixed, segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`), @@ -39,7 +45,7 @@ describe.skipIf(noTools)("diffVideos with segments", () => { ...["-i", mixed, "-vf"], "drawbox=x=20:y=10:w=60:h=40:color=red:t=fill", ]); - const bad = await diffVideos({ + const bad = await diffVerdicts({ reference: ref, yours: boxed, segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`), @@ -62,7 +68,12 @@ describe.skipIf(noTools)("diffVideos with segments", () => { const md = "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\n"; const judge = (yours: string, text: string, name: string) => - diffVideos({ reference: base, yours, segments: parseSegments(text), outDir: join(dir, name) }); + diffVerdicts({ + reference: base, + yours, + segments: parseSegments(text), + outDir: join(dir, name), + }); it("hides a drawn element in a regenerated region without a Drawn line, and flags a box over the whole frame", async () => { expect((await judge(off, md, "drawn-hidden")).segments?.[0]?.pass).toBe(true); @@ -95,7 +106,7 @@ describe.skipIf(noTools)("diffVideos with segments", () => { const right = clip("solo-right.mp4", [...backdrop("0x604030"), "-vf", strokes(6)]); const thin = clip("solo-thin.mp4", [...backdrop("0x604030"), "-vf", strokes(3)]); const box: [number, number, number, number] = [116, 106, 44, 32]; - const good = await diffVideos({ + const good = await diffVerdicts({ reference: base, yours: right, at: [0.5, 1, 1.5, 2], @@ -106,7 +117,7 @@ describe.skipIf(noTools)("diffVideos with segments", () => { expect(good.drawn).toHaveLength(2); expect(good.drawn?.[0]).toMatchObject({ id: "D1", box, window: [0, 3], frames: 4, pass: true }); expect(good.drawn?.[1]).toMatchObject({ id: "D2", window: [1, 3], frames: 3, pass: true }); - const bad = await diffVideos({ + const bad = await diffVerdicts({ reference: base, yours: thin, at: [0.5, 1, 1.5, 2], diff --git a/packages/cli/src/measure/diff.test-helpers.ts b/packages/cli/src/measure/diff.test-helpers.ts index 437c9c60a1..02b3f040aa 100644 --- a/packages/cli/src/measure/diff.test-helpers.ts +++ b/packages/cli/src/measure/diff.test-helpers.ts @@ -1,9 +1,11 @@ import { execFileSync } from "node:child_process"; -import { mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterAll } from "vitest"; +import { cpSync, existsSync, mkdtempSync, renameSync, rmSync } from "node:fs"; +import { tmpdir, userInfo } from "node:os"; +import { basename, join } from "node:path"; +import { afterAll, beforeAll } from "vitest"; import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; +import { diffVideos, type DiffOptions } from "./diff.js"; +import { loadOpenCv, OPENCV_DIR } from "./opencv.js"; export const ffmpeg = findFFmpeg(); export const noTools = !ffmpeg || !findFFprobe(); @@ -29,3 +31,33 @@ export function clipFolder(prefix: string): { }; return { dir, clip }; } + +/** diffVideos for a test that reads verdicts only: no worst-moment pictures, each of which costs seven ffmpeg runs. */ +export const diffVerdicts = (options: DiffOptions) => diffVideos({ worst: 0, ...options }); + +/** opencv.js loaded before a block's tests, so none pays for its download inside the per-test limit. The files come + * from this machine's cache, else a temp copy kept between runs (each has its own home), else the network. */ +export function loadOpenCvFirst(): void { + beforeAll(async () => { + const shared = join(tmpdir(), `hf-test-opencv-${basename(OPENCV_DIR)}`); + const machine = join( + userInfo().homedir, + ".cache", + "hyperframes", + "opencv", + basename(OPENCV_DIR), + ); + for (const from of [machine, shared]) + if (!existsSync(join(OPENCV_DIR, "opencv.js")) && existsSync(join(from, "opencv.js"))) + cpSync(from, OPENCV_DIR, { recursive: true }); + await loadOpenCv(); + if (existsSync(join(shared, "opencv.js"))) return; + const staged = `${shared}-${process.pid}`; + cpSync(OPENCV_DIR, staged, { recursive: true }); + try { + renameSync(staged, shared); + } catch { + rmSync(staged, { recursive: true, force: true }); + } + }, 90_000); +} diff --git a/packages/cli/src/measure/diff.test.ts b/packages/cli/src/measure/diff.test.ts index 86efdfca44..ab30fb1e74 100644 --- a/packages/cli/src/measure/diff.test.ts +++ b/packages/cli/src/measure/diff.test.ts @@ -10,7 +10,13 @@ import { selectFilter, wholeFrameCuts, } from "./diff.js"; -import { clipFolder, noTools, REF_ARGS } from "./diff.test-helpers.js"; +import { + clipFolder, + diffVerdicts, + loadOpenCvFirst, + noTools, + REF_ARGS, +} from "./diff.test-helpers.js"; import { carriedFrame, shotFraming, syncPass } from "./person.js"; import { parseSegments } from "./segments.js"; @@ -190,6 +196,7 @@ describe("frame picks", () => { }); describe.skipIf(noTools)("diffVideos", () => { + loadOpenCvFirst(); const ref = noTools ? "" : clip("ref.mp4", REF_ARGS); it("passes a video against itself", async () => { @@ -245,15 +252,15 @@ describe.skipIf(noTools)("diffVideos", () => { ...["-f", "lavfi", "-i", "color=c=white:s=320x180:r=30:d=3", "-vf"], `${label}:enable='lt(t,1.5)'`, ]); - const r = await diffVideos({ reference: page, yours, outDir: join(dir, "emptied") }); + const r = await diffVerdicts({ reference: page, yours, outDir: join(dir, "emptied") }); expect(r.missing_content.pass).toBe(false); expect(r.missing_content.share).toBeGreaterThan(0.45); expect(r.missing_content.spans[0]![0]).toBeCloseTo(1.5, 1); expect(r.pass).toBe(false); - const same = await diffVideos({ reference: page, yours: page, outDir: join(dir, "page") }); + const same = await diffVerdicts({ reference: page, yours: page, outDir: join(dir, "page") }); expect(same.missing_content).toMatchObject({ share: 0, pass: true }); // A regenerated segment's picture is its own: what it leaves out is not missing. - const regen = await diffVideos({ + const regen = await diffVerdicts({ reference: page, yours, segments: parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n"), @@ -274,7 +281,7 @@ describe.skipIf(noTools)("diffVideos", () => { ...["-f", "lavfi", "-i", "color=c=white:s=320x180:r=30:d=2", "-vf"], "format=gray,geq=lum=235,format=yuv420p", ]); - const r = await diffVideos({ reference: grain, yours: plain, outDir: join(dir, "grain") }); + const r = await diffVerdicts({ reference: grain, yours: plain, outDir: join(dir, "grain") }); expect(r.missing_content).toMatchObject({ share: 0, pass: true }); expect(r.missing_content.texture.share).toBeGreaterThan(0.9); expect(r.warnings.some((w) => w.includes("reported, not failed"))).toBe(true); @@ -285,7 +292,7 @@ describe.skipIf(noTools)("diffVideos", () => { ...["-i", ref, "-vf"], "drawbox=x=0:y=0:w=320:h=180:color=black:t=fill:enable='gte(t,2)'", ]); - const r = await diffVideos({ + const r = await diffVerdicts({ reference: ref, yours, range: [0.5, 1.5], @@ -303,7 +310,7 @@ describe.skipIf(noTools)("diffVideos", () => { ...["-f", "lavfi", "-i", `color=c=0xe0c040:s=320x180:r=30:d=${3 - at}`], ...["-filter_complex", "[0][1]concat=n=2:v=1[v]", "-map", "[v]"], ]); - const r = await diffVideos({ + const r = await diffVerdicts({ reference: two("cut-ref.mp4", 1), yours: two("cut-late.mp4", 1.1), outDir: join(dir, "cuts"), @@ -323,13 +330,13 @@ describe.skipIf(noTools)("diffVideos", () => { ...["-filter_complex", "[0][1]concat=n=2:v=1[v]", "-map", "[v]"], ]); const soft = step("soft-ref.mp4", "0x8b8b8b", 1); - const r = await diffVideos({ + const r = await diffVerdicts({ reference: soft, yours: step("soft-hard.mp4", "0xa0a0a0", 1), outDir: join(dir, "soft"), }); expect(r.cuts).toMatchObject({ reference: [1], missing: [], extra: [], pass: true }); - const late = await diffVideos({ + const late = await diffVerdicts({ reference: soft, yours: step("soft-late.mp4", "0xa0a0a0", 1.2), outDir: join(dir, "soft-late"),