test(cli): keep diff's video tests inside the flake gate's limits

Under the gate's CPU contention every diff test needing face detection or
optical flow paid opencv.js's 11 MB download and WebAssembly start-up, and
every diffVideos call wrote four worst-moment pictures (about 28 ffmpeg runs)
no assertion read. Tests ran 21-28 s against the 20 s limit.

- loadOpenCvFirst() stages opencv.js in a beforeAll (from this machine's
  cache, a temp copy kept between runs, else the network).
- diffVerdicts() runs diffVideos with worst: 0 for tests that only read
  verdicts; the worst-moment tests keep the pictures.
- The carried-text test is split into its dirty and clean halves.

Locally with 8 CPU burners on 8 cores: slowest test 7.7 s, files 24-30 s.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

packages/cli/src/measure/diff-people.test.ts

@@ -2,14 +2,22 @@ import { execFileSync } from "node:child_process";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { describe, expect, it } from "vitest";
-import { diffVideos, soundOffset } from "./diff.js";
-import { clipFolder, ffmpeg, noTools, REF_ARGS } from "./diff.test-helpers.js";
+import { soundOffset } from "./diff.js";
+import {
+  clipFolder,
+  diffVerdicts,
+  ffmpeg,
+  loadOpenCvFirst,
+  noTools,
+  REF_ARGS,
+} from "./diff.test-helpers.js";
 import { probeVideo } from "./media.js";
 import { parseSegments } from "./segments.js";
 
 const { dir, clip } = clipFolder("hf-diff-person-test-");
 
 describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
+  loadOpenCvFirst();
   const ref = noTools ? "" : clip("ref.mp4", REF_ARGS);
 
   it("fails the original's whole frame played where copy is meant to be rebuilt", async () => {
@@ -23,7 +31,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
       }) + "\n",
     );
     const titled = '## S01 · 0–3 s · Title\nPixels: rebuilt\nCopy: "TITLE"\n';
-    const copy = await diffVideos({
+    const copy = await diffVerdicts({
       reference: ref,
       yours: ref,
       segments: parseSegments(titled),
@@ -34,7 +42,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
     expect(copy.segments?.[0]?.pass).toBe(false);
     expect(copy.pass).toBe(false);
     // Footage reused with its burned-in title is a copy too (Miao, 2026-10-23: no whole-frame cut with drawn things).
-    const burned = await diffVideos({
+    const burned = await diffVerdicts({
       reference: ref,
       yours: ref,
       segments: parseSegments(
@@ -45,7 +53,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
     });
     expect(burned.whole_frame_reuse).toMatchObject({ share: 1, drawn: [[0, 3]], pass: false });
     // Footage with nothing drawn on it may be cut whole.
-    const footage = await diffVideos({
+    const footage = await diffVerdicts({
       reference: ref,
       yours: ref,
       segments: parseSegments("## S01 · 0–3 s · Footage\nPixels: reused\n"),
@@ -56,7 +64,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
     expect(footage.pass).toBe(true);
     // A cut that only fed a generation (a motion guide) is not played: yours differs from it.
     const flipped = clip("guided.mp4", [...["-i", ref, "-vf"], "hflip"]);
-    const guided = await diffVideos({
+    const guided = await diffVerdicts({
       reference: ref,
       yours: flipped,
       segments: parseSegments(titled),
@@ -69,7 +77,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
       ...["-i", ref, "-filter_complex"],
       "[0]split[a][b];[b]hflip[f];[a][f]overlay=enable='gte(t\\,0.2)'",
     ]);
-    const start = await diffVideos({
+    const start = await diffVerdicts({
       reference: ref,
       yours: started,
       segments: parseSegments(titled),
@@ -95,7 +103,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
       ...["-i", ref, "-vf"],
       "select=eq(n\\,30),loop=loop=89:size=1,setpts=N/30/TB",
     ]);
-    const r = await diffVideos({
+    const r = await diffVerdicts({
       reference: ref,
       yours: held,
       segments: parseSegments('## S01 · 0–3 s · Collage\nPixels: rebuilt\nCopy: "TITLE"\n'),
@@ -115,10 +123,10 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
         ...["-map", "0:v", "-map", "1:a", "-c:a", "aac"],
       ]);
     const on = voiced("voiced.mp4", 0);
-    const same = await diffVideos({ reference: on, yours: on, outDir: join(dir, "voiced") });
+    const same = await diffVerdicts({ reference: on, yours: on, outDir: join(dir, "voiced") });
     expect(same.duration).toMatchObject({ reference: 3, pass: true });
     expect(same.sound).toMatchObject({ offset_ms: 0, pass: true });
-    const late = await diffVideos({
+    const late = await diffVerdicts({
       reference: on,
       yours: voiced("late.mp4", 0.2),
       outDir: join(dir, "late"),
@@ -155,7 +163,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
       ...["-i", ref, "-filter_complex"],
       "[0]split[a][b];[b]crop=320:90:0:90,trim=end_frame=1,tpad=stop=89:stop_mode=clone[f];[a][f]overlay=0:90",
     ]);
-    const r = await diffVideos({
+    const r = await diffVerdicts({
       reference: ref,
       yours: frozen,
       segments: parseSegments(
@@ -172,55 +180,58 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
     expect(r.pass).toBe(true);
   });
 
-  it("fails a re-shot person whose generated clip carries the original's text", async () => {
-    // The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half.
-    const strokes = (x0: number) =>
-      [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(",");
-    const label = strokes(120);
-    const soft = (delay: number) =>
-      `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`;
+  // The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half.
+  const strokes = (x0: number) =>
+    [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(",");
+  const label = strokes(120);
+  const soft = (delay: number) =>
+    `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`;
+  const carried = join(dir, "carried");
+  // The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the
+  // label over each; over the first, 8 px off, the letters show doubled.
+  const carriedRef = (): string => {
     const base = clip("carried-ref.mp4", [
       ...["-i", ref, "-filter_complex"],
       `${soft(0)},${label}`,
     ]);
-    const project = join(dir, "carried");
-    mkdirSync(join(project, "assets"), { recursive: true });
-    // The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the
-    // label over each; over the first, 8 px off, the letters show doubled.
+    mkdirSync(join(carried, "assets"), { recursive: true });
     clip("carried/assets/dirty.mp4", [...["-i", ref, "-filter_complex"], `${soft(5)},${label}`]);
     clip("carried/assets/clean.mp4", [...["-i", ref, "-filter_complex"], soft(5)]);
     execFileSync(ffmpeg!, [
       ...["-hide_banner", "-loglevel", "error", "-y", "-i", ref, "-filter_complex"],
       `${soft(0)},trim=end_frame=1`,
-      ...["-frames:v", "1", join(project, "assets", "still.png")],
+      ...["-frames:v", "1", join(carried, "assets", "still.png")],
     ]);
-    const ledger = (path: string) => {
-      const file = join(project, `${path}.jsonl`);
-      writeFileSync(
-        file,
-        JSON.stringify({
-          type: "video",
-          path: `assets/${path}.mp4`,
-          provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] },
-        }) + "\n",
-      );
-      return file;
-    };
-    const md =
-      "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n";
-    const judge = (path: string, x0: number) =>
-      diffVideos({
-        reference: base,
-        yours: clip(`carried-yours-${path}.mp4`, [
-          ...["-i", join(project, "assets", `${path}.mp4`), "-vf"],
-          strokes(x0),
-        ]),
-        segments: parseSegments(md),
-        ledger: ledger(path),
-        project,
-        outDir: join(dir, `carried-${path}`),
-      });
-    const dirty = await judge("dirty", 128);
+    return base;
+  };
+  const carriedBase = noTools ? "" : carriedRef();
+  const judgeCarried = (path: string, x0: number) => {
+    const ledger = join(carried, `${path}.jsonl`);
+    writeFileSync(
+      ledger,
+      JSON.stringify({
+        type: "video",
+        path: `assets/${path}.mp4`,
+        provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] },
+      }) + "\n",
+    );
+    return diffVerdicts({
+      reference: carriedBase,
+      yours: clip(`carried-yours-${path}.mp4`, [
+        ...["-i", join(carried, "assets", `${path}.mp4`), "-vf"],
+        strokes(x0),
+      ]),
+      segments: parseSegments(
+        "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n",
+      ),
+      ledger,
+      project: carried,
+      outDir: join(dir, `carried-${path}`),
+    });
+  };
+
+  it("fails a re-shot person whose generated clip carries the original's text", async () => {
+    const dirty = await judgeCarried("dirty", 128);
     expect(dirty.segments?.[0]?.person?.text?.[0]).toMatchObject({
       drawn: "S01.D1",
       carried: true,
@@ -228,7 +239,10 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
     expect(dirty.segments?.[0]?.pass).toBe(false);
     expect(dirty.person).toMatchObject({ gates: { text: false }, failed: { text: ["S01"] } });
     expect(dirty.pass).toBe(false);
-    const clean = await judge("clean", 120);
+  });
+
+  it("passes a re-shot person whose generated clip came from a masked guide", async () => {
+    const clean = await judgeCarried("clean", 120);
     expect(clean.segments?.[0]?.drawn?.[0]?.pass).toBe(true);
     expect(clean.segments?.[0]?.person?.gates.text).toBe(true);
     expect(clean.segments?.[0]?.pass).toBe(true);
@@ -242,7 +256,7 @@ describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => {
   it("judges a regenerated segment by motion and look, not pixels", async () => {
     const shifted = clip("shifted.mp4", [...["-i", ref, "-vf"], "hflip"]);
     const segments = parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n");
-    const r = await diffVideos({
+    const r = await diffVerdicts({
       reference: ref,
       yours: shifted,
       segments,

packages/cli/src/measure/diff-segments.test.ts

@@ -1,12 +1,18 @@
 import { join } from "node:path";
 import { describe, expect, it } from "vitest";
-import { diffVideos } from "./diff.js";
-import { clipFolder, noTools, REF_ARGS } from "./diff.test-helpers.js";
+import {
+  clipFolder,
+  diffVerdicts,
+  loadOpenCvFirst,
+  noTools,
+  REF_ARGS,
+} from "./diff.test-helpers.js";
 import { parseSegments } from "./segments.js";
 
 const { dir, clip } = clipFolder("hf-diff-seg-test-");
 
 describe.skipIf(noTools)("diffVideos with segments", () => {
+  loadOpenCvFirst();
   const ref = noTools ? "" : clip("ref.mp4", REF_ARGS);
 
   it("judges a rebuilt segment's regenerated region by motion and look, the rest by SSIM", async () => {
@@ -16,14 +22,14 @@ describe.skipIf(noTools)("diffVideos with segments", () => {
       "[0]split[a][b];[b]crop=320:90:0:90,hflip[f];[a][f]overlay=0:90",
     ]);
     const md = "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\n";
-    const plain = await diffVideos({
+    const plain = await diffVerdicts({
       reference: ref,
       yours: mixed,
       segments: parseSegments(md),
       outDir: join(dir, "mixed-plain"),
     });
     expect(plain.segments?.[0]?.pass).toBe(false);
-    const r = await diffVideos({
+    const r = await diffVerdicts({
       reference: ref,
       yours: mixed,
       segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`),
@@ -39,7 +45,7 @@ describe.skipIf(noTools)("diffVideos with segments", () => {
       ...["-i", mixed, "-vf"],
       "drawbox=x=20:y=10:w=60:h=40:color=red:t=fill",
     ]);
-    const bad = await diffVideos({
+    const bad = await diffVerdicts({
       reference: ref,
       yours: boxed,
       segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`),
@@ -62,7 +68,12 @@ describe.skipIf(noTools)("diffVideos with segments", () => {
   const md =
     "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\n";
   const judge = (yours: string, text: string, name: string) =>
-    diffVideos({ reference: base, yours, segments: parseSegments(text), outDir: join(dir, name) });
+    diffVerdicts({
+      reference: base,
+      yours,
+      segments: parseSegments(text),
+      outDir: join(dir, name),
+    });
 
   it("hides a drawn element in a regenerated region without a Drawn line, and flags a box over the whole frame", async () => {
     expect((await judge(off, md, "drawn-hidden")).segments?.[0]?.pass).toBe(true);
@@ -95,7 +106,7 @@ describe.skipIf(noTools)("diffVideos with segments", () => {
     const right = clip("solo-right.mp4", [...backdrop("0x604030"), "-vf", strokes(6)]);
     const thin = clip("solo-thin.mp4", [...backdrop("0x604030"), "-vf", strokes(3)]);
     const box: [number, number, number, number] = [116, 106, 44, 32];
-    const good = await diffVideos({
+    const good = await diffVerdicts({
       reference: base,
       yours: right,
       at: [0.5, 1, 1.5, 2],
@@ -106,7 +117,7 @@ describe.skipIf(noTools)("diffVideos with segments", () => {
     expect(good.drawn).toHaveLength(2);
     expect(good.drawn?.[0]).toMatchObject({ id: "D1", box, window: [0, 3], frames: 4, pass: true });
     expect(good.drawn?.[1]).toMatchObject({ id: "D2", window: [1, 3], frames: 3, pass: true });
-    const bad = await diffVideos({
+    const bad = await diffVerdicts({
       reference: base,
       yours: thin,
       at: [0.5, 1, 1.5, 2],

packages/cli/src/measure/diff.test-helpers.ts

@@ -1,9 +1,11 @@
 import { execFileSync } from "node:child_process";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterAll } from "vitest";
+import { cpSync, existsSync, mkdtempSync, renameSync, rmSync } from "node:fs";
+import { tmpdir, userInfo } from "node:os";
+import { basename, join } from "node:path";
+import { afterAll, beforeAll } from "vitest";
 import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
+import { diffVideos, type DiffOptions } from "./diff.js";
+import { loadOpenCv, OPENCV_DIR } from "./opencv.js";
 
 export const ffmpeg = findFFmpeg();
 export const noTools = !ffmpeg || !findFFprobe();
@@ -29,3 +31,33 @@ export function clipFolder(prefix: string): {
   };
   return { dir, clip };
 }
+
+/** diffVideos for a test that reads verdicts only: no worst-moment pictures, each of which costs seven ffmpeg runs. */
+export const diffVerdicts = (options: DiffOptions) => diffVideos({ worst: 0, ...options });
+
+/** opencv.js loaded before a block's tests, so none pays for its download inside the per-test limit. The files come
+ *  from this machine's cache, else a temp copy kept between runs (each has its own home), else the network. */
+export function loadOpenCvFirst(): void {
+  beforeAll(async () => {
+    const shared = join(tmpdir(), `hf-test-opencv-${basename(OPENCV_DIR)}`);
+    const machine = join(
+      userInfo().homedir,
+      ".cache",
+      "hyperframes",
+      "opencv",
+      basename(OPENCV_DIR),
+    );
+    for (const from of [machine, shared])
+      if (!existsSync(join(OPENCV_DIR, "opencv.js")) && existsSync(join(from, "opencv.js")))
+        cpSync(from, OPENCV_DIR, { recursive: true });
+    await loadOpenCv();
+    if (existsSync(join(shared, "opencv.js"))) return;
+    const staged = `${shared}-${process.pid}`;
+    cpSync(OPENCV_DIR, staged, { recursive: true });
+    try {
+      renameSync(staged, shared);
+    } catch {
+      rmSync(staged, { recursive: true, force: true });
+    }
+  }, 90_000);
+}

packages/cli/src/measure/diff.test.ts

@@ -10,7 +10,13 @@ import {
   selectFilter,
   wholeFrameCuts,
 } from "./diff.js";
-import { clipFolder, noTools, REF_ARGS } from "./diff.test-helpers.js";
+import {
+  clipFolder,
+  diffVerdicts,
+  loadOpenCvFirst,
+  noTools,
+  REF_ARGS,
+} from "./diff.test-helpers.js";
 import { carriedFrame, shotFraming, syncPass } from "./person.js";
 import { parseSegments } from "./segments.js";
 
@@ -190,6 +196,7 @@ describe("frame picks", () => {
 });
 
 describe.skipIf(noTools)("diffVideos", () => {
+  loadOpenCvFirst();
   const ref = noTools ? "" : clip("ref.mp4", REF_ARGS);
 
   it("passes a video against itself", async () => {
@@ -245,15 +252,15 @@ describe.skipIf(noTools)("diffVideos", () => {
       ...["-f", "lavfi", "-i", "color=c=white:s=320x180:r=30:d=3", "-vf"],
       `${label}:enable='lt(t,1.5)'`,
     ]);
-    const r = await diffVideos({ reference: page, yours, outDir: join(dir, "emptied") });
+    const r = await diffVerdicts({ reference: page, yours, outDir: join(dir, "emptied") });
     expect(r.missing_content.pass).toBe(false);
     expect(r.missing_content.share).toBeGreaterThan(0.45);
     expect(r.missing_content.spans[0]![0]).toBeCloseTo(1.5, 1);
     expect(r.pass).toBe(false);
-    const same = await diffVideos({ reference: page, yours: page, outDir: join(dir, "page") });
+    const same = await diffVerdicts({ reference: page, yours: page, outDir: join(dir, "page") });
     expect(same.missing_content).toMatchObject({ share: 0, pass: true });
     // A regenerated segment's picture is its own: what it leaves out is not missing.
-    const regen = await diffVideos({
+    const regen = await diffVerdicts({
       reference: page,
       yours,
       segments: parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n"),
@@ -274,7 +281,7 @@ describe.skipIf(noTools)("diffVideos", () => {
       ...["-f", "lavfi", "-i", "color=c=white:s=320x180:r=30:d=2", "-vf"],
       "format=gray,geq=lum=235,format=yuv420p",
     ]);
-    const r = await diffVideos({ reference: grain, yours: plain, outDir: join(dir, "grain") });
+    const r = await diffVerdicts({ reference: grain, yours: plain, outDir: join(dir, "grain") });
     expect(r.missing_content).toMatchObject({ share: 0, pass: true });
     expect(r.missing_content.texture.share).toBeGreaterThan(0.9);
     expect(r.warnings.some((w) => w.includes("reported, not failed"))).toBe(true);
@@ -285,7 +292,7 @@ describe.skipIf(noTools)("diffVideos", () => {
       ...["-i", ref, "-vf"],
       "drawbox=x=0:y=0:w=320:h=180:color=black:t=fill:enable='gte(t,2)'",
     ]);
-    const r = await diffVideos({
+    const r = await diffVerdicts({
       reference: ref,
       yours,
       range: [0.5, 1.5],
@@ -303,7 +310,7 @@ describe.skipIf(noTools)("diffVideos", () => {
         ...["-f", "lavfi", "-i", `color=c=0xe0c040:s=320x180:r=30:d=${3 - at}`],
         ...["-filter_complex", "[0][1]concat=n=2:v=1[v]", "-map", "[v]"],
       ]);
-    const r = await diffVideos({
+    const r = await diffVerdicts({
       reference: two("cut-ref.mp4", 1),
       yours: two("cut-late.mp4", 1.1),
       outDir: join(dir, "cuts"),
@@ -323,13 +330,13 @@ describe.skipIf(noTools)("diffVideos", () => {
         ...["-filter_complex", "[0][1]concat=n=2:v=1[v]", "-map", "[v]"],
       ]);
     const soft = step("soft-ref.mp4", "0x8b8b8b", 1);
-    const r = await diffVideos({
+    const r = await diffVerdicts({
       reference: soft,
       yours: step("soft-hard.mp4", "0xa0a0a0", 1),
       outDir: join(dir, "soft"),
     });
     expect(r.cuts).toMatchObject({ reference: [1], missing: [], extra: [], pass: true });
-    const late = await diffVideos({
+    const late = await diffVerdicts({
       reference: soft,
       yours: step("soft-late.mp4", "0xa0a0a0", 1.2),
       outDir: join(dir, "soft-late"),