diff --git a/packages/cli/src/commands/analyze.ts b/packages/cli/src/commands/analyze.ts index abd566291c..02386d391b 100644 --- a/packages/cli/src/commands/analyze.ts +++ b/packages/cli/src/commands/analyze.ts @@ -1,15 +1,7 @@ // HeyGen's video model watches a video and drafts its timeline as JSON. Its route samples at most 2 fps and reads // one clip of at most 90 s at that rate, so the video goes in parts of up to 30 s split at cuts, each slowed until // 2 fps there is about 6 fps of the original, and every time in the answer is mapped back to the film. -import { - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - statSync, - writeFileSync, -} from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, dirname, extname, join, resolve } from "node:path"; import { defineCommand } from "../utils/defineCommand.js"; @@ -75,7 +67,9 @@ export function underInlineCap( for (const crf of [23, 30, 36]) { const out = join(dir, `${name}-${crf}.mp4`); runFfmpeg(args(crf, out)); - if (statSync(out).size <= INLINE_BYTES) return readFileSync(out); + // Read once and measure the bytes read: a stat then a read could see two different files. + const bytes = readFileSync(out); + if (bytes.length <= INLINE_BYTES) return bytes; } throw new Fix( `a ${round(seconds, 1)} s part stays over HeyGen's 16 MB inline cap`, diff --git a/packages/cli/src/measure/diff-people.test.ts b/packages/cli/src/measure/diff-people.test.ts new file mode 100644 index 0000000000..0fb0933c7f --- /dev/null +++ b/packages/cli/src/measure/diff-people.test.ts @@ -0,0 +1,255 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { diffVideos, soundOffset } from "./diff.js"; +import { clipFolder, ffmpeg, noTools, REF_ARGS } from "./diff.test-helpers.js"; +import { probeVideo } from "./media.js"; +import { parseSegments } from "./segments.js"; + +const { dir, clip } = clipFolder("hf-diff-person-test-"); + +describe.skipIf(noTools)("diffVideos: copies, sound and re-shot people", () => { + const ref = noTools ? "" : clip("ref.mp4", REF_ARGS); + + it("fails the original's whole frame played where copy is meant to be rebuilt", async () => { + const ledger = join(dir, "manifest.jsonl"); + writeFileSync( + ledger, + JSON.stringify({ + type: "video", + path: "assets/all.mp4", + provenance: { provider: "original", window: [0, 3], roi: [0, 0, 320, 180] }, + }) + "\n", + ); + const titled = '## S01 · 0–3 s · Title\nPixels: rebuilt\nCopy: "TITLE"\n'; + const copy = await diffVideos({ + reference: ref, + yours: ref, + segments: parseSegments(titled), + ledger, + outDir: join(dir, "copy"), + }); + expect(copy.whole_frame_reuse).toMatchObject({ spans: [[0, 3]], drawn: [[0, 3]], pass: false }); + expect(copy.segments?.[0]?.pass).toBe(false); + expect(copy.pass).toBe(false); + // Footage reused with its burned-in title is a copy too (Miao, 2026-10-23: no whole-frame cut with drawn things). + const burned = await diffVideos({ + reference: ref, + yours: ref, + segments: parseSegments( + '## S01 · 0–3 s · Title over footage\nPixels: reused\nCopy: "TITLE"\n', + ), + ledger, + outDir: join(dir, "burned"), + }); + expect(burned.whole_frame_reuse).toMatchObject({ share: 1, drawn: [[0, 3]], pass: false }); + // Footage with nothing drawn on it may be cut whole. + const footage = await diffVideos({ + reference: ref, + yours: ref, + segments: parseSegments("## S01 · 0–3 s · Footage\nPixels: reused\n"), + ledger, + outDir: join(dir, "footage"), + }); + expect(footage.whole_frame_reuse).toMatchObject({ share: 1, drawn: [], pass: true }); + expect(footage.pass).toBe(true); + // A cut that only fed a generation (a motion guide) is not played: yours differs from it. + const flipped = clip("guided.mp4", [...["-i", ref, "-vf"], "hflip"]); + const guided = await diffVideos({ + reference: ref, + yours: flipped, + segments: parseSegments(titled), + ledger, + outDir: join(dir, "guided"), + }); + expect(guided.whole_frame_reuse).toMatchObject({ spans: [], pass: true }); + // A generated shot that starts from the original's first frame matches it for a few frames only: not reuse. + const started = clip("started.mp4", [ + ...["-i", ref, "-filter_complex"], + "[0]split[a][b];[b]hflip[f];[a][f]overlay=enable='gte(t\\,0.2)'", + ]); + const start = await diffVideos({ + reference: ref, + yours: started, + segments: parseSegments(titled), + ledger, + outDir: join(dir, "started"), + }); + expect(start.whole_frame_reuse).toMatchObject({ spans: [], pass: true }); + }); + + it("finds a whole-frame still of the original shown in yours", async () => { + const ledger = join(dir, "stills.jsonl"); + writeFileSync( + ledger, + [ + { type: "image", provenance: { provider: "original", at: 1, roi: [0, 0, 320, 180] } }, + { type: "image", provenance: { provider: "original", at: 2.5 } }, + ] + .map((e) => JSON.stringify(e)) + .join("\n"), + ); + // Yours holds the original's frame at 1 s for the whole clip. + const held = clip("held.mp4", [ + ...["-i", ref, "-vf"], + "select=eq(n\\,30),loop=loop=89:size=1,setpts=N/30/TB", + ]); + const r = await diffVideos({ + reference: ref, + yours: held, + segments: parseSegments('## S01 · 0–3 s · Collage\nPixels: rebuilt\nCopy: "TITLE"\n'), + ledger, + outDir: join(dir, "held"), + }); + expect(r.whole_frame_reuse).toMatchObject({ stills: [1], spans: [[0, 3]], pass: false }); + }); + + it("reads sound that starts late as late, and the picture's length as the duration", async () => { + // Clicks at uneven times, with the sound running 0.4 s past the picture. + const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3.4"; + const voiced = (name: string, delay: number) => + clip(name, [ + ...["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"], + ...["-itsoffset", String(delay), "-f", "lavfi", "-i", clicks], + ...["-map", "0:v", "-map", "1:a", "-c:a", "aac"], + ]); + const on = voiced("voiced.mp4", 0); + const same = await diffVideos({ reference: on, yours: on, outDir: join(dir, "voiced") }); + expect(same.duration).toMatchObject({ reference: 3, pass: true }); + expect(same.sound).toMatchObject({ offset_ms: 0, pass: true }); + const late = await diffVideos({ + reference: on, + yours: voiced("late.mp4", 0.2), + outDir: join(dir, "late"), + }); + expect(late.sound).toMatchObject({ offset_ms: 200, pass: false }); + }); + + it("places the sound by its decoded timestamps, not the start time the probe reports", () => { + // ffmpeg 5.1 drops an HE-AAC edit list's skip twice: the first sample it keeps is timestamped 0.1 s in while + // the probe still says the track starts at 0. Here a sound timestamped 0.25 s in, probed as starting at 0, + // against the same sound with 0.25 s of silence baked into its samples: in sync. + const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3"; + const picture = ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"]; + const stamped = clip("stamped.mp4", [ + ...picture, + ...["-itsoffset", "0.25", "-f", "lavfi", "-i", clicks], + ...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"], + ]); + const baked = clip("baked.mp4", [ + ...picture, + ...["-f", "lavfi", "-i", clicks, "-af", "adelay=250:all=1"], + ...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"], + ]); + const probedAtZero = { ...probeVideo(stamped), audioStart: 0 }; + expect(soundOffset(stamped, baked, probedAtZero, probeVideo(baked))).toMatchObject({ + offset_ms: 0, + pass: true, + }); + }); + + it("passes a re-shot person whose motion is off when the components are right", async () => { + // The bottom half (the person) frozen on its first frame: no motion at all. Pixels above match. + const frozen = clip("frozen.mp4", [ + ...["-i", ref, "-filter_complex"], + "[0]split[a][b];[b]crop=320:90:0:90,trim=end_frame=1,tpad=stop=89:stop_mode=clone[f];[a][f]overlay=0:90", + ]); + const r = await diffVideos({ + reference: ref, + yours: frozen, + segments: parseSegments( + "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\nRegenerated: 0, 0.5, 1, 0.5\n", + ), + outDir: join(dir, "frozen"), + }); + const [s] = r.segments!; + expect(s!.person?.energy?.mean_ratio).toBeLessThan(0.5); + expect(s!.person?.gates).toEqual({ framing: null, text: null, sync: null }); + expect(s!.pass).toBe(true); + expect(r.components).toMatchObject({ passed: 1, total: 1 }); + expect(r.person?.energy).toBeLessThan(0.5); + expect(r.pass).toBe(true); + }); + + it("fails a re-shot person whose generated clip carries the original's text", async () => { + // The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half. + const strokes = (x0: number) => + [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(","); + const label = strokes(120); + const soft = (delay: number) => + `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`; + const base = clip("carried-ref.mp4", [ + ...["-i", ref, "-filter_complex"], + `${soft(0)},${label}`, + ]); + const project = join(dir, "carried"); + mkdirSync(join(project, "assets"), { recursive: true }); + // The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the + // label over each; over the first, 8 px off, the letters show doubled. + clip("carried/assets/dirty.mp4", [...["-i", ref, "-filter_complex"], `${soft(5)},${label}`]); + clip("carried/assets/clean.mp4", [...["-i", ref, "-filter_complex"], soft(5)]); + execFileSync(ffmpeg!, [ + ...["-hide_banner", "-loglevel", "error", "-y", "-i", ref, "-filter_complex"], + `${soft(0)},trim=end_frame=1`, + ...["-frames:v", "1", join(project, "assets", "still.png")], + ]); + const ledger = (path: string) => { + const file = join(project, `${path}.jsonl`); + writeFileSync( + file, + JSON.stringify({ + type: "video", + path: `assets/${path}.mp4`, + provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] }, + }) + "\n", + ); + return file; + }; + const md = + "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n"; + const judge = (path: string, x0: number) => + diffVideos({ + reference: base, + yours: clip(`carried-yours-${path}.mp4`, [ + ...["-i", join(project, "assets", `${path}.mp4`), "-vf"], + strokes(x0), + ]), + segments: parseSegments(md), + ledger: ledger(path), + project, + outDir: join(dir, `carried-${path}`), + }); + const dirty = await judge("dirty", 128); + expect(dirty.segments?.[0]?.person?.text?.[0]).toMatchObject({ + drawn: "S01.D1", + carried: true, + }); + expect(dirty.segments?.[0]?.pass).toBe(false); + expect(dirty.person).toMatchObject({ gates: { text: false }, failed: { text: ["S01"] } }); + expect(dirty.pass).toBe(false); + const clean = await judge("clean", 120); + expect(clean.segments?.[0]?.drawn?.[0]?.pass).toBe(true); + expect(clean.segments?.[0]?.person?.gates.text).toBe(true); + expect(clean.segments?.[0]?.pass).toBe(true); + expect(clean.person?.shots_at_cuts).toMatchObject({ + at_cuts: 1, + total: 1, + cuts_without_shot: [], + }); + }); + + it("judges a regenerated segment by motion and look, not pixels", async () => { + const shifted = clip("shifted.mp4", [...["-i", ref, "-vf"], "hflip"]); + const segments = parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n"); + const r = await diffVideos({ + reference: ref, + yours: shifted, + segments, + outDir: join(dir, "regen"), + }); + expect(r.ssim.mean).toBeLessThan(0.9); + expect(r.segments?.[0]?.energy?.mean_ratio).toBeCloseTo(1, 1); + expect(r.segments?.[0]?.pass).toBe(true); + }); +}); diff --git a/packages/cli/src/measure/diff-segments.test.ts b/packages/cli/src/measure/diff-segments.test.ts new file mode 100644 index 0000000000..b2f1861d28 --- /dev/null +++ b/packages/cli/src/measure/diff-segments.test.ts @@ -0,0 +1,121 @@ +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { diffVideos } from "./diff.js"; +import { clipFolder, noTools, REF_ARGS } from "./diff.test-helpers.js"; +import { parseSegments } from "./segments.js"; + +const { dir, clip } = clipFolder("hf-diff-seg-test-"); + +describe.skipIf(noTools)("diffVideos with segments", () => { + const ref = noTools ? "" : clip("ref.mp4", REF_ARGS); + + it("judges a rebuilt segment's regenerated region by motion and look, the rest by SSIM", async () => { + // The bottom half mirrored, as a re-shot presenter differs in pixels but moves the same. + const mixed = clip("mixed.mp4", [ + ...["-i", ref, "-filter_complex"], + "[0]split[a][b];[b]crop=320:90:0:90,hflip[f];[a][f]overlay=0:90", + ]); + const md = "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\n"; + const plain = await diffVideos({ + reference: ref, + yours: mixed, + segments: parseSegments(md), + outDir: join(dir, "mixed-plain"), + }); + expect(plain.segments?.[0]?.pass).toBe(false); + const r = await diffVideos({ + reference: ref, + yours: mixed, + segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`), + outDir: join(dir, "mixed"), + }); + const [s] = r.segments!; + expect(s!.ssim!.mean).toBeGreaterThan(0.99); + expect(s!.regenerated?.roi).toEqual([0, 90, 320, 90]); + expect(s!.person?.energy?.mean_ratio).toBeCloseTo(1, 1); + expect(s!.pass).toBe(true); + // A wrong box outside the region still fails the segment. + const boxed = clip("mixed-box.mp4", [ + ...["-i", mixed, "-vf"], + "drawbox=x=20:y=10:w=60:h=40:color=red:t=fill", + ]); + const bad = await diffVideos({ + reference: ref, + yours: boxed, + segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`), + outDir: join(dir, "mixed-box"), + }); + expect(bad.segments?.[0]?.pass).toBe(false); + }); + + // A label (three white strokes) over a soft bottom half, a regenerated person (here 5 frames late); yours + // draws it 24 px off. Two tests of two comparisons each, so neither nears the per-test limit on a busy runner. + const strokes = (x0: number) => + [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(","); + const soft = (delay: number) => + `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`; + const labelled = (name: string, delay: number, x0: number) => + noTools ? "" : clip(name, [...["-i", ref, "-filter_complex"], `${soft(delay)},${strokes(x0)}`]); + const base = labelled("label-ref.mp4", 0, 120); + const right = labelled("label-right.mp4", 5, 120); + const off = labelled("label-off.mp4", 5, 144); + const md = + "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\n"; + const judge = (yours: string, text: string, name: string) => + diffVideos({ reference: base, yours, segments: parseSegments(text), outDir: join(dir, name) }); + + it("hides a drawn element in a regenerated region without a Drawn line, and flags a box over the whole frame", async () => { + expect((await judge(off, md, "drawn-hidden")).segments?.[0]?.pass).toBe(true); + const whole = await judge(right, md.replace("0, 90, 320, 90", "0, 0, 320, 180"), "drawn-whole"); + expect(whole.warnings.some((w) => w.includes("covers the whole frame"))).toBe(true); + }); + + it("judges each drawn element inside a regenerated region by itself, not by what is behind it", async () => { + const drawn = `${md}Drawn: 118, 108, 42, 28\n`; + const good = (await judge(right, drawn, "drawn-right")).segments![0]!; + expect(good.pass).toBe(true); + expect(good.drawn?.[0]).toMatchObject({ id: "S01.D1", box: [118, 108, 42, 28], pass: true }); + expect(good.scored).toContain("1 drawn elements, each scored by itself"); + const bad = await judge(off, drawn, "drawn-off"); + expect(bad.segments?.[0]?.pass).toBe(false); + expect(bad.segments?.[0]?.drawn?.[0]?.pass).toBe(false); + }); + + it("scores a drawn element by itself without segments, and its ink says when yours is thinner", async () => { + // The same three-stroke label over two different flat backdrops; yours draws it at half the stroke width. + const strokes = (w: number) => + [0, 14, 28].map((x) => `drawbox=x=${120 + x}:y=110:w=${w}:h=24:color=white:t=fill`).join(","); + const backdrop = (colour: string) => [ + "-f", + "lavfi", + "-i", + `color=c=${colour}:s=320x180:d=3:r=30`, + ]; + const base = clip("solo-ref.mp4", [...backdrop("0x305070"), "-vf", strokes(6)]); + const right = clip("solo-right.mp4", [...backdrop("0x604030"), "-vf", strokes(6)]); + const thin = clip("solo-thin.mp4", [...backdrop("0x604030"), "-vf", strokes(3)]); + const box: [number, number, number, number] = [116, 106, 44, 32]; + const good = await diffVideos({ + reference: base, + yours: right, + at: [0.5, 1, 1.5, 2], + drawn: [{ box }, { box, window: [1, 3] }], + outDir: join(dir, "solo-right"), + }); + expect(good.segments).toBeUndefined(); + expect(good.drawn).toHaveLength(2); + expect(good.drawn?.[0]).toMatchObject({ id: "D1", box, window: [0, 3], frames: 4, pass: true }); + expect(good.drawn?.[1]).toMatchObject({ id: "D2", window: [1, 3], frames: 3, pass: true }); + const bad = await diffVideos({ + reference: base, + yours: thin, + at: [0.5, 1, 1.5, 2], + drawn: [{ box }], + outDir: join(dir, "solo-thin"), + }); + expect(good.drawn?.[0]).toMatchObject({ score: 1, offset: [0, 0], ink: 1 }); + // Half-weight letters in the right place: the score barely drops, the ink says it. + expect(bad.drawn?.[0]?.score).toBeLessThan(0.9); + expect(bad.drawn?.[0]?.ink).toBe(0.5); + }); +}); diff --git a/packages/cli/src/measure/diff.test-helpers.ts b/packages/cli/src/measure/diff.test-helpers.ts new file mode 100644 index 0000000000..437c9c60a1 --- /dev/null +++ b/packages/cli/src/measure/diff.test-helpers.ts @@ -0,0 +1,31 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll } from "vitest"; +import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; + +export const ffmpeg = findFFmpeg(); +export const noTools = !ffmpeg || !findFFprobe(); + +/** testsrc2, 3 s at 30 fps: the original most of diff's tests compare against. */ +export const REF_ARGS = ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"]; + +/** A temp folder for one test file's clips, removed after its tests, and a maker of lossless synthetic clips in it, + * so SSIM differences come from the picture, not the encoder. */ +export function clipFolder(prefix: string): { + dir: string; + clip: (name: string, args: string[]) => string; +} { + const dir = mkdtempSync(join(tmpdir(), prefix)); + afterAll(() => rmSync(dir, { recursive: true, force: true })); + const clip = (name: string, args: string[]): string => { + const out = join(dir, name); + execFileSync(ffmpeg!, [ + ...["-hide_banner", "-loglevel", "error", "-y", ...args], + ...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "0", out], + ]); + return out; + }; + return { dir, clip }; +} diff --git a/packages/cli/src/measure/diff.test.ts b/packages/cli/src/measure/diff.test.ts index f25a7536c5..86efdfca44 100644 --- a/packages/cli/src/measure/diff.test.ts +++ b/packages/cli/src/measure/diff.test.ts @@ -1,9 +1,6 @@ -import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { existsSync } from "node:fs"; import { join } from "node:path"; -import { afterAll, describe, expect, it } from "vitest"; -import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; +import { describe, expect, it } from "vitest"; import { cellsOutside, diffVideos, @@ -11,27 +8,13 @@ import { pickedFrames, regionRoi, selectFilter, - soundOffset, wholeFrameCuts, } from "./diff.js"; -import { probeVideo } from "./media.js"; +import { clipFolder, noTools, REF_ARGS } from "./diff.test-helpers.js"; import { carriedFrame, shotFraming, syncPass } from "./person.js"; import { parseSegments } from "./segments.js"; -const ffmpeg = findFFmpeg(); -const noTools = !ffmpeg || !findFFprobe(); -const dir = mkdtempSync(join(tmpdir(), "hf-diff-test-")); -afterAll(() => rmSync(dir, { recursive: true, force: true })); - -/** A lossless synthetic clip, so SSIM differences come from the picture, not the encoder. */ -function clip(name: string, args: string[]): string { - const out = join(dir, name); - execFileSync(ffmpeg!, [ - ...["-hide_banner", "-loglevel", "error", "-y", ...args], - ...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "0", out], - ]); - return out; -} +const { dir, clip } = clipFolder("hf-diff-test-"); describe("parseSegments", () => { it("reads headings and where each window's pixels come from", () => { @@ -207,7 +190,7 @@ describe("frame picks", () => { }); describe.skipIf(noTools)("diffVideos", () => { - const ref = noTools ? "" : clip("ref.mp4", ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"]); + const ref = noTools ? "" : clip("ref.mp4", REF_ARGS); it("passes a video against itself", async () => { const r = await diffVideos({ reference: ref, yours: ref, outDir: join(dir, "same") }); @@ -353,363 +336,4 @@ describe.skipIf(noTools)("diffVideos", () => { }); expect(late.cuts).toMatchObject({ missing: [1], extra: [1.2], pass: false }); }); - - it("judges a rebuilt segment's regenerated region by motion and look, the rest by SSIM", async () => { - // The bottom half mirrored, as a re-shot presenter differs in pixels but moves the same. - const mixed = clip("mixed.mp4", [ - ...["-i", ref, "-filter_complex"], - "[0]split[a][b];[b]crop=320:90:0:90,hflip[f];[a][f]overlay=0:90", - ]); - const md = "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\n"; - const plain = await diffVideos({ - reference: ref, - yours: mixed, - segments: parseSegments(md), - outDir: join(dir, "mixed-plain"), - }); - expect(plain.segments?.[0]?.pass).toBe(false); - const r = await diffVideos({ - reference: ref, - yours: mixed, - segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`), - outDir: join(dir, "mixed"), - }); - const [s] = r.segments!; - expect(s!.ssim!.mean).toBeGreaterThan(0.99); - expect(s!.regenerated?.roi).toEqual([0, 90, 320, 90]); - expect(s!.person?.energy?.mean_ratio).toBeCloseTo(1, 1); - expect(s!.pass).toBe(true); - // A wrong box outside the region still fails the segment. - const boxed = clip("mixed-box.mp4", [ - ...["-i", mixed, "-vf"], - "drawbox=x=20:y=10:w=60:h=40:color=red:t=fill", - ]); - const bad = await diffVideos({ - reference: ref, - yours: boxed, - segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`), - outDir: join(dir, "mixed-box"), - }); - expect(bad.segments?.[0]?.pass).toBe(false); - }); - - it("judges each drawn element inside a regenerated region by itself, not by what is behind it", async () => { - // A label (three white strokes) over a soft bottom half, a regenerated person (here 5 frames late); yours - // draws it 24 px off. - const strokes = (x0: number) => - [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(","); - const label = strokes(120); - const soft = (delay: number) => - `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`; - const base = clip("label-ref.mp4", [...["-i", ref, "-filter_complex"], `${soft(0)},${label}`]); - const right = clip("label-right.mp4", [ - ...["-i", ref, "-filter_complex"], - `${soft(5)},${label}`, - ]); - const off = clip("label-off.mp4", [ - ...["-i", ref, "-filter_complex"], - `${soft(5)},${strokes(144)}`, - ]); - const md = - "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\n"; - const judge = (yours: string, text: string, name: string) => - diffVideos({ - reference: base, - yours, - segments: parseSegments(text), - outDir: join(dir, name), - }); - // Without a Drawn line the label hides in the person's box. - expect((await judge(off, md, "drawn-hidden")).segments?.[0]?.pass).toBe(true); - const drawn = `${md}Drawn: 118, 108, 42, 28\n`; - const good = (await judge(right, drawn, "drawn-right")).segments![0]!; - expect(good.pass).toBe(true); - expect(good.drawn?.[0]).toMatchObject({ id: "S01.D1", box: [118, 108, 42, 28], pass: true }); - expect(good.scored).toContain("1 drawn elements, each scored by itself"); - const bad = await judge(off, drawn, "drawn-off"); - expect(bad.segments?.[0]?.pass).toBe(false); - expect(bad.segments?.[0]?.drawn?.[0]?.pass).toBe(false); - // A box over the whole frame is flagged. - const whole = await judge(right, md.replace("0, 90, 320, 90", "0, 0, 320, 180"), "drawn-whole"); - expect(whole.warnings.some((w) => w.includes("covers the whole frame"))).toBe(true); - }); - - it("scores a drawn element by itself without segments, and its ink says when yours is thinner", async () => { - // The same three-stroke label over two different flat backdrops; yours draws it at half the stroke width. - const strokes = (w: number) => - [0, 14, 28].map((x) => `drawbox=x=${120 + x}:y=110:w=${w}:h=24:color=white:t=fill`).join(","); - const backdrop = (colour: string) => [ - "-f", - "lavfi", - "-i", - `color=c=${colour}:s=320x180:d=3:r=30`, - ]; - const base = clip("solo-ref.mp4", [...backdrop("0x305070"), "-vf", strokes(6)]); - const right = clip("solo-right.mp4", [...backdrop("0x604030"), "-vf", strokes(6)]); - const thin = clip("solo-thin.mp4", [...backdrop("0x604030"), "-vf", strokes(3)]); - const box: [number, number, number, number] = [116, 106, 44, 32]; - const good = await diffVideos({ - reference: base, - yours: right, - at: [0.5, 1, 1.5, 2], - drawn: [{ box }, { box, window: [1, 3] }], - outDir: join(dir, "solo-right"), - }); - expect(good.segments).toBeUndefined(); - expect(good.drawn).toHaveLength(2); - expect(good.drawn?.[0]).toMatchObject({ id: "D1", box, window: [0, 3], frames: 4, pass: true }); - expect(good.drawn?.[1]).toMatchObject({ id: "D2", window: [1, 3], frames: 3, pass: true }); - const bad = await diffVideos({ - reference: base, - yours: thin, - at: [0.5, 1, 1.5, 2], - drawn: [{ box }], - outDir: join(dir, "solo-thin"), - }); - expect(good.drawn?.[0]).toMatchObject({ score: 1, offset: [0, 0], ink: 1 }); - // Half-weight letters in the right place: the score barely drops, the ink says it. - expect(bad.drawn?.[0]?.score).toBeLessThan(0.9); - expect(bad.drawn?.[0]?.ink).toBe(0.5); - }); - - it("fails the original's whole frame played where copy is meant to be rebuilt", async () => { - const ledger = join(dir, "manifest.jsonl"); - writeFileSync( - ledger, - JSON.stringify({ - type: "video", - path: "assets/all.mp4", - provenance: { provider: "original", window: [0, 3], roi: [0, 0, 320, 180] }, - }) + "\n", - ); - const titled = '## S01 · 0–3 s · Title\nPixels: rebuilt\nCopy: "TITLE"\n'; - const copy = await diffVideos({ - reference: ref, - yours: ref, - segments: parseSegments(titled), - ledger, - outDir: join(dir, "copy"), - }); - expect(copy.whole_frame_reuse).toMatchObject({ spans: [[0, 3]], drawn: [[0, 3]], pass: false }); - expect(copy.segments?.[0]?.pass).toBe(false); - expect(copy.pass).toBe(false); - // Footage reused with its burned-in title is a copy too (Miao, 2026-10-23: no whole-frame cut with drawn things). - const burned = await diffVideos({ - reference: ref, - yours: ref, - segments: parseSegments( - '## S01 · 0–3 s · Title over footage\nPixels: reused\nCopy: "TITLE"\n', - ), - ledger, - outDir: join(dir, "burned"), - }); - expect(burned.whole_frame_reuse).toMatchObject({ share: 1, drawn: [[0, 3]], pass: false }); - // Footage with nothing drawn on it may be cut whole. - const footage = await diffVideos({ - reference: ref, - yours: ref, - segments: parseSegments("## S01 · 0–3 s · Footage\nPixels: reused\n"), - ledger, - outDir: join(dir, "footage"), - }); - expect(footage.whole_frame_reuse).toMatchObject({ share: 1, drawn: [], pass: true }); - expect(footage.pass).toBe(true); - // A cut that only fed a generation (a motion guide) is not played: yours differs from it. - const flipped = clip("guided.mp4", [...["-i", ref, "-vf"], "hflip"]); - const guided = await diffVideos({ - reference: ref, - yours: flipped, - segments: parseSegments(titled), - ledger, - outDir: join(dir, "guided"), - }); - expect(guided.whole_frame_reuse).toMatchObject({ spans: [], pass: true }); - // A generated shot that starts from the original's first frame matches it for a few frames only: not reuse. - const started = clip("started.mp4", [ - ...["-i", ref, "-filter_complex"], - "[0]split[a][b];[b]hflip[f];[a][f]overlay=enable='gte(t\\,0.2)'", - ]); - const start = await diffVideos({ - reference: ref, - yours: started, - segments: parseSegments(titled), - ledger, - outDir: join(dir, "started"), - }); - expect(start.whole_frame_reuse).toMatchObject({ spans: [], pass: true }); - }); - - it("finds a whole-frame still of the original shown in yours", async () => { - const ledger = join(dir, "stills.jsonl"); - writeFileSync( - ledger, - [ - { type: "image", provenance: { provider: "original", at: 1, roi: [0, 0, 320, 180] } }, - { type: "image", provenance: { provider: "original", at: 2.5 } }, - ] - .map((e) => JSON.stringify(e)) - .join("\n"), - ); - // Yours holds the original's frame at 1 s for the whole clip. - const held = clip("held.mp4", [ - ...["-i", ref, "-vf"], - "select=eq(n\\,30),loop=loop=89:size=1,setpts=N/30/TB", - ]); - const r = await diffVideos({ - reference: ref, - yours: held, - segments: parseSegments('## S01 · 0–3 s · Collage\nPixels: rebuilt\nCopy: "TITLE"\n'), - ledger, - outDir: join(dir, "held"), - }); - expect(r.whole_frame_reuse).toMatchObject({ stills: [1], spans: [[0, 3]], pass: false }); - }); - - it("reads sound that starts late as late, and the picture's length as the duration", async () => { - // Clicks at uneven times, with the sound running 0.4 s past the picture. - const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3.4"; - const voiced = (name: string, delay: number) => - clip(name, [ - ...["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"], - ...["-itsoffset", String(delay), "-f", "lavfi", "-i", clicks], - ...["-map", "0:v", "-map", "1:a", "-c:a", "aac"], - ]); - const on = voiced("voiced.mp4", 0); - const same = await diffVideos({ reference: on, yours: on, outDir: join(dir, "voiced") }); - expect(same.duration).toMatchObject({ reference: 3, pass: true }); - expect(same.sound).toMatchObject({ offset_ms: 0, pass: true }); - const late = await diffVideos({ - reference: on, - yours: voiced("late.mp4", 0.2), - outDir: join(dir, "late"), - }); - expect(late.sound).toMatchObject({ offset_ms: 200, pass: false }); - }); - - it("places the sound by its decoded timestamps, not the start time the probe reports", () => { - // ffmpeg 5.1 drops an HE-AAC edit list's skip twice: the first sample it keeps is timestamped 0.1 s in while - // the probe still says the track starts at 0. Here a sound timestamped 0.25 s in, probed as starting at 0, - // against the same sound with 0.25 s of silence baked into its samples: in sync. - const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3"; - const picture = ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"]; - const stamped = clip("stamped.mp4", [ - ...picture, - ...["-itsoffset", "0.25", "-f", "lavfi", "-i", clicks], - ...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"], - ]); - const baked = clip("baked.mp4", [ - ...picture, - ...["-f", "lavfi", "-i", clicks, "-af", "adelay=250:all=1"], - ...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"], - ]); - const probedAtZero = { ...probeVideo(stamped), audioStart: 0 }; - expect(soundOffset(stamped, baked, probedAtZero, probeVideo(baked))).toMatchObject({ - offset_ms: 0, - pass: true, - }); - }); - - it("passes a re-shot person whose motion is off when the components are right", async () => { - // The bottom half (the person) frozen on its first frame: no motion at all. Pixels above match. - const frozen = clip("frozen.mp4", [ - ...["-i", ref, "-filter_complex"], - "[0]split[a][b];[b]crop=320:90:0:90,trim=end_frame=1,tpad=stop=89:stop_mode=clone[f];[a][f]overlay=0:90", - ]); - const r = await diffVideos({ - reference: ref, - yours: frozen, - segments: parseSegments( - "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\nRegenerated: 0, 0.5, 1, 0.5\n", - ), - outDir: join(dir, "frozen"), - }); - const [s] = r.segments!; - expect(s!.person?.energy?.mean_ratio).toBeLessThan(0.5); - expect(s!.person?.gates).toEqual({ framing: null, text: null, sync: null }); - expect(s!.pass).toBe(true); - expect(r.components).toMatchObject({ passed: 1, total: 1 }); - expect(r.person?.energy).toBeLessThan(0.5); - expect(r.pass).toBe(true); - }); - - it("fails a re-shot person whose generated clip carries the original's text", async () => { - // The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half. - const strokes = (x0: number) => - [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(","); - const label = strokes(120); - const soft = (delay: number) => - `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`; - const base = clip("carried-ref.mp4", [ - ...["-i", ref, "-filter_complex"], - `${soft(0)},${label}`, - ]); - const project = join(dir, "carried"); - mkdirSync(join(project, "assets"), { recursive: true }); - // The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the - // label over each; over the first, 8 px off, the letters show doubled. - clip("carried/assets/dirty.mp4", [...["-i", ref, "-filter_complex"], `${soft(5)},${label}`]); - clip("carried/assets/clean.mp4", [...["-i", ref, "-filter_complex"], soft(5)]); - execFileSync(ffmpeg!, [ - ...["-hide_banner", "-loglevel", "error", "-y", "-i", ref, "-filter_complex"], - `${soft(0)},trim=end_frame=1`, - ...["-frames:v", "1", join(project, "assets", "still.png")], - ]); - const ledger = (path: string) => { - const file = join(project, `${path}.jsonl`); - writeFileSync( - file, - JSON.stringify({ - type: "video", - path: `assets/${path}.mp4`, - provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] }, - }) + "\n", - ); - return file; - }; - const md = - "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n"; - const judge = (path: string, x0: number) => - diffVideos({ - reference: base, - yours: clip(`carried-yours-${path}.mp4`, [ - ...["-i", join(project, "assets", `${path}.mp4`), "-vf"], - strokes(x0), - ]), - segments: parseSegments(md), - ledger: ledger(path), - project, - outDir: join(dir, `carried-${path}`), - }); - const dirty = await judge("dirty", 128); - expect(dirty.segments?.[0]?.person?.text?.[0]).toMatchObject({ - drawn: "S01.D1", - carried: true, - }); - expect(dirty.segments?.[0]?.pass).toBe(false); - expect(dirty.person).toMatchObject({ gates: { text: false }, failed: { text: ["S01"] } }); - expect(dirty.pass).toBe(false); - const clean = await judge("clean", 120); - expect(clean.segments?.[0]?.drawn?.[0]?.pass).toBe(true); - expect(clean.segments?.[0]?.person?.gates.text).toBe(true); - expect(clean.segments?.[0]?.pass).toBe(true); - expect(clean.person?.shots_at_cuts).toMatchObject({ - at_cuts: 1, - total: 1, - cuts_without_shot: [], - }); - }); - - it("judges a regenerated segment by motion and look, not pixels", async () => { - const shifted = clip("shifted.mp4", [...["-i", ref, "-vf"], "hflip"]); - const segments = parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n"); - const r = await diffVideos({ - reference: ref, - yours: shifted, - segments, - outDir: join(dir, "regen"), - }); - expect(r.ssim.mean).toBeLessThan(0.9); - expect(r.segments?.[0]?.energy?.mean_ratio).toBeCloseTo(1, 1); - expect(r.segments?.[0]?.pass).toBe(true); - }); });