fix(cli): one cut detector that tells real cuts from flashes and fast moves (#6894)

* fix(cli): one cut detector that tells real cuts from flashes and fast moves

* fix(cli): a flash must come back within a few frames; trim comments, format

* test(cli): the cut fixture gets a 60 s timeout; format

* test(cli): drop a history comment

packages/cli/src/commands/measure/cuts.ts

@@ -1,7 +1,7 @@
 import { defineCommand } from "../../utils/defineCommand.js";
 import type { Example } from "../_examples.js";
 import { jsonArg, runMeasure, videoArg, window } from "../../measure/command.js";
-import { findCuts, signatures } from "../../measure/cuts.js";
+import { classifiedCuts, signatures } from "../../measure/cuts.js";
 import { probeVideo, round, videoClip } from "../../measure/media.js";
 
 export const examples: Example[] = [
@@ -13,7 +13,7 @@ export default defineCommand({
   meta: {
     name: "cuts",
     description:
-      "Hard cuts from the frame-to-frame difference (scene scores miss some), and the shots between them",
+      "Hard cuts from the frame-to-frame difference (scene scores miss some), and the shots between them; light flashes, whips and fast moves are listed apart as not_cuts",
   },
   args: {
     video: videoArg,
@@ -26,8 +26,12 @@ export default defineCommand({
       const info = probeVideo(String(args.video));
       const [start, end] = window(info, args.start, args.end);
       const clip = videoClip(info.path, start, end - start, info);
-      const found = findCuts(signatures(clip.input, clip.filters).frames);
+      const all = classifiedCuts(signatures(clip.input, clip.filters).frames);
+      const found = all.filter((c) => !c.kind);
       const f0 = Math.ceil(start * info.fps - 1e-6);
+      const not_cuts = all.flatMap((c) =>
+        c.kind ? [{ t: round((f0 + c.frame) / info.fps, 3), kind: c.kind }] : [],
+      );
       const cuts = found.map((c) => ({
         t: round((f0 + c.frame) / info.fps, 3),
         frame: f0 + c.frame,
@@ -41,6 +45,7 @@ export default defineCommand({
         end: round(end, 3),
         fps: round(info.fps, 3),
         cuts,
+        not_cuts,
         shots: edges.slice(1).map((e, i) => ({
           start: round(edges[i]!, 3),
           end: round(e, 3),

packages/cli/src/commands/media-use/generate.ts

@@ -75,6 +75,11 @@ export default defineCommand({
       type: "string",
       description: "video: a clip whose camera and body motion the shot follows (up to 15 s)",
     },
+    "allow-cuts": {
+      type: "boolean",
+      description: "video: send a --motion clip that reads as several shots (one you have checked)",
+      default: false,
+    },
     mask: {
       type: "string",
       description:
@@ -127,6 +132,7 @@ export default defineCommand({
         firstFrame: args["first-frame"],
         audio: args.audio,
         motion: args.motion,
+        allowCuts: args["allow-cuts"],
         masks: repeated(rawArgs, "--mask").map(parseMask),
         photo: args.photo,
         script: args.script,

packages/cli/src/heygen/generate.test.ts

@@ -14,7 +14,8 @@ vi.mock("../cloud/auth.js", () => ({
 
 vi.stubEnv("HEYGEN_API_KEY", "k");
 
-const { checkRequest, generate, parseMask, structureRatio, urlOf } = await import("./generate.js");
+const { checkRequest, foldShort, generate, parseMask, structureRatio, urlOf } =
+  await import("./generate.js");
 const { checkLimit, creditsOf, priceOf } = await import("./pricing.js");
 const ffmpeg = findFFmpeg();
 const project = mkdtempSync(join(tmpdir(), "hf-generate-test-"));
@@ -335,7 +336,7 @@ describe("generate", () => {
     };
     await expect(generate(request)).rejects.toMatchObject({
       message: `--motion ${guide} has a cut at 1.00 s`,
-      fix: "guide has a cut at 1.00 s: generate 0–1.00 s and 1.00–2.00 s as two shots, each with its own --motion clip",
+      fix: "guide has a cut at 1.00 s: generate 0–1.00 s and 1.00–2.00 s as two shots, each with its own --motion clip, or pass --allow-cuts for a guide you have checked",
     });
     expect(fetched).not.toHaveBeenCalled();
     expect(existsSync(request.out)).toBe(false);
@@ -515,3 +516,12 @@ describe("generate", () => {
     expect(() => checkLimit(null, undefined)).not.toThrow();
   });
 });
+
+describe("foldShort", () => {
+  it("folds a piece under half a second into its neighbour", () => {
+    expect(foldShort([4, 4.2, 8], 10)).toEqual([4.2, 8]);
+    expect(foldShort([0.2, 5], 10)).toEqual([5]);
+    expect(foldShort([5, 9.8], 10)).toEqual([5]);
+    expect(foldShort([2, 2.1, 2.2, 6], 10)).toEqual([2.2, 6]);
+  });
+});

packages/cli/src/heygen/generate.ts

@@ -35,6 +35,8 @@ export interface GenerateRequest {
   audio?: string;
   /** video: a clip whose camera and body motion the shot follows (the original's same stretch, up to 15 s). */
   motion?: string;
+  /** video: skip the one-shot check of the motion clip (a guide a person has looked at). */
+  allowCuts?: boolean;
   /** video: regions of the motion clip blanked before it is sent (its drawn text), so the shot does not copy them. */
   masks?: Mask[];
   /** video: the shot starts from this picture. */
@@ -391,15 +393,29 @@ export function fitAspect(out: string, aspect: string): string | undefined {
   return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`;
 }
 
+const MIN_SHOT_S = 0.5;
+
+export function foldShort(cuts: number[], length: number): number[] {
+  const kept = [...cuts];
+  for (;;) {
+    const edges = [0, ...kept, length];
+    const short = edges.slice(1).findIndex((e, i) => e - edges[i]! < MIN_SHOT_S);
+    if (short < 0 || !kept.length) return kept;
+    kept.splice(short ? short - 1 : 0, 1);
+  }
+}
+
 const COUNT = ["", "one", "two", "three", "four", "five", "six"];
 
 /**
  * Refuses a --motion clip holding more than one shot before anything is sent: HeyGen follows the guide's cut into
  * a cut of its own, so each shot is its own generation.
  */
-function checkGuide(r: Pick<GenerateRequest, "motion">): void {
-  if (!r.motion) return;
-  const { cuts, length } = cutsInside(r.motion, GUIDE_MAX_S);
+function checkGuide(r: Pick<GenerateRequest, "motion" | "allowCuts">): void {
+  if (!r.motion || r.allowCuts) return;
+  const found = cutsInside(r.motion, GUIDE_MAX_S);
+  const { length } = found;
+  const cuts = foldShort(found.cuts, length);
   if (!cuts.length) return;
   const s = (t: number) => t.toFixed(2);
   const and = (items: string[]) => `${items.slice(0, -1).join(", ")} and ${items.at(-1)}`;
@@ -409,7 +425,7 @@ function checkGuide(r: Pick<GenerateRequest, "motion">): void {
   const shots = edges.slice(1).map((e, i) => `${i ? s(edges[i]!) : "0"}–${s(e)} s`);
   throw new Fix(
     `--motion ${r.motion} has ${at}`,
-    `guide has ${at}: generate ${and(shots)} as ${COUNT[shots.length] ?? shots.length} shots, each with its own --motion clip`,
+    `guide has ${at}: generate ${and(shots)} as ${COUNT[shots.length] ?? shots.length} shots, each with its own --motion clip, or pass --allow-cuts for a guide you have checked`,
   );
 }
 

packages/cli/src/measure/cuts.test.ts

@@ -1,5 +1,9 @@
 import { describe, expect, it } from "vitest";
-import { confirmsCut, cutsIn, findCuts, SIG_H, SIG_W } from "./cuts.js";
+import { execFileSync, spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { classifiedCuts, confirmsCut, cutsIn, findCuts, SIG_H, SIG_W, signatures } from "./cuts.js";
 
 /** Frame changes: `base` everywhere, with the given changes at their frames. */
 const changes = (n: number, base: number, at: Record<number, number>) =>
@@ -50,3 +54,67 @@ describe("findCuts", () => {
     expect(findCuts(frames)).toEqual([{ frame: 20, frames: 1, change: 40 }]);
   });
 });
+
+describe("findCuts on moving pictures", () => {
+  const frame = (v: number) => new Uint8Array(SIG_W * SIG_H).fill(v);
+  const still = (v: number, n: number) => Array.from({ length: n }, () => frame(v));
+
+  it("leaves a fast move drawn on twos alone: every other frame changes by 20", () => {
+    expect(cutsIn(Array.from({ length: 60 }, (_, i) => (i % 2 ? 0 : 20)))).toEqual([]);
+  });
+
+  it("still finds a cut in an animation drawn on twos", () => {
+    const d: number[] = Array.from({ length: 60 }, (_, i) => (i % 2 ? 0 : 6));
+    d[30] = 60;
+    expect(cutsIn(d)).toEqual([{ frame: 30, frames: 1, change: 60 }]);
+  });
+
+  it("keeps a quick insert shot as two cuts, not two flashes", () => {
+    const frames = [...still(100, 20), ...still(30, 8), ...still(100, 20)];
+    expect(findCuts(frames).map((c) => c.frame)).toEqual([20, 28]);
+  });
+
+  it("calls a light flash a flash and the cut after it a cut", () => {
+    const frames = [...still(100, 20), ...still(250, 2), ...still(100, 20), ...still(30, 20)];
+    expect(classifiedCuts(frames).map((c) => [c.frame, c.kind])).toEqual([
+      [20, "flash"],
+      [22, "flash"],
+      [42, undefined],
+    ]);
+    expect(findCuts(frames).map((c) => c.frame)).toEqual([42]);
+  });
+});
+
+const ffmpeg = spawnSync("ffmpeg", ["-version"]).status === 0;
+
+describe.skipIf(!ffmpeg)("a clip with a pan, a flash, a dissolve and one hard cut", () => {
+  it("has exactly one cut", () => {
+    const dir = mkdtempSync(join(tmpdir(), "hf-cuts-"));
+    try {
+      const out = join(dir, "clip.mp4");
+      const src = (kind: string, secs: number) => `${kind}=size=160x90:rate=24:duration=${secs}`;
+      // Shot one: a fast pan with a white flash at 2 s. Shot two: a dissolve from one picture to another.
+      execFileSync("ffmpeg", [
+        ...["-v", "error", "-y"],
+        ...["-f", "lavfi", "-i", src("testsrc2", 4)],
+        ...["-f", "lavfi", "-i", src("smptebars", 2)],
+        ...["-f", "lavfi", "-i", src("rgbtestsrc", 2)],
+        ...[
+          "-filter_complex",
+          [
+            "[0]scroll=horizontal=0.015,eq=brightness='if(between(t,2,2.1),1,0)':eval=frame,format=yuv420p[a]",
+            "[1]format=yuv420p[b]",
+            "[2]format=yuv420p[c]",
+            "[b][c]xfade=transition=fade:duration=1:offset=1[bc]",
+            "[a][bc]concat=n=2:v=1[v]",
+          ].join(";"),
+        ],
+        ...["-map", "[v]", "-c:v", "libx264", "-crf", "12", out],
+      ]);
+      const frames = signatures(["-i", out]).frames;
+      expect(findCuts(frames).map((c) => c.frame)).toEqual([96]);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  }, 60_000);
+});

packages/cli/src/measure/cuts.ts

@@ -12,6 +12,11 @@ const SOFT_ALONE = 4;
 const CONFIRM_LEVEL = 6;
 const CONFIRM_SHARE = 0.5;
 const NEIGHBOURS = 12;
+const HELD_CHANGE = 0.5;
+const MIN_MOVING_NEIGHBOURS = 4;
+const MAX_CUT_FRAMES = 3;
+const FLASH_FRAMES = 4;
+const RETURNED = 0.4;
 
 export interface Cut {
   frame: number;
@@ -20,6 +25,8 @@ export interface Cut {
   /** Its largest change, in mean grey levels. */
   change: number;
   soft?: true;
+  /** Not a new shot: a light flash (the picture comes back) or a whip, a fast move or a dissolve. Absent for a cut. */
+  kind?: "flash" | "fast";
 }
 
 /** Every frame of ffmpeg's `input` (after `filters`) as SIG_W x SIG_H grey levels. */
@@ -36,14 +43,20 @@ export function frameChanges(frames: Uint8Array[]): number[] {
   return frames.map((f, i) => (i ? meanAbsDiff(frames[i - 1]!, f) : 0));
 }
 
+const typicalOf = (around: number[]) =>
+  around.length >= MIN_MOVING_NEIGHBOURS ? Math.max(1, median(around)) : 1;
+
 const neighbours = (diffs: number[], i: number) =>
-  diffs.slice(Math.max(1, i - NEIGHBOURS), i).concat(diffs.slice(i + 1, i + NEIGHBOURS + 1));
+  diffs
+    .slice(Math.max(1, i - NEIGHBOURS), i)
+    .concat(diffs.slice(i + 1, i + NEIGHBOURS + 1))
+    .filter((d) => d >= HELD_CHANGE);
 
 export function cutsIn(diffs: number[]): Cut[] {
   const hits = diffs.flatMap((d, i): { frame: number; change: number; soft?: true }[] => {
     if (i === 0) return [];
     const around = neighbours(diffs, i);
-    const typical = Math.max(1, median(around));
+    const typical = typicalOf(around);
     if (d >= CUT_LEVEL && d >= CUT_RATIO * typical) return [{ frame: i, change: d }];
     const alone = d >= SOFT_ALONE * Math.max(0, ...around);
     return d >= SOFT_LEVEL && d >= SOFT_RATIO * typical && alone
@@ -62,9 +75,23 @@ export function cutsIn(diffs: number[]): Cut[] {
   return out;
 }
 
+/** Every change that reads as a cut, each marked a flash or a fast move when it is not a new shot. */
+export function classifiedCuts(frames: Uint8Array[]): Cut[] {
+  return cutsIn(frameChanges(frames)).map((c) => {
+    const end = c.frame + c.frames;
+    const back = (a: number, b: number) =>
+      frames[a] && frames[b] ? meanAbsDiff(frames[a]!, frames[b]!) < RETURNED * c.change : false;
+    if (c.frames > MAX_CUT_FRAMES) return { ...c, kind: "fast" as const };
+    const returns = Array.from({ length: FLASH_FRAMES }, (_, k) => k + 1).some(
+      (k) => back(c.frame - 1, end - 1 + k) || back(c.frame - k, end),
+    );
+    return returns ? { ...c, kind: "flash" as const } : c;
+  });
+}
+
 /** The frame indexes where a new shot starts, each with its change and length. */
 export function findCuts(frames: Uint8Array[]): Cut[] {
-  return cutsIn(frameChanges(frames));
+  return classifiedCuts(frames).filter((c) => !c.kind);
 }
 
 /**
@@ -77,7 +104,7 @@ export function confirmsCut(diffs: number[], cut: Cut): boolean {
     if (i < 1 || d === undefined) return false;
     return (
       d >= Math.max(CONFIRM_LEVEL, CONFIRM_SHARE * cut.change) &&
-      d >= CUT_RATIO * Math.max(1, median(neighbours(diffs, i)))
+      d >= CUT_RATIO * typicalOf(neighbours(diffs, i))
     );
   });
 }

packages/cli/src/measure/diff.ts

@@ -15,7 +15,7 @@ import {
 } from "node:fs";
 import { tmpdir } from "node:os";
 import { basename, dirname, join, resolve } from "node:path";
-import { confirmsCut, cutsIn, frameChanges, signatures } from "./cuts.js";
+import { classifiedCuts, confirmsCut, frameChanges, signatures } from "./cuts.js";
 import {
   drawnFrames,
   drawnVerdict,
@@ -173,6 +173,7 @@ export interface CutReport {
   /** The original's cuts with no cut within a frame of them in yours. */
   missing: number[];
   extra: number[];
+  not_cuts: { t: number; kind: "flash" | "fast" }[];
   pass: boolean;
 }
 
@@ -458,8 +459,10 @@ function outline(px: Uint8Array, w: number, [x, y, bw, bh]: number[], t = 3) {
 }
 
 function cutReport(ref: Clip, yours: Clip, k0: number, fps: number): CutReport {
-  const [da, db] = [ref, yours].map((c) => frameChanges(signatures(c.input, c.filters).frames));
-  const [a, b] = [cutsIn(da!), cutsIn(db!)];
+  const [fa, fb] = [ref, yours].map((c) => signatures(c.input, c.filters).frames);
+  const [da, db] = [frameChanges(fa!), frameChanges(fb!)];
+  const [all, b] = [classifiedCuts(fa!), classifiedCuts(fb!).filter((c) => !c.kind)];
+  const a = all.filter((c) => !c.kind);
   const used = new Set<number>();
   const missing: number[] = [];
   for (const cut of a) {
@@ -475,6 +478,7 @@ function cutReport(ref: Clip, yours: Clip, k0: number, fps: number): CutReport {
     yours: b.map((c) => t(c.frame)),
     missing: missing.map(t),
     extra: extra.map((c) => t(c.frame)),
+    not_cuts: all.flatMap((c) => (c.kind ? [{ t: t(c.frame), kind: c.kind }] : [])),
     pass: missing.length === 0 && extra.length === 0,
   };
 }