From 5259097261a33c8c2aac8bcfa8c2a1b0eae5540f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 31 Oct 2026 21:40:38 +0000 Subject: [PATCH] fix(cli): one cut detector that tells real cuts from flashes and fast moves (#6894) * fix(cli): one cut detector that tells real cuts from flashes and fast moves * fix(cli): a flash must come back within a few frames; trim comments, format * test(cli): the cut fixture gets a 60 s timeout; format * test(cli): drop a history comment --- packages/cli/src/commands/measure/cuts.ts | 11 ++- .../cli/src/commands/media-use/generate.ts | 6 ++ packages/cli/src/heygen/generate.test.ts | 14 +++- packages/cli/src/heygen/generate.ts | 24 +++++-- packages/cli/src/measure/cuts.test.ts | 70 ++++++++++++++++++- packages/cli/src/measure/cuts.ts | 35 ++++++++-- packages/cli/src/measure/diff.ts | 10 ++- 7 files changed, 153 insertions(+), 17 deletions(-) diff --git a/packages/cli/src/commands/measure/cuts.ts b/packages/cli/src/commands/measure/cuts.ts index 53ac6747f8..b47a3fef0d 100644 --- a/packages/cli/src/commands/measure/cuts.ts +++ b/packages/cli/src/commands/measure/cuts.ts @@ -1,7 +1,7 @@ import { defineCommand } from "../../utils/defineCommand.js"; import type { Example } from "../_examples.js"; import { jsonArg, runMeasure, videoArg, window } from "../../measure/command.js"; -import { findCuts, signatures } from "../../measure/cuts.js"; +import { classifiedCuts, signatures } from "../../measure/cuts.js"; import { probeVideo, round, videoClip } from "../../measure/media.js"; export const examples: Example[] = [ @@ -13,7 +13,7 @@ export default defineCommand({ meta: { name: "cuts", description: - "Hard cuts from the frame-to-frame difference (scene scores miss some), and the shots between them", + "Hard cuts from the frame-to-frame difference (scene scores miss some), and the shots between them; light flashes, whips and fast moves are listed apart as not_cuts", }, args: { video: videoArg, @@ -26,8 +26,12 @@ export default defineCommand({ const info = probeVideo(String(args.video)); const [start, end] = window(info, args.start, args.end); const clip = videoClip(info.path, start, end - start, info); - const found = findCuts(signatures(clip.input, clip.filters).frames); + const all = classifiedCuts(signatures(clip.input, clip.filters).frames); + const found = all.filter((c) => !c.kind); const f0 = Math.ceil(start * info.fps - 1e-6); + const not_cuts = all.flatMap((c) => + c.kind ? [{ t: round((f0 + c.frame) / info.fps, 3), kind: c.kind }] : [], + ); const cuts = found.map((c) => ({ t: round((f0 + c.frame) / info.fps, 3), frame: f0 + c.frame, @@ -41,6 +45,7 @@ export default defineCommand({ end: round(end, 3), fps: round(info.fps, 3), cuts, + not_cuts, shots: edges.slice(1).map((e, i) => ({ start: round(edges[i]!, 3), end: round(e, 3), diff --git a/packages/cli/src/commands/media-use/generate.ts b/packages/cli/src/commands/media-use/generate.ts index e5f4fd3fd0..652668ead9 100644 --- a/packages/cli/src/commands/media-use/generate.ts +++ b/packages/cli/src/commands/media-use/generate.ts @@ -75,6 +75,11 @@ export default defineCommand({ type: "string", description: "video: a clip whose camera and body motion the shot follows (up to 15 s)", }, + "allow-cuts": { + type: "boolean", + description: "video: send a --motion clip that reads as several shots (one you have checked)", + default: false, + }, mask: { type: "string", description: @@ -127,6 +132,7 @@ export default defineCommand({ firstFrame: args["first-frame"], audio: args.audio, motion: args.motion, + allowCuts: args["allow-cuts"], masks: repeated(rawArgs, "--mask").map(parseMask), photo: args.photo, script: args.script, diff --git a/packages/cli/src/heygen/generate.test.ts b/packages/cli/src/heygen/generate.test.ts index b7735f020e..034edb65ab 100644 --- a/packages/cli/src/heygen/generate.test.ts +++ b/packages/cli/src/heygen/generate.test.ts @@ -14,7 +14,8 @@ vi.mock("../cloud/auth.js", () => ({ vi.stubEnv("HEYGEN_API_KEY", "k"); -const { checkRequest, generate, parseMask, structureRatio, urlOf } = await import("./generate.js"); +const { checkRequest, foldShort, generate, parseMask, structureRatio, urlOf } = + await import("./generate.js"); const { checkLimit, creditsOf, priceOf } = await import("./pricing.js"); const ffmpeg = findFFmpeg(); const project = mkdtempSync(join(tmpdir(), "hf-generate-test-")); @@ -335,7 +336,7 @@ describe("generate", () => { }; await expect(generate(request)).rejects.toMatchObject({ message: `--motion ${guide} has a cut at 1.00 s`, - fix: "guide has a cut at 1.00 s: generate 0–1.00 s and 1.00–2.00 s as two shots, each with its own --motion clip", + fix: "guide has a cut at 1.00 s: generate 0–1.00 s and 1.00–2.00 s as two shots, each with its own --motion clip, or pass --allow-cuts for a guide you have checked", }); expect(fetched).not.toHaveBeenCalled(); expect(existsSync(request.out)).toBe(false); @@ -515,3 +516,12 @@ describe("generate", () => { expect(() => checkLimit(null, undefined)).not.toThrow(); }); }); + +describe("foldShort", () => { + it("folds a piece under half a second into its neighbour", () => { + expect(foldShort([4, 4.2, 8], 10)).toEqual([4.2, 8]); + expect(foldShort([0.2, 5], 10)).toEqual([5]); + expect(foldShort([5, 9.8], 10)).toEqual([5]); + expect(foldShort([2, 2.1, 2.2, 6], 10)).toEqual([2.2, 6]); + }); +}); diff --git a/packages/cli/src/heygen/generate.ts b/packages/cli/src/heygen/generate.ts index e957982702..8b87af506b 100644 --- a/packages/cli/src/heygen/generate.ts +++ b/packages/cli/src/heygen/generate.ts @@ -35,6 +35,8 @@ export interface GenerateRequest { audio?: string; /** video: a clip whose camera and body motion the shot follows (the original's same stretch, up to 15 s). */ motion?: string; + /** video: skip the one-shot check of the motion clip (a guide a person has looked at). */ + allowCuts?: boolean; /** video: regions of the motion clip blanked before it is sent (its drawn text), so the shot does not copy them. */ masks?: Mask[]; /** video: the shot starts from this picture. */ @@ -391,15 +393,29 @@ export function fitAspect(out: string, aspect: string): string | undefined { return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`; } +const MIN_SHOT_S = 0.5; + +export function foldShort(cuts: number[], length: number): number[] { + const kept = [...cuts]; + for (;;) { + const edges = [0, ...kept, length]; + const short = edges.slice(1).findIndex((e, i) => e - edges[i]! < MIN_SHOT_S); + if (short < 0 || !kept.length) return kept; + kept.splice(short ? short - 1 : 0, 1); + } +} + const COUNT = ["", "one", "two", "three", "four", "five", "six"]; /** * Refuses a --motion clip holding more than one shot before anything is sent: HeyGen follows the guide's cut into * a cut of its own, so each shot is its own generation. */ -function checkGuide(r: Pick): void { - if (!r.motion) return; - const { cuts, length } = cutsInside(r.motion, GUIDE_MAX_S); +function checkGuide(r: Pick): void { + if (!r.motion || r.allowCuts) return; + const found = cutsInside(r.motion, GUIDE_MAX_S); + const { length } = found; + const cuts = foldShort(found.cuts, length); if (!cuts.length) return; const s = (t: number) => t.toFixed(2); const and = (items: string[]) => `${items.slice(0, -1).join(", ")} and ${items.at(-1)}`; @@ -409,7 +425,7 @@ function checkGuide(r: Pick): void { const shots = edges.slice(1).map((e, i) => `${i ? s(edges[i]!) : "0"}–${s(e)} s`); throw new Fix( `--motion ${r.motion} has ${at}`, - `guide has ${at}: generate ${and(shots)} as ${COUNT[shots.length] ?? shots.length} shots, each with its own --motion clip`, + `guide has ${at}: generate ${and(shots)} as ${COUNT[shots.length] ?? shots.length} shots, each with its own --motion clip, or pass --allow-cuts for a guide you have checked`, ); } diff --git a/packages/cli/src/measure/cuts.test.ts b/packages/cli/src/measure/cuts.test.ts index 87102d0185..88944b84f5 100644 --- a/packages/cli/src/measure/cuts.test.ts +++ b/packages/cli/src/measure/cuts.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { confirmsCut, cutsIn, findCuts, SIG_H, SIG_W } from "./cuts.js"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { classifiedCuts, confirmsCut, cutsIn, findCuts, SIG_H, SIG_W, signatures } from "./cuts.js"; /** Frame changes: `base` everywhere, with the given changes at their frames. */ const changes = (n: number, base: number, at: Record) => @@ -50,3 +54,67 @@ describe("findCuts", () => { expect(findCuts(frames)).toEqual([{ frame: 20, frames: 1, change: 40 }]); }); }); + +describe("findCuts on moving pictures", () => { + const frame = (v: number) => new Uint8Array(SIG_W * SIG_H).fill(v); + const still = (v: number, n: number) => Array.from({ length: n }, () => frame(v)); + + it("leaves a fast move drawn on twos alone: every other frame changes by 20", () => { + expect(cutsIn(Array.from({ length: 60 }, (_, i) => (i % 2 ? 0 : 20)))).toEqual([]); + }); + + it("still finds a cut in an animation drawn on twos", () => { + const d: number[] = Array.from({ length: 60 }, (_, i) => (i % 2 ? 0 : 6)); + d[30] = 60; + expect(cutsIn(d)).toEqual([{ frame: 30, frames: 1, change: 60 }]); + }); + + it("keeps a quick insert shot as two cuts, not two flashes", () => { + const frames = [...still(100, 20), ...still(30, 8), ...still(100, 20)]; + expect(findCuts(frames).map((c) => c.frame)).toEqual([20, 28]); + }); + + it("calls a light flash a flash and the cut after it a cut", () => { + const frames = [...still(100, 20), ...still(250, 2), ...still(100, 20), ...still(30, 20)]; + expect(classifiedCuts(frames).map((c) => [c.frame, c.kind])).toEqual([ + [20, "flash"], + [22, "flash"], + [42, undefined], + ]); + expect(findCuts(frames).map((c) => c.frame)).toEqual([42]); + }); +}); + +const ffmpeg = spawnSync("ffmpeg", ["-version"]).status === 0; + +describe.skipIf(!ffmpeg)("a clip with a pan, a flash, a dissolve and one hard cut", () => { + it("has exactly one cut", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-cuts-")); + try { + const out = join(dir, "clip.mp4"); + const src = (kind: string, secs: number) => `${kind}=size=160x90:rate=24:duration=${secs}`; + // Shot one: a fast pan with a white flash at 2 s. Shot two: a dissolve from one picture to another. + execFileSync("ffmpeg", [ + ...["-v", "error", "-y"], + ...["-f", "lavfi", "-i", src("testsrc2", 4)], + ...["-f", "lavfi", "-i", src("smptebars", 2)], + ...["-f", "lavfi", "-i", src("rgbtestsrc", 2)], + ...[ + "-filter_complex", + [ + "[0]scroll=horizontal=0.015,eq=brightness='if(between(t,2,2.1),1,0)':eval=frame,format=yuv420p[a]", + "[1]format=yuv420p[b]", + "[2]format=yuv420p[c]", + "[b][c]xfade=transition=fade:duration=1:offset=1[bc]", + "[a][bc]concat=n=2:v=1[v]", + ].join(";"), + ], + ...["-map", "[v]", "-c:v", "libx264", "-crf", "12", out], + ]); + const frames = signatures(["-i", out]).frames; + expect(findCuts(frames).map((c) => c.frame)).toEqual([96]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/packages/cli/src/measure/cuts.ts b/packages/cli/src/measure/cuts.ts index 725f4bdcb4..7cfcf2088e 100644 --- a/packages/cli/src/measure/cuts.ts +++ b/packages/cli/src/measure/cuts.ts @@ -12,6 +12,11 @@ const SOFT_ALONE = 4; const CONFIRM_LEVEL = 6; const CONFIRM_SHARE = 0.5; const NEIGHBOURS = 12; +const HELD_CHANGE = 0.5; +const MIN_MOVING_NEIGHBOURS = 4; +const MAX_CUT_FRAMES = 3; +const FLASH_FRAMES = 4; +const RETURNED = 0.4; export interface Cut { frame: number; @@ -20,6 +25,8 @@ export interface Cut { /** Its largest change, in mean grey levels. */ change: number; soft?: true; + /** Not a new shot: a light flash (the picture comes back) or a whip, a fast move or a dissolve. Absent for a cut. */ + kind?: "flash" | "fast"; } /** Every frame of ffmpeg's `input` (after `filters`) as SIG_W x SIG_H grey levels. */ @@ -36,14 +43,20 @@ export function frameChanges(frames: Uint8Array[]): number[] { return frames.map((f, i) => (i ? meanAbsDiff(frames[i - 1]!, f) : 0)); } +const typicalOf = (around: number[]) => + around.length >= MIN_MOVING_NEIGHBOURS ? Math.max(1, median(around)) : 1; + const neighbours = (diffs: number[], i: number) => - diffs.slice(Math.max(1, i - NEIGHBOURS), i).concat(diffs.slice(i + 1, i + NEIGHBOURS + 1)); + diffs + .slice(Math.max(1, i - NEIGHBOURS), i) + .concat(diffs.slice(i + 1, i + NEIGHBOURS + 1)) + .filter((d) => d >= HELD_CHANGE); export function cutsIn(diffs: number[]): Cut[] { const hits = diffs.flatMap((d, i): { frame: number; change: number; soft?: true }[] => { if (i === 0) return []; const around = neighbours(diffs, i); - const typical = Math.max(1, median(around)); + const typical = typicalOf(around); if (d >= CUT_LEVEL && d >= CUT_RATIO * typical) return [{ frame: i, change: d }]; const alone = d >= SOFT_ALONE * Math.max(0, ...around); return d >= SOFT_LEVEL && d >= SOFT_RATIO * typical && alone @@ -62,9 +75,23 @@ export function cutsIn(diffs: number[]): Cut[] { return out; } +/** Every change that reads as a cut, each marked a flash or a fast move when it is not a new shot. */ +export function classifiedCuts(frames: Uint8Array[]): Cut[] { + return cutsIn(frameChanges(frames)).map((c) => { + const end = c.frame + c.frames; + const back = (a: number, b: number) => + frames[a] && frames[b] ? meanAbsDiff(frames[a]!, frames[b]!) < RETURNED * c.change : false; + if (c.frames > MAX_CUT_FRAMES) return { ...c, kind: "fast" as const }; + const returns = Array.from({ length: FLASH_FRAMES }, (_, k) => k + 1).some( + (k) => back(c.frame - 1, end - 1 + k) || back(c.frame - k, end), + ); + return returns ? { ...c, kind: "flash" as const } : c; + }); +} + /** The frame indexes where a new shot starts, each with its change and length. */ export function findCuts(frames: Uint8Array[]): Cut[] { - return cutsIn(frameChanges(frames)); + return classifiedCuts(frames).filter((c) => !c.kind); } /** @@ -77,7 +104,7 @@ export function confirmsCut(diffs: number[], cut: Cut): boolean { if (i < 1 || d === undefined) return false; return ( d >= Math.max(CONFIRM_LEVEL, CONFIRM_SHARE * cut.change) && - d >= CUT_RATIO * Math.max(1, median(neighbours(diffs, i))) + d >= CUT_RATIO * typicalOf(neighbours(diffs, i)) ); }); } diff --git a/packages/cli/src/measure/diff.ts b/packages/cli/src/measure/diff.ts index 8c7964d65a..64babf09ba 100644 --- a/packages/cli/src/measure/diff.ts +++ b/packages/cli/src/measure/diff.ts @@ -15,7 +15,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { basename, dirname, join, resolve } from "node:path"; -import { confirmsCut, cutsIn, frameChanges, signatures } from "./cuts.js"; +import { classifiedCuts, confirmsCut, frameChanges, signatures } from "./cuts.js"; import { drawnFrames, drawnVerdict, @@ -173,6 +173,7 @@ export interface CutReport { /** The original's cuts with no cut within a frame of them in yours. */ missing: number[]; extra: number[]; + not_cuts: { t: number; kind: "flash" | "fast" }[]; pass: boolean; } @@ -458,8 +459,10 @@ function outline(px: Uint8Array, w: number, [x, y, bw, bh]: number[], t = 3) { } function cutReport(ref: Clip, yours: Clip, k0: number, fps: number): CutReport { - const [da, db] = [ref, yours].map((c) => frameChanges(signatures(c.input, c.filters).frames)); - const [a, b] = [cutsIn(da!), cutsIn(db!)]; + const [fa, fb] = [ref, yours].map((c) => signatures(c.input, c.filters).frames); + const [da, db] = [frameChanges(fa!), frameChanges(fb!)]; + const [all, b] = [classifiedCuts(fa!), classifiedCuts(fb!).filter((c) => !c.kind)]; + const a = all.filter((c) => !c.kind); const used = new Set(); const missing: number[] = []; for (const cut of a) { @@ -475,6 +478,7 @@ function cutReport(ref: Clip, yours: Clip, k0: number, fps: number): CutReport { yours: b.map((c) => t(c.frame)), missing: missing.map(t), extra: extra.map((c) => t(c.frame)), + not_cuts: all.flatMap((c) => (c.kind ? [{ t: t(c.frame), kind: c.kind }] : [])), pass: missing.length === 0 && extra.length === 0, }; }