From 170265cf241ec7fd07a8382c055e4452d15d1b66 Mon Sep 17 00:00:00 2001 From: Miao Yang Date: Fri, 23 Oct 2026 23:56:24 +0800 Subject: [PATCH] feat(cli): hyperframes measure and hyperframes diff read a video file as numbers measure has 18 subcommands (probe, cuts, events, frames, trail, extract, box, ink, curve, rhythm, objects, energy, look, track, flow, face, sync, continuity), each printing JSON. diff scores a render or a project against the original frame by frame. Ported from the Studio's viral-clone scripts and hf-media so Desktop and the CLI share one implementation. Co-Authored-By: Claude Opus 5.5 --- packages/cli/src/cli.ts | 2 + packages/cli/src/commands/diff.ts | 166 ++++ packages/cli/src/commands/measure.ts | 46 ++ packages/cli/src/commands/measure/box.ts | 75 ++ .../cli/src/commands/measure/continuity.ts | 53 ++ packages/cli/src/commands/measure/curve.ts | 80 ++ packages/cli/src/commands/measure/cuts.ts | 52 ++ packages/cli/src/commands/measure/energy.ts | 66 ++ packages/cli/src/commands/measure/events.ts | 190 +++++ packages/cli/src/commands/measure/extract.ts | 274 +++++++ packages/cli/src/commands/measure/face.ts | 50 ++ packages/cli/src/commands/measure/flow.ts | 91 +++ packages/cli/src/commands/measure/frames.ts | 338 ++++++++ packages/cli/src/commands/measure/ink.ts | 58 ++ packages/cli/src/commands/measure/look.ts | 114 +++ packages/cli/src/commands/measure/objects.ts | 38 + packages/cli/src/commands/measure/probe.ts | 129 +++ packages/cli/src/commands/measure/rhythm.ts | 64 ++ packages/cli/src/commands/measure/sync.ts | 53 ++ packages/cli/src/commands/measure/track.ts | 96 +++ packages/cli/src/commands/measure/trail.ts | 269 ++++++ packages/cli/src/commands/snapshot.ts | 11 +- packages/cli/src/help.ts | 5 + packages/cli/src/measure/box.test.ts | 49 ++ packages/cli/src/measure/box.ts | 431 ++++++++++ packages/cli/src/measure/command.ts | 91 +++ packages/cli/src/measure/continuity.test.ts | 50 ++ packages/cli/src/measure/continuity.ts | 329 ++++++++ packages/cli/src/measure/curve.test.ts | 43 + packages/cli/src/measure/curve.ts | 303 +++++++ packages/cli/src/measure/cuts.ts | 44 + packages/cli/src/measure/diff.test.ts | 171 ++++ packages/cli/src/measure/diff.ts | 767 ++++++++++++++++++ packages/cli/src/measure/energy.ts | 42 + packages/cli/src/measure/events.test.ts | 107 +++ packages/cli/src/measure/events.ts | 436 ++++++++++ packages/cli/src/measure/face.ts | 103 +++ packages/cli/src/measure/flow.ts | 285 +++++++ packages/cli/src/measure/ink.test.ts | 27 + packages/cli/src/measure/ink.ts | 73 ++ packages/cli/src/measure/look.ts | 83 ++ packages/cli/src/measure/media.ts | 330 ++++++++ packages/cli/src/measure/objects.test.ts | 54 ++ packages/cli/src/measure/objects.ts | 198 +++++ packages/cli/src/measure/opencv.test.ts | 109 +++ packages/cli/src/measure/opencv.ts | 175 ++++ packages/cli/src/measure/rhythm.test.ts | 32 + packages/cli/src/measure/rhythm.ts | 219 +++++ packages/cli/src/measure/segments.ts | 48 ++ packages/cli/src/measure/stats.ts | 14 + packages/cli/src/measure/sync.test.ts | 42 + packages/cli/src/measure/sync.ts | 130 +++ packages/cli/src/measure/track.ts | 123 +++ .../cli/src/media-use/lib/index-gen.d.mts | 1 + packages/cli/src/media-use/lib/manifest.d.mts | 21 + 55 files changed, 7247 insertions(+), 3 deletions(-) create mode 100644 packages/cli/src/commands/diff.ts create mode 100644 packages/cli/src/commands/measure.ts create mode 100644 packages/cli/src/commands/measure/box.ts create mode 100644 packages/cli/src/commands/measure/continuity.ts create mode 100644 packages/cli/src/commands/measure/curve.ts create mode 100644 packages/cli/src/commands/measure/cuts.ts create mode 100644 packages/cli/src/commands/measure/energy.ts create mode 100644 packages/cli/src/commands/measure/events.ts create mode 100644 packages/cli/src/commands/measure/extract.ts create mode 100644 packages/cli/src/commands/measure/face.ts create mode 100644 packages/cli/src/commands/measure/flow.ts create mode 100644 packages/cli/src/commands/measure/frames.ts create mode 100644 packages/cli/src/commands/measure/ink.ts create mode 100644 packages/cli/src/commands/measure/look.ts create mode 100644 packages/cli/src/commands/measure/objects.ts create mode 100644 packages/cli/src/commands/measure/probe.ts create mode 100644 packages/cli/src/commands/measure/rhythm.ts create mode 100644 packages/cli/src/commands/measure/sync.ts create mode 100644 packages/cli/src/commands/measure/track.ts create mode 100644 packages/cli/src/commands/measure/trail.ts create mode 100644 packages/cli/src/measure/box.test.ts create mode 100644 packages/cli/src/measure/box.ts create mode 100644 packages/cli/src/measure/command.ts create mode 100644 packages/cli/src/measure/continuity.test.ts create mode 100644 packages/cli/src/measure/continuity.ts create mode 100644 packages/cli/src/measure/curve.test.ts create mode 100644 packages/cli/src/measure/curve.ts create mode 100644 packages/cli/src/measure/cuts.ts create mode 100644 packages/cli/src/measure/diff.test.ts create mode 100644 packages/cli/src/measure/diff.ts create mode 100644 packages/cli/src/measure/energy.ts create mode 100644 packages/cli/src/measure/events.test.ts create mode 100644 packages/cli/src/measure/events.ts create mode 100644 packages/cli/src/measure/face.ts create mode 100644 packages/cli/src/measure/flow.ts create mode 100644 packages/cli/src/measure/ink.test.ts create mode 100644 packages/cli/src/measure/ink.ts create mode 100644 packages/cli/src/measure/look.ts create mode 100644 packages/cli/src/measure/media.ts create mode 100644 packages/cli/src/measure/objects.test.ts create mode 100644 packages/cli/src/measure/objects.ts create mode 100644 packages/cli/src/measure/opencv.test.ts create mode 100644 packages/cli/src/measure/opencv.ts create mode 100644 packages/cli/src/measure/rhythm.test.ts create mode 100644 packages/cli/src/measure/rhythm.ts create mode 100644 packages/cli/src/measure/segments.ts create mode 100644 packages/cli/src/measure/stats.ts create mode 100644 packages/cli/src/measure/sync.test.ts create mode 100644 packages/cli/src/measure/sync.ts create mode 100644 packages/cli/src/measure/track.ts create mode 100644 packages/cli/src/media-use/lib/index-gen.d.mts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index afae651083..92b4ce98b7 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -175,6 +175,8 @@ const commandLoaders = { import("./commands/media-treatment.js").then((m) => m.mediaTreatmentCommand), "grade-compare": () => import("./commands/grade-compare.js").then((m) => m.default), compare: () => import("./commands/compare.js").then((m) => m.default), + diff: () => import("./commands/diff.js").then((m) => m.default), + measure: () => import("./commands/measure.js").then((m) => m.default), capture: () => import("./commands/capture.js").then((m) => m.default), lambda: () => import("./commands/lambda.js").then((m) => m.default), cloudrun: () => import("./commands/cloudrun.js").then((m) => m.default), diff --git a/packages/cli/src/commands/diff.ts b/packages/cli/src/commands/diff.ts new file mode 100644 index 0000000000..9f42d5f882 --- /dev/null +++ b/packages/cli/src/commands/diff.ts @@ -0,0 +1,166 @@ +import { basename, extname, join, resolve } from "node:path"; +import { defineCommand } from "citty"; +import type { Example } from "./_examples.js"; +import { c } from "../ui/colors.js"; +import { failCommand, setCommandExitCode } from "../utils/commandResult.js"; +import { normalizeErrorMessage } from "../utils/errorMessage.js"; +import { withMeta } from "../utils/updateCheck.js"; +import { + DEFAULT_MIN_CELL, + DEFAULT_MIN_MEAN, + DEFAULT_MIN_P05, + diffVideos, + type DiffResult, +} from "../measure/diff.js"; +import { Fix, formatJson, outDir, parseNumbers, parseRange } from "../measure/media.js"; +import { readSegments } from "../measure/segments.js"; + +export const examples: Example[] = [ + [ + "Score a render against the original, frame by frame", + "hyperframes diff .reverse/source.mp4 renders/final.mp4", + ], + [ + "Score one segment of a project without rendering it", + "hyperframes diff .reverse/source.mp4 . --range 2.1,4.5", + ], + [ + "Score chosen moments of a project without rendering it", + "hyperframes diff .reverse/source.mp4 . --at 0.5,2.4,3.1", + ], + [ + "Judge each REVERSE.md segment of the final render and write a side-by-side", + "hyperframes diff .reverse/source.mp4 renders/final.mp4 --segments REVERSE.md --sbs", + ], +]; + +function number(value: unknown, flag: string): number | undefined { + if (value === undefined) return undefined; + const [n] = parseNumbers(value, flag); + return n; +} + +// fallow-ignore-next-line complexity +function printHuman(r: DiffResult) { + const mark = r.pass ? c.success("✓ pass") : c.error("✗ fail"); + console.log( + `${mark} SSIM mean ${r.ssim.mean}, p05 ${r.ssim.p05}, worst cell p05 ${r.ssim.cell_p05} over ${r.ssim.frames} frames (${r.window[0]}–${r.window[1]} s)`, + ); + if (!r.missing_content.pass) + console.log( + ` missing content in ${Math.round(r.missing_content.share * 100)}% of frames: ${r.missing_content.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}`, + ); + for (const m of r.worst) + console.log(` worst ${m.t}s ssim ${m.ssim} cell ${m.cell.roi.join(",")} at ${m.cell.ssim}`); + if (r.cuts) + console.log( + ` cuts: ${r.cuts.reference.length} in the original${r.cuts.pass ? ", all matched" : `; missing ${r.cuts.missing.join(", ") || "none"}, extra ${r.cuts.extra.join(", ") || "none"}`}`, + ); + if (r.duration) + console.log( + ` duration: original ${r.duration.reference} s, yours ${r.duration.yours} s${r.duration.pass ? "" : " (differ)"}`, + ); + if (r.sound) + console.log( + ` sound: ${"offset_ms" in r.sound ? (r.sound.offset_ms === null ? (r.sound.note ?? "not aligned") : `${r.sound.offset_ms} ms off`) : `original ${r.sound.original ? "has" : "has no"} sound, yours ${r.sound.yours ? "has" : "has no"}`}${r.sound.pass ? "" : " (fails)"}`, + ); + console.log(` every frame: ${r.frames}`); + for (const s of r.segments ?? []) { + const state = s.pass === null ? c.dim("—") : s.pass ? c.success("✓") : c.error("✗"); + const detail = s.ssim + ? `ssim ${s.ssim.mean} / p05 ${s.ssim.p05} / cell ${s.ssim.cell_p05}` + : s.energy + ? `motion ×${s.energy.mean_ratio}, luma ×${s.look?.luma_ratio}` + : (s.note ?? ""); + console.log(` ${state} ${s.id} ${s.window[0]}–${s.window[1]} s ${s.pixels}: ${detail}`); + } + for (const w of r.warnings) console.log(` ${c.warn("⚠")} ${w}`); + if (r.sheet) console.log(` sheet ${r.sheet}`); + if (r.sbs) console.log(` side by side ${r.sbs}`); +} + +export default defineCommand({ + meta: { + name: "diff", + description: + "Score a render or a project against the original video frame by frame: SSIM, worst moments, cuts, segments", + }, + args: { + reference: { type: "positional", description: "The original video", required: true }, + yours: { + type: "positional", + description: + "A rendered video, or a project folder (captured at the original's frame times, no render)", + required: true, + }, + at: { type: "string", description: "Compare only these seconds, e.g. 0.5,2.4" }, + range: { type: "string", description: "Compare every frame of start,end seconds" }, + segments: { + type: "string", + description: + "REVERSE.md: a verdict per `## S01 · a–b s` segment (Pixels: regenerated compares motion and look)", + }, + "min-mean": { type: "string", description: `Mean SSIM to pass (default ${DEFAULT_MIN_MEAN})` }, + "min-p05": { + type: "string", + description: `5th-percentile SSIM to pass (default ${DEFAULT_MIN_P05})`, + }, + "min-cell": { + type: "string", + description: `5th percentile of each frame's worst 4x4 cell to pass (default ${DEFAULT_MIN_CELL})`, + }, + worst: { type: "string", description: "Worst moments to show (default 4)" }, + sbs: { + type: "boolean", + description: "Also write an original | yours side-by-side video", + default: false, + }, + out: { + type: "string", + description: "Folder for the sheet and log (default .hyperframes/diff/)", + }, + timeout: { + type: "string", + description: "Render-ready timeout in ms for a project folder (default 5000)", + }, + json: { type: "boolean", description: "Print one JSON result", default: false }, + }, + // fallow-ignore-next-line complexity + async run({ args }) { + const json = args.json === true; + try { + if (args.at !== undefined && args.range !== undefined) + throw new Fix("--at and --range both given", "pass one of them"); + const yours = resolve(String(args.yours)); + const stem = basename(yours, extname(yours)) || "project"; + const result = await diffVideos({ + reference: resolve(String(args.reference)), + yours, + at: args.at === undefined ? undefined : parseNumbers(args.at, "--at"), + range: args.range === undefined ? undefined : parseRange(args.range), + segments: + args.segments === undefined ? undefined : readSegments(resolve(String(args.segments))), + thresholds: { + mean: number(args["min-mean"], "--min-mean"), + p05: number(args["min-p05"], "--min-p05"), + cell: number(args["min-cell"], "--min-cell"), + }, + worst: number(args.worst, "--worst"), + sbs: args.sbs === true, + outDir: outDir(args.out, join("diff", stem)), + timeoutMs: number(args.timeout, "--timeout"), + log: (message) => console.error(message), + }); + if (json) console.log(formatJson(withMeta({ ok: true, ...result }))); + else printHuman(result); + if (!result.pass) setCommandExitCode(1); + } catch (err) { + const fix = err instanceof Fix ? err.fix : undefined; + const message = normalizeErrorMessage(err); + if (json) + console.log(formatJson(withMeta({ ok: false, error: message, ...(fix ? { fix } : {}) }))); + else console.error(`${c.error("✗")} ${message}${fix ? `\n Fix: ${fix}` : ""}`); + failCommand(); + } + }, +}); diff --git a/packages/cli/src/commands/measure.ts b/packages/cli/src/commands/measure.ts new file mode 100644 index 0000000000..7888f8d3d3 --- /dev/null +++ b/packages/cli/src/commands/measure.ts @@ -0,0 +1,46 @@ +import { defineCommand } from "citty"; +import type { Example } from "./_examples.js"; + +export const examples: Example[] = [ + [ + "Size, frame rate and what to fix before rebuilding a video", + "hyperframes measure probe ref.mp4", + ], + [ + "Everything that changes when, with the sound's hits", + "hyperframes measure events ref.mp4 --map", + ], + ["The frames around every event, as strips", "hyperframes measure frames ref.mp4 --events"], + [ + "One element's box, colours, text lines and moves", + "hyperframes measure box ref.mp4 --start 1 --end 2.5 --roi 200,300,600,200 --track", + ], +]; + +export default defineCommand({ + meta: { + name: "measure", + description: + "Measure a video file as JSON: clock, cuts, events, frames, motion trails, crops, element boxes, eases, text, rhythm, look, tracking, flow, faces, lip sync, continuity", + }, + subCommands: { + probe: () => import("./measure/probe.js").then((m) => m.default), + cuts: () => import("./measure/cuts.js").then((m) => m.default), + events: () => import("./measure/events.js").then((m) => m.default), + frames: () => import("./measure/frames.js").then((m) => m.default), + trail: () => import("./measure/trail.js").then((m) => m.default), + extract: () => import("./measure/extract.js").then((m) => m.default), + box: () => import("./measure/box.js").then((m) => m.default), + ink: () => import("./measure/ink.js").then((m) => m.default), + curve: () => import("./measure/curve.js").then((m) => m.default), + rhythm: () => import("./measure/rhythm.js").then((m) => m.default), + objects: () => import("./measure/objects.js").then((m) => m.default), + energy: () => import("./measure/energy.js").then((m) => m.default), + look: () => import("./measure/look.js").then((m) => m.default), + track: () => import("./measure/track.js").then((m) => m.default), + flow: () => import("./measure/flow.js").then((m) => m.default), + face: () => import("./measure/face.js").then((m) => m.default), + sync: () => import("./measure/sync.js").then((m) => m.default), + continuity: () => import("./measure/continuity.js").then((m) => m.default), + }, +}); diff --git a/packages/cli/src/commands/measure/box.ts b/packages/cli/src/commands/measure/box.ts new file mode 100644 index 0000000000..899e6a7fa8 --- /dev/null +++ b/packages/cli/src/commands/measure/box.ts @@ -0,0 +1,75 @@ +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { + num, + jsonArg, + outArg, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { boxAt, boxRange } from "../../measure/box.js"; +import { Fix, outDir } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "One graphic's box, colours, text lines and cap height at one time", + "hyperframes measure box .reverse/source.mp4 --at 1 --roi 240,280,600,540", + ], + [ + "When it is on screen over a range, and each move it makes with its ease", + "hyperframes measure box .reverse/source.mp4 --start 3.5 --end 4.5 --roi 240,280,600,540 --track", + ], +]; + +export default defineCommand({ + meta: { + name: "box", + description: + "Where one graphic sits, found by its colour: box, ink, colours, text lines, cap height; over a range its moves", + }, + args: { + video: videoArg, + at: { type: "string", description: "One time in seconds" }, + start: { type: "string", description: "Range start in seconds" }, + end: { type: "string", description: "Range end in seconds" }, + roi: { + ...roiArg, + description: "Region round the graphic with some of its ground (default: whole frame)", + }, + color: { + type: "string", + description: + "Its pixels: auto (the commonest colour unlike the ROI border), white, dark, alpha or #rrggbb (default auto)", + default: "auto", + }, + tol: { type: "string", description: "RGB distance that still matches the colour (40)" }, + track: { + type: "boolean", + description: "Over a range: each move of its centre, size and ink, with the ease", + default: false, + }, + out: outArg, + json: jsonArg, + }, + run: ({ args }) => + runMeasure(() => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const opts = { + video: info.path, + info, + roi, + color: args.color ?? "auto", + tol: num(args.tol, "--tol", 40), + dir: outDir(args.out, "measure/box"), + }; + if (args.at !== undefined) + return { video: info.path, roi, ...boxAt(opts, num(args.at, "--at", 0)) }; + if (args.start === undefined || args.end === undefined) + throw new Fix("give --at t, or --start and --end", "--start 3.5 --end 4.5"); + const [start, end] = window(info, args.start, args.end); + return { video: info.path, roi, ...boxRange(opts, start, end, args.track === true) }; + }), +}); diff --git a/packages/cli/src/commands/measure/continuity.ts b/packages/cli/src/commands/measure/continuity.ts new file mode 100644 index 0000000000..53d200c75c --- /dev/null +++ b/packages/cli/src/commands/measure/continuity.ts @@ -0,0 +1,53 @@ +import { resolve } from "node:path"; +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { jsonArg, num, outArg, runMeasure } from "../../measure/command.js"; +import { continuity } from "../../measure/continuity.js"; +import { outDir } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "Seams a chained shot has that the original does not", + "hyperframes measure continuity .reverse/source.mp4 renders/final.mp4 --start 4 --end 12", + ], +]; + +export default defineCommand({ + meta: { + name: "continuity", + description: + "Seams yours has and the original does not: jumps, background flicker, and extra or missing shapes, on the original's clock", + }, + args: { + reference: { type: "positional", description: "The original", required: true }, + yours: { type: "positional", description: "The rebuild's render", required: true }, + start: { type: "string", description: "First second (default 0)" }, + end: { type: "string", description: "Last second (default: the end)" }, + out: outArg, + json: jsonArg, + }, + async run({ args }) { + await runMeasure(() => { + const r = continuity(resolve(String(args.reference)), resolve(String(args.yours)), { + start: num(args.start, "--start"), + end: num(args.end, "--end"), + outDir: outDir(args.out, "measure/continuity"), + }); + const count = (kind: string) => r.events.filter((e) => e.kind === kind).length; + return { + reference: resolve(String(args.reference)), + yours: resolve(String(args.yours)), + frames: r.frames, + counts: { jump: count("jump"), flicker: count("flicker"), structure: count("structure") }, + events: r.events, + thresholds: r.thresholds, + reading: + "jump: a whole-frame change the original lacks; flicker: a change over background the original keeps still; structure: an extra, missing or misplaced shape at `at` (source pixels); open each sheet (original over yours, peak marked) before acting", + limitations: [ + "frames pair by time on the original's clock, so a retimed or offset rebuild reads as structure everywhere", + "events are candidates to look at, not verdicts", + ], + }; + }); + }, +}); diff --git a/packages/cli/src/commands/measure/curve.ts b/packages/cli/src/commands/measure/curve.ts new file mode 100644 index 0000000000..be3dc219eb --- /dev/null +++ b/packages/cli/src/commands/measure/curve.ts @@ -0,0 +1,80 @@ +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { + compareArgs, + jsonArg, + num, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { compareCurves, curves, printable } from "../../measure/curve.js"; +import { Fix, parseRoi, probeVideo } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "The ease, duration and overshoot of one element's move", + "hyperframes measure curve .reverse/source.mp4 --start 2.1 --end 3 --roi 240,280,600,540", + ], + [ + "The same move in your render, paired with the original's", + "hyperframes measure curve .reverse/source.mp4 --start 2.1 --end 3 --roi 240,280,600,540 --compare renders/final.mp4 --compare-start 2.1", + ], +]; + +/** "#rrggbb" as RGB. */ +function hex(text: string): number[] { + const m = /^#?([0-9a-f]{6})$/i.exec(text); + if (!m) throw new Fix(`--bg must be #rrggbb, got "${text}"`, "--bg #101820"); + return [0, 2, 4].map((i) => parseInt(m[1]!.slice(i, i + 2), 16)); +} + +export default defineCommand({ + meta: { + name: "curve", + description: + "Each move of one element on a flat background: start, duration, overshoot and the closest GSAP ease", + }, + args: { + video: videoArg, + start: { type: "string", description: "Window start in seconds (default 0)" }, + end: { type: "string", description: "Window end in seconds (default: the end)" }, + roi: { + ...roiArg, + description: "Region holding the whole move, overshoot included, and nothing else that moves", + }, + bg: { type: "string", description: "Background #rrggbb (default: the ROI border's median)" }, + thr: { + type: "string", + description: "Pixel distance from the background that counts as the element (24)", + }, + ...compareArgs("A second video (your render) to measure the same way"), + json: jsonArg, + }, + run: ({ args }) => + runMeasure(() => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const [start, end] = window(info, args.start, args.end); + const opts = { bg: args.bg ? hex(args.bg) : null, thr: num(args.thr, "--thr", 24) }; + const ref = curves(info.path, start, end - start, roi, opts); + if (!args.compare) return { video: info.path, ...ref, moves: printable(ref.moves) }; + const other = probeVideo(args.compare); + const yoursRoi = args["compare-roi"] + ? parseRoi(args["compare-roi"], other.width, other.height) + : roi; + const yours = curves( + other.path, + num(args["compare-start"], "--compare-start", start), + end - start, + yoursRoi, + opts, + ); + return { + compare: compareCurves(ref, yours), + reference: printable(ref.moves), + yours: printable(yours.moves), + }; + }), +}); diff --git a/packages/cli/src/commands/measure/cuts.ts b/packages/cli/src/commands/measure/cuts.ts new file mode 100644 index 0000000000..e148bca0b6 --- /dev/null +++ b/packages/cli/src/commands/measure/cuts.ts @@ -0,0 +1,52 @@ +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { jsonArg, runMeasure, videoArg, window } from "../../measure/command.js"; +import { findCuts, signatures } from "../../measure/cuts.js"; +import { probeVideo, round, videoClip } from "../../measure/media.js"; + +export const examples: Example[] = [ + ["Every hard cut and the shots between them", "hyperframes measure cuts ref.mp4"], + ["Cuts in the first 20 seconds", "hyperframes measure cuts ref.mp4 --start 0 --end 20"], +]; + +export default defineCommand({ + meta: { + name: "cuts", + description: + "Hard cuts from the frame-to-frame difference (scene scores miss some), and the shots between them", + }, + args: { + video: videoArg, + start: { type: "string", description: "First second to read (default 0)" }, + end: { type: "string", description: "Last second to read (default: the end)" }, + json: jsonArg, + }, + async run({ args }) { + await runMeasure(() => { + const info = probeVideo(String(args.video)); + const [start, end] = window(info, args.start, args.end); + const clip = videoClip(info.path, start, end - start, info); + const found = findCuts(signatures(clip.input, clip.filters).frames); + const f0 = Math.ceil(start * info.fps - 1e-6); + const cuts = found.map((c) => ({ + t: round((f0 + c.frame) / info.fps, 3), + frame: f0 + c.frame, + ...(c.frames > 1 ? { frames: c.frames } : {}), + change: round(c.change, 1), + })); + const edges = [start, ...cuts.map((c) => c.t), end]; + return { + video: info.path, + start, + end: round(end, 3), + fps: round(info.fps, 3), + cuts, + shots: edges.slice(1).map((e, i) => ({ + start: round(edges[i]!, 3), + end: round(e, 3), + seconds: round(e - edges[i]!, 2), + })), + }; + }); + }, +}); diff --git a/packages/cli/src/commands/measure/energy.ts b/packages/cli/src/commands/measure/energy.ts new file mode 100644 index 0000000000..73595da846 --- /dev/null +++ b/packages/cli/src/commands/measure/energy.ts @@ -0,0 +1,66 @@ +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { jsonArg, num, runMeasure } from "../../measure/command.js"; +import { compareEnergy, energyCurve } from "../../measure/energy.js"; +import { Fix, parseRoi, probeVideo, round, videoClip, type Roi } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "Does a regenerated shot move as much as the original's, and when", + "hyperframes measure energy ref.mp4 renders/v1.mp4 --start 4 --seconds 5", + ], +]; + +export default defineCommand({ + meta: { + name: "energy", + description: + "How much two clips move, and whether they move at the same moments, when their pixels differ", + }, + args: { + reference: { type: "positional", description: "The original", required: true }, + yours: { type: "positional", description: "The other video", required: true }, + start: { type: "string", description: "Second the window starts in the original (default 0)" }, + "yours-start": { type: "string", description: "Second it starts in yours (default: --start)" }, + seconds: { type: "string", description: "Window length (default: the rest of the original)" }, + roi: { type: "string", description: "Region of the original, x,y,w,h" }, + "yours-roi": { + type: "string", + description: "Region of yours (default: --roi scaled by the widths)", + }, + json: jsonArg, + }, + async run({ args }) { + await runMeasure(() => { + const [a, b] = [probeVideo(String(args.reference)), probeVideo(String(args.yours))]; + const start = num(args.start, "--start", 0); + const yoursStart = num(args["yours-start"], "--yours-start", start); + const seconds = num(args.seconds, "--seconds", a.duration - start); + if (seconds <= 0.5) throw new Fix("the window is under half a second", "--seconds 2 or more"); + const roi = args.roi === undefined ? undefined : parseRoi(args.roi, a.width, a.height); + const k = b.width / a.width; + const scaled = (r: Roi): Roi => [ + Math.round(r[0] * k), + Math.round(r[1] * k), + Math.max(2, Math.round(r[2] * k)), + Math.max(2, Math.round(r[3] * k)), + ]; + const yoursRoi = + args["yours-roi"] !== undefined + ? parseRoi(args["yours-roi"], b.width, b.height) + : roi && scaled(roi); + const r = compareEnergy( + energyCurve(videoClip(a.path, start, seconds, a), roi), + energyCurve(videoClip(b.path, yoursStart, seconds, b), yoursRoi), + ); + return { + reference: a.path, + yours: b.path, + window_s: round(seconds, 3), + ...r, + reading: + "mean_ratio is how much yours moves (0.8–1.25 is the same amount); r near 1 at a small lag_s is the same action at the same time", + }; + }); + }, +}); diff --git a/packages/cli/src/commands/measure/events.ts b/packages/cli/src/commands/measure/events.ts new file mode 100644 index 0000000000..e894872577 --- /dev/null +++ b/packages/cli/src/commands/measure/events.ts @@ -0,0 +1,190 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { + jsonArg, + num, + outArg, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { + countEvents, + detectEvents, + drawStrip, + lookWindows, + type ScreenEvent, +} from "../../measure/events.js"; +import { Fix, outDir, round, type VideoInfo } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "The film's event map: cuts, flashes, what appears where, holds, and the sound's hits, grouped into windows to look at", + "hyperframes measure events ref.mp4 --map", + ], + [ + "When one region changes, frame by frame", + "hyperframes measure events ref.mp4 --roi 0,800,1080,300 --start 2 --end 6", + ], +]; + +/** "6x10" as columns and rows. */ +function parseGrid(text: unknown): { cols: number; rows: number } { + const m = /^(\d+)x(\d+)$/.exec(String(text)); + if (!m || Number(m[1]) < 1 || Number(m[2]) < 1) + throw new Fix(`--grid must be COLSxROWS, got "${String(text)}"`, "--grid 6x10"); + return { cols: Number(m[1]), rows: Number(m[2]) }; +} + +const isRecord = (v: unknown): v is Record => + typeof v === "object" && v !== null && !Array.isArray(v); +const numbers = (v: unknown): number[] => + Array.isArray(v) ? v.filter((x): x is number => typeof x === "number") : []; + +interface Sound { + bpm: number | null; + beats: number[]; + onsets: { t: number; strength: number }[]; +} + +/** The sound's onsets, beats and tempo, measured on this machine. */ +async function soundOf(info: VideoInfo, start: number, end: number): Promise { + if (!info.audio) return null; + const { analyzeTargets, previewTarget } = await import("../../audio/analysis/run.js"); + const report = await analyzeTargets([await previewTarget(info.path, info.path, null)], { + capabilities: ["onsets", "beats", "bpm"], + quality: "local", + }); + const item = report.items[0]; + if (!item) return null; + const inside = (t: number) => t >= start && t <= end; + const onsets = + isRecord(item.onsets) && Array.isArray(item.onsets.events) ? item.onsets.events : []; + return { + bpm: isRecord(item.bpm) && typeof item.bpm.value === "number" ? round(item.bpm.value, 1) : null, + beats: numbers(isRecord(item.beats) ? item.beats.times : undefined).filter(inside), + onsets: onsets + .filter(isRecord) + .flatMap((e) => + typeof e.t === "number" && typeof e.strength === "number" && inside(e.t) + ? [{ t: e.t, strength: e.strength }] + : [], + ), + }; +} + +const LIST_MAX = 64; + +export default defineCommand({ + meta: { + name: "events", + description: + "When things change on screen: cuts, flashes, appears, disappears, changes and holds; --map adds the sound and the windows worth looking at", + }, + args: { + video: videoArg, + map: { + type: "boolean", + description: + "The whole window on a 4x4 grid with the sound's onsets and beats, grouped into windows to look at", + default: false, + }, + roi: roiArg, + grid: { + type: "string", + description: "Split the frame into COLSxROWS cells, e.g. 6x10 (default with --map: 4x4)", + }, + start: { type: "string", description: "First second (default 0)" }, + end: { type: "string", description: "Last second (default: the end)" }, + thr: { + type: "string", + description: "Mean grey-level change of an appear or a cut (default 5)", + }, + out: outArg, + json: jsonArg, + }, + async run({ args }) { + // fallow-ignore-next-line complexity + await runMeasure(async () => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const [start, end] = window(info, args.start, args.end); + const grid = + args.grid !== undefined + ? parseGrid(args.grid) + : args.map && args.roi === undefined + ? { cols: 4, rows: 4 } + : undefined; + const thr = num(args.thr, "--thr", 5); + const found = detectEvents({ info, start, end, roi, grid, thr }); + if (found.frames < 3) + throw new Fix(`under 3 frames between ${start}s and ${end}s`, "widen --start/--end"); + const dir = outDir(args.out, "measure/events"); + const name = `events-${start}-${round(end, 2)}s`; + const strip = found.shown.length ? drawStrip(found, join(dir, `${name}.png`)) : null; + const holds = found.events.filter((e) => e.type === "hold"); + const head = { + video: info.path, + ...(grid ? { grid: `${grid.cols}x${grid.rows}` } : { roi }), + start, + end: round(end, 3), + fps: round(info.fps, 3), + }; + if (!args.map) { + const lengths = holds.map((e) => e.seconds ?? 0).sort((a, b) => a - b); + return { + ...head, + frames: found.frames, + thr, + counts: countEvents(found.events), + holds: { + longest_s: lengths.at(-1) ?? 0, + median_s: lengths[(lengths.length - 1) >> 1] ?? 0, + count: lengths.length, + }, + strip, + strip_events: `${found.shown.length} of ${found.events.length - holds.length} (cuts and flashes, then the largest steps, then changes)`, + events: found.events, + }; + } + const sound = await soundOf(info, start, end); + const cuts = found.events.filter((e) => e.type === "cut").map((e) => e.t); + const edges = [start, ...cuts, end]; + const windows = lookWindows(found.events, info.fps, sound?.onsets.map((o) => o.t) ?? []); + const file = join(dir, `${name}-map.json`); + writeFileSync( + file, + JSON.stringify({ ...head, events: found.events, sound, windows }, null, 1), + ); + const strong = sound?.onsets.filter((o) => o.strength >= 0.5).map((o) => round(o.t, 3)) ?? []; + const longHolds = holds + .filter((e: ScreenEvent) => (e.seconds ?? 0) >= 0.5) + .map((e) => ({ t: e.t, seconds: e.seconds })); + return { + ...head, + duration: round(end - start, 3), + counts: countEvents(found.events), + shots: edges.slice(1).map((e, i) => ({ start: round(edges[i]!, 3), end: round(e, 3) })), + holds: longHolds, + sound: sound + ? { + bpm: sound.bpm, + beats: sound.beats.length, + ...(sound.beats.length + ? { beat_times: sound.beats.slice(0, LIST_MAX).map((t) => round(t, 3)) } + : {}), + onsets: sound.onsets.length, + strong_onsets: strong.slice(0, LIST_MAX), + } + : "none", + windows, + strip, + file, + next: "look at each window's frames: npx hyperframes measure frames