diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index afae651083..92b4ce98b7 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -175,6 +175,8 @@ const commandLoaders = { import("./commands/media-treatment.js").then((m) => m.mediaTreatmentCommand), "grade-compare": () => import("./commands/grade-compare.js").then((m) => m.default), compare: () => import("./commands/compare.js").then((m) => m.default), + diff: () => import("./commands/diff.js").then((m) => m.default), + measure: () => import("./commands/measure.js").then((m) => m.default), capture: () => import("./commands/capture.js").then((m) => m.default), lambda: () => import("./commands/lambda.js").then((m) => m.default), cloudrun: () => import("./commands/cloudrun.js").then((m) => m.default), diff --git a/packages/cli/src/commands/diff.ts b/packages/cli/src/commands/diff.ts new file mode 100644 index 0000000000..9f42d5f882 --- /dev/null +++ b/packages/cli/src/commands/diff.ts @@ -0,0 +1,166 @@ +import { basename, extname, join, resolve } from "node:path"; +import { defineCommand } from "citty"; +import type { Example } from "./_examples.js"; +import { c } from "../ui/colors.js"; +import { failCommand, setCommandExitCode } from "../utils/commandResult.js"; +import { normalizeErrorMessage } from "../utils/errorMessage.js"; +import { withMeta } from "../utils/updateCheck.js"; +import { + DEFAULT_MIN_CELL, + DEFAULT_MIN_MEAN, + DEFAULT_MIN_P05, + diffVideos, + type DiffResult, +} from "../measure/diff.js"; +import { Fix, formatJson, outDir, parseNumbers, parseRange } from "../measure/media.js"; +import { readSegments } from "../measure/segments.js"; + +export const examples: Example[] = [ + [ + "Score a render against the original, frame by frame", + "hyperframes diff .reverse/source.mp4 renders/final.mp4", + ], + [ + "Score one segment of a project without rendering it", + "hyperframes diff .reverse/source.mp4 . --range 2.1,4.5", + ], + [ + "Score chosen moments of a project without rendering it", + "hyperframes diff .reverse/source.mp4 . --at 0.5,2.4,3.1", + ], + [ + "Judge each REVERSE.md segment of the final render and write a side-by-side", + "hyperframes diff .reverse/source.mp4 renders/final.mp4 --segments REVERSE.md --sbs", + ], +]; + +function number(value: unknown, flag: string): number | undefined { + if (value === undefined) return undefined; + const [n] = parseNumbers(value, flag); + return n; +} + +// fallow-ignore-next-line complexity +function printHuman(r: DiffResult) { + const mark = r.pass ? c.success("✓ pass") : c.error("✗ fail"); + console.log( + `${mark} SSIM mean ${r.ssim.mean}, p05 ${r.ssim.p05}, worst cell p05 ${r.ssim.cell_p05} over ${r.ssim.frames} frames (${r.window[0]}–${r.window[1]} s)`, + ); + if (!r.missing_content.pass) + console.log( + ` missing content in ${Math.round(r.missing_content.share * 100)}% of frames: ${r.missing_content.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}`, + ); + for (const m of r.worst) + console.log(` worst ${m.t}s ssim ${m.ssim} cell ${m.cell.roi.join(",")} at ${m.cell.ssim}`); + if (r.cuts) + console.log( + ` cuts: ${r.cuts.reference.length} in the original${r.cuts.pass ? ", all matched" : `; missing ${r.cuts.missing.join(", ") || "none"}, extra ${r.cuts.extra.join(", ") || "none"}`}`, + ); + if (r.duration) + console.log( + ` duration: original ${r.duration.reference} s, yours ${r.duration.yours} s${r.duration.pass ? "" : " (differ)"}`, + ); + if (r.sound) + console.log( + ` sound: ${"offset_ms" in r.sound ? (r.sound.offset_ms === null ? (r.sound.note ?? "not aligned") : `${r.sound.offset_ms} ms off`) : `original ${r.sound.original ? "has" : "has no"} sound, yours ${r.sound.yours ? "has" : "has no"}`}${r.sound.pass ? "" : " (fails)"}`, + ); + console.log(` every frame: ${r.frames}`); + for (const s of r.segments ?? []) { + const state = s.pass === null ? c.dim("—") : s.pass ? c.success("✓") : c.error("✗"); + const detail = s.ssim + ? `ssim ${s.ssim.mean} / p05 ${s.ssim.p05} / cell ${s.ssim.cell_p05}` + : s.energy + ? `motion ×${s.energy.mean_ratio}, luma ×${s.look?.luma_ratio}` + : (s.note ?? ""); + console.log(` ${state} ${s.id} ${s.window[0]}–${s.window[1]} s ${s.pixels}: ${detail}`); + } + for (const w of r.warnings) console.log(` ${c.warn("⚠")} ${w}`); + if (r.sheet) console.log(` sheet ${r.sheet}`); + if (r.sbs) console.log(` side by side ${r.sbs}`); +} + +export default defineCommand({ + meta: { + name: "diff", + description: + "Score a render or a project against the original video frame by frame: SSIM, worst moments, cuts, segments", + }, + args: { + reference: { type: "positional", description: "The original video", required: true }, + yours: { + type: "positional", + description: + "A rendered video, or a project folder (captured at the original's frame times, no render)", + required: true, + }, + at: { type: "string", description: "Compare only these seconds, e.g. 0.5,2.4" }, + range: { type: "string", description: "Compare every frame of start,end seconds" }, + segments: { + type: "string", + description: + "REVERSE.md: a verdict per `## S01 · a–b s` segment (Pixels: regenerated compares motion and look)", + }, + "min-mean": { type: "string", description: `Mean SSIM to pass (default ${DEFAULT_MIN_MEAN})` }, + "min-p05": { + type: "string", + description: `5th-percentile SSIM to pass (default ${DEFAULT_MIN_P05})`, + }, + "min-cell": { + type: "string", + description: `5th percentile of each frame's worst 4x4 cell to pass (default ${DEFAULT_MIN_CELL})`, + }, + worst: { type: "string", description: "Worst moments to show (default 4)" }, + sbs: { + type: "boolean", + description: "Also write an original | yours side-by-side video", + default: false, + }, + out: { + type: "string", + description: "Folder for the sheet and log (default .hyperframes/diff/)", + }, + timeout: { + type: "string", + description: "Render-ready timeout in ms for a project folder (default 5000)", + }, + json: { type: "boolean", description: "Print one JSON result", default: false }, + }, + // fallow-ignore-next-line complexity + async run({ args }) { + const json = args.json === true; + try { + if (args.at !== undefined && args.range !== undefined) + throw new Fix("--at and --range both given", "pass one of them"); + const yours = resolve(String(args.yours)); + const stem = basename(yours, extname(yours)) || "project"; + const result = await diffVideos({ + reference: resolve(String(args.reference)), + yours, + at: args.at === undefined ? undefined : parseNumbers(args.at, "--at"), + range: args.range === undefined ? undefined : parseRange(args.range), + segments: + args.segments === undefined ? undefined : readSegments(resolve(String(args.segments))), + thresholds: { + mean: number(args["min-mean"], "--min-mean"), + p05: number(args["min-p05"], "--min-p05"), + cell: number(args["min-cell"], "--min-cell"), + }, + worst: number(args.worst, "--worst"), + sbs: args.sbs === true, + outDir: outDir(args.out, join("diff", stem)), + timeoutMs: number(args.timeout, "--timeout"), + log: (message) => console.error(message), + }); + if (json) console.log(formatJson(withMeta({ ok: true, ...result }))); + else printHuman(result); + if (!result.pass) setCommandExitCode(1); + } catch (err) { + const fix = err instanceof Fix ? err.fix : undefined; + const message = normalizeErrorMessage(err); + if (json) + console.log(formatJson(withMeta({ ok: false, error: message, ...(fix ? { fix } : {}) }))); + else console.error(`${c.error("✗")} ${message}${fix ? `\n Fix: ${fix}` : ""}`); + failCommand(); + } + }, +}); diff --git a/packages/cli/src/commands/measure.ts b/packages/cli/src/commands/measure.ts new file mode 100644 index 0000000000..7888f8d3d3 --- /dev/null +++ b/packages/cli/src/commands/measure.ts @@ -0,0 +1,46 @@ +import { defineCommand } from "citty"; +import type { Example } from "./_examples.js"; + +export const examples: Example[] = [ + [ + "Size, frame rate and what to fix before rebuilding a video", + "hyperframes measure probe ref.mp4", + ], + [ + "Everything that changes when, with the sound's hits", + "hyperframes measure events ref.mp4 --map", + ], + ["The frames around every event, as strips", "hyperframes measure frames ref.mp4 --events"], + [ + "One element's box, colours, text lines and moves", + "hyperframes measure box ref.mp4 --start 1 --end 2.5 --roi 200,300,600,200 --track", + ], +]; + +export default defineCommand({ + meta: { + name: "measure", + description: + "Measure a video file as JSON: clock, cuts, events, frames, motion trails, crops, element boxes, eases, text, rhythm, look, tracking, flow, faces, lip sync, continuity", + }, + subCommands: { + probe: () => import("./measure/probe.js").then((m) => m.default), + cuts: () => import("./measure/cuts.js").then((m) => m.default), + events: () => import("./measure/events.js").then((m) => m.default), + frames: () => import("./measure/frames.js").then((m) => m.default), + trail: () => import("./measure/trail.js").then((m) => m.default), + extract: () => import("./measure/extract.js").then((m) => m.default), + box: () => import("./measure/box.js").then((m) => m.default), + ink: () => import("./measure/ink.js").then((m) => m.default), + curve: () => import("./measure/curve.js").then((m) => m.default), + rhythm: () => import("./measure/rhythm.js").then((m) => m.default), + objects: () => import("./measure/objects.js").then((m) => m.default), + energy: () => import("./measure/energy.js").then((m) => m.default), + look: () => import("./measure/look.js").then((m) => m.default), + track: () => import("./measure/track.js").then((m) => m.default), + flow: () => import("./measure/flow.js").then((m) => m.default), + face: () => import("./measure/face.js").then((m) => m.default), + sync: () => import("./measure/sync.js").then((m) => m.default), + continuity: () => import("./measure/continuity.js").then((m) => m.default), + }, +}); diff --git a/packages/cli/src/commands/measure/box.ts b/packages/cli/src/commands/measure/box.ts new file mode 100644 index 0000000000..899e6a7fa8 --- /dev/null +++ b/packages/cli/src/commands/measure/box.ts @@ -0,0 +1,75 @@ +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { + num, + jsonArg, + outArg, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { boxAt, boxRange } from "../../measure/box.js"; +import { Fix, outDir } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "One graphic's box, colours, text lines and cap height at one time", + "hyperframes measure box .reverse/source.mp4 --at 1 --roi 240,280,600,540", + ], + [ + "When it is on screen over a range, and each move it makes with its ease", + "hyperframes measure box .reverse/source.mp4 --start 3.5 --end 4.5 --roi 240,280,600,540 --track", + ], +]; + +export default defineCommand({ + meta: { + name: "box", + description: + "Where one graphic sits, found by its colour: box, ink, colours, text lines, cap height; over a range its moves", + }, + args: { + video: videoArg, + at: { type: "string", description: "One time in seconds" }, + start: { type: "string", description: "Range start in seconds" }, + end: { type: "string", description: "Range end in seconds" }, + roi: { + ...roiArg, + description: "Region round the graphic with some of its ground (default: whole frame)", + }, + color: { + type: "string", + description: + "Its pixels: auto (the commonest colour unlike the ROI border), white, dark, alpha or #rrggbb (default auto)", + default: "auto", + }, + tol: { type: "string", description: "RGB distance that still matches the colour (40)" }, + track: { + type: "boolean", + description: "Over a range: each move of its centre, size and ink, with the ease", + default: false, + }, + out: outArg, + json: jsonArg, + }, + run: ({ args }) => + runMeasure(() => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const opts = { + video: info.path, + info, + roi, + color: args.color ?? "auto", + tol: num(args.tol, "--tol", 40), + dir: outDir(args.out, "measure/box"), + }; + if (args.at !== undefined) + return { video: info.path, roi, ...boxAt(opts, num(args.at, "--at", 0)) }; + if (args.start === undefined || args.end === undefined) + throw new Fix("give --at t, or --start and --end", "--start 3.5 --end 4.5"); + const [start, end] = window(info, args.start, args.end); + return { video: info.path, roi, ...boxRange(opts, start, end, args.track === true) }; + }), +}); diff --git a/packages/cli/src/commands/measure/continuity.ts b/packages/cli/src/commands/measure/continuity.ts new file mode 100644 index 0000000000..53d200c75c --- /dev/null +++ b/packages/cli/src/commands/measure/continuity.ts @@ -0,0 +1,53 @@ +import { resolve } from "node:path"; +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { jsonArg, num, outArg, runMeasure } from "../../measure/command.js"; +import { continuity } from "../../measure/continuity.js"; +import { outDir } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "Seams a chained shot has that the original does not", + "hyperframes measure continuity .reverse/source.mp4 renders/final.mp4 --start 4 --end 12", + ], +]; + +export default defineCommand({ + meta: { + name: "continuity", + description: + "Seams yours has and the original does not: jumps, background flicker, and extra or missing shapes, on the original's clock", + }, + args: { + reference: { type: "positional", description: "The original", required: true }, + yours: { type: "positional", description: "The rebuild's render", required: true }, + start: { type: "string", description: "First second (default 0)" }, + end: { type: "string", description: "Last second (default: the end)" }, + out: outArg, + json: jsonArg, + }, + async run({ args }) { + await runMeasure(() => { + const r = continuity(resolve(String(args.reference)), resolve(String(args.yours)), { + start: num(args.start, "--start"), + end: num(args.end, "--end"), + outDir: outDir(args.out, "measure/continuity"), + }); + const count = (kind: string) => r.events.filter((e) => e.kind === kind).length; + return { + reference: resolve(String(args.reference)), + yours: resolve(String(args.yours)), + frames: r.frames, + counts: { jump: count("jump"), flicker: count("flicker"), structure: count("structure") }, + events: r.events, + thresholds: r.thresholds, + reading: + "jump: a whole-frame change the original lacks; flicker: a change over background the original keeps still; structure: an extra, missing or misplaced shape at `at` (source pixels); open each sheet (original over yours, peak marked) before acting", + limitations: [ + "frames pair by time on the original's clock, so a retimed or offset rebuild reads as structure everywhere", + "events are candidates to look at, not verdicts", + ], + }; + }); + }, +}); diff --git a/packages/cli/src/commands/measure/curve.ts b/packages/cli/src/commands/measure/curve.ts new file mode 100644 index 0000000000..be3dc219eb --- /dev/null +++ b/packages/cli/src/commands/measure/curve.ts @@ -0,0 +1,80 @@ +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { + compareArgs, + jsonArg, + num, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { compareCurves, curves, printable } from "../../measure/curve.js"; +import { Fix, parseRoi, probeVideo } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "The ease, duration and overshoot of one element's move", + "hyperframes measure curve .reverse/source.mp4 --start 2.1 --end 3 --roi 240,280,600,540", + ], + [ + "The same move in your render, paired with the original's", + "hyperframes measure curve .reverse/source.mp4 --start 2.1 --end 3 --roi 240,280,600,540 --compare renders/final.mp4 --compare-start 2.1", + ], +]; + +/** "#rrggbb" as RGB. */ +function hex(text: string): number[] { + const m = /^#?([0-9a-f]{6})$/i.exec(text); + if (!m) throw new Fix(`--bg must be #rrggbb, got "${text}"`, "--bg #101820"); + return [0, 2, 4].map((i) => parseInt(m[1]!.slice(i, i + 2), 16)); +} + +export default defineCommand({ + meta: { + name: "curve", + description: + "Each move of one element on a flat background: start, duration, overshoot and the closest GSAP ease", + }, + args: { + video: videoArg, + start: { type: "string", description: "Window start in seconds (default 0)" }, + end: { type: "string", description: "Window end in seconds (default: the end)" }, + roi: { + ...roiArg, + description: "Region holding the whole move, overshoot included, and nothing else that moves", + }, + bg: { type: "string", description: "Background #rrggbb (default: the ROI border's median)" }, + thr: { + type: "string", + description: "Pixel distance from the background that counts as the element (24)", + }, + ...compareArgs("A second video (your render) to measure the same way"), + json: jsonArg, + }, + run: ({ args }) => + runMeasure(() => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const [start, end] = window(info, args.start, args.end); + const opts = { bg: args.bg ? hex(args.bg) : null, thr: num(args.thr, "--thr", 24) }; + const ref = curves(info.path, start, end - start, roi, opts); + if (!args.compare) return { video: info.path, ...ref, moves: printable(ref.moves) }; + const other = probeVideo(args.compare); + const yoursRoi = args["compare-roi"] + ? parseRoi(args["compare-roi"], other.width, other.height) + : roi; + const yours = curves( + other.path, + num(args["compare-start"], "--compare-start", start), + end - start, + yoursRoi, + opts, + ); + return { + compare: compareCurves(ref, yours), + reference: printable(ref.moves), + yours: printable(yours.moves), + }; + }), +}); diff --git a/packages/cli/src/commands/measure/cuts.ts b/packages/cli/src/commands/measure/cuts.ts new file mode 100644 index 0000000000..e148bca0b6 --- /dev/null +++ b/packages/cli/src/commands/measure/cuts.ts @@ -0,0 +1,52 @@ +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { jsonArg, runMeasure, videoArg, window } from "../../measure/command.js"; +import { findCuts, signatures } from "../../measure/cuts.js"; +import { probeVideo, round, videoClip } from "../../measure/media.js"; + +export const examples: Example[] = [ + ["Every hard cut and the shots between them", "hyperframes measure cuts ref.mp4"], + ["Cuts in the first 20 seconds", "hyperframes measure cuts ref.mp4 --start 0 --end 20"], +]; + +export default defineCommand({ + meta: { + name: "cuts", + description: + "Hard cuts from the frame-to-frame difference (scene scores miss some), and the shots between them", + }, + args: { + video: videoArg, + start: { type: "string", description: "First second to read (default 0)" }, + end: { type: "string", description: "Last second to read (default: the end)" }, + json: jsonArg, + }, + async run({ args }) { + await runMeasure(() => { + const info = probeVideo(String(args.video)); + const [start, end] = window(info, args.start, args.end); + const clip = videoClip(info.path, start, end - start, info); + const found = findCuts(signatures(clip.input, clip.filters).frames); + const f0 = Math.ceil(start * info.fps - 1e-6); + const cuts = found.map((c) => ({ + t: round((f0 + c.frame) / info.fps, 3), + frame: f0 + c.frame, + ...(c.frames > 1 ? { frames: c.frames } : {}), + change: round(c.change, 1), + })); + const edges = [start, ...cuts.map((c) => c.t), end]; + return { + video: info.path, + start, + end: round(end, 3), + fps: round(info.fps, 3), + cuts, + shots: edges.slice(1).map((e, i) => ({ + start: round(edges[i]!, 3), + end: round(e, 3), + seconds: round(e - edges[i]!, 2), + })), + }; + }); + }, +}); diff --git a/packages/cli/src/commands/measure/energy.ts b/packages/cli/src/commands/measure/energy.ts new file mode 100644 index 0000000000..73595da846 --- /dev/null +++ b/packages/cli/src/commands/measure/energy.ts @@ -0,0 +1,66 @@ +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { jsonArg, num, runMeasure } from "../../measure/command.js"; +import { compareEnergy, energyCurve } from "../../measure/energy.js"; +import { Fix, parseRoi, probeVideo, round, videoClip, type Roi } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "Does a regenerated shot move as much as the original's, and when", + "hyperframes measure energy ref.mp4 renders/v1.mp4 --start 4 --seconds 5", + ], +]; + +export default defineCommand({ + meta: { + name: "energy", + description: + "How much two clips move, and whether they move at the same moments, when their pixels differ", + }, + args: { + reference: { type: "positional", description: "The original", required: true }, + yours: { type: "positional", description: "The other video", required: true }, + start: { type: "string", description: "Second the window starts in the original (default 0)" }, + "yours-start": { type: "string", description: "Second it starts in yours (default: --start)" }, + seconds: { type: "string", description: "Window length (default: the rest of the original)" }, + roi: { type: "string", description: "Region of the original, x,y,w,h" }, + "yours-roi": { + type: "string", + description: "Region of yours (default: --roi scaled by the widths)", + }, + json: jsonArg, + }, + async run({ args }) { + await runMeasure(() => { + const [a, b] = [probeVideo(String(args.reference)), probeVideo(String(args.yours))]; + const start = num(args.start, "--start", 0); + const yoursStart = num(args["yours-start"], "--yours-start", start); + const seconds = num(args.seconds, "--seconds", a.duration - start); + if (seconds <= 0.5) throw new Fix("the window is under half a second", "--seconds 2 or more"); + const roi = args.roi === undefined ? undefined : parseRoi(args.roi, a.width, a.height); + const k = b.width / a.width; + const scaled = (r: Roi): Roi => [ + Math.round(r[0] * k), + Math.round(r[1] * k), + Math.max(2, Math.round(r[2] * k)), + Math.max(2, Math.round(r[3] * k)), + ]; + const yoursRoi = + args["yours-roi"] !== undefined + ? parseRoi(args["yours-roi"], b.width, b.height) + : roi && scaled(roi); + const r = compareEnergy( + energyCurve(videoClip(a.path, start, seconds, a), roi), + energyCurve(videoClip(b.path, yoursStart, seconds, b), yoursRoi), + ); + return { + reference: a.path, + yours: b.path, + window_s: round(seconds, 3), + ...r, + reading: + "mean_ratio is how much yours moves (0.8–1.25 is the same amount); r near 1 at a small lag_s is the same action at the same time", + }; + }); + }, +}); diff --git a/packages/cli/src/commands/measure/events.ts b/packages/cli/src/commands/measure/events.ts new file mode 100644 index 0000000000..e894872577 --- /dev/null +++ b/packages/cli/src/commands/measure/events.ts @@ -0,0 +1,190 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { defineCommand } from "citty"; +import type { Example } from "../_examples.js"; +import { + jsonArg, + num, + outArg, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { + countEvents, + detectEvents, + drawStrip, + lookWindows, + type ScreenEvent, +} from "../../measure/events.js"; +import { Fix, outDir, round, type VideoInfo } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "The film's event map: cuts, flashes, what appears where, holds, and the sound's hits, grouped into windows to look at", + "hyperframes measure events ref.mp4 --map", + ], + [ + "When one region changes, frame by frame", + "hyperframes measure events ref.mp4 --roi 0,800,1080,300 --start 2 --end 6", + ], +]; + +/** "6x10" as columns and rows. */ +function parseGrid(text: unknown): { cols: number; rows: number } { + const m = /^(\d+)x(\d+)$/.exec(String(text)); + if (!m || Number(m[1]) < 1 || Number(m[2]) < 1) + throw new Fix(`--grid must be COLSxROWS, got "${String(text)}"`, "--grid 6x10"); + return { cols: Number(m[1]), rows: Number(m[2]) }; +} + +const isRecord = (v: unknown): v is Record => + typeof v === "object" && v !== null && !Array.isArray(v); +const numbers = (v: unknown): number[] => + Array.isArray(v) ? v.filter((x): x is number => typeof x === "number") : []; + +interface Sound { + bpm: number | null; + beats: number[]; + onsets: { t: number; strength: number }[]; +} + +/** The sound's onsets, beats and tempo, measured on this machine. */ +async function soundOf(info: VideoInfo, start: number, end: number): Promise { + if (!info.audio) return null; + const { analyzeTargets, previewTarget } = await import("../../audio/analysis/run.js"); + const report = await analyzeTargets([await previewTarget(info.path, info.path, null)], { + capabilities: ["onsets", "beats", "bpm"], + quality: "local", + }); + const item = report.items[0]; + if (!item) return null; + const inside = (t: number) => t >= start && t <= end; + const onsets = + isRecord(item.onsets) && Array.isArray(item.onsets.events) ? item.onsets.events : []; + return { + bpm: isRecord(item.bpm) && typeof item.bpm.value === "number" ? round(item.bpm.value, 1) : null, + beats: numbers(isRecord(item.beats) ? item.beats.times : undefined).filter(inside), + onsets: onsets + .filter(isRecord) + .flatMap((e) => + typeof e.t === "number" && typeof e.strength === "number" && inside(e.t) + ? [{ t: e.t, strength: e.strength }] + : [], + ), + }; +} + +const LIST_MAX = 64; + +export default defineCommand({ + meta: { + name: "events", + description: + "When things change on screen: cuts, flashes, appears, disappears, changes and holds; --map adds the sound and the windows worth looking at", + }, + args: { + video: videoArg, + map: { + type: "boolean", + description: + "The whole window on a 4x4 grid with the sound's onsets and beats, grouped into windows to look at", + default: false, + }, + roi: roiArg, + grid: { + type: "string", + description: "Split the frame into COLSxROWS cells, e.g. 6x10 (default with --map: 4x4)", + }, + start: { type: "string", description: "First second (default 0)" }, + end: { type: "string", description: "Last second (default: the end)" }, + thr: { + type: "string", + description: "Mean grey-level change of an appear or a cut (default 5)", + }, + out: outArg, + json: jsonArg, + }, + async run({ args }) { + // fallow-ignore-next-line complexity + await runMeasure(async () => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const [start, end] = window(info, args.start, args.end); + const grid = + args.grid !== undefined + ? parseGrid(args.grid) + : args.map && args.roi === undefined + ? { cols: 4, rows: 4 } + : undefined; + const thr = num(args.thr, "--thr", 5); + const found = detectEvents({ info, start, end, roi, grid, thr }); + if (found.frames < 3) + throw new Fix(`under 3 frames between ${start}s and ${end}s`, "widen --start/--end"); + const dir = outDir(args.out, "measure/events"); + const name = `events-${start}-${round(end, 2)}s`; + const strip = found.shown.length ? drawStrip(found, join(dir, `${name}.png`)) : null; + const holds = found.events.filter((e) => e.type === "hold"); + const head = { + video: info.path, + ...(grid ? { grid: `${grid.cols}x${grid.rows}` } : { roi }), + start, + end: round(end, 3), + fps: round(info.fps, 3), + }; + if (!args.map) { + const lengths = holds.map((e) => e.seconds ?? 0).sort((a, b) => a - b); + return { + ...head, + frames: found.frames, + thr, + counts: countEvents(found.events), + holds: { + longest_s: lengths.at(-1) ?? 0, + median_s: lengths[(lengths.length - 1) >> 1] ?? 0, + count: lengths.length, + }, + strip, + strip_events: `${found.shown.length} of ${found.events.length - holds.length} (cuts and flashes, then the largest steps, then changes)`, + events: found.events, + }; + } + const sound = await soundOf(info, start, end); + const cuts = found.events.filter((e) => e.type === "cut").map((e) => e.t); + const edges = [start, ...cuts, end]; + const windows = lookWindows(found.events, info.fps, sound?.onsets.map((o) => o.t) ?? []); + const file = join(dir, `${name}-map.json`); + writeFileSync( + file, + JSON.stringify({ ...head, events: found.events, sound, windows }, null, 1), + ); + const strong = sound?.onsets.filter((o) => o.strength >= 0.5).map((o) => round(o.t, 3)) ?? []; + const longHolds = holds + .filter((e: ScreenEvent) => (e.seconds ?? 0) >= 0.5) + .map((e) => ({ t: e.t, seconds: e.seconds })); + return { + ...head, + duration: round(end - start, 3), + counts: countEvents(found.events), + shots: edges.slice(1).map((e, i) => ({ start: round(edges[i]!, 3), end: round(e, 3) })), + holds: longHolds, + sound: sound + ? { + bpm: sound.bpm, + beats: sound.beats.length, + ...(sound.beats.length + ? { beat_times: sound.beats.slice(0, LIST_MAX).map((t) => round(t, 3)) } + : {}), + onsets: sound.onsets.length, + strong_onsets: strong.slice(0, LIST_MAX), + } + : "none", + windows, + strip, + file, + next: "look at each window's frames: npx hyperframes measure frames