feat(cli): hyperframes measure and hyperframes diff read a video file as numbers

measure has 18 subcommands (probe, cuts, events, frames, trail, extract, box, ink,
curve, rhythm, objects, energy, look, track, flow, face, sync, continuity), each
printing JSON. diff scores a render or a project against the original frame by
frame. Ported from the Studio's viral-clone scripts and hf-media so Desktop and
the CLI share one implementation.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Miao Yang
2026-10-23 23:56:24 +08:00
co-authored by Claude Opus 5.5
parent bd4cae7e35
commit 170265cf24
55 changed files with 7247 additions and 3 deletions
+2
View File
@@ -175,6 +175,8 @@ const commandLoaders = {
import("./commands/media-treatment.js").then((m) => m.mediaTreatmentCommand),
"grade-compare": () => import("./commands/grade-compare.js").then((m) => m.default),
compare: () => import("./commands/compare.js").then((m) => m.default),
diff: () => import("./commands/diff.js").then((m) => m.default),
measure: () => import("./commands/measure.js").then((m) => m.default),
capture: () => import("./commands/capture.js").then((m) => m.default),
lambda: () => import("./commands/lambda.js").then((m) => m.default),
cloudrun: () => import("./commands/cloudrun.js").then((m) => m.default),
+166
View File
@@ -0,0 +1,166 @@
import { basename, extname, join, resolve } from "node:path";
import { defineCommand } from "citty";
import type { Example } from "./_examples.js";
import { c } from "../ui/colors.js";
import { failCommand, setCommandExitCode } from "../utils/commandResult.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { withMeta } from "../utils/updateCheck.js";
import {
DEFAULT_MIN_CELL,
DEFAULT_MIN_MEAN,
DEFAULT_MIN_P05,
diffVideos,
type DiffResult,
} from "../measure/diff.js";
import { Fix, formatJson, outDir, parseNumbers, parseRange } from "../measure/media.js";
import { readSegments } from "../measure/segments.js";
export const examples: Example[] = [
[
"Score a render against the original, frame by frame",
"hyperframes diff .reverse/source.mp4 renders/final.mp4",
],
[
"Score one segment of a project without rendering it",
"hyperframes diff .reverse/source.mp4 . --range 2.1,4.5",
],
[
"Score chosen moments of a project without rendering it",
"hyperframes diff .reverse/source.mp4 . --at 0.5,2.4,3.1",
],
[
"Judge each REVERSE.md segment of the final render and write a side-by-side",
"hyperframes diff .reverse/source.mp4 renders/final.mp4 --segments REVERSE.md --sbs",
],
];
function number(value: unknown, flag: string): number | undefined {
if (value === undefined) return undefined;
const [n] = parseNumbers(value, flag);
return n;
}
// fallow-ignore-next-line complexity
function printHuman(r: DiffResult) {
const mark = r.pass ? c.success("✓ pass") : c.error("✗ fail");
console.log(
`${mark} SSIM mean ${r.ssim.mean}, p05 ${r.ssim.p05}, worst cell p05 ${r.ssim.cell_p05} over ${r.ssim.frames} frames (${r.window[0]}–${r.window[1]} s)`,
);
if (!r.missing_content.pass)
console.log(
` missing content in ${Math.round(r.missing_content.share * 100)}% of frames: ${r.missing_content.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}`,
);
for (const m of r.worst)
console.log(` worst ${m.t}s ssim ${m.ssim} cell ${m.cell.roi.join(",")} at ${m.cell.ssim}`);
if (r.cuts)
console.log(
` cuts: ${r.cuts.reference.length} in the original${r.cuts.pass ? ", all matched" : `; missing ${r.cuts.missing.join(", ") || "none"}, extra ${r.cuts.extra.join(", ") || "none"}`}`,
);
if (r.duration)
console.log(
` duration: original ${r.duration.reference} s, yours ${r.duration.yours} s${r.duration.pass ? "" : " (differ)"}`,
);
if (r.sound)
console.log(
` sound: ${"offset_ms" in r.sound ? (r.sound.offset_ms === null ? (r.sound.note ?? "not aligned") : `${r.sound.offset_ms} ms off`) : `original ${r.sound.original ? "has" : "has no"} sound, yours ${r.sound.yours ? "has" : "has no"}`}${r.sound.pass ? "" : " (fails)"}`,
);
console.log(` every frame: ${r.frames}`);
for (const s of r.segments ?? []) {
const state = s.pass === null ? c.dim("—") : s.pass ? c.success("✓") : c.error("✗");
const detail = s.ssim
? `ssim ${s.ssim.mean} / p05 ${s.ssim.p05} / cell ${s.ssim.cell_p05}`
: s.energy
? `motion ×${s.energy.mean_ratio}, luma ×${s.look?.luma_ratio}`
: (s.note ?? "");
console.log(` ${state} ${s.id} ${s.window[0]}–${s.window[1]} s ${s.pixels}: ${detail}`);
}
for (const w of r.warnings) console.log(` ${c.warn("⚠")} ${w}`);
if (r.sheet) console.log(` sheet ${r.sheet}`);
if (r.sbs) console.log(` side by side ${r.sbs}`);
}
export default defineCommand({
meta: {
name: "diff",
description:
"Score a render or a project against the original video frame by frame: SSIM, worst moments, cuts, segments",
},
args: {
reference: { type: "positional", description: "The original video", required: true },
yours: {
type: "positional",
description:
"A rendered video, or a project folder (captured at the original's frame times, no render)",
required: true,
},
at: { type: "string", description: "Compare only these seconds, e.g. 0.5,2.4" },
range: { type: "string", description: "Compare every frame of start,end seconds" },
segments: {
type: "string",
description:
"REVERSE.md: a verdict per `## S01 · a–b s` segment (Pixels: regenerated compares motion and look)",
},
"min-mean": { type: "string", description: `Mean SSIM to pass (default ${DEFAULT_MIN_MEAN})` },
"min-p05": {
type: "string",
description: `5th-percentile SSIM to pass (default ${DEFAULT_MIN_P05})`,
},
"min-cell": {
type: "string",
description: `5th percentile of each frame's worst 4x4 cell to pass (default ${DEFAULT_MIN_CELL})`,
},
worst: { type: "string", description: "Worst moments to show (default 4)" },
sbs: {
type: "boolean",
description: "Also write an original | yours side-by-side video",
default: false,
},
out: {
type: "string",
description: "Folder for the sheet and log (default .hyperframes/diff/<yours>)",
},
timeout: {
type: "string",
description: "Render-ready timeout in ms for a project folder (default 5000)",
},
json: { type: "boolean", description: "Print one JSON result", default: false },
},
// fallow-ignore-next-line complexity
async run({ args }) {
const json = args.json === true;
try {
if (args.at !== undefined && args.range !== undefined)
throw new Fix("--at and --range both given", "pass one of them");
const yours = resolve(String(args.yours));
const stem = basename(yours, extname(yours)) || "project";
const result = await diffVideos({
reference: resolve(String(args.reference)),
yours,
at: args.at === undefined ? undefined : parseNumbers(args.at, "--at"),
range: args.range === undefined ? undefined : parseRange(args.range),
segments:
args.segments === undefined ? undefined : readSegments(resolve(String(args.segments))),
thresholds: {
mean: number(args["min-mean"], "--min-mean"),
p05: number(args["min-p05"], "--min-p05"),
cell: number(args["min-cell"], "--min-cell"),
},
worst: number(args.worst, "--worst"),
sbs: args.sbs === true,
outDir: outDir(args.out, join("diff", stem)),
timeoutMs: number(args.timeout, "--timeout"),
log: (message) => console.error(message),
});
if (json) console.log(formatJson(withMeta({ ok: true, ...result })));
else printHuman(result);
if (!result.pass) setCommandExitCode(1);
} catch (err) {
const fix = err instanceof Fix ? err.fix : undefined;
const message = normalizeErrorMessage(err);
if (json)
console.log(formatJson(withMeta({ ok: false, error: message, ...(fix ? { fix } : {}) })));
else console.error(`${c.error("✗")} ${message}${fix ? `\n Fix: ${fix}` : ""}`);
failCommand();
}
},
});
+46
View File
@@ -0,0 +1,46 @@
import { defineCommand } from "citty";
import type { Example } from "./_examples.js";
export const examples: Example[] = [
[
"Size, frame rate and what to fix before rebuilding a video",
"hyperframes measure probe ref.mp4",
],
[
"Everything that changes when, with the sound's hits",
"hyperframes measure events ref.mp4 --map",
],
["The frames around every event, as strips", "hyperframes measure frames ref.mp4 --events"],
[
"One element's box, colours, text lines and moves",
"hyperframes measure box ref.mp4 --start 1 --end 2.5 --roi 200,300,600,200 --track",
],
];
export default defineCommand({
meta: {
name: "measure",
description:
"Measure a video file as JSON: clock, cuts, events, frames, motion trails, crops, element boxes, eases, text, rhythm, look, tracking, flow, faces, lip sync, continuity",
},
subCommands: {
probe: () => import("./measure/probe.js").then((m) => m.default),
cuts: () => import("./measure/cuts.js").then((m) => m.default),
events: () => import("./measure/events.js").then((m) => m.default),
frames: () => import("./measure/frames.js").then((m) => m.default),
trail: () => import("./measure/trail.js").then((m) => m.default),
extract: () => import("./measure/extract.js").then((m) => m.default),
box: () => import("./measure/box.js").then((m) => m.default),
ink: () => import("./measure/ink.js").then((m) => m.default),
curve: () => import("./measure/curve.js").then((m) => m.default),
rhythm: () => import("./measure/rhythm.js").then((m) => m.default),
objects: () => import("./measure/objects.js").then((m) => m.default),
energy: () => import("./measure/energy.js").then((m) => m.default),
look: () => import("./measure/look.js").then((m) => m.default),
track: () => import("./measure/track.js").then((m) => m.default),
flow: () => import("./measure/flow.js").then((m) => m.default),
face: () => import("./measure/face.js").then((m) => m.default),
sync: () => import("./measure/sync.js").then((m) => m.default),
continuity: () => import("./measure/continuity.js").then((m) => m.default),
},
});
+75
View File
@@ -0,0 +1,75 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
num,
jsonArg,
outArg,
roiArg,
runMeasure,
videoAndRoi,
videoArg,
window,
} from "../../measure/command.js";
import { boxAt, boxRange } from "../../measure/box.js";
import { Fix, outDir } from "../../measure/media.js";
export const examples: Example[] = [
[
"One graphic's box, colours, text lines and cap height at one time",
"hyperframes measure box .reverse/source.mp4 --at 1 --roi 240,280,600,540",
],
[
"When it is on screen over a range, and each move it makes with its ease",
"hyperframes measure box .reverse/source.mp4 --start 3.5 --end 4.5 --roi 240,280,600,540 --track",
],
];
export default defineCommand({
meta: {
name: "box",
description:
"Where one graphic sits, found by its colour: box, ink, colours, text lines, cap height; over a range its moves",
},
args: {
video: videoArg,
at: { type: "string", description: "One time in seconds" },
start: { type: "string", description: "Range start in seconds" },
end: { type: "string", description: "Range end in seconds" },
roi: {
...roiArg,
description: "Region round the graphic with some of its ground (default: whole frame)",
},
color: {
type: "string",
description:
"Its pixels: auto (the commonest colour unlike the ROI border), white, dark, alpha or #rrggbb (default auto)",
default: "auto",
},
tol: { type: "string", description: "RGB distance that still matches the colour (40)" },
track: {
type: "boolean",
description: "Over a range: each move of its centre, size and ink, with the ease",
default: false,
},
out: outArg,
json: jsonArg,
},
run: ({ args }) =>
runMeasure(() => {
const { info, roi } = videoAndRoi(args.video, args.roi);
const opts = {
video: info.path,
info,
roi,
color: args.color ?? "auto",
tol: num(args.tol, "--tol", 40),
dir: outDir(args.out, "measure/box"),
};
if (args.at !== undefined)
return { video: info.path, roi, ...boxAt(opts, num(args.at, "--at", 0)) };
if (args.start === undefined || args.end === undefined)
throw new Fix("give --at t, or --start and --end", "--start 3.5 --end 4.5");
const [start, end] = window(info, args.start, args.end);
return { video: info.path, roi, ...boxRange(opts, start, end, args.track === true) };
}),
});
@@ -0,0 +1,53 @@
import { resolve } from "node:path";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, outArg, runMeasure } from "../../measure/command.js";
import { continuity } from "../../measure/continuity.js";
import { outDir } from "../../measure/media.js";
export const examples: Example[] = [
[
"Seams a chained shot has that the original does not",
"hyperframes measure continuity .reverse/source.mp4 renders/final.mp4 --start 4 --end 12",
],
];
export default defineCommand({
meta: {
name: "continuity",
description:
"Seams yours has and the original does not: jumps, background flicker, and extra or missing shapes, on the original's clock",
},
args: {
reference: { type: "positional", description: "The original", required: true },
yours: { type: "positional", description: "The rebuild's render", required: true },
start: { type: "string", description: "First second (default 0)" },
end: { type: "string", description: "Last second (default: the end)" },
out: outArg,
json: jsonArg,
},
async run({ args }) {
await runMeasure(() => {
const r = continuity(resolve(String(args.reference)), resolve(String(args.yours)), {
start: num(args.start, "--start"),
end: num(args.end, "--end"),
outDir: outDir(args.out, "measure/continuity"),
});
const count = (kind: string) => r.events.filter((e) => e.kind === kind).length;
return {
reference: resolve(String(args.reference)),
yours: resolve(String(args.yours)),
frames: r.frames,
counts: { jump: count("jump"), flicker: count("flicker"), structure: count("structure") },
events: r.events,
thresholds: r.thresholds,
reading:
"jump: a whole-frame change the original lacks; flicker: a change over background the original keeps still; structure: an extra, missing or misplaced shape at `at` (source pixels); open each sheet (original over yours, peak marked) before acting",
limitations: [
"frames pair by time on the original's clock, so a retimed or offset rebuild reads as structure everywhere",
"events are candidates to look at, not verdicts",
],
};
});
},
});
@@ -0,0 +1,80 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
compareArgs,
jsonArg,
num,
roiArg,
runMeasure,
videoAndRoi,
videoArg,
window,
} from "../../measure/command.js";
import { compareCurves, curves, printable } from "../../measure/curve.js";
import { Fix, parseRoi, probeVideo } from "../../measure/media.js";
export const examples: Example[] = [
[
"The ease, duration and overshoot of one element's move",
"hyperframes measure curve .reverse/source.mp4 --start 2.1 --end 3 --roi 240,280,600,540",
],
[
"The same move in your render, paired with the original's",
"hyperframes measure curve .reverse/source.mp4 --start 2.1 --end 3 --roi 240,280,600,540 --compare renders/final.mp4 --compare-start 2.1",
],
];
/** "#rrggbb" as RGB. */
function hex(text: string): number[] {
const m = /^#?([0-9a-f]{6})$/i.exec(text);
if (!m) throw new Fix(`--bg must be #rrggbb, got "${text}"`, "--bg #101820");
return [0, 2, 4].map((i) => parseInt(m[1]!.slice(i, i + 2), 16));
}
export default defineCommand({
meta: {
name: "curve",
description:
"Each move of one element on a flat background: start, duration, overshoot and the closest GSAP ease",
},
args: {
video: videoArg,
start: { type: "string", description: "Window start in seconds (default 0)" },
end: { type: "string", description: "Window end in seconds (default: the end)" },
roi: {
...roiArg,
description: "Region holding the whole move, overshoot included, and nothing else that moves",
},
bg: { type: "string", description: "Background #rrggbb (default: the ROI border's median)" },
thr: {
type: "string",
description: "Pixel distance from the background that counts as the element (24)",
},
...compareArgs("A second video (your render) to measure the same way"),
json: jsonArg,
},
run: ({ args }) =>
runMeasure(() => {
const { info, roi } = videoAndRoi(args.video, args.roi);
const [start, end] = window(info, args.start, args.end);
const opts = { bg: args.bg ? hex(args.bg) : null, thr: num(args.thr, "--thr", 24) };
const ref = curves(info.path, start, end - start, roi, opts);
if (!args.compare) return { video: info.path, ...ref, moves: printable(ref.moves) };
const other = probeVideo(args.compare);
const yoursRoi = args["compare-roi"]
? parseRoi(args["compare-roi"], other.width, other.height)
: roi;
const yours = curves(
other.path,
num(args["compare-start"], "--compare-start", start),
end - start,
yoursRoi,
opts,
);
return {
compare: compareCurves(ref, yours),
reference: printable(ref.moves),
yours: printable(yours.moves),
};
}),
});
+52
View File
@@ -0,0 +1,52 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, runMeasure, videoArg, window } from "../../measure/command.js";
import { findCuts, signatures } from "../../measure/cuts.js";
import { probeVideo, round, videoClip } from "../../measure/media.js";
export const examples: Example[] = [
["Every hard cut and the shots between them", "hyperframes measure cuts ref.mp4"],
["Cuts in the first 20 seconds", "hyperframes measure cuts ref.mp4 --start 0 --end 20"],
];
export default defineCommand({
meta: {
name: "cuts",
description:
"Hard cuts from the frame-to-frame difference (scene scores miss some), and the shots between them",
},
args: {
video: videoArg,
start: { type: "string", description: "First second to read (default 0)" },
end: { type: "string", description: "Last second to read (default: the end)" },
json: jsonArg,
},
async run({ args }) {
await runMeasure(() => {
const info = probeVideo(String(args.video));
const [start, end] = window(info, args.start, args.end);
const clip = videoClip(info.path, start, end - start, info);
const found = findCuts(signatures(clip.input, clip.filters).frames);
const f0 = Math.ceil(start * info.fps - 1e-6);
const cuts = found.map((c) => ({
t: round((f0 + c.frame) / info.fps, 3),
frame: f0 + c.frame,
...(c.frames > 1 ? { frames: c.frames } : {}),
change: round(c.change, 1),
}));
const edges = [start, ...cuts.map((c) => c.t), end];
return {
video: info.path,
start,
end: round(end, 3),
fps: round(info.fps, 3),
cuts,
shots: edges.slice(1).map((e, i) => ({
start: round(edges[i]!, 3),
end: round(e, 3),
seconds: round(e - edges[i]!, 2),
})),
};
});
},
});
@@ -0,0 +1,66 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, runMeasure } from "../../measure/command.js";
import { compareEnergy, energyCurve } from "../../measure/energy.js";
import { Fix, parseRoi, probeVideo, round, videoClip, type Roi } from "../../measure/media.js";
export const examples: Example[] = [
[
"Does a regenerated shot move as much as the original's, and when",
"hyperframes measure energy ref.mp4 renders/v1.mp4 --start 4 --seconds 5",
],
];
export default defineCommand({
meta: {
name: "energy",
description:
"How much two clips move, and whether they move at the same moments, when their pixels differ",
},
args: {
reference: { type: "positional", description: "The original", required: true },
yours: { type: "positional", description: "The other video", required: true },
start: { type: "string", description: "Second the window starts in the original (default 0)" },
"yours-start": { type: "string", description: "Second it starts in yours (default: --start)" },
seconds: { type: "string", description: "Window length (default: the rest of the original)" },
roi: { type: "string", description: "Region of the original, x,y,w,h" },
"yours-roi": {
type: "string",
description: "Region of yours (default: --roi scaled by the widths)",
},
json: jsonArg,
},
async run({ args }) {
await runMeasure(() => {
const [a, b] = [probeVideo(String(args.reference)), probeVideo(String(args.yours))];
const start = num(args.start, "--start", 0);
const yoursStart = num(args["yours-start"], "--yours-start", start);
const seconds = num(args.seconds, "--seconds", a.duration - start);
if (seconds <= 0.5) throw new Fix("the window is under half a second", "--seconds 2 or more");
const roi = args.roi === undefined ? undefined : parseRoi(args.roi, a.width, a.height);
const k = b.width / a.width;
const scaled = (r: Roi): Roi => [
Math.round(r[0] * k),
Math.round(r[1] * k),
Math.max(2, Math.round(r[2] * k)),
Math.max(2, Math.round(r[3] * k)),
];
const yoursRoi =
args["yours-roi"] !== undefined
? parseRoi(args["yours-roi"], b.width, b.height)
: roi && scaled(roi);
const r = compareEnergy(
energyCurve(videoClip(a.path, start, seconds, a), roi),
energyCurve(videoClip(b.path, yoursStart, seconds, b), yoursRoi),
);
return {
reference: a.path,
yours: b.path,
window_s: round(seconds, 3),
...r,
reading:
"mean_ratio is how much yours moves (0.8–1.25 is the same amount); r near 1 at a small lag_s is the same action at the same time",
};
});
},
});
+190
View File
@@ -0,0 +1,190 @@
import { writeFileSync } from "node:fs";
import { join } from "node:path";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
num,
outArg,
roiArg,
runMeasure,
videoAndRoi,
videoArg,
window,
} from "../../measure/command.js";
import {
countEvents,
detectEvents,
drawStrip,
lookWindows,
type ScreenEvent,
} from "../../measure/events.js";
import { Fix, outDir, round, type VideoInfo } from "../../measure/media.js";
export const examples: Example[] = [
[
"The film's event map: cuts, flashes, what appears where, holds, and the sound's hits, grouped into windows to look at",
"hyperframes measure events ref.mp4 --map",
],
[
"When one region changes, frame by frame",
"hyperframes measure events ref.mp4 --roi 0,800,1080,300 --start 2 --end 6",
],
];
/** "6x10" as columns and rows. */
function parseGrid(text: unknown): { cols: number; rows: number } {
const m = /^(\d+)x(\d+)$/.exec(String(text));
if (!m || Number(m[1]) < 1 || Number(m[2]) < 1)
throw new Fix(`--grid must be COLSxROWS, got "${String(text)}"`, "--grid 6x10");
return { cols: Number(m[1]), rows: Number(m[2]) };
}
const isRecord = (v: unknown): v is Record<string, unknown> =>
typeof v === "object" && v !== null && !Array.isArray(v);
const numbers = (v: unknown): number[] =>
Array.isArray(v) ? v.filter((x): x is number => typeof x === "number") : [];
interface Sound {
bpm: number | null;
beats: number[];
onsets: { t: number; strength: number }[];
}
/** The sound's onsets, beats and tempo, measured on this machine. */
async function soundOf(info: VideoInfo, start: number, end: number): Promise<Sound | null> {
if (!info.audio) return null;
const { analyzeTargets, previewTarget } = await import("../../audio/analysis/run.js");
const report = await analyzeTargets([await previewTarget(info.path, info.path, null)], {
capabilities: ["onsets", "beats", "bpm"],
quality: "local",
});
const item = report.items[0];
if (!item) return null;
const inside = (t: number) => t >= start && t <= end;
const onsets =
isRecord(item.onsets) && Array.isArray(item.onsets.events) ? item.onsets.events : [];
return {
bpm: isRecord(item.bpm) && typeof item.bpm.value === "number" ? round(item.bpm.value, 1) : null,
beats: numbers(isRecord(item.beats) ? item.beats.times : undefined).filter(inside),
onsets: onsets
.filter(isRecord)
.flatMap((e) =>
typeof e.t === "number" && typeof e.strength === "number" && inside(e.t)
? [{ t: e.t, strength: e.strength }]
: [],
),
};
}
const LIST_MAX = 64;
export default defineCommand({
meta: {
name: "events",
description:
"When things change on screen: cuts, flashes, appears, disappears, changes and holds; --map adds the sound and the windows worth looking at",
},
args: {
video: videoArg,
map: {
type: "boolean",
description:
"The whole window on a 4x4 grid with the sound's onsets and beats, grouped into windows to look at",
default: false,
},
roi: roiArg,
grid: {
type: "string",
description: "Split the frame into COLSxROWS cells, e.g. 6x10 (default with --map: 4x4)",
},
start: { type: "string", description: "First second (default 0)" },
end: { type: "string", description: "Last second (default: the end)" },
thr: {
type: "string",
description: "Mean grey-level change of an appear or a cut (default 5)",
},
out: outArg,
json: jsonArg,
},
async run({ args }) {
// fallow-ignore-next-line complexity
await runMeasure(async () => {
const { info, roi } = videoAndRoi(args.video, args.roi);
const [start, end] = window(info, args.start, args.end);
const grid =
args.grid !== undefined
? parseGrid(args.grid)
: args.map && args.roi === undefined
? { cols: 4, rows: 4 }
: undefined;
const thr = num(args.thr, "--thr", 5);
const found = detectEvents({ info, start, end, roi, grid, thr });
if (found.frames < 3)
throw new Fix(`under 3 frames between ${start}s and ${end}s`, "widen --start/--end");
const dir = outDir(args.out, "measure/events");
const name = `events-${start}-${round(end, 2)}s`;
const strip = found.shown.length ? drawStrip(found, join(dir, `${name}.png`)) : null;
const holds = found.events.filter((e) => e.type === "hold");
const head = {
video: info.path,
...(grid ? { grid: `${grid.cols}x${grid.rows}` } : { roi }),
start,
end: round(end, 3),
fps: round(info.fps, 3),
};
if (!args.map) {
const lengths = holds.map((e) => e.seconds ?? 0).sort((a, b) => a - b);
return {
...head,
frames: found.frames,
thr,
counts: countEvents(found.events),
holds: {
longest_s: lengths.at(-1) ?? 0,
median_s: lengths[(lengths.length - 1) >> 1] ?? 0,
count: lengths.length,
},
strip,
strip_events: `${found.shown.length} of ${found.events.length - holds.length} (cuts and flashes, then the largest steps, then changes)`,
events: found.events,
};
}
const sound = await soundOf(info, start, end);
const cuts = found.events.filter((e) => e.type === "cut").map((e) => e.t);
const edges = [start, ...cuts, end];
const windows = lookWindows(found.events, info.fps, sound?.onsets.map((o) => o.t) ?? []);
const file = join(dir, `${name}-map.json`);
writeFileSync(
file,
JSON.stringify({ ...head, events: found.events, sound, windows }, null, 1),
);
const strong = sound?.onsets.filter((o) => o.strength >= 0.5).map((o) => round(o.t, 3)) ?? [];
const longHolds = holds
.filter((e: ScreenEvent) => (e.seconds ?? 0) >= 0.5)
.map((e) => ({ t: e.t, seconds: e.seconds }));
return {
...head,
duration: round(end - start, 3),
counts: countEvents(found.events),
shots: edges.slice(1).map((e, i) => ({ start: round(edges[i]!, 3), end: round(e, 3) })),
holds: longHolds,
sound: sound
? {
bpm: sound.bpm,
beats: sound.beats.length,
...(sound.beats.length
? { beat_times: sound.beats.slice(0, LIST_MAX).map((t) => round(t, 3)) }
: {}),
onsets: sound.onsets.length,
strong_onsets: strong.slice(0, LIST_MAX),
}
: "none",
windows,
strip,
file,
next: "look at each window's frames: npx hyperframes measure frames <video> --events",
};
});
},
});
@@ -0,0 +1,274 @@
// Pixels from the original, as project media: a still at one moment or a clip over a window, cropped to a region,
// frame-exact, recorded in the media ledger as cut from the original. A rebuild reuses footage, photos and people
// this way instead of drawing or generating them.
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, relative, resolve } from "node:path";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, roiArg, runMeasure, videoAndRoi, videoArg } from "../../measure/command.js";
import { Fix, frameAt, round, runFfmpeg, type Roi, type VideoInfo } from "../../measure/media.js";
export const examples: Example[] = [
[
"A photo from the original at 3.2 s, cropped to its box",
"hyperframes measure extract .reverse/source.mp4 --at 3.2 --roi 120,340,840,620 --out assets/photo.png",
],
[
"The original's whole soundtrack",
"hyperframes measure extract .reverse/source.mp4 --out assets/sound.m4a",
],
[
"The talking head from 4.0 s to 9.5 s with its sound",
"hyperframes measure extract .reverse/source.mp4 --from 4 --to 9.5 --roi 0,0,1080,1350 --audio --out assets/presenter.mp4",
],
];
/** H.264 needs even sizes: shrink an odd width or height by a pixel. */
const even = ([x, y, w, h]: Roi): Roi => [x, y, w - (w % 2), h - (h % 2)];
/** The nearest folder at or above `from` holding index.html: the project the file belongs to. */
function projectOf(from: string): string | null {
for (let dir = resolve(from); ; dir = dirname(dir)) {
if (existsSync(join(dir, "index.html"))) return dir;
if (dirname(dir) === dir) return null;
}
}
async function record(
out: string,
type: "image" | "video" | "bgm",
provenance: Record<string, unknown>,
duration?: number,
) {
const project = projectOf(dirname(out));
if (!project) return null;
const { recordInPlace } = await import("../../media-use/lib/manifest.mjs");
const { regenerateIndex } = await import("../../media-use/lib/index-gen.mjs");
const rel = (p: string) => relative(project, p).split("\\").join("/");
const path = rel(out);
const from = String(provenance.from);
if (!rel(from).startsWith("..")) provenance = { ...provenance, from: rel(from) };
const entry = recordInPlace(project, {
type,
path,
source: "extracted",
description: `cut from the original ${String(provenance.window ?? provenance.at)}`,
...(duration !== undefined ? { duration } : {}),
provenance,
});
regenerateIndex(project);
return entry.id;
}
const crop = (roi: Roi) => `crop=${roi[2]}:${roi[3]}:${roi[0]}:${roi[1]}`;
/** The frame shown at `at` seconds, cropped, as a picture. */
export async function cutStill(info: VideoInfo, at: number, roi: Roi, out: string) {
const k = frameAt(at, info.fps);
const t = k / info.fps;
const { extractVideoFrameToBuffer } = await import("../snapshot.js");
const png = await extractVideoFrameToBuffer(info.path, t);
if (!png)
throw new Fix(`no frame at ${round(t, 3)}s`, `pick a time before ${round(info.duration, 2)}s`);
const work = mkdtempSync(join(tmpdir(), "hf-extract-"));
try {
const full = join(work, "frame.png");
writeFileSync(full, png);
runFfmpeg(["-y", "-i", full, "-vf", crop(roi), "-frames:v", "1", out]);
} finally {
rmSync(work, { recursive: true, force: true });
}
return { at: round(t, 3), frame: k };
}
/** The frames shown from `from` to `to` seconds, cropped (to even sizes), as an H.264 clip. */
export function cutClip(
info: VideoInfo,
from: number,
to: number,
asked: Roi,
out: string,
opts: { audio?: boolean; crf?: number } = {},
) {
const k0 = frameAt(from, info.fps);
const k1 = Math.min(info.frames, Math.ceil(to * info.fps - 1e-6));
if (k1 <= k0) throw new Fix("--to is not after --from", "--from 4 --to 9.5");
const roi = even(asked);
const [a, n] = [k0 / info.fps, k1 - k0];
const sound = Boolean(opts.audio && info.audio);
// Frames picked by index, as diff and the event map count them: a seek keeps whichever frame is showing at the
// seek time, which at some rates is the one before k0.
runFfmpeg([
...["-y", "-i", info.path],
...[
"-vf",
`trim=start_frame=${k0}:end_frame=${k1},setpts=PTS-STARTPTS,${crop(roi)},format=yuv420p`,
],
...["-r", info.fpsText, "-c:v", "libx264", "-crf", String(opts.crf ?? 14), "-preset", "medium"],
...(sound
? [
"-af",
`atrim=start=${a}:end=${k1 / info.fps},asetpts=PTS-STARTPTS`,
"-c:a",
"aac",
"-b:a",
"192k",
]
: ["-an"]),
...["-movflags", "+faststart", out],
]);
return {
window: [round(a, 3), round(k1 / info.fps, 3)],
frames: n,
duration: round(n / info.fps, 3),
roi,
audio: sound,
};
}
/** The sound from `from` to `to` seconds as an audio file, on the same clock as the picture. */
function cutSound(info: VideoInfo, from: number, to: number, out: string) {
if (!info.audio) throw new Fix(`${info.path} has no sound`, "leave <audio> out of the rebuild");
const [a, b] = [Math.max(0, from), Math.min(info.duration, to)];
if (b <= a) throw new Fix("--to is not after --from", "--from 0 --to 12");
const codec = /\.wav$/i.test(out)
? ["-c:a", "pcm_s16le"]
: /\.mp3$/i.test(out)
? ["-c:a", "libmp3lame", "-q:a", "2"]
: ["-c:a", "aac", "-b:a", "256k"];
runFfmpeg(["-y", "-ss", String(a), "-i", info.path, "-t", String(b - a), "-vn", ...codec, out]);
return { window: [round(a, 3), round(b, 3)], duration: round(b - a, 3) };
}
export default defineCommand({
meta: {
name: "extract",
description:
"Cut a still (--at) or a clip (--from/--to) out of a video, cropped to a region, as project media",
},
args: {
video: videoArg,
out: {
type: "string",
description: "The file to write: .png/.jpg for a still, .mp4 for a clip",
required: true,
},
at: { type: "string", description: "A still of the frame shown at this second" },
from: { type: "string", description: "A clip's first second" },
to: { type: "string", description: "A clip's end second" },
audio: {
type: "boolean",
description: "Keep the clip's sound (default: picture only)",
default: false,
},
crf: { type: "string", description: "Clip quality, lower is better (default 14)" },
roi: roiArg,
json: jsonArg,
},
async run({ args }) {
// fallow-ignore-next-line complexity
await runMeasure(async () => {
const { info, roi: asked } = videoAndRoi(args.video, args.roi);
const out = resolve(String(args.out));
mkdirSync(dirname(out), { recursive: true });
if (/\.(m4a|aac|mp3|wav)$/i.test(out)) {
const sound = cutSound(
info,
num(args.from, "--from", 0),
num(args.to, "--to", info.duration),
out,
);
const ledger = await record(
out,
"bgm",
{ provider: "original", from: info.path, window: sound.window },
sound.duration,
);
return {
out,
kind: "sound",
...sound,
...(ledger ? { ledger } : {}),
place: {
"data-start": sound.window[0],
"data-duration": sound.duration,
note: "an <audio id> with this src",
},
};
}
const isClip = args.from !== undefined || args.to !== undefined;
if (isClip === (args.at !== undefined))
throw new Fix(
"pass --at for a still, or --from and --to for a clip",
"--at 3.2, or --from 4 --to 9.5",
);
if (!isClip) {
if (!/\.(png|jpe?g|webp)$/i.test(out))
throw new Fix(`${out} is not a picture`, "--out assets/name.png");
const { at, frame: k } = await cutStill(info, num(args.at, "--at", 0), asked, out);
const ledger = await record(out, "image", {
provider: "original",
from: info.path,
at,
roi: asked,
});
return {
out,
kind: "image",
at,
frame: k,
roi: asked,
...(ledger ? { ledger } : {}),
place: { left: asked[0], top: asked[1], width: asked[2], height: asked[3] },
};
}
if (!/\.(mp4|mov|webm)$/i.test(out))
throw new Fix(`${out} is not a video`, "--out assets/name.mp4");
const clip = cutClip(
info,
num(args.from, "--from", 0),
num(args.to, "--to", info.duration),
asked,
out,
{
audio: args.audio,
crf: num(args.crf, "--crf", 14),
},
);
const { window, duration, roi } = clip;
const askedEnd = num(args.to, "--to", info.duration);
const ledger = await record(
out,
"video",
{ provider: "original", from: info.path, window, roi, audio: clip.audio },
duration,
);
return {
out,
kind: "clip",
window,
frames: clip.frames,
duration,
roi,
audio: clip.audio,
...(askedEnd > info.duration + 0.5 / info.fps
? {
note: `--to ${askedEnd} is past the end; the clip stops at ${round(info.duration, 3)} s`,
}
: {}),
...(ledger ? { ledger } : {}),
place: {
"data-start": window[0],
"data-duration": duration,
left: roi[0],
top: roi[1],
width: roi[2],
height: roi[3],
note: "a <video> clip at the original's time and box; muted unless it carries the film's sound",
},
};
});
},
});
+50
View File
@@ -0,0 +1,50 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, runMeasure, videoArg, window } from "../../measure/command.js";
import { faceBoxes, faceSummary } from "../../measure/face.js";
import { probeVideo } from "../../measure/media.js";
export const examples: Example[] = [
[
"How big and where the presenter's face is in a shot",
"hyperframes measure face ref.mp4 --start 4 --end 9.5",
],
];
export default defineCommand({
meta: {
name: "face",
description:
"How big the face is and where, over a window: its box per sampled frame and whether a generated presenter can keep the likeness at that size",
},
args: {
video: videoArg,
start: { type: "string", description: "First second (default 0)" },
end: { type: "string", description: "Last second (default: the end)" },
samples: {
type: "string",
description: "Frames sampled, evenly, the last one included (default 12)",
},
cast: {
type: "boolean",
description: "The person is one face in a crowd: grade the smallest face",
default: false,
},
json: jsonArg,
},
async run({ args }) {
await runMeasure(async () => {
const info = probeVideo(String(args.video));
const [a, b] = window(info, args.start, args.end);
const samples = await faceBoxes(info, a, b, Math.max(1, num(args.samples, "--samples", 12)));
return {
video: info.path,
start: a,
end: b,
...faceSummary(samples, args.cast === true),
reading:
"face_height is the face's height over the frame's; boxes are x, y, w, h as shares of the frame",
};
});
},
});
+91
View File
@@ -0,0 +1,91 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
num,
roiArg,
runMeasure,
videoAndRoi,
videoArg,
window,
} from "../../measure/command.js";
import { compareFlow, flowMoves, flowSeries } from "../../measure/flow.js";
import { parseRoi, probeVideo, type Roi } from "../../measure/media.js";
export const examples: Example[] = [
[
"A card's moves from optical flow, as tweens",
"hyperframes measure flow ref.mp4 --start 1 --end 2.4 --roi 300,200,640,400",
],
[
"Does the rebuild's element move like the original's",
"hyperframes measure flow ref.mp4 --start 1 --end 2.4 --roi 300,200,640,400 --compare renders/v1.mp4",
],
];
export default defineCommand({
meta: {
name: "flow",
description:
"One element's motion from optical flow inside its box: its moves as tweens, or compared with another video's (energy, timing, path, scale, rotation)",
},
args: {
video: videoArg,
start: { type: "string", description: "First second", required: true },
end: { type: "string", description: "Last second", required: true },
roi: roiArg,
compare: {
type: "string",
description: "Another video over the same window, on the same clock",
},
"compare-start": {
type: "string",
description: "Second the window starts in it (default: --start)",
},
"compare-roi": { type: "string", description: "Its box (default: --roi scaled by the widths)" },
json: jsonArg,
},
async run({ args }) {
await runMeasure(async () => {
const { info, roi } = videoAndRoi(args.video, args.roi);
const [a, b] = window(info, args.start, args.end);
if (args.compare === undefined) {
const rows = await flowSeries(info, a, b, roi, 320);
return {
video: info.path,
start: a,
end: b,
roi,
moves: flowMoves(rows, info.fps, roi, a),
};
}
const other = probeVideo(String(args.compare));
// Each axis scales on its own: a generated shot is rarely the original's exact aspect (768x1344 for 9:16).
const [kx, ky] = [other.width / info.width, other.height / info.height];
const scaled: Roi = [
Math.round(roi[0] * kx),
Math.round(roi[1] * ky),
Math.round(roi[2] * kx),
Math.round(roi[3] * ky),
];
const otherRoi =
args["compare-roi"] !== undefined
? parseRoi(args["compare-roi"], other.width, other.height)
: scaled;
const oa = num(args["compare-start"], "--compare-start", a);
const [ref, yours] = await Promise.all([
flowSeries(info, a, b, roi, 160),
flowSeries(other, oa, oa + (b - a), otherRoi, 160),
]);
return {
video: info.path,
compare: other.path,
start: a,
end: b,
...compareFlow(ref, yours),
reading:
"energy_ratio near 1 moves as much; motion_corr near 1 at lag_frames 0 moves at the same moments; path, scale and rotation errors are per frame",
};
});
},
});
+338
View File
@@ -0,0 +1,338 @@
import { mkdtempSync, readdirSync, renameSync, rmSync } from "node:fs";
import { join } from "node:path";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
num,
outArg,
roiArg,
runMeasure,
videoAndRoi,
videoArg,
window,
} from "../../measure/command.js";
import { detectEvents, lookWindows } from "../../measure/events.js";
import {
Fix,
frameAt,
outDir,
parseNumbers,
parseRange,
readClip,
round,
runFfmpeg,
videoClip,
type Frames,
type Roi,
type VideoInfo,
} from "../../measure/media.js";
export const examples: Example[] = [
["An overview sheet, one frame a second", "hyperframes measure frames ref.mp4 --every 1"],
[
"The consecutive frames around every event, one strip per window",
"hyperframes measure frames ref.mp4 --events",
],
[
"Every frame of a transition, at full size",
"hyperframes measure frames ref.mp4 --range 2.1,2.6 --width 0",
],
[
"Two moments, cropped to read the text",
"hyperframes measure frames ref.mp4 --at 1.2,3.4 --roi 100,800,880,200",
],
];
const TILE_W = 360;
const SHEET_TILES = 24;
const STRIP_MAX = 12;
const WINDOWS_MAX = 60;
/** Runs of consecutive indexes, for a short select expression. */
function runsOf(ks: number[]): [number, number][] {
const out: [number, number][] = [];
for (const k of ks) {
const last = out.at(-1);
if (last && k === last[1] + 1) last[1] = k;
else out.push([k, k]);
}
return out;
}
/** The frames at these indexes (sorted, unique), in order, from one decode pass. */
function framesAt(info: VideoInfo, ks: number[], roi: Roi, width: number): Frames {
const expr = runsOf(ks)
.map(([a, b]) => (a === b ? `eq(n\\,${a})` : `between(n\\,${a}\\,${b})`))
.join("+");
// Renumbered so the raw output is exactly the picked frames, no gap filled by a repeat.
const clip = {
...videoClip(info.path, 0, undefined, info),
filters: [`select=${expr}`, `setpts=N/(${info.fpsText})/TB`],
};
const full = roi[0] === 0 && roi[1] === 0 && roi[2] === info.width && roi[3] === info.height;
return readClip(clip, { ...(full ? {} : { roi }), width: width || roi[2] });
}
/** Writes frames as numbered PNGs in one ffmpeg call; returns their paths in order. */
function writeFrames(
f: Frames,
from: number,
count: number,
dir: string,
prefix: string,
): string[] {
// Beside the output, so the move below stays on one filesystem (a container's /tmp often is not the project's).
const work = mkdtempSync(join(dir, ".frames-"));
try {
const raw = Buffer.concat(
f.frames
.slice(from, from + count)
.map((px) => Buffer.from(px.buffer, px.byteOffset, px.byteLength)),
);
runFfmpeg(
[
"-y",
"-f",
"rawvideo",
"-pix_fmt",
"rgb24",
"-s",
`${f.w}x${f.h}`,
"-i",
"-",
"-start_number",
"0",
join(work, "%04d.png"),
],
raw,
);
return readdirSync(work)
.sort()
.map((name, i) => {
const to = join(dir, `${prefix}-${String(i).padStart(2, "0")}.png`);
renameSync(join(work, name), to);
return to;
});
} finally {
rmSync(work, { recursive: true, force: true });
}
}
async function sheet(
paths: string[],
labels: string[],
out: string,
width: number,
): Promise<string | null> {
const { createContactSheet } = await import("../../capture/contactSheet.js");
return createContactSheet(paths, out, {
cols: Math.min(6, paths.length),
maxImages: paths.length,
cellWidth: width,
labelMode: "custom",
labels,
});
}
/**
* Up to `max` of a window's frames: all when they fit, else the frame before, at and after each event's onset
* (uniform thinning turns a cut into a dissolve and drops one-frame flashes), the rest evenly spaced.
*/
function spread(
k0: number,
k1: number,
max: number,
onsets: number[] = [],
): { ks: number[]; step: number } {
const n = k1 - k0 + 1;
if (n <= max) return { ks: Array.from({ length: n }, (_, i) => k0 + i), step: 1 };
const must = new Set(onsets.flatMap((k) => [k - 1, k, k + 1]).filter((k) => k >= k0 && k <= k1));
// At least half the strip stays evenly spaced, so what happens between the onsets is seen too.
const room = Math.max(Math.ceil(max / 2), max - must.size);
for (let i = 0; i < room; i++) must.add(k0 + Math.round((i * (n - 1)) / (room - 1)));
const ks = [...must].sort((a, b) => a - b);
return { ks, step: Math.ceil((n - 1) / Math.max(1, ks.length - 1)) };
}
const label = (k: number, fps: number) => `f${k} · ${round(k / fps, 3)}s`;
export default defineCommand({
meta: {
name: "frames",
description:
"Frames of a video as pictures: chosen moments, an overview sheet, consecutive native frames of a range, or strips around every event",
},
args: {
video: videoArg,
at: { type: "string", description: "Seconds, e.g. 1.2,3.4: one picture each" },
every: {
type: "string",
description: "An overview: one frame every N seconds on sheets of 24",
},
range: {
type: "string",
description: "start,end seconds: consecutive native frames as one strip",
},
events: {
type: "boolean",
description: "One strip of consecutive frames per window from measure events --map",
default: false,
},
start: { type: "string", description: "With --events or --every: first second (default 0)" },
end: {
type: "string",
description: "With --events or --every: last second (default: the end)",
},
max: {
type: "string",
description: `Frames per strip (default ${STRIP_MAX}; longer ranges are evenly thinned)`,
},
width: {
type: "string",
description: `Picture width in pixels (default ${TILE_W}; 0 keeps the ROI's size)`,
},
roi: roiArg,
out: outArg,
json: jsonArg,
},
async run({ args }) {
// fallow-ignore-next-line complexity
await runMeasure(async () => {
const modes = [args.at, args.every, args.range, args.events || undefined].filter(
(v) => v !== undefined,
);
if (modes.length !== 1)
throw new Fix(
"pass exactly one of --at, --every, --range or --events",
"npx hyperframes measure frames ref.mp4 --every 1",
);
const { info, roi } = videoAndRoi(args.video, args.roi);
const dir = outDir(args.out, "measure/frames");
const width = num(args.width, "--width", TILE_W);
const max = num(args.max, "--max", STRIP_MAX);
const last = info.frames - 1;
const head = {
video: info.path,
fps: round(info.fps, 3),
...(args.roi !== undefined ? { roi } : {}),
};
if (args.at !== undefined) {
const ks = [
...new Set(
parseNumbers(args.at, "--at").map((t) => Math.min(last, frameAt(t, info.fps))),
),
].sort((a, b) => a - b);
const f = framesAt(info, ks, roi, num(args.width, "--width", 0));
const paths = writeFrames(f, 0, ks.length, dir, "at");
return {
...head,
frames: ks.map((k, i) => ({ t: round(k / info.fps, 3), frame: k, path: paths[i] })),
};
}
if (args.every !== undefined) {
const step = num(args.every, "--every", 1);
const [a, b] = window(info, args.start, args.end);
const ks = [
...new Set(
Array.from({ length: Math.floor((b - a) / step) + 1 }, (_, i) =>
Math.min(last, frameAt(a + i * step, info.fps)),
),
),
];
const f = framesAt(info, ks, roi, width);
const sheets: string[] = [];
for (let s = 0; s < ks.length; s += SHEET_TILES) {
const part = ks.slice(s, s + SHEET_TILES);
const paths = writeFrames(f, s, part.length, dir, `every-${s / SHEET_TILES}`);
const out = await sheet(
paths,
part.map((k) => `${round(k / info.fps, 2)}s`),
join(dir, `every-${step}s-${s / SHEET_TILES + 1}.jpg`),
f.w,
);
for (const p of paths) rmSync(p, { force: true });
if (out) sheets.push(out);
}
return { ...head, every_s: step, frames: ks.length, sheets };
}
if (args.range !== undefined) {
const [a, b] = parseRange(args.range);
const k0 = frameAt(a, info.fps);
const k1 = Math.min(last, Math.max(k0, Math.ceil(b * info.fps - 1e-6) - 1));
const { ks, step } = spread(k0, k1, max);
const f = framesAt(info, ks, roi, width);
const paths = writeFrames(f, 0, ks.length, dir, "range");
const out = await sheet(
paths,
ks.map((k) => label(k, info.fps)),
join(dir, `range-${a}-${b}s.jpg`),
f.w,
);
for (const p of paths) rmSync(p, { force: true });
return { ...head, start: a, end: b, frames: ks.length, every_nth_frame: step, strip: out };
}
// --events: the windows the event map groups, each as a strip of its consecutive frames.
const [a, b] = window(info, args.start, args.end);
const found = detectEvents({
info,
start: a,
end: b,
roi: [0, 0, info.width, info.height],
grid: { cols: 4, rows: 4 },
});
const windows = lookWindows(found.events, info.fps).slice(0, WINDOWS_MAX);
const picks = windows.map((wnd) =>
spread(
frameAt(wnd.start, info.fps),
Math.min(last, frameAt(wnd.end, info.fps)),
max,
wnd.onset_frames,
),
);
const all = [...new Set(picks.flatMap((p) => p.ks))].sort((x, y) => x - y);
if (!all.length)
return {
...head,
windows: [],
note: "nothing changes on screen: one overview frame is enough (--every)",
};
const f = framesAt(info, all, roi, width);
const strips = [];
for (const [i, wnd] of windows.entries()) {
const { ks, step } = picks[i]!;
const from = all.indexOf(ks[0]!);
const paths = writeFrames(f, from, ks.length, dir, `w${i + 1}`);
const out = await sheet(
paths,
ks.map((k) => label(k, info.fps)),
join(dir, `window-${String(i + 1).padStart(2, "0")}-${wnd.start}s.jpg`),
f.w,
);
for (const p of paths) rmSync(p, { force: true });
strips.push({
start: wnd.start,
end: wnd.end,
what: wnd.what,
frames: ks.length,
...(step > 1 ? { thinned: `onsets kept, about every ${step}th frame between` } : {}),
strip: out,
});
}
const total = lookWindows(found.events, info.fps).length;
return {
...head,
windows: strips,
...(total > WINDOWS_MAX
? {
note: `${total} windows; the first ${WINDOWS_MAX} shown: use --start/--end for the rest`,
}
: {}),
};
});
},
});
+58
View File
@@ -0,0 +1,58 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, runMeasure, videoAndRoi, videoArg, window } from "../../measure/command.js";
import { scan, summary } from "../../measure/ink.js";
import { Fix } from "../../measure/media.js";
export const examples: Example[] = [
[
"When a caption is on screen and how tall its letters are",
"hyperframes measure ink .reverse/source.mp4 --roi 60,1500,960,260 --start 0 --end 6",
],
[
"The ink box of dark text at one time",
"hyperframes measure ink .reverse/source.mp4 --roi 100,80,880,200 --at 2.4 --ink dark",
],
];
export default defineCommand({
meta: {
name: "ink",
description:
"Where text ink sits in a region: first and last visible time, widest box and median height (for font size)",
},
args: {
video: videoArg,
roi: { type: "string", description: "Region x,y,w,h holding the text", required: true },
at: { type: "string", description: "One time in seconds" },
start: { type: "string", description: "Range start in seconds (default 0)" },
end: { type: "string", description: "Range end in seconds (default: the end)" },
fps: { type: "string", description: "Samples per second over a range (10)" },
ink: { type: "string", description: "light, dark or R,G,B (default light)", default: "light" },
tol: { type: "string", description: "Distance that still counts as ink (60)" },
json: jsonArg,
},
run: ({ args }) =>
runMeasure(() => {
const { info, roi } = videoAndRoi(args.video, args.roi);
const opts = { ink: args.ink ?? "light", tol: num(args.tol, "--tol", 60) };
if (args.at !== undefined) {
if (args.start !== undefined)
throw new Fix("--at and --start both given", "pass one of them");
const t = num(args.at, "--at", 0);
const [row] = scan(info.path, t, 1.5 / info.fps, roi, { ...opts, fps: 0 });
if (!row) throw new Fix(`no frame at ${t}s`, "pick a time inside the video");
return { video: info.path, roi, ...row };
}
const [start, end] = window(info, args.start, args.end);
const fps = num(args.fps, "--fps", 10);
return {
video: info.path,
roi,
start,
end,
fps,
...summary(scan(info.path, start, end - start, roi, { ...opts, fps })),
};
}),
});
+114
View File
@@ -0,0 +1,114 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
num,
roiArg,
runMeasure,
videoAndRoi,
videoArg,
window,
} from "../../measure/command.js";
import { compareLook, look, sameLook } from "../../measure/look.js";
import { Fix, parseRoi, probeVideo, readFrameAt, videoClip } from "../../measure/media.js";
export const examples: Example[] = [
["The grade of the first 5 seconds", "hyperframes measure look ref.mp4 --start 0 --end 5"],
[
"Does the rebuild's grade match the original's",
"hyperframes measure look ref.mp4 --compare renders/v1.mp4",
],
];
export default defineCommand({
meta: {
name: "look",
description:
"Brightness, contrast, saturation, black-and-white share and warmth of a window, or compared with another video",
},
args: {
video: videoArg,
start: { type: "string", description: "First second (default 0)" },
end: { type: "string", description: "Last second (default: the end)" },
fps: { type: "string", description: "Frames sampled per second (default 2)" },
roi: roiArg,
at: { type: "string", description: "With --grid: the moment to read" },
grid: {
type: "string",
description: "With --at: the mean colour of each cell, COLSxROWS, e.g. 8x6",
},
compare: { type: "string", description: "Another video measured over the same window" },
"compare-start": {
type: "string",
description: "Second the window starts in it (default: --start)",
},
"compare-roi": { type: "string", description: "Region of it (default: the whole frame)" },
json: jsonArg,
},
async run({ args }) {
await runMeasure(() => {
const { info, roi } = videoAndRoi(args.video, args.roi);
if (args.grid !== undefined || args.at !== undefined) {
const m = /^(\d+)x(\d+)$/.exec(String(args.grid ?? ""));
if (!m || args.at === undefined)
throw new Fix("--grid and --at go together", "--at 2 --grid 8x6");
const [cols, rows] = [Number(m[1]), Number(m[2])];
const at = num(args.at, "--at", 0);
const { frames, w, h } = readFrameAt(info, at, { roi, width: Math.min(640, roi[2]) });
const px = frames[0];
if (!px) throw new Fix(`no frame at ${at}s`, "pick a time inside the video");
const hex = (v: number[]) =>
`#${v.map((c) => Math.round(c).toString(16).padStart(2, "0")).join("")}`;
const colors = Array.from({ length: rows }, (_, r) =>
Array.from({ length: cols }, (_, c) => {
const [x0, x1, y0, y1] = [
Math.floor((c * w) / cols),
Math.floor(((c + 1) * w) / cols),
Math.floor((r * h) / rows),
Math.floor(((r + 1) * h) / rows),
];
const sum = [0, 0, 0];
for (let y = y0; y < y1; y++)
for (let x = x0; x < x1; x++)
for (let k = 0; k < 3; k++) sum[k]! += px[(y * w + x) * 3 + k]!;
const n = Math.max(1, (x1 - x0) * (y1 - y0));
return hex(sum.map((v) => v / n));
}),
);
return {
video: info.path,
at,
roi,
grid: `${cols}x${rows}`,
colors,
reading: "rows top to bottom, each cell's mean colour",
};
}
const [a, b] = window(info, args.start, args.end);
const fps = num(args.fps, "--fps", 2);
const ref = look(videoClip(info.path, a, b - a, info), fps, roi);
if (args.compare === undefined) return { video: info.path, start: a, end: b, ...ref };
const other = probeVideo(String(args.compare));
const otherRoi =
args["compare-roi"] === undefined
? undefined
: parseRoi(args["compare-roi"], other.width, other.height);
const theirs = look(
videoClip(other.path, num(args["compare-start"], "--compare-start", a), b - a, other),
fps,
otherRoi,
);
const diff = compareLook(ref, theirs);
return {
video: info.path,
compare: other.path,
start: a,
end: b,
reference: ref,
other: theirs,
...diff,
same_look: sameLook(diff),
};
});
},
});
@@ -0,0 +1,38 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, roiArg, runMeasure, videoAndRoi, videoArg } from "../../measure/command.js";
import { objects } from "../../measure/objects.js";
export const examples: Example[] = [
[
"How many elements are on screen and how they are arranged (ring, grid, row, stack)",
"hyperframes measure objects .reverse/source.mp4 --at 2.4 --roi 300,200,1320,680",
],
];
export default defineCommand({
meta: {
name: "objects",
description:
"Separate elements at one time: count, sizes and arrangement (ring, grid, row, column, stack) as shares of the frame",
},
args: {
video: videoArg,
at: { type: "string", description: "Time in seconds", required: true },
roi: roiArg,
tol: {
type: "string",
description: "Distance from the background that counts as an element (40)",
},
json: jsonArg,
},
run: ({ args }) =>
runMeasure(() => {
const { info, roi } = videoAndRoi(args.video, args.roi);
return {
video: info.path,
roi,
...objects(info.path, info, num(args.at, "--at", 0), roi, num(args.tol, "--tol", 40)),
};
}),
});
+129
View File
@@ -0,0 +1,129 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, runMeasure, videoArg } from "../../measure/command.js";
import {
ffmpegBin,
Fix,
probeVideo,
round,
runFfmpeg,
type VideoInfo,
} from "../../measure/media.js";
import { execFileSync } from "node:child_process";
import { mkdirSync } from "node:fs";
import { dirname, resolve } from "node:path";
export const examples: Example[] = [
[
"Size, frame rate, sound, and what to fix before rebuilding",
"hyperframes measure probe ref.mp4",
],
[
"Make a constant-rate SDR working copy of a phone or HDR video",
"hyperframes measure probe ref.mp4 --normalize .reverse/source.mp4",
],
];
// Sections longer than this are analysed and checked one at a time.
const SECTION_S = 60;
const STANDARD = ["24000/1001", "24", "25", "30000/1001", "30", "50", "60000/1001", "60"];
const rateOf = (text: string) => {
const [a, b] = text.split("/").map(Number);
return (a ?? 0) / (b || 1);
};
/** The standard frame rate closest to a variable-rate video's average. */
const standardRate = (info: VideoInfo) =>
STANDARD.reduce((best, r) =>
Math.abs(rateOf(r) - info.avgFps) < Math.abs(rateOf(best) - info.avgFps) ? r : best,
);
const hasFilter = (name: string) =>
new RegExp(`\\s${name}\\s`).test(
execFileSync(ffmpegBin(), ["-hide_banner", "-filters"], { windowsHide: true }).toString(),
);
/** A constant-rate SDR H.264 copy: what the measuring tools, diff and the render all read the same way. */
function normalize(info: VideoInfo, out: string): { path: string; fps: string } {
const fps = info.vfr ? standardRate(info) : info.fpsText;
const filters = [`fps=${fps}`];
if (info.hdr) {
if (!hasFilter("zscale"))
throw new Fix(
"this ffmpeg cannot tone-map HDR (no zscale filter)",
"install an ffmpeg built with libzimg, e.g. brew install ffmpeg",
);
filters.push(
"zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv",
);
}
mkdirSync(dirname(out), { recursive: true });
runFfmpeg([
...["-y", "-i", info.path, "-vf", [...filters, "format=yuv420p"].join(",")],
...["-c:v", "libx264", "-crf", "12", "-preset", "medium", "-c:a", "aac", "-b:a", "192k", out],
]);
return { path: out, fps };
}
export default defineCommand({
meta: {
name: "probe",
description:
"Size (as displayed), frame rate, duration, sound, and what to fix before measuring or rebuilding",
},
args: {
video: videoArg,
normalize: {
type: "string",
description:
"Write a constant-frame-rate SDR copy here (for variable-rate or HDR videos); the original is left alone",
},
json: jsonArg,
},
async run({ args }) {
// fallow-ignore-next-line complexity
await runMeasure(() => {
const info = probeVideo(String(args.video));
const notes: string[] = [];
if (info.vfr)
notes.push(
"variable frame rate: frame times drift, so measure and rebuild a constant-rate copy (--normalize)",
);
if (info.hdr) notes.push("HDR: --normalize tone-maps an SDR copy; the render is SDR");
if (info.duration > SECTION_S)
notes.push(
`longer than ${SECTION_S} s: analyse and check it in sections of at most ${SECTION_S} s, split at cuts (measure cuts)`,
);
if (!info.audio) notes.push("no sound: the rebuild has no <audio>");
const made = args.normalize ? normalize(info, resolve(String(args.normalize))) : undefined;
return {
video: info.path,
width: info.width,
height: info.height,
fps: round(info.fps, 3),
fps_text: info.fpsText,
duration: round(info.duration, 3),
frames: info.frames,
picture_duration: round(info.frames / info.fps, 3),
audio: info.audio,
vfr: info.vfr,
hdr: info.hdr,
codec: info.codec,
canvas: {
width: info.width,
height: info.height,
fps: made?.fps ?? info.fpsText,
duration: round(info.frames / info.fps, 3),
note: "data-width, data-height and data-duration (the picture's length) on the root composition; render with --fps at this rate",
},
...(made ? { normalized: made.path } : {}),
notes,
...((info.vfr || info.hdr) && !made
? {
fix: `npx hyperframes measure probe ${String(args.video)} --normalize <working-copy.mp4>`,
}
: {}),
};
});
},
});
@@ -0,0 +1,64 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
compareArgs,
jsonArg,
num,
roiArg,
runMeasure,
videoAndRoi,
videoArg,
window,
} from "../../measure/command.js";
import { parseRoi, probeVideo } from "../../measure/media.js";
import { compareRhythm, rhythm } from "../../measure/rhythm.js";
export const examples: Example[] = [
[
"How the film moves: holds, update rate, cuts and camera per shot",
"hyperframes measure rhythm .reverse/source.mp4",
],
[
"The same numbers for your render, as ratios to the original's",
"hyperframes measure rhythm .reverse/source.mp4 --start 4 --end 9 --compare renders/final.mp4",
],
];
export default defineCommand({
meta: {
name: "rhythm",
description:
"How a film moves: moving share, holds, update rate (ones, twos), cuts, and each shot's zoom, pan or handheld",
},
args: {
video: videoArg,
start: { type: "string", description: "Window start in seconds (default 0)" },
end: { type: "string", description: "Window end in seconds (default: the end)" },
roi: roiArg,
...compareArgs("A second video (your render) measured at the original's still level"),
json: jsonArg,
},
run: ({ args }) =>
runMeasure(() => {
const { info, roi } = videoAndRoi(args.video, args.roi);
const [start, end] = window(info, args.start, args.end);
const region = args.roi === undefined ? undefined : roi;
const ref = rhythm(info.path, start, end - start, region);
if (!args.compare) return { video: info.path, ...ref };
const other = probeVideo(args.compare);
const yoursRoi = args["compare-roi"]
? parseRoi(args["compare-roi"], other.width, other.height)
: region;
const from = num(args["compare-start"], "--compare-start", start);
const yours = rhythm(other.path, from, end - start, yoursRoi, ref.still_threshold);
const { shots: refShots, ...refSummary } = ref;
const { shots: yoursShots, ...yoursSummary } = yours;
return {
reference: refSummary,
yours: yoursSummary,
ratio: compareRhythm(ref, yours),
reference_shots: refShots,
yours_shots: yoursShots,
};
}),
});
+53
View File
@@ -0,0 +1,53 @@
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
num,
roiArg,
runMeasure,
videoAndRoi,
videoArg,
window,
} from "../../measure/command.js";
import { Fix } from "../../measure/media.js";
import { avSync } from "../../measure/sync.js";
export const examples: Example[] = [
[
"How far the presenter's mouth trails the voice",
"hyperframes measure sync renders/v1.mp4 --roi 470,620,140,90 --start 2 --end 8",
],
];
export default defineCommand({
meta: {
name: "sync",
description:
"Picture against sound for a person speaking: the lag between mouth motion and the voice",
},
args: {
video: videoArg,
roi: { ...roiArg, description: "The mouth, x,y,w,h in the video's pixels", required: true },
start: { type: "string", description: "First second (default 0)" },
end: { type: "string", description: "Last second (default: the end)" },
"max-lag-ms": { type: "string", description: "Lags searched each way (default 400)" },
json: jsonArg,
},
async run({ args }) {
await runMeasure(() => {
if (args.roi === undefined)
throw new Fix("--roi is required", "--roi x,y,w,h round the mouth");
const { info, roi } = videoAndRoi(args.video, args.roi);
const [start, end] = window(info, args.start, args.end);
return {
video: info.path,
roi,
start,
end,
...avSync(info, roi, start, end, num(args["max-lag-ms"], "--max-lag-ms", 400)),
reading:
"positive best_lag_ms means the picture is late; trust high confidence, repeat a medium one on another window without cuts",
};
});
},
});
@@ -0,0 +1,96 @@
import { join } from "node:path";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, outArg, runMeasure, videoArg } from "../../measure/command.js";
import { Fix, outDir, parseRoi, probeVideo, round, runFfmpeg } from "../../measure/media.js";
import { trackTemplate } from "../../measure/track.js";
export const examples: Example[] = [
[
"Follow the cursor from 2.0 s to 5.5 s, starting from its box at 2.0 s",
"hyperframes measure track ref.mp4 --at 2 --to 5.5 --roi 840,410,36,48",
],
];
const PATH_POINTS = 24;
export default defineCommand({
meta: {
name: "track",
description:
"Follow one textured thing (a cursor, a hand, a product) from its box at one moment, frame by frame, until a cut or a lost match",
},
args: {
video: videoArg,
at: { type: "string", description: "The second its box is taken at", required: true },
roi: {
type: "string",
description: "Its box at --at, x,y,w,h in the video's pixels",
required: true,
},
to: { type: "string", description: "Last second to follow it to (default: the end)" },
radius: {
type: "string",
description: "How far it can move between frames, in pixels (default 48)",
},
"min-score": { type: "string", description: "Weakest match kept, 0–1 (default 0.6)" },
out: outArg,
json: jsonArg,
},
async run({ args }) {
await runMeasure(async () => {
const info = probeVideo(String(args.video));
const at = num(args.at, "--at", 0);
const to = Math.min(info.duration, num(args.to, "--to", info.duration));
if (to <= at + 1 / info.fps) throw new Fix("--to is not after --at", "--at 2 --to 5.5");
const box = parseRoi(args.roi, info.width, info.height);
const r = await trackTemplate({
info,
at,
box,
to,
radius: num(args.radius, "--radius"),
minScore: num(args["min-score"], "--min-score"),
});
const step = Math.max(1, Math.ceil(r.rows.length / PATH_POINTS));
const path = r.rows.filter((_, i) => i % step === 0 || i === r.rows.length - 1);
// The path drawn on the first frame, so one look shows where it went.
const dir = outDir(args.out, "measure/track");
const picture = join(dir, `track-${at}-${round(to, 2)}s.png`);
const draw = path.flatMap((p, i) =>
i
? [
`drawbox=x=${Math.round(p.x! - 3)}:y=${Math.round(p.y! - 3)}:w=6:h=6:color=red@0.9:t=fill`,
]
: [],
);
runFfmpeg([
...["-y", "-ss", String(at), "-i", info.path, "-frames:v", "1"],
...[
"-vf",
[
`drawbox=x=${box[0]}:y=${box[1]}:w=${box[2]}:h=${box[3]}:color=yellow:t=3`,
...draw,
].join(","),
picture,
],
]);
const scores = r.scores.slice(1);
return {
video: info.path,
template: { at, box },
followed_s: [round(at, 3), r.rows.at(-1)!.t],
frames: r.rows.length,
lost: r.lost,
mean_score: scores.length
? round(scores.reduce((s, v) => s + v, 0) / scores.length, 3)
: null,
path: path.map((p) => ({ t: p.t, x: Math.round(p.x!), y: Math.round(p.y!) })),
moves: r.moves,
picture,
reading:
"x and y are the box centre in video pixels; moves are its eases; a lost track needs a new --at and --roi where it reappears",
};
});
},
});
+269
View File
@@ -0,0 +1,269 @@
// Motion as one picture, for an agent that reads still images: the frames of a window laid over each other as an
// onion skin (earliest faintest), the moving part's path with numbered points, and a heat map of where the picture
// changed. One look tells a slide from a scale from a mask reveal; `keyframes --shot` does the same for a project.
import { join } from "node:path";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
num,
outArg,
roiArg,
runMeasure,
videoAndRoi,
videoArg,
window,
} from "../../measure/command.js";
import { Fix, grey, outDir, readClip, round, videoClip, writeImage } from "../../measure/media.js";
export const examples: Example[] = [
[
"How a title enters between 2.1 s and 2.9 s",
"hyperframes measure trail ref.mp4 --from 2.1 --to 2.9",
],
[
"One card's path, 10 ghosts",
"hyperframes measure trail ref.mp4 --from 4 --to 5.2 --roi 300,200,900,600 --ghosts 10",
],
];
const WORK_W = 640;
const MOVING = 24;
// 3x5 digits, drawn without a font so the picture comes out the same on every machine.
const DIGITS = [
"111101101101111",
"010110010010111",
"111001111100111",
"111001111001111",
"101101111001001",
"111100111001111",
"111100111101111",
"111001001001001",
"111101111101111",
"111101111001111",
];
function put(px: Uint8Array, w: number, h: number, x: number, y: number, rgb: number[]) {
if (x < 0 || y < 0 || x >= w || y >= h) return;
px.set(rgb, (y * w + x) * 3);
}
function dot(
px: Uint8Array,
w: number,
h: number,
cx: number,
cy: number,
r: number,
rgb: number[],
) {
for (let y = -r; y <= r; y++)
for (let x = -r; x <= r; x++)
if (x * x + y * y <= r * r) put(px, w, h, Math.round(cx + x), Math.round(cy + y), rgb);
}
function line(
px: Uint8Array,
w: number,
h: number,
[x0, y0]: number[],
[x1, y1]: number[],
rgb: number[],
) {
const n = Math.max(1, Math.ceil(Math.hypot(x1! - x0!, y1! - y0!)));
for (let i = 0; i <= n; i++)
dot(px, w, h, x0! + ((x1! - x0!) * i) / n, y0! + ((y1! - y0!) * i) / n, 1, rgb);
}
function digits(
px: Uint8Array,
w: number,
h: number,
text: string,
x: number,
y: number,
s: number,
) {
[...text].forEach((ch, i) => {
const g = DIGITS[Number(ch)] ?? "";
for (let k = 0; k < g.length; k++)
if (g[k] === "1")
for (let yy = 0; yy < s; yy++)
for (let xx = 0; xx < s; xx++)
put(
px,
w,
h,
x + (i * 4 + (k % 3)) * s + xx,
y + Math.floor(k / 3) * s + yy,
[255, 255, 255],
);
});
}
/** Per pixel, the median of the frames' grey: the still background the moving part passes over. */
function medianGrey(gs: Uint8Array[]): Uint8Array {
const out = new Uint8Array(gs[0]!.length);
const v = new Array<number>(gs.length);
for (let i = 0; i < out.length; i++) {
for (let k = 0; k < gs.length; k++) v[k] = gs[k]![i]!;
v.sort((a, b) => a - b);
out[i] = v[gs.length >> 1]!;
}
return out;
}
const heat = (u: number): number[] =>
u < 0.5
? [Math.round(510 * u), 0, 0]
: [255, Math.round(510 * (u - 0.5)), Math.round(Math.max(0, 255 * (u - 0.8) * 5))];
export default defineCommand({
meta: {
name: "trail",
description:
"Motion as one picture: an onion skin of a window's frames, the moving part's path, and a heat map of where it changed",
},
args: {
video: videoArg,
from: { type: "string", description: "First second", required: true },
to: { type: "string", description: "Last second", required: true },
ghosts: { type: "string", description: "Frames laid over each other (default 8)" },
roi: roiArg,
out: outArg,
json: jsonArg,
},
async run({ args }) {
// fallow-ignore-next-line complexity
await runMeasure(() => {
const { info, roi } = videoAndRoi(args.video, args.roi);
const [a, b] = window(info, args.from, args.to, ["--from", "--to"]);
const count = Math.max(2, num(args.ghosts, "--ghosts", 8));
const { frames, w, h } = readClip(videoClip(info.path, a, b - a, info), {
roi,
width: Math.min(WORK_W, roi[2]),
});
if (frames.length < 2)
throw new Fix(`under 2 frames between ${a}s and ${b}s`, "widen --from/--to");
const picks = [
...new Set(
Array.from({ length: count }, (_, i) =>
Math.round((i * (frames.length - 1)) / (count - 1)),
),
),
];
const gs = frames.map(grey);
const base = medianGrey(picks.map((i) => gs[i]!));
// The onion skin: the background dimmed, each ghost's moving pixels over it, later ones stronger.
const skin = new Uint8Array(w * h * 3);
for (let i = 0; i < w * h; i++) skin.fill(Math.round(base[i]! * 0.45), i * 3, i * 3 + 3);
const [kx, ky] = [roi[2] / w, roi[3] / h];
const path = picks.map((fi, order) => {
const alpha = 0.3 + (0.7 * order) / Math.max(1, picks.length - 1);
const [f, g] = [frames[fi]!, gs[fi]!];
let [sx, sy, n] = [0, 0, 0];
for (let i = 0; i < w * h; i++) {
if (Math.abs(g[i]! - base[i]!) <= MOVING) continue;
for (let c = 0; c < 3; c++)
skin[i * 3 + c] = Math.round(skin[i * 3 + c]! * (1 - alpha) + f[i * 3 + c]! * alpha);
sx += i % w;
sy += Math.floor(i / w);
n++;
}
const t = round(a + fi / info.fps, 3);
return n
? { t, x: sx / n, y: sy / n, share: n / (w * h) }
: { t, x: null, y: null, share: 0 };
});
const seen = path.flatMap((p, i) =>
p.x !== null && p.y !== null ? [{ ...p, x: p.x, y: p.y, i }] : [],
);
for (let i = 1; i < seen.length; i++)
line(
skin,
w,
h,
[seen[i - 1]!.x, seen[i - 1]!.y],
[seen[i]!.x, seen[i]!.y],
[255, 200, 40],
);
const scale = Math.max(1, Math.round(w / 320));
for (const p of seen) {
dot(skin, w, h, p.x, p.y, 2 + scale, [255, 60, 60]);
digits(
skin,
w,
h,
String(p.i + 1),
Math.round(p.x) + 4 + scale,
Math.round(p.y) - 6 * scale,
scale,
);
}
// The heat map: every frame's change from the one before, summed, over the dimmed first frame.
const sum = new Float32Array(w * h);
for (let k = 1; k < gs.length; k++)
for (let i = 0; i < w * h; i++) sum[i]! += Math.abs(gs[k]![i]! - gs[k - 1]![i]!);
const top = sum.reduce((m, v) => Math.max(m, v), 0) || 1;
const map = new Uint8Array(w * h * 3);
let [x0, y0, x1, y1] = [w, h, -1, -1];
for (let i = 0; i < w * h; i++) {
const u = Math.min(1, sum[i]! / top);
const [r, g, bl] = heat(u);
const under = gs[0]![i]! * 0.35;
map.set(
[Math.max(r!, under), Math.max(g!, under), Math.max(bl!, under)].map(Math.round),
i * 3,
);
if (u > 0.15) {
const [x, y] = [i % w, Math.floor(i / w)];
[x0, y0, x1, y1] = [Math.min(x0, x), Math.min(y0, y), Math.max(x1, x), Math.max(y1, y)];
}
}
const dir = outDir(args.out, "measure/trail");
const name = `trail-${a}-${b}s`;
const first = seen[0];
const final = seen.at(-1);
return {
video: info.path,
from: a,
to: b,
frames: frames.length,
ghosts: picks.length,
trail: writeImage(join(dir, `${name}.png`), skin, w, h),
heat: writeImage(join(dir, `${name}-heat.png`), map, w, h),
path: path.map((p, i) => ({
n: i + 1,
t: p.t,
...(p.x !== null && p.y !== null
? {
x: Math.round(roi[0] + p.x * kx),
y: Math.round(roi[1] + p.y * ky),
moving_share: round(p.share, 3),
}
: { moving_share: 0 }),
})),
...(first && final
? {
net_move: {
dx: Math.round((final.x - first.x) * kx),
dy: Math.round((final.y - first.y) * ky),
},
size_ratio: round(final.share / (first.share || 1e-9), 2),
}
: {}),
changed_box:
x1 >= 0
? [roi[0] + x0 * kx, roi[1] + y0 * ky, (x1 - x0 + 1) * kx, (y1 - y0 + 1) * ky].map(
Math.round,
)
: null,
reading:
"ghosts fade from first (faint) to last (solid); numbered red points are the moving part's centre; in the heat map white-yellow changed most",
};
});
},
});
+8 -3
View File
@@ -495,7 +495,7 @@ export function serveCapturePage(projectDir: string, html: string, proxy: boolea
* Render key frames from a composition as PNG screenshots.
* The agent can Read these to verify its output visually.
*/
async function captureSnapshots(
export async function captureSnapshots(
projectDir: string,
opts: {
frames?: number;
@@ -518,12 +518,17 @@ async function captureSnapshots(
review?: boolean;
/** Progress lines; `--json` sends them to stderr so stdout stays one envelope. */
log?: (message: string) => void;
/** False for working frames another command reads and discards: nothing enters project history. */
record?: boolean;
},
): Promise<CapturedSnapshots> {
const log = opts.log ?? ((message: string) => console.log(message));
const { bundleWithLocalizedFonts } = await import("../utils/bundleWithLocalizedFonts.js");
const capture = await captureArtifactRevision(projectDir).catch(warnSnapshotsNotRecorded);
const capture =
opts.record === false
? undefined
: await captureArtifactRevision(projectDir).catch(warnSnapshotsNotRecorded);
const invocation = randomUUID();
const artifacts: Array<{
id: string;
@@ -1000,7 +1005,7 @@ async function captureSnapshots(
log,
)
: false;
if (!recorded) {
if (!recorded && opts.record !== false) {
const paths = [...replaced, ...artifacts.map((artifact) => artifact.image)];
if (paths.length > 0) await forgetRecordedArtifacts(projectDir, paths).catch(() => undefined);
}
+5
View File
@@ -58,6 +58,11 @@ const GROUPS: Group[] = [
"Render candidate color grades onto a reference frame as one labeled comparison PNG",
],
["compare", "Render composition variants into one labeled comparison sheet"],
["diff", "Score a render or a project against the original video, frame by frame"],
[
"measure",
"Measure a video file: clock, cuts, events, frames, motion, crops, boxes, eases, look",
],
["info", "Print project metadata"],
["compositions", "List all compositions in a project"],
["timeline", "Print the project's tracks and clips"],
+49
View File
@@ -0,0 +1,49 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { boxAt, boxRange } from "./box.js";
import { probeVideo, type Roi } from "./media.js";
const ffmpeg = findFFmpeg();
const dir = mkdtempSync(join(tmpdir(), "hf-box-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
function clip(name: string, overlayX: string): string {
const out = join(dir, name);
execFileSync(ffmpeg!, [
...["-nostdin", "-v", "error", "-y"],
...["-f", "lavfi", "-i", "color=0x101820:s=320x180:d=1.2:r=30"],
...["-f", "lavfi", "-i", "color=0xff8000:s=60x30:d=1.2:r=30"],
...["-filter_complex", `[0][1]overlay=x='${overlayX}':y=40`],
...["-pix_fmt", "yuv444p", "-c:v", "libx264", "-crf", "0", out],
]);
return out;
}
const options = (video: string) => {
const roi: Roi = [0, 0, 320, 180];
return { video, info: probeVideo(video), roi, color: "auto", tol: 40, dir };
};
describe.skipIf(!ffmpeg || !findFFprobe())("box", () => {
it("finds a coloured box by its colour at one time", () => {
const r = boxAt(options(clip("still.mp4", "100")), 0.5);
expect(r.color).toMatch(/^#f[0-9a-f]8/);
const want = { x: 100, y: 40, w: 60, h: 30 };
for (const k of ["x", "y", "w", "h"] as const)
expect(Math.abs(r.box[k] - want[k])).toBeLessThanOrEqual(2);
});
it("tracks a linear slide as one x move", () => {
const r = boxRange(options(clip("slide.mp4", "20+400*min(max(t-0.3,0),0.5)")), 0, 1.2, true);
expect(r.first_seen_s).toBe(0);
const moves = "moves" in r ? r.moves : [];
const xs = moves.filter((m) => m.prop === "x");
expect(xs).toHaveLength(1);
expect(moves.filter((m) => m.prop === "w" || m.prop === "h")).toHaveLength(0);
expect(xs[0]!.start_s).toBeCloseTo(0.3, 1);
});
});
+431
View File
@@ -0,0 +1,431 @@
// Where one graphic sits, found by its colour rather than by what moves, so a presenter who never stops does not
// pull the box (ported from the Studio's box.mjs): at one time its box, ink, dominant colours, text line bands and
// cap height; over a range when it is on screen; with track, each move of its centre, size and ink, fitted to the
// closest GSAP ease.
import { join } from "node:path";
import { fitMoves, printable, type Row } from "./curve.js";
import {
Fix,
frameAt,
readFrameAt,
readFrames,
round,
runFfmpeg,
type Roi,
type VideoInfo,
} from "./media.js";
// A range is read at this width (a 1080 px frame at 3 px a step); one time is read at full size.
const RANGE_W = 360;
const BLOB = 0.02;
const MIN_BLOB = 4;
// Over a range a frame shows the graphic when it holds this share of the range's most ink; less is noise.
const PRESENT = 0.05;
const BORDER = 2;
const HOLD_S = 0.3;
export interface Frame {
px: Uint8Array;
w: number;
h: number;
}
export type Match = (r: number, g: number, b: number) => boolean;
export interface Blob {
n: number;
x0: number;
y0: number;
x1: number;
y1: number;
}
interface Cluster {
rgb: number[];
n: number;
}
const hexOf = (c: number[]) =>
`#${c.map((v) => Math.round(v).toString(16).padStart(2, "0")).join("")}`;
const dist2 = (a: number[], b: number[]) =>
(a[0]! - b[0]!) ** 2 + (a[1]! - b[1]!) ** 2 + (a[2]! - b[2]!) ** 2;
/** Colour clusters of the pixels in `rect` (x0,y0,x1,y1 inclusive) that `keep` passes: 16-level bins, each merged
* into the first larger cluster within `radius`, largest first. */
// fallow-ignore-next-line complexity
function clusters(
{ px, w }: Frame,
[x0, y0, x1, y1]: number[],
keep: (x: number, y: number, o: number) => boolean,
radius: number,
): Cluster[] {
const bins = new Map<number, number[]>();
for (let y = y0!; y <= y1!; y++)
for (let x = x0!; x <= x1!; x++) {
const o = (y * w + x) * 3;
if (!keep(x, y, o)) continue;
const key = ((px[o]! >> 4) << 8) | ((px[o + 1]! >> 4) << 4) | (px[o + 2]! >> 4);
let bin = bins.get(key);
if (!bin) bins.set(key, (bin = [0, 0, 0, 0]));
bin[0]!++;
for (let c = 0; c < 3; c++) bin[c + 1]! += px[o + c]!;
}
const out: { seed: number[]; n: number; sum: number[] }[] = [];
for (const [n, ...sum] of [...bins.values()].sort((a, b) => b[0]! - a[0]!)) {
const rgb = sum.map((s) => s / n!);
const home = out.find((c) => dist2(c.seed, rgb) <= radius * radius);
if (!home) out.push({ seed: rgb, n: n!, sum });
else {
home.n += n!;
sum.forEach((s, c) => (home.sum[c]! += s));
}
}
return out.map((c) => ({ rgb: c.sum.map((s) => s / c.n), n: c.n })).sort((a, b) => b.n - a.n);
}
/** --color auto: the commonest cluster among pixels that match no colour holding 2% of the ROI's border ring. */
function autoColor(f: Frame, tol: number): Cluster | null {
const { w, h } = f;
const ring = (x: number, y: number) =>
x < BORDER || y < BORDER || x >= w - BORDER || y >= h - BORDER;
const border = clusters(f, [0, 0, w - 1, h - 1], ring, tol);
const total = border.reduce((s, c) => s + c.n, 0);
const ground = border.filter((c) => c.n >= 0.02 * total).map((c) => c.rgb);
const unlike = (x: number, y: number, o: number) =>
!ring(x, y) &&
ground.every((c) => dist2(c, [f.px[o]!, f.px[o + 1]!, f.px[o + 2]!]) > tol * tol);
return clusters(f, [0, 0, w - 1, h - 1], unlike, tol)[0] ?? null;
}
/** The pixel test for --color. */
function matcher(spec: string, tol: number): Match {
if (spec === "white") return (r, g, b) => Math.min(r, g, b) >= 255 - tol;
if (spec === "dark") return (r, g, b) => Math.max(r, g, b) <= tol;
// what a solo render on #00ff00 leaves once the green is keyed out
if (spec === "alpha") return (r, g, b) => r + (255 - g) + b >= 90;
const m = /^#?([0-9a-f]{6})$/i.exec(spec);
if (!m)
throw new Fix(
`--color must be #rrggbb, white, dark, auto or alpha, got "${spec}"`,
"--color #fce7bc",
);
const c = [0, 2, 4].map((i) => parseInt(m[1]!.slice(i, i + 2), 16));
return (r, g, b) => dist2(c, [r, g, b]) <= tol * tol;
}
/** The 8-connected blobs of the pixels `match` passes: each one's size and box, and every pixel's blob (0: none). */
// fallow-ignore-next-line complexity
export function blobs({ px, w, h }: Frame, match: Match): { id: Int32Array; list: Blob[] } {
const mask = new Uint8Array(w * h);
for (let i = 0; i < w * h; i++)
mask[i] = match(px[3 * i]!, px[3 * i + 1]!, px[3 * i + 2]!) ? 1 : 0;
const id = new Int32Array(w * h);
const list: Blob[] = [{ n: 0, x0: w, y0: h, x1: -1, y1: -1 }];
for (let s = 0; s < w * h; s++) {
if (!mask[s] || id[s]) continue;
const blob = { n: 0, x0: w, y0: h, x1: -1, y1: -1 };
const c = list.push(blob) - 1;
const stack = [s];
id[s] = c;
for (let p = stack.pop(); p !== undefined; p = stack.pop()) {
const x = p % w;
const y = (p - x) / w;
blob.n++;
[blob.x0, blob.y0] = [Math.min(blob.x0, x), Math.min(blob.y0, y)];
[blob.x1, blob.y1] = [Math.max(blob.x1, x), Math.max(blob.y1, y)];
for (let ny = Math.max(0, y - 1); ny <= Math.min(h - 1, y + 1); ny++)
for (let nx = Math.max(0, x - 1); nx <= Math.min(w - 1, x + 1); nx++) {
const q = ny * w + nx;
if (!mask[q] || id[q]) continue;
id[q] = c;
stack.push(q);
}
}
}
return { id, list };
}
/** Blobs under BLOB of the graphic's largest are speckle (a highlight on a jacket, a slice of other text). */
const floorOf = (...lists: Blob[][]) =>
Math.max(MIN_BLOB, BLOB * lists.flat().reduce((m, b) => Math.max(m, b.n), 0));
/** The box and pixel count of the blobs holding `floor` pixels or more, or null. */
function graphic(list: Blob[], floor: number) {
const kept = list.filter((b) => b.n >= floor);
if (!kept.length) return null;
const edge = (k: "x0" | "y0" | "x1" | "y1", f: (...v: number[]) => number) =>
f(...kept.map((b) => b[k]));
return {
x0: edge("x0", Math.min),
y0: edge("y0", Math.min),
x1: edge("x1", Math.max),
y1: edge("y1", Math.max),
ink: kept.reduce((s, b) => s + b.n, 0),
};
}
type Graphic = NonNullable<ReturnType<typeof graphic>>;
/** One frame's graphic, with its text line bands: runs of rows holding it, a one-row gap bridged, under 3 rows
* dropped. */
function measure(f: Frame, match: Match) {
const { id, list } = blobs(f, match);
const floor = floorOf(list);
const g = graphic(list, floor);
if (!g) return null;
const lines: number[][] = [];
for (let y = g.y0; y <= g.y1; y++) {
let on = false;
for (let x = g.x0; x <= g.x1 && !on; x++)
on = id[y * f.w + x]! > 0 && list[id[y * f.w + x]!]!.n >= floor;
if (!on) continue;
const last = lines.at(-1);
if (last && y - last[1]! <= 2) last[1] = y;
else lines.push([y, y]);
}
return { ...g, lines: lines.filter(([a, b]) => b! - a! + 1 >= 3) };
}
/** Frames of `roi` from `start` for `seconds`, every one, at `width` (0: full size). */
function read(video: string, start: number, seconds: number, roi: Roi, width: number): Frame[] {
const { frames, w, h } = readFrames(video, { start, seconds, roi, width });
return frames.map((px) => ({ px, w, h }));
}
/** One frame with the box drawn (and the union in yellow), cropped round it. */
function picture(
video: string,
t: number,
b: number[],
[W, H]: number[],
out: string,
union: number[] | null = null,
) {
const outer = union ?? b;
const pad = Math.max(16, Math.round(0.25 * Math.max(outer[2]!, outer[3]!)));
const cx = Math.max(0, outer[0]! - pad);
const cy = Math.max(0, outer[1]! - pad);
const cw = Math.min(W!, outer[0]! + outer[2]! + pad) - cx;
const ch = Math.min(H!, outer[1]! + outer[3]! + pad) - cy;
const draw = ([x, y, w, h]: number[], color: string, thick: number) =>
`drawbox=x=${x! - cx}:y=${y! - cy}:w=${w}:h=${h}:color=${color}:t=${thick}`;
const vf = [
`crop=${cw}:${ch}:${cx}:${cy}`,
...(union ? [draw(union, "yellow", 1)] : []),
draw(b, "red", 2),
];
runFfmpeg(["-y", "-ss", String(t), "-i", video, "-frames:v", "1", "-vf", vf.join(","), out]);
return out;
}
/** The --color spec as a test, choosing auto's colour on the sample of `fs` where it covers most. */
function choose(spec: string, tol: number, fs: Frame[]) {
if (spec !== "auto") return { match: matcher(spec, tol), color: spec };
const picks = [0, 0.25, 0.5, 0.75, 1].map((p) =>
autoColor(fs[Math.round(p * (fs.length - 1))]!, tol),
);
const best = picks.reduce<Cluster | null>((a, b) => ((b?.n ?? 0) > (a?.n ?? 0) ? b : a), null);
if (!best)
throw new Fix(
"--color auto found nothing unlike the ROI's border",
"pass --color #rrggbb or white",
);
return { match: matcher(hexOf(best.rgb), tol), color: hexOf(best.rgb) };
}
/** The no-pixels error, naming what the ROI does hold. */
function nothing(f: Frame, color: string, tol: number) {
const top = clusters(f, [0, 0, f.w - 1, f.h - 1], () => true, tol).slice(0, 3);
const names = top
.map((c) => `${hexOf(c.rgb)} ${Math.round((100 * c.n) / (f.w * f.h))}%`)
.join(", ");
const pick = autoColor(f, tol);
return new Fix(
`no pixels of ${color} (within --tol ${tol}) in the ROI; it holds ${names}`,
pick
? `pass --color ${hexOf(pick.rgb)} (what --color auto picks here) or one of those colours`
: "draw --roi round the graphic with some of its ground, then --color auto",
);
}
/** In video pixels. */
const toVideo = (m: Graphic, [rx, ry]: Roi, kx: number, ky: number) => ({
x: Math.round(rx + m.x0 * kx),
y: Math.round(ry + m.y0 * ky),
w: Math.round((m.x1 - m.x0 + 1) * kx),
h: Math.round((m.y1 - m.y0 + 1) * ky),
});
export interface BoxOptions {
video: string;
info: VideoInfo;
roi: Roi;
color: string;
tol: number;
dir: string;
}
const frameAtStart = (t: number, fps: number) => Math.ceil(t * fps - 1e-6);
/** The graphic at one time. */
export function boxAt(o: BoxOptions, t: number) {
const one = readFrameAt(o.info, t, { roi: o.roi });
const f: Frame | undefined = one.frames[0] && { px: one.frames[0], w: one.w, h: one.h };
if (!f)
throw new Fix(
`no frame at ${t}s`,
`pick a time before ${round(o.info.duration - 1 / o.info.fps, 2)}s`,
);
const { match, color } = choose(o.color, o.tol, [f]);
const m = measure(f, match);
if (!m) throw nothing(f, color, o.tol);
const box = toVideo(m, o.roi, 1, 1);
const inBox = clusters(f, [m.x0, m.y0, m.x1, m.y1], () => true, o.tol);
const area = box.w * box.h;
const lines = m.lines.map(([a, b]) => ({ y: o.roi[1] + a!, h: b! - a! + 1 }));
const heights = lines.map((l) => l.h).sort((a, b) => a - b);
return {
t,
frame: frameAt(t, o.info.fps),
color,
box,
ink_px: m.ink,
colors: inBox.slice(0, 3).map((c) => ({ hex: hexOf(c.rgb), share: round(c.n / area, 2) })),
lines,
cap_height_px: heights[(heights.length - 1) >> 1] ?? null,
image: picture(
o.video,
t,
[box.x, box.y, box.w, box.h],
[o.info.width, o.info.height],
join(o.dir, `box-${t}s.png`),
),
};
}
interface RangeRow extends Row {
frame: number;
box: { x: number; y: number; w: number; h: number } | null;
}
/** Runs where the box is on screen and none of x, y, w, h moves past `quant`, 0.3 s or longer. */
function holds(rows: RangeRow[], quant: number, fps: number) {
const still = rows.map((r, i) => {
const prev = rows[i - 1];
if (!prev || r.x == null || prev.x == null || r.y == null || prev.y == null) return false;
return (
Math.abs(r.x - prev.x) <= quant &&
Math.abs(r.y - prev.y) <= quant &&
Math.abs(r.w - prev.w) <= quant &&
Math.abs(r.h - prev.h) <= quant
);
});
const out: { start_s: number; seconds: number }[] = [];
for (let i = 0, run = 0; i <= still.length; i++) {
if (still[i]) run++;
else {
if (run + 1 >= HOLD_S * fps)
out.push({ start_s: rows[i - run - 1]!.t, seconds: round((run + 1) / fps, 2) });
run = 0;
}
}
return out;
}
/** Over a range: when it is seen, its union and median box; with `track`, each move fitted and the still holds. */
export function boxRange(opts: BoxOptions, start: number, end: number, track: boolean) {
const { info, roi } = opts;
const width = Math.min(RANGE_W, roi[2]);
const fs = read(opts.video, start, end - start, roi, width);
if (!fs.length)
throw new Fix(
`no frames between ${start}s and ${end}s`,
`keep --end at most ${round(info.duration, 2)}`,
);
const kx = roi[2] / width;
const ky = roi[3] / fs[0]!.h;
const { match, color } = choose(opts.color, opts.tol, fs);
// speckle is judged against the graphic's largest blob in the whole range, so a sliver entering is not swamped
const found = fs.map((f) => blobs(f, match).list);
const floor = floorOf(...found);
const frame0 = frameAtStart(start, info.fps);
const empty = (t: number, frame: number): RangeRow => ({
t,
frame,
x: null,
y: null,
w: 0,
h: 0,
ink: 0,
box: null,
});
const rows = found.map((list, i): RangeRow => {
const frame = frame0 + i;
const t = round(frame / info.fps, 3);
const m = graphic(list, floor);
if (!m) return empty(t, frame);
const b = toVideo(m, roi, kx, ky);
return {
t,
frame,
x: round(b.x + b.w / 2, 1),
y: round(b.y + b.h / 2, 1),
w: b.w,
h: b.h,
ink: Math.round(m.ink * kx * ky),
box: b,
};
});
const most = Math.max(...rows.map((r) => r.ink));
const present = rows.map((r) => (r.box && r.ink < PRESENT * most ? empty(r.t, r.frame) : r));
const seen = present.filter((r) => r.box);
if (!seen.length) throw nothing(fs[fs.length >> 1]!, color, opts.tol);
const med = (v: number[]) => [...v].sort((a, b) => a - b)[(v.length - 1) >> 1]!;
const boxes = seen.flatMap((r) => (r.box ? [r.box] : []));
const x0 = Math.min(...boxes.map((b) => b.x));
const y0 = Math.min(...boxes.map((b) => b.y));
const union = [
x0,
y0,
Math.max(...boxes.map((b) => b.x + b.w)) - x0,
Math.max(...boxes.map((b) => b.y + b.h)) - y0,
];
const last = seen.at(-1)!;
const lastBox = boxes.at(-1)!;
const out = {
start,
end,
fps: round(info.fps, 3),
color,
frames: present.length,
seen_frames: seen.length,
first_seen_s: seen[0]!.t,
first_seen_frame: seen[0]!.frame,
last_seen_s: last.t,
last_seen_frame: last.frame,
box_union: union,
box_median: (["x", "y", "w", "h"] as const).map((k) => med(boxes.map((b) => b[k]))),
ink_px_median: med(seen.map((r) => r.ink)),
image: picture(
opts.video,
last.t,
[lastBox.x, lastBox.y, lastBox.w, lastBox.h],
[info.width, info.height],
join(opts.dir, `box-${start}-${end}s.png`),
union,
),
};
if (!track) return out;
// Each move of x, y (the box's centre), w, h and ink, and the still holds.
const quant = Math.max(kx, ky);
const minPx = Math.max(2 * quant, 0.02 * Math.max(roi[2], roi[3]));
const maxInk = Math.max(...present.map((r) => r.ink));
const moves = (["x", "y", "w", "h", "ink"] as const)
.flatMap((k) =>
k === "ink"
? fitMoves(present, k, info.fps, 0.1 * maxInk, 0.02 * maxInk)
: fitMoves(present, k, info.fps, minPx, quant),
)
.sort((a, b) => a.start_s - b.start_s);
return {
...out,
moves: printable(moves).map((m) => ({ ...m, next: undefined })),
holds: holds(present, quant, info.fps),
};
}
+91
View File
@@ -0,0 +1,91 @@
// What every `hyperframes measure <op>` shares: one JSON object on stdout (summaries, never per-frame arrays;
// pictures as file paths), progress on stderr, and an error that says what to run instead.
import { c } from "../ui/colors.js";
import { failCommand } from "../utils/commandResult.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { withMeta } from "../utils/updateCheck.js";
import {
Fix,
formatJson,
parseNumbers,
parseRoi,
probeVideo,
type Roi,
type VideoInfo,
} from "./media.js";
export async function runMeasure(work: () => object | Promise<object>): Promise<void> {
try {
console.log(formatJson(withMeta({ ok: true, ...(await work()) })));
} catch (err) {
const fix = err instanceof Fix ? err.fix : undefined;
console.log(
formatJson(
withMeta({ ok: false, error: normalizeErrorMessage(err), ...(fix ? { fix } : {}) }),
),
);
if (process.stdout.isTTY) console.error(`${c.error("✗")} ${normalizeErrorMessage(err)}`);
failCommand();
}
}
/** Arguments most ops take. */
export const videoArg = {
type: "positional",
description: "The video file",
required: true,
} as const;
export const roiArg = {
type: "string",
description: "Region x,y,w,h in the video's pixels (default: whole frame)",
} as const;
export const outArg = {
type: "string",
description: "Folder for pictures (default .hyperframes/measure/<op>)",
} as const;
/** --compare, --compare-start and --compare-roi: a second video measured over the same window. */
export const compareArgs = (description: string) =>
({
compare: { type: "string", description },
"compare-start": { type: "string", description: "Its window start (default: --start)" },
"compare-roi": { type: "string", description: "Its region (default: --roi)" },
}) as const;
export const jsonArg = {
type: "boolean",
description: "Accepted for symmetry; output is always JSON",
default: false,
} as const;
/** The video's probe and its ROI (whole frame by default). */
export function videoAndRoi(video: unknown, roi: unknown): { info: VideoInfo; roi: Roi } {
const info = probeVideo(String(video));
return {
info,
roi:
roi === undefined ? [0, 0, info.width, info.height] : parseRoi(roi, info.width, info.height),
};
}
/** One number from a flag, or `fallback` when it is absent. */
export function num(value: unknown, flag: string, fallback: number): number;
export function num(value: unknown, flag: string): number | undefined;
export function num(value: unknown, flag: string, fallback?: number): number | undefined {
return value === undefined ? fallback : parseNumbers(value, flag)[0];
}
/** start,end seconds from --start/--end (or --from/--to), clamped to the video. */
export function window(
info: VideoInfo,
start: unknown,
end: unknown,
flags = ["--start", "--end"],
) {
const a = num(start, flags[0]!, 0);
const b = Math.min(info.duration, num(end, flags[1]!, info.duration));
if (!(b > a))
throw new Fix(
`${flags[1]} ${b} is not after ${flags[0]} ${a}`,
`${flags[0]} 0 ${flags[1]} ${Math.min(6, info.duration)}`,
);
return [a, b] as const;
}
@@ -0,0 +1,50 @@
import { execFileSync } from "node:child_process";
import { existsSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { continuity } from "./continuity.js";
const ffmpeg = findFFmpeg();
const dir = mkdtempSync(join(tmpdir(), "hf-continuity-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
function clip(name: string, args: string[]): string {
const out = join(dir, name);
execFileSync(ffmpeg!, [
...["-v", "error", "-y", ...args],
...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-qp", "0", out],
]);
return out;
}
describe.skipIf(!ffmpeg || !findFFprobe())("continuity", () => {
it("flags a frame the original does not have as a jump", () => {
const original = clip("orig.mp4", ["-f", "lavfi", "-i", "testsrc2=d=2:r=30:s=160x90"]);
const yours = clip("jump.mp4", [
...["-i", original, "-vf"],
"drawbox=x=0:y=0:w=160:h=90:color=red:t=fill:enable='eq(n,30)'",
]);
const r = continuity(original, yours, { outDir: dir });
const jumps = r.events.filter((e) => e.kind === "jump");
expect(jumps).toHaveLength(1);
expect(jumps[0]!.peak_frame).toBeGreaterThanOrEqual(30);
expect(jumps[0]!.peak_frame).toBeLessThanOrEqual(31);
expect(existsSync(jumps[0]!.sheet)).toBe(true);
});
it("flags a shape drawn over a still background as structure", () => {
const original = clip("still.mp4", ["-f", "lavfi", "-i", "color=c=0x305070:s=160x90:r=30:d=2"]);
const yours = clip("box.mp4", [
...["-i", original, "-vf"],
"drawbox=x=100:y=20:w=24:h=24:color=white:t=fill:enable='between(n,20,29)'",
]);
const r = continuity(original, yours, { outDir: dir });
const shapes = r.events.filter((e) => e.kind === "structure");
expect(shapes.length).toBeGreaterThan(0);
expect(shapes[0]!.peak_frame).toBeGreaterThanOrEqual(20);
expect(shapes[0]!.peak_frame).toBeLessThanOrEqual(29);
expect(shapes[0]!.at![0]).toBeGreaterThan(80);
});
});
+329
View File
@@ -0,0 +1,329 @@
// Seams a rebuild has and its original does not, ported from the Studio's hf-media continuity: on the original's
// clock, a jump (a whole-frame change the original lacks), flicker (a change over background the original keeps
// still) and structure (an extra, missing or misplaced shape, from blurred edge maps). Screening candidates: open each
// event's sheet before acting. The per-pixel work is plain TypeScript so it needs no OpenCV download.
import { join } from "node:path";
import { decode, Fix, frameAt, probeVideo, round, writeImage, type VideoInfo } from "./media.js";
const WIDTH = 160;
const NEIGHBOURS = 2; // a spike must beat the original's max within ±this many frames
const STATIC_WINDOW = 12; // still mask: the original's per-pixel change stays low for ±this many frames ...
const STATIC_LEVEL = 6; // ... below this level
const STATIC_MIN_SHARE = 0.05; // fewer still pixels than this share: no flicker reading at that frame
const [JUMP_ABS, JUMP_RATIO] = [3, 2];
const [FLICKER_ABS, FLICKER_RATIO, FLICKER_LOCAL] = [2, 3, 1.8];
const STRUCTURE_K = 4; // a structure spike is above the film's median + k × MAD ...
const STRUCTURE_LOCAL = 1.5; // ... above this multiple of its median within ±1 s ...
const STRUCTURE_MIN = 1.5; // ... and above this floor (mean-normalised edge difference)
const STRUCTURE_BOX = 0.15; // local window of the structure difference, share of the width
const EVENT_GAP = 2;
export type ContinuityKind = "jump" | "flicker" | "structure";
/** Convolves rows then columns with kernel `k` (odd length), clamping at the edges and dividing by its sum. */
function separable(src: Float32Array, w: number, h: number, k: number[]): Float32Array {
const r = (k.length - 1) / 2;
const sum = k.reduce((s, v) => s + v, 0);
const pass = (a: Float32Array, horizontal: boolean) => {
const out = new Float32Array(a.length);
for (let y = 0; y < h; y++)
for (let x = 0; x < w; x++) {
let s = 0;
for (let i = -r; i <= r; i++) {
const xx = horizontal ? Math.min(w - 1, Math.max(0, x + i)) : x;
const yy = horizontal ? y : Math.min(h - 1, Math.max(0, y + i));
s += a[yy * w + xx]! * k[i + r]!;
}
out[y * w + x] = s / sum;
}
return out;
};
return pass(pass(src, true), false);
}
/** Separable Gaussian with clamped edges; cv2.GaussianBlur with sigma and an automatic kernel, near enough. */
function gaussian(src: Float32Array, w: number, h: number, sigma: number): Float32Array {
const r = Math.ceil(3 * sigma);
const k = Array.from({ length: 2 * r + 1 }, (_, i) =>
Math.exp(-((i - r) ** 2) / (2 * sigma * sigma)),
);
return separable(src, w, h, k);
}
/** Box mean over a size×size window with clamped edges (cv2.blur). */
function box(src: Float32Array, w: number, h: number, size: number): Float32Array {
return separable(src, w, h, new Array<number>(2 * Math.floor(size / 2) + 1).fill(1));
}
/** Blurred edge strength: Gaussian σ1, 3×3 Sobel magnitude, Gaussian σ2. */
function edges(g: Uint8Array, w: number, h: number): Float32Array {
const b = gaussian(Float32Array.from(g), w, h, 1);
const at = (x: number, y: number) =>
b[Math.min(h - 1, Math.max(0, y)) * w + Math.min(w - 1, Math.max(0, x))]!;
const mag = new Float32Array(w * h);
for (let y = 0; y < h; y++)
for (let x = 0; x < w; x++) {
const gx =
at(x + 1, y - 1) +
2 * at(x + 1, y) +
at(x + 1, y + 1) -
at(x - 1, y - 1) -
2 * at(x - 1, y) -
at(x - 1, y + 1);
const gy =
at(x - 1, y + 1) +
2 * at(x, y + 1) +
at(x + 1, y + 1) -
at(x - 1, y - 1) -
2 * at(x, y - 1) -
at(x + 1, y - 1);
mag[y * w + x] = Math.hypot(gx, gy);
}
return gaussian(mag, w, h, 2);
}
const mean = (a: ArrayLike<number>) => {
let s = 0;
for (let i = 0; i < a.length; i++) s += a[i]!;
return s / (a.length || 1);
};
const median = (v: number[]) => {
const s = [...v].sort((a, b) => a - b);
return s[Math.floor(s.length / 2)] ?? 0;
};
interface Row {
frame: number;
original_change: number | null;
change: number | null;
original_static_change: number | null;
static_change: number | null;
structure: number;
at: [number, number];
}
export interface ContinuityEvent {
kind: ContinuityKind;
start_s: number;
end_s: number;
peak_s: number;
peak_frame: number;
value: number;
original_value: number | null;
at?: [number, number];
sheet: string;
}
/** The original's frames [a, b) and yours on the original's clock, grey at WIDTH. */
function frames(path: string, a: number, b: number, h: number, fps?: string): Uint8Array[] {
const select = `select=between(n\\,${a}\\,${b - 1})`;
const filters = [
...(fps ? [`fps=${fps}`] : []),
select,
`setpts=N/(${fps ?? "1"})/TB`,
`scale=${WIDTH}:${h}`,
];
return decode(["-i", path], filters, WIDTH, h, true).frames;
}
/** One sheet: the original's frames over yours, the peak marked by a white bar under it. */
function sheet(
orig: Uint8Array[],
yours: Uint8Array[],
w: number,
h: number,
peak: number,
out: string,
) {
const W = orig.length * (w + 4);
const H = 2 * h + 12;
const px = new Uint8Array(W * H).fill(30);
orig.forEach((f, i) => {
for (let y = 0; y < h; y++) {
px.set(f.subarray(y * w, (y + 1) * w), y * W + i * (w + 4));
px.set(yours[i]!.subarray(y * w, (y + 1) * w), (h + 6 + y) * W + i * (w + 4));
}
if (i === peak)
for (let y = 2 * h + 8; y < H; y++)
px.fill(255, y * W + i * (w + 4), y * W + i * (w + 4) + w);
});
return writeImage(out, px, W, H, true);
}
// fallow-ignore-next-line complexity
export function continuity(
reference: string,
yoursPath: string,
opts: { start?: number; end?: number; outDir: string },
): { events: ContinuityEvent[]; thresholds: Record<string, number>; frames: number } {
const ref: VideoInfo = probeVideo(reference);
const yoursInfo = probeVideo(yoursPath);
const h = Math.max(2, Math.round((ref.height * WIDTH) / ref.width / 2) * 2);
const k0 = frameAt(opts.start ?? 0, ref.fps);
const k1 = Math.min(ref.frames, Math.ceil((opts.end ?? ref.duration) * ref.fps - 1e-6));
const yours = frames(yoursInfo.path, k0, k1, h, ref.fpsText);
if (yours.length < 3)
throw new Fix(
"fewer than three frames of yours fall in the window",
"widen --start/--end, or check yours plays that long",
);
const low = Math.max(0, k0 - STATIC_WINDOW - 1);
const high = Math.min(ref.frames, k0 + yours.length + STATIC_WINDOW + 1);
const stack = frames(ref.path, low, high, h, ref.fpsText);
const n = WIDTH * h;
const moved = stack.map((f, k) => {
const d = new Uint8Array(n);
if (k) for (let i = 0; i < n; i++) d[i] = Math.abs(f[i]! - stack[k - 1]![i]!);
return d;
});
const boxSize = Math.max(3, Math.round(STRUCTURE_BOX * WIDTH));
const scale = ref.width / WIDTH;
const rows: Row[] = [];
for (let m = 0; m < yours.length; m++) {
const j = k0 + m;
const k = j - low;
if (!stack[k]) break;
// Still pixels: the original's change stays under STATIC_LEVEL for ±STATIC_WINDOW frames.
const mask = new Uint8Array(n).fill(1);
for (
let q = Math.max(0, k - STATIC_WINDOW);
q <= Math.min(stack.length - 1, k + STATIC_WINDOW);
q++
)
for (let i = 0; i < n; i++) if (moved[q]![i]! >= STATIC_LEVEL) mask[i] = 0;
const [a, b] = [edges(stack[k]!, WIDTH, h), edges(yours[m]!, WIDTH, h)];
const norm = (mean(a) + mean(b)) / 2 + 1e-6;
const diff = box(
a.map((v, i) => Math.abs(v - b[i]!) / norm),
WIDTH,
h,
boxSize,
);
let best = 0;
for (let i = 1; i < n; i++) if (diff[i]! > diff[best]!) best = i;
const row: Row = {
frame: j,
original_change: j > 0 && k > 0 ? mean(moved[k]!) : null,
change: null,
original_static_change: null,
static_change: null,
structure: diff[best]!,
at: [Math.round((best % WIDTH) * scale), Math.round(Math.floor(best / WIDTH) * scale)],
};
if (m > 0 && k > 0) {
const prev = yours[m - 1]!;
let [all, still, stillOrig, count] = [0, 0, 0, 0];
for (let i = 0; i < n; i++) {
const d = Math.abs(yours[m]![i]! - prev[i]!);
all += d;
if (mask[i]) {
still += d;
stillOrig += moved[k]![i]!;
count++;
}
}
row.change = all / n;
if (count / n >= STATIC_MIN_SHARE) {
row.static_change = still / count;
row.original_static_change = stillOrig / count;
}
}
rows.push(row);
}
const around = (key: keyof Row, at: number, reach: number, reduce: (v: number[]) => number) => {
const v = rows
.slice(Math.max(0, at - reach), at + reach + 1)
.map((r) => r[key])
.filter((x): x is number => typeof x === "number");
return v.length ? reduce(v) : null;
};
const structures = rows.map((r) => r.structure);
const med = median(structures);
const mad = Math.max(median(structures.map((s) => Math.abs(s - med))), 0.05 * med);
const structureThreshold = Math.max(STRUCTURE_MIN, med + STRUCTURE_K * mad);
const localReach = Math.max(1, Math.round(ref.fps));
const flagged: { at: number; kind: ContinuityKind; value: number; base: number | null }[] = [];
// fallow-ignore-next-line complexity
rows.forEach((r, i) => {
if (r.change !== null) {
const base = around("original_change", i, NEIGHBOURS, (v) => Math.max(...v)) ?? 0;
if (r.change > Math.max(base + JUMP_ABS, base * JUMP_RATIO))
flagged.push({ at: i, kind: "jump", value: r.change, base });
else if (r.static_change !== null) {
const sb = around("original_static_change", i, NEIGHBOURS, (v) => Math.max(...v)) ?? 0;
const own = around("static_change", i, STATIC_WINDOW, median) ?? 0;
if (r.static_change > Math.max(sb + FLICKER_ABS, sb * FLICKER_RATIO, own * FLICKER_LOCAL))
flagged.push({ at: i, kind: "flicker", value: r.static_change, base: sb });
}
}
if (
r.structure > structureThreshold &&
r.structure > STRUCTURE_LOCAL * (around("structure", i, localReach, median) ?? 0)
)
flagged.push({ at: i, kind: "structure", value: r.structure, base: med });
});
// Flagged frames of one kind within EVENT_GAP of each other are one event, peaking at its largest value.
const groups: {
kind: ContinuityKind;
first: number;
last: number;
peak: number;
value: number;
base: number | null;
}[] = [];
for (const kind of ["jump", "flicker", "structure"] as const) {
let open: (typeof groups)[number] | null = null;
for (const f of flagged.filter((x) => x.kind === kind)) {
if (open && f.at - open.last <= EVENT_GAP) {
open.last = f.at;
if (f.value > open.value) Object.assign(open, { peak: f.at, value: f.value, base: f.base });
continue;
}
open = { kind, first: f.at, last: f.at, peak: f.at, value: f.value, base: f.base };
groups.push(open);
}
}
groups.sort((x, y) => x.first - y.first);
const t = (i: number) => round(rows[i]!.frame / ref.fps, 3);
const events = groups.map((g, number) => {
const shown = [g.peak - 2, g.peak - 1, g.peak, g.peak + 1].filter(
(i) => i >= 0 && i < rows.length,
);
const path = sheet(
shown.map((i) => stack[rows[i]!.frame - low]!),
shown.map((i) => yours[i]!),
WIDTH,
h,
shown.indexOf(g.peak),
join(
opts.outDir,
`event-${String(number + 1).padStart(2, "0")}-${g.kind}-${rows[g.peak]!.frame}.png`,
),
);
return {
kind: g.kind,
start_s: t(g.first),
end_s: t(g.last),
peak_s: t(g.peak),
peak_frame: rows[g.peak]!.frame,
value: round(g.value, 2),
original_value: g.base === null ? null : round(g.base, 2),
...(g.kind === "structure" ? { at: rows[g.peak]!.at } : {}),
sheet: path,
};
});
return {
events,
frames: rows.length,
thresholds: {
jump_abs: JUMP_ABS,
jump_ratio: JUMP_RATIO,
flicker_abs: FLICKER_ABS,
flicker_ratio: FLICKER_RATIO,
structure: round(structureThreshold, 3),
structure_median: round(med, 3),
},
};
}
+43
View File
@@ -0,0 +1,43 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { compareCurves, curves } from "./curve.js";
const ffmpeg = findFFmpeg();
const dir = mkdtempSync(join(tmpdir(), "hf-curve-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
/** A white box sliding 0.5 s with back.out(1.7) from x=20 to x=220, starting at 0.3 s + `shift`. */
function slide(name: string, shift: number): string {
const q = `min(max((t-0.3)/0.5,0),1)-1`;
const x = `20+200*(1+2.7*pow(${q},3)+1.7*pow(${q},2))`.replaceAll("t", `(t-${shift})`);
const out = join(dir, name);
execFileSync(ffmpeg!, [
...["-nostdin", "-v", "error", "-y"],
...["-f", "lavfi", "-i", "color=black:s=320x120:d=1.2:r=30"],
...["-f", "lavfi", "-i", "color=white:s=40x40:d=1.2:r=30"],
...["-filter_complex", `[0][1]overlay=x='${x}':y=40`],
...["-pix_fmt", "yuv444p", "-c:v", "libx264", "-crf", "0", out],
]);
return out;
}
describe.skipIf(!ffmpeg || !findFFprobe())("curves", () => {
it("reads a back.out slide's ease, length and overshoot, and pairs it with a later copy", () => {
const a = curves(slide("a.mp4", 0), 0, 1.2, [0, 0, 320, 120]);
const b = curves(slide("b.mp4", 0.1), 0, 1.2, [0, 0, 320, 120]);
expect(a.moves).toHaveLength(1);
const x = a.moves[0];
if (!x || x.pulse) throw new Error("expected one fitted move");
expect(x.prop).toBe("x");
expect(x.ease).toMatch(/^back\.out\(1\.[5-9]\)/);
expect(Math.abs(x.duration_s - 0.5)).toBeLessThanOrEqual(0.07);
expect(Math.abs(x.overshoot_pct - 10)).toBeLessThanOrEqual(3);
const c = compareCurves(a, b).find((m) => m.prop === "x")!;
expect(Math.abs(c.start_offset_s! - 0.1)).toBeLessThanOrEqual(0.04);
expect(c.curve_rmse!).toBeLessThan(0.05);
});
});
+303
View File
@@ -0,0 +1,303 @@
// The ease of one graphic move as numbers, ported from the Studio's motion-curve.mjs: inside an ROI, the element's
// box (centre, width, height) and its ink (mean difference from the background) on every frame, then for each
// property that changes, when the move starts and settles, the overshoot past its end value and the closest GSAP
// ease. Compared with a second window, a spring, a back-out or a slow expo is checked rather than eyeballed.
import { Fix, probeVideo, readFrames, round, type Roi } from "./media.js";
const W = 240;
export type Prop = "x" | "y" | "w" | "h" | "ink";
const PROPS: Prop[] = ["x", "y", "w", "h", "ink"];
/** One frame of an element: its box centre and size (video pixels) and ink; x and y are null when it is absent. */
export interface Row {
t: number;
x: number | null;
y: number | null;
w: number;
h: number;
ink: number;
}
type Ease = [name: string, f: (u: number) => number];
const pw = (k: number): Ease => [`power${k}.in`, (u) => u ** (k + 1)];
const pwo = (k: number): Ease => [`power${k}.out`, (u) => 1 - (1 - u) ** (k + 1)];
const pwio = (k: number): Ease => [
`power${k}.inOut`,
(u) => (u < 0.5 ? 2 ** k * u ** (k + 1) : 1 - (-2 * u + 2) ** (k + 1) / 2),
];
const back = (s: number): Ease => [
`back.out(${s.toFixed(1)})`,
(u) => 1 + (s + 1) * (u - 1) ** 3 + s * (u - 1) ** 2,
];
const elastic = (p: number): Ease => [
`elastic.out(1,${p})`,
(u) => (u <= 0 ? 0 : u >= 1 ? 1 : 2 ** (-10 * u) * Math.sin(((u - p / 4) * 2 * Math.PI) / p) + 1),
];
/** GSAP's named eases as functions of normalized time. */
export const EASES: Ease[] = [
["none", (u) => u],
...[1, 2, 3, 4].flatMap((k) => [pw(k), pwo(k), pwio(k)]),
["sine.out", (u) => Math.sin((u * Math.PI) / 2)],
["sine.inOut", (u) => -(Math.cos(Math.PI * u) - 1) / 2],
["expo.out", (u) => (u >= 1 ? 1 : 1 - 2 ** (-10 * u))],
["circ.out", (u) => Math.sqrt(1 - (u - 1) ** 2)],
...Array.from({ length: 26 }, (_, i) => back(0.5 + i * 0.1)),
...[0.2, 0.25, 0.3, 0.4, 0.5].map(elastic),
];
export interface FittedMove {
prop: Prop;
pulse?: undefined;
from: number;
to: number;
start_s: number;
duration_s: number;
frames: number;
overshoot_pct: number;
anticipation_pct: number;
ease: string;
rmse: number;
next: string[];
/** 21 samples of normalized progress, for comparing two moves' shapes. */
curve: number[];
}
/** A stretch that returns to where it began: its peak and length, no ease. */
export interface PulseMove {
prop: Prop;
pulse: true;
from: number;
peak: number;
start_s: number;
duration_s: number;
frames: number;
}
export type Move = FittedMove | PulseMove;
/** Box and ink per frame of `roi`, against a flat background (the ROI border's median on the first frame, or `bg`). */
// fallow-ignore-next-line complexity
function track(
path: string,
start: number,
seconds: number,
roi: Roi,
{ bg = null, thr = 24 }: { bg?: number[] | null; thr?: number } = {},
): { fps: number; bg: number[]; rows: Row[] } {
const fps = probeVideo(path).fps;
const width = Math.min(W, roi[2]);
const { frames: fs, h } = readFrames(path, { start, seconds, roi, width });
const first = fs[0];
if (!first)
throw new Fix(
`no frames between ${start}s and ${start + seconds}s`,
"pick a window inside the video",
);
const k = roi[2] / width;
let ref = bg;
if (!ref) {
const border: number[][] = [[], [], []];
for (let y = 0; y < h; y++)
for (let x = 0; x < width; x++)
if (x < 2 || y < 2 || x >= width - 2 || y >= h - 2)
for (let c = 0; c < 3; c++) border[c]!.push(first[(y * width + x) * 3 + c]!);
ref = border.map((v) => v.sort((a, b) => a - b)[v.length >> 1]!);
}
const [r0, g0, b0] = [ref[0]!, ref[1]!, ref[2]!];
const rows = fs.map((f, i): Row => {
const xs: number[] = [];
const ys: number[] = [];
let ink = 0;
for (let y = 0; y < h; y++)
for (let x = 0; x < width; x++) {
const j = (y * width + x) * 3;
const d = Math.max(
Math.abs(f[j]! - r0),
Math.abs(f[j + 1]! - g0),
Math.abs(f[j + 2]! - b0),
);
ink += d;
if (d > thr) {
xs.push(x);
ys.push(y);
}
}
const t = round(start + i / fps, 3);
if (xs.length < 4) return { t, x: null, y: null, w: 0, h: 0, ink: round(ink / (width * h), 2) };
xs.sort((a, b) => a - b);
ys.sort((a, b) => a - b);
const q = (v: number[], p: number) => v[Math.min(v.length - 1, Math.floor(v.length * p))]!;
const [x0, x1, y0, y1] = [q(xs, 0.01), q(xs, 0.99), q(ys, 0.01), q(ys, 0.99)];
return {
t,
x: round(roi[0] + ((x0 + x1 + 1) / 2) * k, 1),
y: round(roi[1] + ((y0 + y1 + 1) / 2) * k, 1),
w: round((x1 - x0 + 1) * k, 1),
h: round((y1 - y0 + 1) * k, 1),
ink: round(ink / (width * h), 2),
};
});
return { fps, bg: ref, rows };
}
/** Fit one stretch of values from its start value to its settled end value. */
function fit(
rows: readonly Row[],
idx: number[],
v: number[],
prop: Prop,
fps: number,
a: number,
b: number,
): FittedMove {
const [v0, v1] = [v[a]!, v[b]!];
const span = v1 - v0;
const p = v.slice(a, b + 1).map((x) => (x - v0) / span);
const u = p.map((_, i) => i / Math.max(1, p.length - 1));
const fits = EASES.map(([name, f]) => ({
name,
rmse: Math.sqrt(p.reduce((s, x, i) => s + (x - f(u[i]!)) ** 2, 0) / p.length),
})).sort((m, n) => m.rmse - n.rmse);
return {
prop,
from: round(v0, 1),
to: round(v1, 1),
start_s: rows[idx[a]!]!.t,
duration_s: round((idx[b]! - idx[a]!) / fps, 3),
frames: idx[b]! - idx[a]!,
overshoot_pct: round(Math.max(0, Math.max(...p) - 1) * 100, 1),
anticipation_pct: round(Math.max(0, -Math.min(...p)) * 100, 1),
ease: fits[0]!.name,
rmse: round(fits[0]!.rmse, 3),
next: fits.slice(1, 3).map((f) => `${f.name} ${round(f.rmse, 3)}`),
curve: Array.from({ length: 21 }, (_, i) =>
round(p[Math.round((i / 20) * (p.length - 1))]!, 3),
),
};
}
/** Every move of one property: stretches of change separated by holds of 0.15 s or more (shorter stills are the
* turn at an overshoot's peak). A stretch that returns to where it began is a pulse (peak and length, no ease). */
// fallow-ignore-next-line complexity
export function fitMoves(
rows: readonly Row[],
prop: Prop,
fps: number,
minChange: number,
quant: number,
): Move[] {
const idx = rows.flatMap((r, i) => (r[prop] == null ? [] : [i]));
const v = idx.map((i) => rows[i]![prop] ?? 0);
const out: Move[] = [];
for (let i = 0; i < v.length - 1; ) {
if (Math.abs(v[i + 1]! - v[i]!) <= quant) {
i++;
continue;
}
const hold = Math.max(3, Math.round(0.15 * fps));
let [b, still] = [i + 1, 0];
for (let j = i + 1; j < v.length - 1 && still < hold; j++)
if (Math.abs(v[j + 1]! - v[j]!) <= quant) still++;
else [still, b] = [0, j + 1];
const span = v[b]! - v[i]!;
const peak = v
.slice(i, b + 1)
.reduce((m, x) => (Math.abs(x - v[i]!) > Math.abs(m - v[i]!) ? x : m), v[i]!);
// a creep of under 3% of the move before it really starts is compression noise, not the move
let a = i;
while (a < b - 2 && Math.abs(v[a + 1]! - v[a]!) < 0.03 * Math.abs(span)) a++;
if (Math.abs(span) >= minChange) out.push(fit(rows, idx, v, prop, fps, a, b));
else if (Math.abs(peak - v[i]!) >= minChange)
out.push({
prop,
pulse: true,
from: round(v[i]!, 1),
peak: round(peak, 1),
start_s: rows[idx[i]!]!.t,
duration_s: round((idx[b]! - idx[i]!) / fps, 3),
frames: idx[b]! - idx[i]!,
});
i = b;
}
return out;
}
export interface Curves {
start: number;
seconds: number;
roi: Roi;
fps: number;
bg: number[];
moves: Move[];
}
/** Every move of the element in `roi`, sorted by start. */
export function curves(
path: string,
start: number,
seconds: number,
roi: Roi,
opts: { bg?: number[] | null; thr?: number } = {},
): Curves {
const { fps, bg, rows } = track(path, start, seconds, roi, opts);
const minPx = 0.02 * Math.max(roi[2], roi[3]);
const step = roi[2] / Math.min(W, roi[2]);
const moves = PROPS.flatMap((k) =>
k === "ink" ? fitMoves(rows, k, fps, 2, 0.5) : fitMoves(rows, k, fps, minPx, step),
).sort((m, n) => m.start_s - n.start_s);
return { start, seconds, roi, fps: round(fps, 2), bg, moves };
}
export interface MoveComparison {
prop: Prop;
at_s: number;
yours?: "no matching move";
start_offset_s?: number;
duration_frames?: string;
pulse?: true;
peak?: string;
ease?: string;
same_family?: boolean;
overshoot_pct?: string;
curve_rmse?: number;
}
/** Pair each reference move with your move of the same property that starts closest on the window's clock. */
export function compareCurves(ref: Curves, yours: Curves): MoveComparison[] {
const family = (s: string) => s.replace(/\(.*\)/, "").replace(/\d/, "");
return ref.moves.map((m) => {
const rel = m.start_s - ref.start;
const d = yours.moves
.filter((x) => x.prop === m.prop && !!x.pulse === !!m.pulse)
.sort(
(x, y) => Math.abs(x.start_s - yours.start - rel) - Math.abs(y.start_s - yours.start - rel),
)[0];
const head = { prop: m.prop, at_s: round(rel, 2) };
if (!d) return { ...head, yours: "no matching move" };
const common = {
...head,
start_offset_s: round(d.start_s - yours.start - rel, 3),
duration_frames: `${m.frames} → ${d.frames}`,
};
if (m.pulse || d.pulse)
return {
...common,
pulse: true,
peak: `${m.pulse ? m.peak : m.to} → ${d.pulse ? d.peak : d.to}`,
};
return {
...common,
ease: `${m.ease} → ${d.ease}`,
same_family: family(m.ease) === family(d.ease),
overshoot_pct: `${m.overshoot_pct} → ${d.overshoot_pct}`,
curve_rmse: round(
Math.sqrt(m.curve.reduce((s, x, i) => s + (x - d.curve[i]!) ** 2, 0) / m.curve.length),
3,
),
};
});
}
/** Moves as printed: the shape samples stay internal. */
export const printable = (moves: Move[]) =>
moves.map((m) => (m.pulse ? m : { ...m, curve: undefined }));
+44
View File
@@ -0,0 +1,44 @@
// Cuts from the frame-to-frame difference itself, ported from the Studio's videoShots.mjs: ffmpeg's scene score
// missed two of the James Blake source's cuts at 0.25, so a cut is a frame far unlike the one before and several
// times unlike its neighbours' changes.
import { decode, meanAbsDiff, type Frames } from "./media.js";
// Small enough to read a minute quickly, large enough to tell a cut from motion.
const SIG_W = 64;
const SIG_H = 36;
const CUT_LEVEL = 12;
const CUT_RATIO = 4;
/** Every frame of ffmpeg's `input` (after `filters`) as SIG_W x SIG_H grey levels. */
export function signatures(input: string[], filters: string[] = []): Frames {
return decode(input, [...filters, `scale=${SIG_W}:${SIG_H}:flags=area`], SIG_W, SIG_H, true);
}
const median = (values: number[]) => {
const sorted = [...values].sort((x, y) => x - y);
return sorted[Math.floor(sorted.length / 2)] ?? 0;
};
/**
* The frame indexes where a new shot starts, each with its change (mean grey levels) and length: a whip or a
* two-frame dissolve changes on consecutive frames, which is one cut `frames` long, not a shot of one frame.
*/
export function findCuts(
frames: Uint8Array[],
): { frame: number; frames: number; change: number }[] {
const diffs = frames.map((f, i) => (i ? meanAbsDiff(frames[i - 1]!, f) : 0));
const hits = diffs.flatMap((d, i) => {
if (i === 0 || d < CUT_LEVEL) return [];
const around = diffs.slice(Math.max(1, i - 12), i).concat(diffs.slice(i + 1, i + 13));
return d >= CUT_RATIO * Math.max(1, median(around)) ? [{ frame: i, change: d }] : [];
});
const out: { frame: number; frames: number; change: number }[] = [];
for (const h of hits) {
const last = out.at(-1);
if (last && h.frame === last.frame + last.frames) {
last.frames++;
last.change = Math.max(last.change, h.change);
} else out.push({ ...h, frames: 1 });
}
return out;
}
+171
View File
@@ -0,0 +1,171 @@
import { execFileSync } from "node:child_process";
import { existsSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { diffVideos, pickedFrames, selectFilter } from "./diff.js";
import { parseSegments } from "./segments.js";
const ffmpeg = findFFmpeg();
const noTools = !ffmpeg || !findFFprobe();
const dir = mkdtempSync(join(tmpdir(), "hf-diff-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
/** A lossless synthetic clip, so SSIM differences come from the picture, not the encoder. */
function clip(name: string, args: string[]): string {
const out = join(dir, name);
execFileSync(ffmpeg!, [
...["-hide_banner", "-loglevel", "error", "-y", ...args],
...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "0", out],
]);
return out;
}
describe("parseSegments", () => {
it("reads headings and where each window's pixels come from", () => {
const md = [
"# Film — reverse",
"## S01 · 0.00–2.40 s · Title card",
"Pixels: rebuilt",
"## S02 · 2.40–5.00 s · Talking head",
"Pixels: reused",
"## S03 · 5.00–6.5 s · Product shot",
"Pixels: regenerated",
"## S04 · 6.5–8 s",
"## Unknowns",
].join("\n");
expect(parseSegments(md)).toEqual([
{ id: "S01", start: 0, end: 2.4, title: "Title card", pixels: "rebuilt" },
{ id: "S02", start: 2.4, end: 5, title: "Talking head", pixels: "reused" },
{ id: "S03", start: 5, end: 6.5, title: "Product shot", pixels: "regenerated" },
{ id: "S04", start: 6.5, end: 8, title: "", pixels: "rebuilt" },
]);
});
});
describe("frame picks", () => {
it("keeps a range's frames end-exclusive and renumbers them on the original's clock", () => {
expect(pickedFrames({ kind: "range", k0: 3, k1: 6 }, 90)).toEqual([3, 4, 5]);
expect(selectFilter({ kind: "range", k0: 3, k1: 6 }, "30000/1001")).toEqual([
"select=between(n\\,3\\,5)",
"setpts=N/(30000/1001)/TB",
]);
expect(selectFilter({ kind: "at", ks: [2, 9] }, "30")[0]).toBe("select=eq(n\\,2)+eq(n\\,9)");
expect(selectFilter({ kind: "all" }, "30")).toEqual([]);
});
});
describe.skipIf(noTools)("diffVideos", () => {
const ref = noTools ? "" : clip("ref.mp4", ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"]);
it("passes a video against itself", async () => {
const r = await diffVideos({ reference: ref, yours: ref, outDir: join(dir, "same") });
expect(r.pass).toBe(true);
expect(r.ssim.frames).toBe(90);
expect(r.ssim.mean).toBeGreaterThan(0.999);
expect(r.worst).toEqual([]);
});
it("finds a wrong element by its cell when the whole frame still scores high", async () => {
// An 80x60 box over a 320x180 frame for one second: about 0.94 whole-frame SSIM.
const yours = clip("box.mp4", [
...["-i", ref, "-vf"],
"drawbox=x=40:y=40:w=80:h=60:color=red:t=fill:enable='between(t,1,2)'",
]);
const r = await diffVideos({ reference: ref, yours, outDir: join(dir, "box") });
expect(r.ssim.mean).toBeGreaterThan(0.92);
expect(r.ssim.cell_p05).toBeLessThan(0.8);
expect(r.pass).toBe(false);
const [first] = r.worst;
expect(first!.t).toBeGreaterThanOrEqual(1);
expect(first!.t).toBeLessThanOrEqual(2);
const [x, y, w, h] = first!.cell.roi;
expect(x < 120 && x + w > 40 && y < 100 && y + h > 40).toBe(true);
expect(existsSync(r.sheet!)).toBe(true);
});
it("draws the worst cell of a tall frame scaled for the sheet", async () => {
// 1080x1920 scales to 960x1706 for the sheet; the bottom row's box rounds a pixel past the edge.
const tall = clip("tall.mp4", ["-f", "lavfi", "-i", "testsrc2=d=1:r=30:s=1080x1920"]);
const yours = clip("tall-box.mp4", [
...["-i", tall, "-vf"],
"drawbox=x=0:y=1500:w=1080:h=420:color=red:t=fill",
]);
const r = await diffVideos({ reference: tall, yours, outDir: join(dir, "tall") });
expect(r.worst.length).toBeGreaterThan(0);
expect(r.worst[0]!.cell.roi[1]).toBe(1440);
});
it("fails a scene left empty on a plain background", async () => {
// A small dark label on white, gone from yours after 1.5 s: still about 0.97 whole-frame SSIM.
const label = "drawbox=x=40:y=20:w=36:h=12:color=0x303030:t=fill";
const page = clip("page.mp4", [
"-f",
"lavfi",
"-i",
"color=c=white:s=320x180:r=30:d=3",
"-vf",
label,
]);
const yours = clip("emptied.mp4", [
...["-f", "lavfi", "-i", "color=c=white:s=320x180:r=30:d=3", "-vf"],
`${label}:enable='lt(t,1.5)'`,
]);
const r = await diffVideos({ reference: page, yours, outDir: join(dir, "emptied") });
expect(r.missing_content.pass).toBe(false);
expect(r.missing_content.share).toBeGreaterThan(0.45);
expect(r.missing_content.spans[0]![0]).toBeCloseTo(1.5, 1);
expect(r.pass).toBe(false);
const same = await diffVideos({ reference: page, yours: page, outDir: join(dir, "page") });
expect(same.missing_content).toMatchObject({ share: 0, pass: true });
});
it("scores only a range, on the original's frame numbers", async () => {
const yours = clip("late.mp4", [
...["-i", ref, "-vf"],
"drawbox=x=0:y=0:w=320:h=180:color=black:t=fill:enable='gte(t,2)'",
]);
const r = await diffVideos({
reference: ref,
yours,
range: [0.5, 1.5],
outDir: join(dir, "range"),
});
expect(r.ssim.frames).toBe(30);
expect(r.window).toEqual([0.5, 1.5]);
expect(r.pass).toBe(true);
});
it("reports a cut a few frames late as missing and extra", async () => {
const two = (name: string, at: number) =>
clip(name, [
...["-f", "lavfi", "-i", `color=c=0x2050c0:s=320x180:r=30:d=${at}`],
...["-f", "lavfi", "-i", `color=c=0xe0c040:s=320x180:r=30:d=${3 - at}`],
...["-filter_complex", "[0][1]concat=n=2:v=1[v]", "-map", "[v]"],
]);
const r = await diffVideos({
reference: two("cut-ref.mp4", 1),
yours: two("cut-late.mp4", 1.1),
outDir: join(dir, "cuts"),
});
expect(r.cuts?.reference).toEqual([1]);
expect(r.cuts?.missing).toEqual([1]);
expect(r.cuts?.extra).toEqual([1.1]);
expect(r.pass).toBe(false);
});
it("judges a regenerated segment by motion and look, not pixels", async () => {
const shifted = clip("shifted.mp4", [...["-i", ref, "-vf"], "hflip"]);
const segments = parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n");
const r = await diffVideos({
reference: ref,
yours: shifted,
segments,
outDir: join(dir, "regen"),
});
expect(r.ssim.mean).toBeLessThan(0.9);
expect(r.segments?.[0]?.energy?.mean_ratio).toBeCloseTo(1, 1);
expect(r.segments?.[0]?.pass).toBe(true);
});
});
+767
View File
@@ -0,0 +1,767 @@
// A rebuild against its original, frame by frame: per-frame SSIM on the original's clock for the whole frame and
// each cell of a 4x4 grid, the worst moments with their worst cell boxed, cuts on both sides, and per-segment
// verdicts. The second side is a video file (a render) or a project folder, whose frames are captured at the
// original's frame times without rendering.
import {
copyFileSync,
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { findCuts, signatures } from "./cuts.js";
import { compareEnergy, energyCurve, type EnergyComparison } from "./energy.js";
import { compareLook, look, sameLook, type LookComparison } from "./look.js";
import {
decode,
Fix,
frameAt,
probeVideo,
round,
runFfmpeg,
videoClip,
writeImage,
type Clip,
type VideoInfo,
} from "./media.js";
import type { Segment } from "./segments.js";
// To be calibrated on the reverse bench; light-reverse's rebuilt moments scored 0.885–0.958 mean SSIM.
export const DEFAULT_MIN_MEAN = 0.92;
export const DEFAULT_MIN_P05 = 0.85;
/**
* The worst cell of most frames: a missing or misplaced element drags one cell, not the frame. Provisional: four
* no-API rebuilds scored 0.67–0.98 and synthetic wrong elements 0.71–0.76.
*/
export const DEFAULT_MIN_CELL = 0.75;
// A regenerated shot moves about as much as the original's.
const ENERGY_RATIO: [number, number] = [0.8, 1.25];
const GRID = 4;
const WORST_GAP_S = 0.5;
const SHEET_W = 960;
const CUT_SLACK = 1;
// Missing content: cells are read at this width, from at most this many frames. A cell whose luma spread in the
// original reaches MISSING_DETAIL while yours is flat (under MISSING_FLAT and MISSING_KEEP of the original's) is
// empty; over MISSING_MAX_SHARE of frames with an empty cell fails. A weaker blur or a missing small label is not
// empty: SSIM and its cell gate grade those (provisional, calibrated on the reverse bench).
const MISSING_W = 192;
const MISSING_SAMPLES = 1800;
const MISSING_DETAIL = 8;
const MISSING_KEEP = 0.35;
const MISSING_FLAT = 4;
const MISSING_MAX_SHARE = 0.02;
/** Which of the original's frames are compared. */
export type Pick =
| { kind: "all" }
| { kind: "range"; k0: number; k1: number }
| { kind: "at"; ks: number[] };
export interface Thresholds {
mean: number;
p05: number;
cell: number;
}
export interface DiffOptions {
reference: string;
/** A video file, or a project folder holding index.html. */
yours: string;
at?: number[];
range?: [number, number];
segments?: Segment[];
thresholds?: Partial<Thresholds>;
worst?: number;
sbs?: boolean;
outDir: string;
/** Render-ready timeout for a project folder's capture. */
timeoutMs?: number;
log?: (message: string) => void;
}
export interface Scores {
frames: number;
mean: number;
p05: number;
min: number;
/** The 5th percentile of each frame's worst cell. */
cell_p05: number;
}
export interface Moment {
t: number;
frame: number;
ssim: number;
/** The frame's worst cell of a 4x4 grid, in the original's pixels, and both sides of it at full size. */
cell: { roi: [number, number, number, number]; ssim: number; picture: string };
}
export interface SegmentVerdict {
id: string;
window: [number, number];
pixels: Segment["pixels"];
pass: boolean | null;
ssim?: Scores;
energy?: EnergyComparison;
look?: LookComparison;
note?: string;
}
export interface CutReport {
reference: number[];
yours: number[];
/** The original's cuts with no cut within a frame of them in yours. */
missing: number[];
extra: number[];
pass: boolean;
}
export interface MissingContent {
/** Share of compared frames with a cell the original fills and yours leaves flat. */
share: number;
/** The longest such stretches, in seconds. */
spans: [number, number][];
pass: boolean;
}
export interface DiffResult {
pass: boolean;
reference: string;
yours: string;
mode: "video" | "project";
window: [number, number];
thresholds: Thresholds;
ssim: Scores;
missing_content: MissingContent;
worst: Moment[];
cuts?: CutReport;
duration?: { reference: number; yours: number; pass: boolean };
sound?: ReturnType<typeof soundOffset>;
segments?: SegmentVerdict[];
sheet?: string;
sbs?: string;
log: string;
/** Every compared frame's SSIM and worst cell. */
frames: string;
warnings: string[];
}
interface Series {
/** Whole-frame SSIM per compared frame. */
frame: number[];
/** Each frame's 16 cell scores, row by row. */
cells: number[][];
}
const pct = (sorted: number[], p: number) =>
sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] ?? 0;
function scores(frame: number[], worstCell: number[]): Scores {
const sorted = [...frame].sort((a, b) => a - b);
const n = sorted.length;
return {
frames: n,
mean: round(n ? sorted.reduce((s, v) => s + v, 0) / n : 0, 4),
p05: round(pct(sorted, 0.05), 4),
min: round(sorted[0] ?? 0, 4),
cell_p05: round(
pct(
[...worstCell].sort((a, b) => a - b),
0.05,
),
4,
),
};
}
const passes = (s: Scores, t: Thresholds) =>
s.frames > 0 && s.mean >= t.mean && s.p05 >= t.p05 && s.cell_p05 >= t.cell;
/** The original's frame indexes a pick compares, in order. */
export function pickedFrames(pick: Pick, total: number): number[] {
if (pick.kind === "at") return pick.ks;
const [k0, k1] = pick.kind === "range" ? [pick.k0, pick.k1] : [0, total];
return Array.from({ length: Math.max(0, k1 - k0) }, (_, i) => k0 + i);
}
/** The filters that keep a pick's frames of a stream, renumbered from 0 on the original's clock. */
export function selectFilter(pick: Pick, fpsText: string): string[] {
const pts = `setpts=N/(${fpsText})/TB`;
if (pick.kind === "all") return [];
if (pick.kind === "range") return [`select=between(n\\,${pick.k0}\\,${pick.k1 - 1})`, pts];
return [`select=${pick.ks.map((k) => `eq(n\\,${k})`).join("+")}`, pts];
}
/** Even cell sizes keep 4:2:0 chroma aligned; the last row and column drop at most a pixel or two. */
const cellSize = (w: number, h: number): [number, number] => [
Math.floor(w / (2 * GRID)) * 2,
Math.floor(h / (2 * GRID)) * 2,
];
/**
* Per-frame SSIM (ffmpeg's All: luma and chroma) of yours against the original, in the pick's order, for the
* whole frame and each cell: a wrong box over an eighth of the frame still scores 0.94 whole-frame.
*/
function ssimSeries(ref: Clip, yours: Clip, logPath: string): Series {
const side = (filters: string[]) => [...filters, "setsar=1", "format=yuv420p"].join(",");
const n = GRID * GRID + 1;
const tags = (p: string) => Array.from({ length: n }, (_, i) => `[${p}${i}]`).join("");
const [cw, ch] = cellSize(ref.width, ref.height);
const graph = [
`[0:v]${side(ref.filters)},split=${n}${tags("r")}`,
`[1:v]${side([...yours.filters, `scale=${ref.width}:${ref.height}:flags=bicubic`])},split=${n}${tags("o")}`,
"[o0][r0]ssim=stats_file=-:shortest=1[s0]",
...Array.from({ length: GRID * GRID }, (_, i) => {
const crop = `crop=${cw}:${ch}:${(i % GRID) * cw}:${Math.floor(i / GRID) * ch}`;
const k = i + 1;
return `[r${k}]${crop}[rc${k}];[o${k}]${crop}[oc${k}];[oc${k}][rc${k}]ssim=stats_file=cell${i}.log:shortest=1[s${k}]`;
}),
].join(";");
// Cell logs are written beside ffmpeg's working folder, so their names need no filtergraph escaping.
const work = mkdtempSync(join(tmpdir(), "hf-diff-cells-"));
const all = (text: string) => [...text.matchAll(/All:([\d.]+)/g)].map((m) => Number(m[1]));
try {
const maps = Array.from({ length: n }, (_, i) => ["-map", `[s${i}]`]).flat();
const text = runFfmpeg(
[...ref.input, ...yours.input, "-filter_complex", graph, ...maps, "-f", "null", "-"],
undefined,
work,
).toString();
writeFileSync(logPath, text);
const frame = all(text);
const perCell = Array.from({ length: GRID * GRID }, (_, i) =>
all(readFileSync(join(work, `cell${i}.log`), "utf8")),
);
return { frame, cells: frame.map((_, f) => perCell.map((cell) => cell[f] ?? 1)) };
} finally {
rmSync(work, { recursive: true, force: true });
}
}
/**
* Stretches where a cell the original fills is flat in yours. SSIM scores two near-blank patches alike, so a scene
* left empty on a plain background passes SSIM and its cell gate; this compares each cell's luma spread instead.
*/
function missingContent(ref: Clip, yours: Clip, ks: number[], fps: number): MissingContent {
const stride = Math.max(1, Math.ceil(ks.length / MISSING_SAMPLES));
const w = MISSING_W;
const h = Math.max(GRID * 2, Math.round((w * ref.height) / ref.width / GRID) * GRID);
const thin =
stride > 1 ? [`select=not(mod(n\\,${stride}))`, `setpts=N/(${fps / stride})/TB`] : [];
const spreads = (clip: Clip) =>
decode(clip.input, [...clip.filters, ...thin, `scale=${w}:${h}`], w, h, true).frames.map((px) =>
cellSpreads(px, w, h),
);
const [a, b] = [spreads(ref), spreads(yours)];
// The thinning select can pass one frame past the pick; times come from ks, so stop with it.
const n = Math.min(a.length, b.length, Math.ceil(ks.length / stride));
const empty = Array.from({ length: n }, (_, j) =>
a[j]!.some(
(s, c) => s >= MISSING_DETAIL && b[j]![c]! < Math.min(MISSING_FLAT, MISSING_KEEP * s),
),
);
const spans: [number, number][] = [];
empty.forEach((e, j) => {
if (!e) return;
const t = round(ks[j * stride]! / fps, 3);
const last = spans.at(-1);
if (last && empty[j - 1])
last[1] = round((ks[Math.min(ks.length - 1, (j + 1) * stride - 1)]! + 1) / fps, 3);
else spans.push([t, round((ks[Math.min(ks.length - 1, (j + 1) * stride - 1)]! + 1) / fps, 3)]);
});
const share = n ? empty.filter(Boolean).length / n : 0;
return {
share: round(share, 4),
spans: spans
.sort((x, y) => y[1] - y[0] - (x[1] - x[0]))
.slice(0, 8)
.sort((x, y) => x[0] - y[0]),
pass: share <= MISSING_MAX_SHARE,
};
}
/** Each grid cell's luma standard deviation, row by row. */
function cellSpreads(px: Uint8Array, w: number, h: number): number[] {
const [cw, ch] = [Math.floor(w / GRID), Math.floor(h / GRID)];
return Array.from({ length: GRID * GRID }, (_, c) => {
const [x0, y0] = [(c % GRID) * cw, Math.floor(c / GRID) * ch];
let [s, ss] = [0, 0];
for (let y = y0; y < y0 + ch; y++)
for (let x = x0; x < x0 + cw; x++) {
const v = px[y * w + x]!;
s += v;
ss += v * v;
}
const m = s / (cw * ch);
return Math.sqrt(Math.max(0, ss / (cw * ch) - m * m));
});
}
/** The `count` frames with the lowest worst cell, at least WORST_GAP_S apart; near-perfect frames are skipped. */
function worstIndexes(worstCell: number[], ks: number[], fps: number, count: number): number[] {
const order = worstCell
.map((_, i) => i)
.filter((i) => worstCell[i]! < 0.98)
.sort((a, b) => worstCell[a]! - worstCell[b]!);
const out: number[] = [];
for (const i of order) {
if (out.length >= count) break;
if (out.every((j) => Math.abs(ks[i]! - ks[j]!) / fps >= WORST_GAP_S)) out.push(i);
}
return out.sort((a, b) => ks[a]! - ks[b]!);
}
/** One frame of a clip (its i-th after the clip's own filters) at `w`x`h` RGB. */
function frameOf(clip: Clip, i: number, w: number, h: number): Uint8Array {
const { frames } = decode(
clip.input,
[...clip.filters, `select=eq(n\\,${i})`, `scale=${w}:${h}`],
w,
h,
false,
1,
);
if (!frames[0]) throw new Fix(`could not read frame ${i}`, "check both videos play to their end");
return frames[0];
}
/** A red rectangle drawn into RGB bytes, clipped to the picture (a scaled box can round a pixel past its edge). */
function outline(px: Uint8Array, w: number, [x, y, bw, bh]: number[], t = 3) {
const h = px.length / 3 / w;
for (let yy = y!; yy < Math.min(h, y! + bh!); yy++)
for (let xx = x!; xx < Math.min(w, x! + bw!); xx++) {
const edge = yy - y! < t || y! + bh! - 1 - yy < t || xx - x! < t || x! + bw! - 1 - xx < t;
if (edge) px.set([255, 40, 40], (yy * w + xx) * 3);
}
}
function cutReport(ref: Clip, yours: Clip, k0: number, fps: number): CutReport {
const a = findCuts(signatures(ref.input, ref.filters).frames).map((c) => c.frame);
const b = findCuts(signatures(yours.input, yours.filters).frames).map((c) => c.frame);
const used = new Set<number>();
const missing: number[] = [];
for (const f of a) {
const hit = b.find((g) => !used.has(g) && Math.abs(g - f) <= CUT_SLACK);
if (hit === undefined) missing.push(f);
else used.add(hit);
}
const t = (f: number) => round((k0 + f) / fps, 3);
return {
reference: a.map(t),
yours: b.map(t),
missing: missing.map(t),
extra: b.filter((g) => !used.has(g)).map(t),
pass: missing.length === 0 && b.length === used.size,
};
}
/** Captures a project's frames at the original's frame times as a numbered picture sequence. */
async function captureProject(
projectDir: string,
times: number[],
opts: DiffOptions,
ref: VideoInfo,
): Promise<{ clip: Clip; dir: string }> {
const { captureSnapshots } = await import("../commands/snapshot.js");
const shots = mkdtempSync(join(tmpdir(), "hf-diff-shots-"));
const seq = mkdtempSync(join(tmpdir(), "hf-diff-seq-"));
try {
const captured = await captureSnapshots(projectDir, {
at: times,
includeEnd: false,
outputDir: shots,
timeout: opts.timeoutMs,
log: opts.log ?? (() => {}),
record: false,
});
if (captured.paths.length !== times.length)
throw new Fix(
`captured ${captured.paths.length} of ${times.length} frames from ${projectDir}`,
"run npx hyperframes check on the project, then diff again",
);
captured.paths.forEach((p, i) =>
copyFileSync(resolve(projectDir, p), join(seq, `${String(i).padStart(6, "0")}.png`)),
);
} catch (err) {
rmSync(seq, { recursive: true, force: true });
throw err;
} finally {
rmSync(shots, { recursive: true, force: true });
}
return {
dir: seq,
clip: {
input: ["-framerate", ref.fpsText, "-i", join(seq, "%06d.png")],
filters: [],
width: ref.width,
height: ref.height,
fps: ref.fps,
},
};
}
function regeneratedVerdict(s: Segment, reference: string, yours: string, ref: VideoInfo) {
const seconds = s.end - s.start;
const [a, b] = [videoClip(reference, s.start, seconds, ref), videoClip(yours, s.start, seconds)];
const energy = compareEnergy(energyCurve(a), energyCurve(b));
const looks = compareLook(look(a), look(b));
const moves = energy.mean_ratio >= ENERGY_RATIO[0] && energy.mean_ratio <= ENERGY_RATIO[1];
return { pass: moves && sameLook(looks), energy, look: looks };
}
function segmentVerdicts(
segments: Segment[],
series: Series,
worstCell: number[],
ks: number[],
ctx: {
reference: string;
yours: string;
ref: VideoInfo;
mode: DiffResult["mode"];
t: Thresholds;
},
): SegmentVerdict[] {
return segments.map((s) => {
const head = { id: s.id, window: [s.start, s.end] as [number, number], pixels: s.pixels };
if (s.pixels === "regenerated")
return ctx.mode === "video"
? { ...head, ...regeneratedVerdict(s, ctx.reference, ctx.yours, ctx.ref) }
: {
...head,
pass: null,
note: "regenerated pixels are judged by motion and look: diff a render",
};
const [a, b] = [frameAt(s.start, ctx.ref.fps), frameAt(s.end, ctx.ref.fps)];
const mine = ks.flatMap((k, i) => (k >= a && k < b && i < series.frame.length ? [i] : []));
if (!mine.length)
return { ...head, pass: null, note: "no compared frame falls in this segment" };
const st = scores(
mine.map((i) => series.frame[i]!),
mine.map((i) => worstCell[i]!),
);
return { ...head, pass: passes(st, ctx.t), ssim: st };
});
}
async function worstMoments(
picks: number[],
ctx: { refClip: Clip; yours: Clip; series: Series; ks: number[]; ref: VideoInfo; outDir: string },
): Promise<{ worst: Moment[]; sheet?: string }> {
const { ref, series } = ctx;
const sw = Math.min(SHEET_W, ref.width);
const sh = Math.max(2, Math.round((ref.height * sw) / ref.width / 2) * 2);
const [cw, ch] = cellSize(ref.width, ref.height);
const worst: Moment[] = [];
const images: string[] = [];
const labels: string[] = [];
for (const [n, i] of picks.entries()) {
const cells = series.cells[i]!;
const c = cells.indexOf(Math.min(...cells));
const roi: Moment["cell"]["roi"] = [(c % GRID) * cw, Math.floor(c / GRID) * ch, cw, ch];
const scaled = roi.map((v, j) =>
Math.round((v * (j % 2 ? sh : sw)) / (j % 2 ? ref.height : ref.width)),
);
const t = round(ctx.ks[i]! / ref.fps, 3);
const tag = `worst-${String(n + 1).padStart(2, "0")}`;
// The failing cell at full size, original | yours, to read text and edges.
const [fa, fb] = [
frameOf(ctx.refClip, i, ref.width, ref.height),
frameOf(ctx.yours, i, ref.width, ref.height),
];
const pair = new Uint8Array((2 * cw + 4) * ch * 3).fill(255);
for (let y = 0; y < ch; y++)
[fa, fb].forEach((f, side) =>
pair.set(
f.subarray(
((roi[1] + y) * ref.width + roi[0]) * 3,
((roi[1] + y) * ref.width + roi[0] + cw) * 3,
),
(y * (2 * cw + 4) + side * (cw + 4)) * 3,
),
);
const cellPicture = writeImage(join(ctx.outDir, `${tag}-cell.png`), pair, 2 * cw + 4, ch);
const [a, b] = [frameOf(ctx.refClip, i, sw, sh), frameOf(ctx.yours, i, sw, sh)];
outline(a, sw, scaled);
outline(b, sw, scaled);
images.push(writeImage(join(ctx.outDir, `${tag}-original.png`), a, sw, sh));
images.push(writeImage(join(ctx.outDir, `${tag}-yours.png`), b, sw, sh));
labels.push(
`original ${t}s`,
`yours ${t}s · ssim ${round(series.frame[i]!, 3)} · cell ${round(cells[c]!, 2)}`,
);
worst.push({
t,
frame: ctx.ks[i]!,
ssim: round(series.frame[i]!, 4),
cell: { roi, ssim: round(cells[c]!, 3), picture: cellPicture },
});
}
if (!images.length) return { worst };
const { createContactSheet } = await import("../capture/contactSheet.js");
const sheet = await createContactSheet(images, join(ctx.outDir, "worst.jpg"), {
cols: 2,
maxImages: images.length,
cellWidth: 640,
labelMode: "custom",
labels,
});
return { worst, ...(sheet ? { sheet } : {}) };
}
/** Loudness every 10 ms of the first 20 s, as a rising-edge curve: where sounds start. */
function onsetCurve(path: string): number[] {
// Padded from time 0, so a track that starts late reads late.
const raw = runFfmpeg([
"-i",
path,
"-vn",
"-af",
"aresample=async=1:first_pts=0",
"-t",
"20",
"-ac",
"1",
"-ar",
"8000",
"-f",
"f32le",
"-",
]);
const samples = new Float32Array(raw.buffer, raw.byteOffset, Math.floor(raw.byteLength / 4));
const rms: number[] = [];
for (let i = 0; i + 80 <= samples.length; i += 80) {
let s = 0;
for (let j = i; j < i + 80; j++) s += samples[j]! ** 2;
rms.push(Math.sqrt(s / 80));
}
return rms.map((v, i) => Math.max(0, v - (rms[i - 1] ?? v)));
}
/** How far yours' sound sits from the original's (positive: yours is late), from where sounds start. */
// fallow-ignore-next-line complexity
function soundOffset(reference: string, yours: string, ref: VideoInfo, mine: VideoInfo) {
if (!ref.audio || !mine.audio)
return { original: ref.audio, yours: mine.audio, pass: ref.audio === mine.audio };
const [a, b] = [onsetCurve(reference), onsetCurve(yours)];
let best: [number, number] = [-2, 0];
for (let lag = -50; lag <= 50; lag++) {
const n = Math.min(a.length, b.length) - Math.abs(lag);
if (n < 50) continue;
let [sab, sa, sb, ma, mb] = [0, 0, 0, 0, 0];
for (let i = 0; i < n; i++) {
ma += a[i + Math.max(0, -lag)]!;
mb += b[i + Math.max(0, lag)]!;
}
[ma, mb] = [ma / n, mb / n];
for (let i = 0; i < n; i++) {
const [x, y] = [a[i + Math.max(0, -lag)]! - ma, b[i + Math.max(0, lag)]! - mb];
sab += x * y;
sa += x * x;
sb += y * y;
}
const r = sa && sb ? sab / Math.sqrt(sa * sb) : -2;
if (r > best[0]) best = [r, lag];
}
const offset = best[1] * 10;
// Steady sound (a held pad, room tone) has too few starts to align: say so rather than fail it.
if (best[0] <= 0.3)
return {
offset_ms: null,
match: round(best[0], 2),
pass: true,
note: "too few distinct sounds to align",
};
return { offset_ms: offset, match: round(best[0], 2), pass: Math.abs(offset) <= 1000 / ref.fps };
}
function sideBySide(reference: string, yours: string, ref: VideoInfo, out: string): string {
runFfmpeg([
...["-y", "-i", reference, "-i", yours, "-filter_complex"],
`[0:v]setsar=1[a];[1:v]scale=${ref.width}:${ref.height},setsar=1[b];[a][b]hstack=inputs=2,pad=ceil(iw/2)*2:ceil(ih/2)*2[v]`,
...["-map", "[v]", "-map", "0:a?", "-c:v", "libx264", "-crf", "20", "-pix_fmt", "yuv420p"],
...["-c:a", "aac", "-shortest", out],
]);
return out;
}
function choosePick(opts: DiffOptions, ref: VideoInfo): Pick {
if (opts.at)
return {
kind: "at",
ks: [...new Set(opts.at.map((t) => frameAt(t, ref.fps)))].sort((x, y) => x - y),
};
if (opts.range)
return {
kind: "range",
k0: frameAt(opts.range[0], ref.fps),
k1: Math.min(ref.frames, Math.ceil(opts.range[1] * ref.fps - 1e-6)),
};
return { kind: "all" };
}
// fallow-ignore-next-line complexity
export async function diffVideos(options: DiffOptions): Promise<DiffResult> {
// ffmpeg runs in a scratch folder (see ssimSeries), so every path it reads is absolute.
const opts = { ...options, reference: resolve(options.reference), yours: resolve(options.yours) };
const ref = probeVideo(opts.reference);
const warnings: string[] = [];
if (ref.vfr)
warnings.push(
"the original's frame rate varies, so frame times drift: npx hyperframes measure probe makes a constant-rate copy",
);
const mode = existsSync(opts.yours) && statSync(opts.yours).isDirectory() ? "project" : "video";
if (mode === "project" && !existsSync(join(opts.yours, "index.html")))
throw new Fix(`${opts.yours} has no index.html`, "pass a project folder or a rendered video");
if (mode === "project" && !opts.at && !opts.range && opts.segments?.length)
opts.range = [
Math.min(...opts.segments.map((s) => s.start)),
Math.max(...opts.segments.map((s) => s.end)),
];
if (mode === "project" && !opts.at && !opts.range)
throw new Fix(
"a project folder is compared at chosen times, not whole",
"pass --at t1,t2 or --range start,end, or render it and diff the render",
);
const pick = choosePick(opts, ref);
const ks = pickedFrames(pick, ref.frames);
if (!ks.length || ks.some((k) => k >= ref.frames))
throw new Fix(
`the original has ${ref.frames} frames (${round(ref.duration, 2)} s)`,
"pick times inside it",
);
const select = selectFilter(pick, ref.fpsText);
const refClip: Clip = { ...videoClip(opts.reference, 0, undefined, ref), filters: select };
let yours: Clip;
let seqDir: string | undefined;
let yoursInfo: VideoInfo | undefined;
if (mode === "project") {
const captured = await captureProject(
opts.yours,
ks.map((k) => k / ref.fps),
opts,
ref,
);
[yours, seqDir] = [captured.clip, captured.dir];
} else {
yoursInfo = probeVideo(opts.yours);
yours = {
...videoClip(opts.yours, 0, undefined, yoursInfo),
filters: [`fps=${ref.fpsText}`, ...select],
};
}
try {
mkdirSync(opts.outDir, { recursive: true });
for (const f of readdirSync(opts.outDir))
if (/^worst-/.test(f)) rmSync(join(opts.outDir, f), { force: true });
const logPath = join(opts.outDir, "ssim.log");
const series = ssimSeries(refClip, yours, logPath);
if (!series.frame.length)
throw new Fix("ffmpeg compared no frames", "check both videos play and overlap in time");
if (series.frame.length < ks.length)
warnings.push(`yours ends early: ${series.frame.length} of ${ks.length} frames compared`);
const worstCell = series.cells.map((cells) => Math.min(...cells));
// Every compared frame, to find the stretches that fail.
writeFileSync(
join(opts.outDir, "frames.tsv"),
[
"t\tframe\tssim\tworst_cell\tcell",
...series.frame.map((v, i) => {
const c = series.cells[i]!.indexOf(worstCell[i]!);
return [
round(ks[i]! / ref.fps, 3),
ks[i],
round(v, 4),
round(worstCell[i]!, 4),
`r${Math.floor(c / GRID)}c${c % GRID}`,
].join("\t");
}),
].join("\n") + "\n",
);
const t: Thresholds = {
mean: opts.thresholds?.mean ?? DEFAULT_MIN_MEAN,
p05: opts.thresholds?.p05 ?? DEFAULT_MIN_P05,
cell: opts.thresholds?.cell ?? DEFAULT_MIN_CELL,
};
const overall = scores(series.frame, worstCell);
const missing = missingContent(refClip, yours, ks, ref.fps);
if (!missing.pass)
warnings.push(
`${Math.round(missing.share * 100)}% of frames leave empty what the original fills (${missing.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}): build what is missing there`,
);
const picks = worstIndexes(worstCell, ks, ref.fps, opts.worst ?? 4);
const { worst, sheet } = await worstMoments(picks, {
refClip,
yours,
series,
ks,
ref,
outDir: opts.outDir,
});
const cuts = pick.kind === "at" ? undefined : cutReport(refClip, yours, ks[0]!, ref.fps);
const duration =
yoursInfo && pick.kind === "all"
? {
reference: round(ref.duration, 3),
yours: round(yoursInfo.duration, 3),
pass: Math.abs(ref.duration - yoursInfo.duration) <= 1.5 / ref.fps,
}
: undefined;
const segments = opts.segments
? segmentVerdicts(opts.segments, series, worstCell, ks, {
reference: opts.reference,
yours: opts.yours,
ref,
mode,
t,
})
: undefined;
const sound =
yoursInfo && pick.kind === "all"
? soundOffset(opts.reference, opts.yours, ref, yoursInfo)
: undefined;
const sbs =
opts.sbs && mode === "video"
? sideBySide(opts.reference, opts.yours, ref, join(opts.outDir, "sbs.mp4"))
: undefined;
const judged = segments ? segments.every((s) => s.pass !== false) : passes(overall, t);
const complete = series.frame.length === ks.length;
return {
pass:
judged &&
complete &&
missing.pass &&
(cuts?.pass ?? true) &&
(duration?.pass ?? true) &&
(sound?.pass ?? true),
reference: opts.reference,
yours: opts.yours,
mode,
window: [round(ks[0]! / ref.fps, 3), round((ks.at(-1)! + 1) / ref.fps, 3)],
thresholds: t,
ssim: overall,
missing_content: missing,
worst,
...(cuts ? { cuts } : {}),
...(duration ? { duration } : {}),
...(sound ? { sound } : {}),
...(segments ? { segments } : {}),
...(sheet ? { sheet } : {}),
...(sbs ? { sbs } : {}),
log: logPath,
frames: join(opts.outDir, "frames.tsv"),
warnings,
};
} finally {
if (seqDir) rmSync(seqDir, { recursive: true, force: true });
}
}
+42
View File
@@ -0,0 +1,42 @@
// Body and camera motion where the pixels differ (a regenerated shot, another person, another room), ported from
// the Studio's motion-energy.mjs. Both clips are sampled on one 12 fps grid (mixed 24/30 fps skews per-frame
// differences), ROI scaled to 270 px wide, luma. Energy(t) = mean |frame_t - frame_t-1|.
import { meanAbsDiff, readClip, round, type Clip, type Roi } from "./media.js";
import { pearson } from "./stats.js";
const ENERGY_RATE = 12;
export function energyCurve(clip: Clip, roi?: Roi): number[] {
const { frames } = readClip(clip, { fps: ENERGY_RATE, width: 270, gray: true, roi });
return frames.slice(1).map((f, i) => meanAbsDiff(f, frames[i]!));
}
export interface EnergyComparison {
/** Pearson r of the two curves at the best lag within ±1 s; only meaningful when the action is the same. */
r: number;
lag_s: number;
peak_ratio: number;
/** Amount of movement, yours over the reference's. */
mean_ratio: number;
}
export function compareEnergy(ref: number[], yours: number[]): EnergyComparison {
let best: [number, number] = [-2, 0];
for (let lag = -ENERGY_RATE; lag <= ENERGY_RATE; lag++) {
const [aa, bb] =
lag >= 0
? [ref.slice(0, ref.length - lag), yours.slice(lag)]
: [ref.slice(-lag), yours.slice(0, yours.length + lag)];
if (Math.min(aa.length, bb.length) > 8) {
const r = pearson(aa, bb) ?? 0;
if (r > best[0]) best = [r, lag];
}
}
const mean = (v: number[]) => v.reduce((s, x) => s + x, 0) / (v.length || 1);
return {
r: round(best[0] === -2 ? 0 : best[0]),
lag_s: round(best[1] / ENERGY_RATE, 2),
peak_ratio: round(Math.max(...yours, 0) / (Math.max(...ref, 0) || 1e-9), 2),
mean_ratio: round(mean(yours) / (mean(ref) || 1e-9), 2),
};
}
+107
View File
@@ -0,0 +1,107 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { cutClip, cutStill } from "../commands/measure/extract.js";
import { detectEvents, lookWindows } from "./events.js";
import { probeVideo, readFrames } from "./media.js";
const ffmpeg = findFFmpeg();
const noTools = !ffmpeg || !findFFprobe();
const dir = mkdtempSync(join(tmpdir(), "hf-events-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
function clip(name: string, args: string[]): string {
const out = join(dir, name);
execFileSync(ffmpeg!, ["-hide_banner", "-loglevel", "error", "-y", ...args, out]);
return out;
}
describe.skipIf(noTools)("detectEvents", () => {
// 25 fps, 3 s: a box appears at 0.52 s, the whole frame flashes white on frame 30 (1.2 s) only, the background
// cuts from dark blue to dark green at 2.0 s.
const video = noTools
? ""
: clip("events.mp4", [
...["-f", "lavfi", "-i", "color=c=0x101830:s=320x180:r=25:d=2"],
...["-f", "lavfi", "-i", "color=c=0x103018:s=320x180:r=25:d=1"],
"-filter_complex",
"[0][1]concat=n=2:v=1[bg];[bg]" +
"drawbox=x=200:y=100:w=60:h=40:color=white:t=fill:enable='gte(n,13)'," +
"drawbox=x=0:y=0:w=320:h=180:color=white:t=fill:enable='eq(n,30)'[v]",
...["-map", "[v]", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "0"],
]);
it("tells an appear, a one-frame flash, a cut and a hold apart, each on its frame", () => {
const info = probeVideo(video);
const found = detectEvents({
info,
start: 0,
end: 3,
roi: [0, 0, 320, 180],
grid: { cols: 4, rows: 4 },
});
const of = (type: string) => found.events.filter((e) => e.type === type);
expect(of("appear").map((e) => e.frame)).toEqual([13]);
expect(of("appear")[0]!.cells).toEqual(["r2c2", "r2c3", "r3c2", "r3c3"]);
expect(of("flash").map((e) => [e.frame, e.frames])).toEqual([[30, 1]]);
expect(of("cut").map((e) => e.frame)).toEqual([50]);
expect(of("hold").length).toBeGreaterThan(0);
const windows = lookWindows(found.events, info.fps);
expect(windows.map((w) => w.what[0]!.split(" ")[0])).toEqual(["appear", "flash", "cut"]);
expect(windows[1]!.onset_frames).toEqual([30]);
});
});
describe.skipIf(noTools)("extract", () => {
// Every frame's luma is 40 + 3 × its frame number, with B-frames so the first frame does not start at 0.
const video = noTools
? ""
: clip("counter.mp4", [
...[
"-f",
"lavfi",
"-i",
"nullsrc=s=64x36:r=25:d=2",
"-vf",
"geq=lum='40+3*N':cb=128:cr=128",
],
...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-qp", "0", "-bf", "3"],
]);
// Grey output is full range; the clip's luma is limited range (16–235).
const levelOf = (path: string) => {
const { frames } = readFrames(path, { gray: true });
return frames.map((f) =>
Math.round((((f.reduce((s, v) => s + v, 0) / f.length) * 219) / 255 + 16 - 40) / 3),
);
};
it("cuts the still shown at a time", async () => {
const out = join(dir, "still.png");
const r = await cutStill(probeVideo(video), 0.52, [8, 8, 32, 16], out);
expect(r.frame).toBe(13);
expect(levelOf(out)).toEqual([13]);
});
it("cuts a clip of exactly the frames in its window at 30 fps too", () => {
const thirty = clip("counter30.mp4", [
...["-f", "lavfi", "-i", "nullsrc=s=64x36:r=30:d=4", "-vf", "geq=lum='40+3*N':cb=128:cr=128"],
...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-qp", "0", "-bf", "3"],
]);
const out = join(dir, "clip30.mp4");
const r = cutClip(probeVideo(thirty), 31 / 30, 37 / 30, [0, 0, 32, 16], out, { crf: 0 });
expect(r.frames).toBe(6);
expect(levelOf(out)).toEqual([31, 32, 33, 34, 35, 36]);
});
it("cuts a clip of exactly the frames in its window", () => {
const out = join(dir, "clip.mp4");
const r = cutClip(probeVideo(video), 0.4, 0.8, [0, 0, 33, 17], out, { crf: 0 });
expect(r.frames).toBe(10);
expect(r.roi).toEqual([0, 0, 32, 16]);
expect(levelOf(out)).toEqual([10, 11, 12, 13, 14, 15, 16, 17, 18, 19]);
});
});
+436
View File
@@ -0,0 +1,436 @@
// When things change on screen, ported from the Studio's viral-clone events.mjs: from the frame-to-frame grey
// difference inside an ROI or each cell of a grid, cuts (most of the frame changes at once), flashes (one or two
// frames that return to the picture before), appears and disappears (a region steps from one steady look to
// another), holds (still for 0.3 s or more) and other motion, each with its frame number. The map adds the sound's
// onsets and beats and groups everything into the windows worth looking at frame by frame.
import { findCuts } from "./cuts.js";
import { grey, readClip, round, videoClip, writeImage, type Roi, type VideoInfo } from "./media.js";
// The width motion-rhythm reads motion at, so its still threshold carries over.
const EVENT_W = 160;
const CUT_SHARE = 0.7;
const CUT_GRID = 4;
const HOLD_S = 0.3;
const STEP_MAX_S = 0.5;
const FLASH_FRAMES = 2;
const STRIP_MAX = 24;
const STRIP_W = 1200;
// Events closer than this share one window to look at, up to WINDOW_MAX_S: a talking head changes somewhere all
// the time, and one 20 s window shows nothing frame by frame.
const WINDOW_GAP_S = 0.25;
const WINDOW_MAX_S = 1.5;
/** A grid cell; rect is x0,y0,x1,y1 in working pixels, end exclusive. */
interface Cell {
name: string;
rect: [number, number, number, number];
}
export type EventType = "cut" | "flash" | "appear" | "disappear" | "change" | "hold";
export interface ScreenEvent {
type: EventType;
t: number;
frame: number;
frames?: number;
seconds?: number;
cells?: string[];
roi?: number[];
magnitude?: number;
}
/** An event with the frames and box its strip pair shows (working pixels). */
interface Shown extends ScreenEvent {
before: number;
after: number;
rect: number[] | null;
}
function cells(cols: number, rows: number, w: number, h: number): Cell[] {
return Array.from({ length: rows * cols }, (_, i) => {
const [r, c] = [Math.floor(i / cols), i % cols];
return {
name: `r${r}c${c}`,
rect: [
Math.floor((c * w) / cols),
Math.floor((r * h) / rows),
Math.floor(((c + 1) * w) / cols),
Math.floor(((r + 1) * h) / rows),
],
};
});
}
/** Mean absolute difference of two frames inside `rect`. */
function diff(a: Uint8Array, b: Uint8Array, w: number, [x0, y0, x1, y1]: number[]): number {
let s = 0;
for (let y = y0!; y < y1!; y++)
for (let x = x0!; x < x1!; x++) s += Math.abs(a[y * w + x]! - b[y * w + x]!);
return s / Math.max(1, (x1! - x0!) * (y1! - y0!));
}
/** Mean and standard deviation of grey inside `rect`. */
function level(f: Uint8Array, w: number, [x0, y0, x1, y1]: number[]) {
let [s, s2] = [0, 0];
for (let y = y0!; y < y1!; y++)
for (let x = x0!; x < x1!; x++) {
s += f[y * w + x]!;
s2 += f[y * w + x]! ** 2;
}
const n = Math.max(1, (x1! - x0!) * (y1! - y0!));
return { mean: s / n, std: Math.sqrt(Math.max(0, s2 / n - (s / n) ** 2)) };
}
/** Each frame's change from the one before inside `rect` (frame 0: 0). */
const changes = (g: Uint8Array[], w: number, rect: number[]) =>
g.map((f, i) => (i ? diff(f, g[i - 1]!, w, rect) : 0));
/** Runs of frames changing by over twice the series' 25th percentile and over 2 grey levels, one quiet frame bridged. */
function bursts(d: number[]): [number, number][] {
const q = d.slice(1).sort((a, b) => a - b);
const act = Math.max(2, 2 * (q[Math.floor(q.length / 4)] ?? 0));
const out: [number, number][] = [];
for (let i = 1; i < d.length; i++) {
if (d[i]! <= act) continue;
const last = out.at(-1);
if (last && i - last[1] <= 2) last[1] = i;
else out.push([i, i]);
}
return out;
}
/** Still runs of HOLD_S or more by motion-rhythm's still threshold: [first frame of the held picture, its frames]. */
function stills(d: number[], fps: number): [number, number][] {
const q = d.slice(1).sort((a, b) => a - b);
const floor = Math.min(1.5, Math.max(0.5, 2 * (q[Math.floor(q.length / 10)] ?? 0)));
const out: [number, number][] = [];
for (let i = 1, run = 0; i <= d.length; i++) {
if (i < d.length && d[i]! <= floor) run++;
else {
if (run + 1 >= HOLD_S * fps) out.push([i - run - 1, run + 1]);
run = 0;
}
}
return out;
}
/** Joins items whose `near(a, b)` holds, transitively. */
function groups<T>(items: T[], near: (a: T, b: T) => boolean): T[][] {
const root = items.map((_, i) => i);
const find = (i: number): number => (root[i] === i ? i : (root[i] = find(root[i]!)));
for (let i = 0; i < items.length; i++)
for (let j = i + 1; j < items.length; j++)
if (near(items[i]!, items[j]!)) root[find(i)] = find(j);
const out = new Map<number, T[]>();
items.forEach((it, i) => out.set(find(i), [...(out.get(find(i)) ?? []), it]));
return [...out.values()];
}
/**
* Frames where CUT_SHARE of the cells change by `thr` or more at once. Two such frames a frame or two apart whose
* second returns to the picture before the first are a flash, not two cuts.
*/
function cutFrames(g: Uint8Array[], w: number, cutCells: Cell[], thr: number) {
const d = cutCells.map((c) => changes(g, w, c.rect));
const hits = g
.map((_, i) => i)
.filter((i) => i > 0 && d.filter((x) => x[i]! >= thr).length >= CUT_SHARE * cutCells.length);
const cuts: [number, number][] = [];
const flashes: [number, number][] = [];
for (let k = 0; k < hits.length; k++) {
const [i, j] = [hits[k]!, hits[k + 1]];
const last = cuts.at(-1);
if (
j !== undefined &&
j - i <= FLASH_FRAMES &&
diff(g[j]!, g[i - 1]!, w, [0, 0, w, g[0]!.length / w]) < thr
) {
flashes.push([i, j - 1]);
k++;
} else if (last && i === last[1] + 1)
last[1] = i; // a whip or a short dissolve: one cut over several frames
else cuts.push([i, i]);
}
return { cuts, flashes, d };
}
interface Burst {
ci: number;
c: Cell;
a: number;
b: number;
net: number;
peak: number;
}
/** Each region's bursts outside the cuts and flashes, split into steps, local flashes and other motion. */
// fallow-ignore-next-line complexity
function bursting(
g: Uint8Array[],
w: number,
regions: Cell[],
skip: (a: number, b: number) => boolean,
thr: number,
fps: number,
) {
const settle = Math.round(HOLD_S * fps);
const steps: Burst[] = [];
const flashes: Burst[] = [];
const moves: Burst[] = [];
for (const [ci, c] of regions.entries()) {
const d = changes(g, w, c.rect);
for (const [a, b] of bursts(d)) {
if (skip(a, b)) continue;
const net = diff(g[b]!, g[a - 1]!, w, c.rect);
const peak = Math.max(...d.slice(a, b + 1));
const burst = { ci, c, a, b, net, peak };
// a flash lights up for a frame or two and the region returns to how it was
if (
b - a + 1 <= FLASH_FRAMES &&
peak >= 2 * thr &&
b + 1 < g.length &&
diff(g[b + 1]!, g[a - 1]!, w, c.rect) < thr
) {
flashes.push(burst);
continue;
}
// a step joins two looks that each last: the region moves under a third as much in the 0.3 s either side
const was = diff(g[a - 1]!, g[Math.max(0, a - 1 - settle)]!, w, c.rect);
const now = diff(g[b]!, g[Math.min(g.length - 1, b + settle)]!, w, c.rect);
const step = net >= thr && b - a + 1 <= STEP_MAX_S * fps && Math.max(was, now) <= net / 3;
(step ? steps : moves).push(burst);
}
}
return { steps, flashes, moves };
}
export interface DetectOptions {
info: VideoInfo;
start: number;
end: number;
/** One region, or (when `grid` is given) the whole frame split into cells. */
roi: Roi;
grid?: { cols: number; rows: number };
thr?: number;
}
export interface Detected {
fps: number;
frames: number;
events: ScreenEvent[];
/** Working-size RGB frames and the events a strip should show, for drawing. */
rgb: Uint8Array[];
w: number;
h: number;
shown: Shown[];
}
/** Every cut, flash, step, change and hold of a window of a video. */
export function detectEvents(opts: DetectOptions): Detected {
const { info, start, end, roi, grid } = opts;
const thr = opts.thr ?? 5;
const fps = info.fps;
const whole: Roi = [0, 0, info.width, info.height];
const sameAsWhole = roi.every((v, i) => v === whole[i]);
const clip = videoClip(info.path, start, end - start, info);
const { frames: rgb, w, h } = readClip(clip, { roi, width: EVENT_W });
const g = rgb.map(grey);
const wholeG = sameAsWhole ? g : readClip(clip, { width: EVENT_W, gray: true }).frames;
const wholeH = wholeG[0] ? wholeG[0].length / EVENT_W : h;
const f0 = Math.ceil(start * fps - 1e-6);
const at = (i: number) => ({ frame: f0 + i, t: round((f0 + i) / fps, 3) });
if (g.length < 3) return { fps, frames: g.length, events: [], rgb, w, h, shown: [] };
// Cuts are read on the whole frame: the grid's own cells, or with an ROI a 4x4 grid of the whole frame.
const cutCells = cells(grid?.cols ?? CUT_GRID, grid?.rows ?? CUT_GRID, EVENT_W, wholeH);
const cut = cutFrames(wholeG, EVENT_W, cutCells, thr);
// A fast handheld move or whip changes most cells on frame after frame: motion, not a run of cuts. A cut is one
// the brightness test diff uses also finds (cuts.ts), or a change that stands alone in time.
const real = new Set(
findCuts(wholeG).flatMap((c) => Array.from({ length: c.frames }, (_, i) => c.frame + i)),
);
const alone = ([i, j]: [number, number]) =>
j === i && cut.cuts.every(([a, b]) => (a === i && b === j) || b < i - 3 || a > j + 3);
cut.cuts = cut.cuts.filter((c) => real.has(c[0]) || alone(c));
const single: Cell = { name: "roi", rect: [0, 0, w, h] };
const regions = grid ? cells(grid.cols, grid.rows, w, h) : [single];
const covered = (a: number, b: number) =>
[...cut.cuts, ...cut.flashes].some(([x, y]) => x <= b && a <= y);
const { steps, flashes, moves } = bursting(g, w, regions, covered, thr, fps);
const cols = grid?.cols ?? 1;
const touching = (s: Burst, o: Burst) =>
Math.abs(Math.floor(s.ci / cols) - Math.floor(o.ci / cols)) <= 1 &&
Math.abs((s.ci % cols) - (o.ci % cols)) <= 1;
const overlap = (s: Burst, o: Burst) => s.a <= o.b + 2 && o.a <= s.b + 2;
const [kx, ky] = [roi[2] / w, roi[3] / h];
const event = (grp: Burst[], type: EventType, size: number): Shown => {
const [a, b] = [Math.min(...grp.map((s) => s.a)), Math.max(...grp.map((s) => s.b))];
const rect = [0, 1, 2, 3].map((k) =>
(k < 2 ? Math.min : Math.max)(...grp.map((s) => s.c.rect[k]!)),
);
const where = grid
? {
cells: [...new Set(grp.map((s) => s.c.name))].sort(),
roi: [
rect[0]! * kx + roi[0],
rect[1]! * ky + roi[1],
(rect[2]! - rect[0]!) * kx,
(rect[3]! - rect[1]!) * ky,
].map(Math.round),
}
: {};
return {
type,
...at(a),
frames: b - a + 1,
...where,
magnitude: round(size, 1),
before: a - 1,
after: b,
rect: grid ? rect : null,
};
};
const stepEvents = groups(steps, (s, o) => overlap(s, o) && touching(s, o)).map((grp) => {
const e = event(grp, "change", grp.reduce((m, s) => m + s.net, 0) / grp.length);
const box = e.rect ?? [0, 0, w, h];
const [was, now] = [level(g[e.before]!, w, box), level(g[e.after]!, w, box)];
const up = Math.abs(now.std - was.std) >= 2 ? now.std > was.std : now.mean > was.mean;
return { ...e, type: up ? ("appear" as const) : ("disappear" as const) };
});
const flashEvents = [
...groups(flashes, (s, o) => overlap(s, o) && touching(s, o)).map((grp) =>
event(grp, "flash", Math.max(...grp.map((s) => s.peak))),
),
...cut.flashes.map(([i, j]) => ({
type: "flash" as const,
...at(i),
frames: j - i + 1,
magnitude: round(cut.d.reduce((m, d) => m + d[i]!, 0) / cut.d.length, 1),
before: i - 1,
after: i,
rect: null,
})),
];
const moveEvents = groups(moves, overlap).map((grp) =>
event(grp, "change", Math.max(...grp.map((s) => s.peak))),
);
const cutEvents: Shown[] = cut.cuts.map(([i, j]) => ({
type: "cut",
...at(i),
...(j > i ? { frames: j - i + 1 } : {}),
magnitude: round(cut.d.reduce((m, d) => m + d[i]!, 0) / cut.d.length, 1),
before: i - 1,
after: j,
rect: null,
}));
const still = changes(grid ? wholeG : g, EVENT_W, [0, 0, EVENT_W, grid ? wholeH : h]);
const held: ScreenEvent[] = stills(still, fps).map(([i, n]) => ({
type: "hold",
...at(i),
seconds: round(n / fps, 2),
}));
// The strip keeps the biggest: cuts and flashes, then steps by cells × magnitude, then changes by magnitude.
const weight = (e: Shown) => (e.cells?.length ?? 1) * (e.magnitude ?? 0);
const order = (x: ScreenEvent, y: ScreenEvent) => x.frame - y.frame;
const shown = [
...cutEvents,
...flashEvents,
...[...stepEvents].sort((x, y) => weight(y) - weight(x)),
...[...moveEvents].sort((x, y) => (y.magnitude ?? 0) - (x.magnitude ?? 0)),
]
.slice(0, STRIP_MAX)
.sort(order);
const plain = ({ before: _b, after: _a, rect: _r, ...e }: Shown): ScreenEvent => e;
const events = [...cutEvents, ...flashEvents, ...stepEvents, ...moveEvents].map(plain);
return { fps, frames: g.length, events: [...events, ...held].sort(order), rgb, w, h, shown };
}
/** One picture of each shown event's before|after pair, its region boxed in red. */
export function drawStrip(d: Detected, out: string): string {
const { rgb, w, h, shown } = d;
const per = Math.max(1, Math.floor(STRIP_W / (2 * w + 12)));
const cw = per * (2 * w + 12);
const ch = Math.ceil(shown.length / per) * (h + 8);
const px = new Uint8Array(cw * ch * 3).fill(255);
shown.forEach((e, k) => {
const [ox, oy] = [(k % per) * (2 * w + 12), Math.floor(k / per) * (h + 8)];
[e.before, e.after].forEach((f, side) => {
const x0 = ox + side * (w + 2);
for (let y = 0; y < h; y++)
px.set(rgb[f]!.subarray(y * w * 3, (y + 1) * w * 3), ((oy + y) * cw + x0) * 3);
if (!e.rect) return;
const [rx0, ry0, rx1, ry1] = e.rect;
for (let y = ry0!; y < ry1!; y++)
for (let x = rx0!; x < rx1!; x++)
if (y === ry0 || y === ry1! - 1 || x === rx0 || x === rx1! - 1)
px.set([255, 0, 0], ((oy + y) * cw + x0 + x) * 3);
});
});
return writeImage(out, px, cw, ch);
}
export interface LookWindow {
start: number;
end: number;
/** What happens, e.g. "cut", "appear r1c2,r1c3", "flash 1f". */
what: string[];
/** Sound onsets inside the window, seconds. */
onsets?: number[];
/** The frames its events start on: a strip of the window must show these. */
onset_frames: number[];
}
/** Every event but holds, merged into the windows worth looking at frame by frame, two frames padded either side. */
export function lookWindows(
events: ScreenEvent[],
fps: number,
onsets: number[] = [],
): LookWindow[] {
const pad = 2 / fps;
const spans = events
.filter((e) => e.type !== "hold")
.map((e) => {
const where = e.cells ? ` ${e.cells.join(",")}` : "";
const len = e.type === "flash" ? ` ${e.frames}f` : "";
return {
a: e.t - pad,
b: e.t + ((e.frames ?? 1) - 1) / fps + pad,
what: `${e.type}${where}${len}`,
frame: e.frame,
};
})
.sort((x, y) => x.a - y.a);
const out: Omit<LookWindow, "onsets">[] = [];
for (const s of spans) {
const last = out.at(-1);
if (last && s.a - last.end <= WINDOW_GAP_S && s.b - last.start <= WINDOW_MAX_S) {
last.end = Math.max(last.end, s.b);
last.what.push(s.what);
last.onset_frames.push(s.frame);
} else out.push({ start: s.a, end: s.b, what: [s.what], onset_frames: [s.frame] });
}
return out.map((wnd) => {
const hits = onsets.filter((t) => t >= wnd.start && t <= wnd.end);
return {
start: round(Math.max(0, wnd.start), 3),
end: round(wnd.end, 3),
what: wnd.what,
...(hits.length ? { onsets: hits.map((t) => round(t, 3)) } : {}),
onset_frames: [...new Set(wnd.onset_frames)].sort((x, y) => x - y),
};
});
}
/** How many of each type, for a summary line. */
export function countEvents(events: ScreenEvent[]): Record<EventType, number> {
const count = (type: EventType) => events.filter((e) => e.type === type).length;
return {
cut: count("cut"),
flash: count("flash"),
appear: count("appear"),
disappear: count("disappear"),
change: count("change"),
hold: count("hold"),
};
}
+103
View File
@@ -0,0 +1,103 @@
// How big the face is and where, ported from hf-media's face_size: OpenCV's frontal and profile Haar cascades on
// evenly spaced frames, graded by face height over frame height. A regenerated presenter keeps the likeness from
// FACE_HOLDS up; below FACE_FLOOR a video model cannot carry one at all (the harness's face-size eval, 2026-10-08).
import { frameAt, readClip, round, videoClip, type VideoInfo } from "./media.js";
import { greyMat, loadOpenCv } from "./opencv.js";
const FACE_HOLDS = 0.25;
const FACE_SLIDES = 0.15;
const FACE_FLOOR = 0.06;
const WORK_W = 960;
type Verdict = "pass" | "warn" | "fail" | "unmeasured";
const ORDER: Verdict[] = ["fail", "unmeasured", "warn", "pass"];
export function faceVerdict(height: number | null, cast = false): Verdict {
if (height === null) return "unmeasured";
if (height >= FACE_HOLDS) return "pass";
return height < (cast ? FACE_SLIDES : FACE_FLOOR) ? "fail" : "warn";
}
/** Each sampled frame's faces as [x, y, w, h] shares of the frame. */
export async function faceBoxes(info: VideoInfo, start: number, end: number, samples: number) {
const cv = await loadOpenCv((m) => console.error(m));
const [k0, k1] = [
frameAt(start, info.fps),
Math.max(frameAt(start, info.fps), Math.ceil(end * info.fps - 1e-6) - 1),
];
const n = Math.min(samples, k1 - k0 + 1);
const ks = [
...new Set(
Array.from({ length: n }, (_, i) => k0 + Math.round((i * (k1 - k0)) / Math.max(1, n - 1))),
),
];
const clip = {
...videoClip(info.path, 0, undefined, info),
filters: [`select=${ks.map((k) => `eq(n\\,${k})`).join("+")}`, `setpts=N/(${info.fpsText})/TB`],
};
const { frames, w, h } = readClip(clip, { width: Math.min(WORK_W, info.width), gray: true });
const cascades = ["haarcascade_frontalface_alt2.xml", "haarcascade_profileface.xml"].map(
(name) => {
const c = new cv.CascadeClassifier();
c.load(name);
return c;
},
);
try {
return frames.map((px, i) => {
const gray = greyMat(cv, px, w, h);
const boxes: number[][] = [];
try {
for (const c of cascades) {
const found = new cv.RectVector();
c.detectMultiScale(gray, found, 1.1, 5, 0, new cv.Size(20, 20));
for (let j = 0; j < found.size(); j++) {
const r = found.get(j);
boxes.push([r.x / w, r.y / h, r.width / w, r.height / h]);
}
found.delete();
}
} finally {
gray.delete();
}
return { frame: ks[i]!, t: round(ks[i]! / info.fps, 3), boxes };
});
} finally {
for (const c of cascades) c.delete();
}
}
/** The subject's face (largest, or the smallest when the person is cast in a crowd), its median and last height. */
export function faceSummary(
samples: { frame: number; t: number; boxes: number[][] }[],
cast = false,
) {
const pick = (boxes: number[][]) =>
boxes.reduce<number[] | null>(
(a, b) => (!a || (cast ? b[3]! < a[3]! : b[3]! > a[3]!) ? b : a),
null,
);
const seen = samples.flatMap((s) => {
const b = pick(s.boxes);
return b ? [{ t: s.t, box: b.map((v) => round(v, 4)) }] : [];
});
const heights = seen.map((s) => s.box[3]!).sort((a, b) => a - b);
const height = heights.length ? heights[heights.length >> 1]! : null;
const lastFace = pick(samples.at(-1)?.boxes ?? []);
const last = lastFace ? lastFace[3]! : null;
const verdicts = [faceVerdict(height, cast), faceVerdict(last, cast)];
return {
frames_sampled: samples.length,
frames_with_face: seen.length,
face_height: height === null ? null : round(height, 3),
last_face_height: last === null ? null : round(last, 3),
verdict: verdicts.reduce((a, b) => (ORDER.indexOf(b) < ORDER.indexOf(a) ? b : a)),
bands: { pass: FACE_HOLDS, cast_fail_below: FACE_SLIDES, fail_below: FACE_FLOOR },
boxes: seen,
...(last === null
? {
note: "no face at the last sampled frame (turned away, covered or missed): look at the frames",
}
: {}),
};
}
+285
View File
@@ -0,0 +1,285 @@
// Dense optical flow inside one element's box, ported from hf-media's flow.py. Two uses: comparing a rebuild with
// its original on one clock (motion energy, the shape of the motion over time, path, scale and rotation; fonts and
// content do not move these numbers as they do SSIM), and reversing a move into tweens. hf-media's DIS flow is not in
// opencv.js, so this is Farneback's; thresholds carry over but are calibrated on the reverse bench.
import { EASES } from "./curve.js";
import { readClip, round, videoClip, type Roi, type VideoInfo } from "./media.js";
import { greyMat, loadOpenCv } from "./opencv.js";
// Below this mean flow (pixels a frame at the analysis size) a frame counts as still.
const STILL = 0.05;
// Flow under this magnitude is noise for the affine fit.
const FIT_MIN = 0.3;
// Flow needs room for its windows: thin boxes are read taller.
const MIN_HEIGHT = 64;
export interface FlowRow {
mag: number;
share: number;
tx: number;
ty: number;
scale: number;
rot: number;
}
/** Per pair of frames: mean flow, the share of moving pixels, and the affine of the moving part. */
// fallow-ignore-next-line complexity
export async function flowSeries(
info: VideoInfo,
start: number,
end: number,
roi: Roi,
maxWidth: number,
): Promise<FlowRow[]> {
const cv = await loadOpenCv((m) => console.error(m));
let width = Math.min(maxWidth, roi[2]);
if ((width * roi[3]) / roi[2] < MIN_HEIGHT) width = Math.ceil((MIN_HEIGHT * roi[2]) / roi[3]);
const { frames, w, h } = readClip(videoClip(info.path, start, end - start, info), {
roi,
width,
gray: true,
});
const rows: FlowRow[] = [];
const grid: number[] = [];
for (let y = 0; y < h; y += 4) for (let x = 0; x < w; x += 4) grid.push(x, y);
for (let i = 1; i < frames.length; i++) {
const [a, b, flow] = [
greyMat(cv, frames[i - 1]!, w, h),
greyMat(cv, frames[i]!, w, h),
new cv.Mat(),
];
try {
cv.calcOpticalFlowFarneback(a, b, flow, 0.5, 5, 25, 3, 7, 1.5, 0);
const f = flow.data32F;
let [sum, moving] = [0, 0];
for (let p = 0; p < w * h; p++) {
const m = Math.hypot(f[2 * p]!, f[2 * p + 1]!);
sum += m;
if (m > FIT_MIN) moving++;
}
const row: FlowRow = {
mag: sum / (w * h),
share: moving / (w * h),
tx: 0,
ty: 0,
scale: 1,
rot: 0,
};
const from: number[] = [];
const to: number[] = [];
for (let g = 0; g < grid.length; g += 2) {
const p = grid[g + 1]! * w + grid[g]!;
const [dx, dy] = [f[2 * p]!, f[2 * p + 1]!];
if (Math.hypot(dx, dy) <= FIT_MIN) continue;
from.push(grid[g]!, grid[g + 1]!);
to.push(grid[g]! + dx, grid[g + 1]! + dy);
}
if (from.length >= 24) {
const fit = similarity(from, to);
if (fit)
Object.assign(row, { tx: fit.tx / w, ty: fit.ty / h, scale: fit.scale, rot: fit.rot });
}
rows.push(row);
} finally {
for (const mat of [a, b, flow]) mat.delete();
}
}
return rows;
}
interface Similarity {
a: number;
b: number;
tx: number;
ty: number;
}
/** Least squares in closed form over the points `keep`. */
function fitSimilarity(from: number[], to: number[], keep: number[]): Similarity | null {
let [px, py, qx, qy] = [0, 0, 0, 0];
for (const i of keep) {
px += from[2 * i]!;
py += from[2 * i + 1]!;
qx += to[2 * i]!;
qy += to[2 * i + 1]!;
}
[px, py, qx, qy] = [px / keep.length, py / keep.length, qx / keep.length, qy / keep.length];
let [sa, sb, ss] = [0, 0, 0];
for (const i of keep) {
const [ux, uy, vx, vy] = [
from[2 * i]! - px,
from[2 * i + 1]! - py,
to[2 * i]! - qx,
to[2 * i + 1]! - qy,
];
sa += ux * vx + uy * vy;
sb += ux * vy - uy * vx;
ss += ux * ux + uy * uy;
}
if (ss < 1e-9) return null;
const [a, b] = [sa / ss, sb / ss];
return { a, b, tx: qx - (a * px - b * py), ty: qy - (b * px + a * py) };
}
const residual = (f: Similarity, from: number[], to: number[], i: number) =>
Math.hypot(
f.a * from[2 * i]! - f.b * from[2 * i + 1]! + f.tx - to[2 * i]!,
f.b * from[2 * i]! + f.a * from[2 * i + 1]! + f.ty - to[2 * i + 1]!,
);
/**
* The scale, rotation and translation the most points agree on (`from` and `to` are flat x,y lists): RANSAC over
* two-point hypotheses with a fixed seed, then least squares on the inliers. opencv.js has no
* estimateAffinePartial2D, and a plain fit is pulled off by the smeared flow at a moving element's edges.
*/
export function similarity(
from: number[],
to: number[],
): { tx: number; ty: number; scale: number; rot: number } | null {
const n = from.length / 2;
let seed = 1709;
const pick = () => {
seed = (seed * 1103515245 + 12345) % 2147483648;
return seed % n;
};
let best: number[] = [];
for (let k = 0; k < 200; k++) {
const [i, j] = [pick(), pick()];
if (i === j) continue;
const f = fitSimilarity(from, to, [i, j]);
if (!f) continue;
const inliers: number[] = [];
for (let p = 0; p < n; p++) if (residual(f, from, to, p) <= 1) inliers.push(p);
if (inliers.length > best.length) best = inliers;
}
if (best.length < Math.max(6, 0.2 * n)) return null;
const fit = fitSimilarity(from, to, best);
if (!fit) return null;
return {
tx: fit.tx,
ty: fit.ty,
scale: Math.hypot(fit.a, fit.b),
rot: (Math.atan2(fit.b, fit.a) * 180) / Math.PI,
};
}
/** The best Pearson r of two curves within ±3 frames, and its lag. */
function corr(a: number[], b: number[]): [number | null, number] {
let best: [number | null, number] = [null, 0];
for (let lag = -3; lag <= 3; lag++) {
const x = a.slice(Math.max(0, -lag), a.length - Math.max(0, lag));
const y = b.slice(Math.max(0, lag), b.length - Math.max(0, -lag));
const n = Math.min(x.length, y.length);
if (n < 4) continue;
const mx = x.slice(0, n).reduce((s, v) => s + v, 0) / n;
const my = y.slice(0, n).reduce((s, v) => s + v, 0) / n;
let [sxy, sx, sy] = [0, 0, 0];
for (let i = 0; i < n; i++) {
sxy += (x[i]! - mx) * (y[i]! - my);
sx += (x[i]! - mx) ** 2;
sy += (y[i]! - my) ** 2;
}
if (sx < 1e-18 || sy < 1e-18) continue;
const c = sxy / Math.sqrt(sx * sy);
if (best[0] === null || c > best[0]) best = [c, lag];
}
return best;
}
/** Yours against the original, frame for frame on one clock. */
export function compareFlow(ref: FlowRow[], yours: FlowRow[]) {
const n = Math.min(ref.length, yours.length);
const [a, b] = [ref.slice(0, n), yours.slice(0, n)];
const energy = (rows: FlowRow[]) =>
rows.reduce((s, r) => s + r.mag, 0) / Math.max(1, rows.length);
const [re, ye] = [energy(a), energy(b)];
const still = re < STILL;
const ratio = still ? (ye < STILL ? 1 : null) : ye / re;
const [c, lag] = corr(
a.map((r) => r.mag),
b.map((r) => r.mag),
);
const mean = (v: number[]) => v.reduce((s, x) => s + x, 0) / Math.max(1, v.length);
return {
frames: n,
original_still: still,
energy_ratio: ratio === null ? null : round(ratio, 3),
motion_corr: c === null ? null : round(c, 3),
lag_frames: lag,
path_error: round(
mean(a.map((r, i) => Math.abs(r.tx - b[i]!.tx) + Math.abs(r.ty - b[i]!.ty))),
4,
),
scale_error: round(mean(a.map((r, i) => Math.abs(r.scale - b[i]!.scale))), 4),
rotation_error: round(mean(a.map((r, i) => Math.abs(r.rot - b[i]!.rot))), 3),
};
}
function fitEase(progress: number[]): [string, number] {
const n = progress.length;
let best: [string, number] = ["none", Infinity];
for (const [name, f] of EASES) {
const err = progress.reduce((s, p, i) => s + (f(i / Math.max(1, n - 1)) - p) ** 2, 0) / n;
if (err < best[1]) best = [name, err];
}
return [best[0], round(Math.sqrt(best[1]), 4)];
}
/** Moves between holds, each as a tween: when, how far (source pixels), scale, rotation, ease and overshoot. */
// fallow-ignore-next-line complexity
export function flowMoves(rows: FlowRow[], fps: number, [, , w, h]: Roi, start: number) {
const pos: [number, number, number, number][] = [[0, 0, 1, 0]];
for (const r of rows) {
const [x, y, s, a] = pos.at(-1)!;
pos.push([x + r.tx * w, y + r.ty * h, s * r.scale, a + r.rot]);
}
// A frame belongs to a move when the element itself shifts, scales or turns, however slowly it eases out.
const active = rows.map(
(r) =>
Math.hypot(r.tx * w, r.ty * h) >= 0.25 ||
Math.abs(r.scale - 1) >= 0.003 ||
Math.abs(r.rot) >= 0.15,
);
const moves: Record<string, unknown>[] = [];
for (let i = 0; i < active.length; ) {
if (!active[i]) {
i++;
continue;
}
let j = i;
while (j + 1 < active.length && (active[j + 1] || (j + 2 < active.length && active[j + 2])))
j++;
const [s0, s1] = [pos[i]!, pos[j + 1]!];
const [dx, dy] = [s1[0] - s0[0], s1[1] - s0[1]];
const along = pos
.slice(i, j + 2)
.map((p) => (Math.abs(dx) >= Math.abs(dy) ? p[0] - s0[0] : p[1] - s0[1]));
const total = Math.abs(dx) >= Math.abs(dy) ? dx : dy;
const ds = s1[2] - s0[2];
const progress =
Math.abs(total) >= 1
? along.map((v) => v / total)
: pos.slice(i, j + 2).map((p) => (Math.abs(ds) > 1e-3 ? (p[2] - s0[2]) / ds : 0));
const head = {
start_s: round(start + i / fps, 3),
x: round(dx, 1),
y: round(dy, 1),
scale: round(s1[2] / s0[2], 3),
};
if (j === i) moves.push({ kind: "jump", ...head });
else {
const [ease, err] = fitEase(progress);
moves.push({
kind: "tween",
...head,
duration_s: round((j + 1 - i) / fps, 3),
rotation: round(s1[3] - s0[3], 2),
ease,
ease_error: err,
overshoot: round(Math.max(0, Math.max(...progress) - 1), 3),
});
}
i = j + 1;
}
return moves;
}
+27
View File
@@ -0,0 +1,27 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { scan, summary } from "./ink.js";
const ffmpeg = findFFmpeg();
const dir = mkdtempSync(join(tmpdir(), "hf-ink-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
describe.skipIf(!ffmpeg || !findFFprobe())("ink scan", () => {
it("finds when a light box is on screen and where", () => {
const clip = join(dir, "t.mp4");
execFileSync(ffmpeg!, [
...["-nostdin", "-v", "error", "-y", "-f", "lavfi", "-i", "color=black:s=320x180:d=2:r=10"],
...["-vf", "drawbox=x=100:y=50:w=60:h=20:color=white:t=fill:enable='between(t,0.5,1.45)'"],
...["-pix_fmt", "yuv444p", "-c:v", "libx264", "-crf", "0", clip],
]);
const s = summary(scan(clip, 0, 2, [0, 0, 320, 180]));
expect(s.first_visible).toBe(0.5);
expect(s.last_visible).toBe(1.4);
const [row] = scan(clip, 1, 0.15, [0, 0, 320, 180], { fps: 0 });
row!.box!.forEach((v, i) => expect(Math.abs(v - [100, 50, 60, 20][i]!)).toBeLessThanOrEqual(1));
});
});
+73
View File
@@ -0,0 +1,73 @@
// Where text ink sits in an ROI over time, ported from the Studio's ink-scan.mjs: its box in frame pixels per
// sample, summarised as first and last visible time (cue timing against the heard words), the widest box (caption
// maximum width) and the median box height, which solves font size: render a candidate font at a known size,
// scan the same text there, and size = candidate size × median height here / median height there.
import { Fix, readFrames, round, type Roi } from "./media.js";
export interface InkSample {
t: number;
box: [number, number, number, number] | null;
ink_px: number;
}
/** "light", "dark" or "R,G,B" as a pixel test within `tol`. */
function isInk(spec: string, tol: number): (r: number, g: number, b: number) => boolean {
if (spec === "light") return (r, g, b) => 0.299 * r + 0.587 * g + 0.114 * b >= 255 - tol;
if (spec === "dark") return (r, g, b) => 0.299 * r + 0.587 * g + 0.114 * b <= tol;
const c = spec.split(",").map(Number);
if (c.length !== 3 || c.some((v) => !Number.isFinite(v)))
throw new Fix(`--ink must be light, dark or R,G,B, got "${spec}"`, "--ink 255,214,0");
const [cr, cg, cb] = [c[0]!, c[1]!, c[2]!];
return (r, g, b) => Math.hypot(r - cr, g - cg, b - cb) <= tol;
}
/** Samples at `fps` from `start` for `seconds`; `fps` 0 reads one frame at `start`. */
export function scan(
path: string,
start: number,
seconds: number,
[x, y, w, h]: Roi,
{
fps = 10,
ink = "light",
tol = 60,
min = 12,
}: { fps?: number; ink?: string; tol?: number; min?: number } = {},
): InkSample[] {
const test = isInk(ink, tol);
const { frames } = readFrames(path, { start, seconds, roi: [x, y, w, h], fps });
// fallow-ignore-next-line complexity
return (fps ? frames : frames.slice(0, 1)).map((f, i) => {
let [x0, y0, x1, y1, n] = [w, h, -1, -1, 0];
for (let py = 0; py < h; py++)
for (let px = 0; px < w; px++) {
const o = (py * w + px) * 3;
if (!test(f[o]!, f[o + 1]!, f[o + 2]!)) continue;
n++;
if (px < x0) x0 = px;
if (px > x1) x1 = px;
if (py < y0) y0 = py;
if (py > y1) y1 = py;
}
const t = round(start + (fps ? i / fps : 0));
return n < min
? { t, box: null, ink_px: n }
: { t, box: [x + x0, y + y0, x1 - x0 + 1, y1 - y0 + 1], ink_px: n };
});
}
export function summary(rows: InkSample[]) {
const on = rows.flatMap((r) => (r.box ? [r.box] : []));
const times = rows.filter((r) => r.box).map((r) => r.t);
const med = (v: number[]) => [...v].sort((a, b) => a - b)[v.length >> 1]!;
if (!on.length) return { samples: rows.length, first_visible: null };
return {
samples: rows.length,
visible_samples: on.length,
first_visible: times[0]!,
last_visible: times.at(-1)!,
median_box_h: med(on.map((b) => b[3])),
max_box_w: Math.max(...on.map((b) => b[2])),
median_box: [0, 1, 2, 3].map((k) => med(on.map((b) => b[k]!))),
};
}
+83
View File
@@ -0,0 +1,83 @@
// The look of a stretch of film as numbers, ported from the Studio's look-stats.mjs: mean luma and contrast
// (0–255), mean saturation (0–1, HSV), the share of black-and-white frames (under 2% of mid-tone pixels carry
// colour, so a colour face in a white room is not black and white) and warmth (mean R − B, negative is cool).
import { readClip, round, type Clip, type Roi } from "./media.js";
export interface Look {
frames: number;
luma: number;
contrast: number;
saturation: number;
bw_share: number;
warmth: number;
}
/** Sampled at `fps`, 160 px wide; `roi` measures one region, e.g. a person. */
export function look(clip: Clip, fps = 2, roi?: Roi): Look {
const { frames } = readClip(clip, { fps, width: 160, roi });
const per = frames.map((f) => {
let [l, l2, s, warm, mid, tinted] = [0, 0, 0, 0, 0, 0];
const n = f.length / 3;
for (let i = 0; i < f.length; i += 3) {
const [r, g, b] = [f[i]!, f[i + 1]!, f[i + 2]!];
const y = 0.299 * r + 0.587 * g + 0.114 * b;
const [mx, mn] = [Math.max(r, g, b), Math.min(r, g, b)];
l += y;
l2 += y * y;
s += mx ? (mx - mn) / mx : 0;
warm += r - b;
if (y > 40 && y < 220) {
mid++;
if (mx - mn > 25) tinted++;
}
}
const mean = l / n;
return {
luma: mean,
contrast: Math.sqrt(Math.max(0, l2 / n - mean * mean)),
sat: s / n,
warm: warm / n,
bw: tinted < 0.02 * Math.max(1, mid),
};
});
const avg = (k: "luma" | "contrast" | "sat" | "warm") =>
per.reduce((a, p) => a + p[k], 0) / (per.length || 1);
return {
frames: per.length,
luma: round(avg("luma"), 1),
contrast: round(avg("contrast"), 1),
saturation: round(avg("sat"), 3),
bw_share: round(per.filter((p) => p.bw).length / (per.length || 1), 2),
warmth: round(avg("warm"), 1),
};
}
export interface LookComparison {
luma_ratio: number;
contrast_ratio: number;
saturation_ratio: number;
bw_share_diff: number;
warmth_diff: number;
}
/** Yours over the reference's: ratios near 1 and differences near 0 are the same look. */
export function compareLook(ref: Look, yours: Look): LookComparison {
const ratio = (k: "luma" | "contrast" | "saturation") => round(yours[k] / (ref[k] || 1e-9), 2);
return {
luma_ratio: ratio("luma"),
contrast_ratio: ratio("contrast"),
saturation_ratio: ratio("saturation"),
bw_share_diff: round(yours.bw_share - ref.bw_share, 2),
warmth_diff: round(yours.warmth - ref.warmth, 1),
};
}
/** The same look within a regenerated shot's tolerance (to be calibrated on the bench). */
export function sameLook(c: LookComparison): boolean {
return (
Math.abs(c.luma_ratio - 1) <= 0.15 &&
Math.abs(c.contrast_ratio - 1) <= 0.25 &&
Math.abs(c.saturation_ratio - 1) <= 0.25 &&
Math.abs(c.warmth_diff) <= 12
);
}
+330
View File
@@ -0,0 +1,330 @@
// Decode with ffmpeg into raw arrays, so measuring and comparing need nothing but node and ffmpeg.
// Ported from the Studio's viral-clone media.mjs and tool.mjs.
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync } from "node:fs";
import { join, resolve } from "node:path";
import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
const MAX_BUFFER = 1 << 30;
/** An error whose message says what to run instead; commands print `fix` as its own line. */
export class Fix extends Error {
constructor(
problem: string,
readonly fix: string,
) {
super(problem);
this.name = "Fix";
}
}
export function ffmpegBin(): string {
const bin = findFFmpeg();
if (!bin) throw new Fix("ffmpeg is not installed", getFFmpegInstallHint());
return bin;
}
function ffprobeBin(): string {
const bin = findFFprobe();
if (!bin) throw new Fix("ffprobe is not installed", getFFmpegInstallHint());
return bin;
}
/** Runs ffmpeg and returns its stdout; `input` is piped to stdin. */
export function runFfmpeg(args: string[], input?: Buffer, cwd?: string): Buffer {
return execFileSync(ffmpegBin(), ["-nostdin", "-v", "error", ...args], {
maxBuffer: MAX_BUFFER,
windowsHide: true,
...(input ? { input } : {}),
...(cwd ? { cwd } : {}),
});
}
export interface VideoInfo {
path: string;
width: number;
height: number;
fps: number;
/** The frame rate as ffmpeg writes it, e.g. 30000/1001. */
fpsText: string;
duration: number;
frames: number;
audio: boolean;
/** Frames over seconds, which a variable-rate video's nominal rate does not tell. */
avgFps: number;
/** The container's frame rate and the stream's average differ: frames are not evenly spaced. */
vfr: boolean;
hdr: boolean;
codec: string;
}
interface ProbeStream {
codec_type?: string;
codec_name?: string;
width?: number;
height?: number;
r_frame_rate?: string;
avg_frame_rate?: string;
nb_frames?: string;
duration?: string;
color_transfer?: string;
side_data_list?: { rotation?: number }[];
tags?: { rotate?: string };
}
const rate = (text: string | undefined): number => {
const [a, b] = String(text ?? "0/1")
.split("/")
.map(Number);
return b ? (a ?? 0) / b : (a ?? 0);
};
/** Size (as displayed, rotation applied), clock and sound of a video. */
// fallow-ignore-next-line complexity
export function probeVideo(path: string): VideoInfo {
if (!existsSync(path))
throw new Fix(`no file at ${path}`, "pass the video's path from the project folder");
let json: { streams?: ProbeStream[]; format?: { duration?: string } };
try {
json = JSON.parse(
execFileSync(
ffprobeBin(),
["-v", "error", "-show_streams", "-show_format", "-of", "json", path],
{ maxBuffer: MAX_BUFFER, windowsHide: true },
).toString(),
);
} catch {
throw new Fix(`ffprobe cannot read ${path}`, "pass a video file (mp4, mov, webm)");
}
const streams = json.streams ?? [];
const video = streams.find((s) => s.codec_type === "video");
if (!video) throw new Fix(`${path} has no video stream`, "pass a video file, not audio");
const rotation = Math.abs(
Number(video.side_data_list?.find((d) => d.rotation !== undefined)?.rotation ?? 0) ||
Number(video.tags?.rotate ?? 0),
);
const turned = rotation % 180 === 90;
const fpsText = video.r_frame_rate && video.r_frame_rate !== "0/0" ? video.r_frame_rate : "30/1";
const fps = rate(fpsText);
const avg = rate(video.avg_frame_rate);
const duration = Number(json.format?.duration ?? video.duration ?? 0);
const counted = Number(video.nb_frames ?? 0);
return {
path,
width: (turned ? video.height : video.width) ?? 0,
height: (turned ? video.width : video.height) ?? 0,
fps,
fpsText: fpsText.endsWith("/1") ? fpsText.slice(0, -2) : fpsText,
duration,
frames: counted > 0 ? counted : Math.round(duration * fps),
audio: streams.some((s) => s.codec_type === "audio"),
avgFps: avg || fps,
vfr: avg > 0 && Math.abs(avg - fps) / fps > 0.01,
hdr: /smpte2084|arib-std-b67/.test(video.color_transfer ?? ""),
codec: video.codec_name ?? "",
};
}
export type Roi = [x: number, y: number, w: number, h: number];
export interface ReadOptions {
start?: number;
/** Seconds to read; omit for the rest of the file. */
seconds?: number;
roi?: Roi;
/** Output width (height follows the ROI's aspect, even); 0 or omitted keeps the ROI's size. */
width?: number;
gray?: boolean;
/** Resample to this rate; 0 or omitted reads every frame. */
fps?: number;
/** Stop after this many frames. */
limit?: number;
}
export interface Frames {
frames: Uint8Array[];
w: number;
h: number;
}
/**
* Frames from anywhere ffmpeg reads: a window of a video file, or a numbered picture sequence. `filters` run
* before any crop or scale, so a clip can pick frames (select, fps) without changing their size.
*/
export interface Clip {
input: string[];
filters: string[];
width: number;
height: number;
fps: number;
}
/** A window of a video file as a clip; `info` saves a probe when the caller has one. */
export function videoClip(
path: string,
start = 0,
seconds?: number,
info = probeVideo(path),
): Clip {
return {
input: [
...(start > 0 ? ["-ss", String(start)] : []),
"-i",
path,
...(seconds !== undefined ? ["-t", String(seconds)] : []),
],
filters: [],
width: info.width,
height: info.height,
fps: info.fps,
};
}
/** A clip's frames of one ROI as raw RGB (or grey) bytes. */
export function readClip(clip: Clip, opts: Omit<ReadOptions, "start" | "seconds"> = {}): Frames {
const { roi, width = 0, gray = false, fps = 0, limit } = opts;
const filters = [...clip.filters];
if (fps) filters.push(`fps=${fps}`);
let [w, h] = [clip.width, clip.height];
if (roi) {
filters.push(`crop=${roi[2]}:${roi[3]}:${roi[0]}:${roi[1]}`);
[w, h] = [roi[2], roi[3]];
}
if (width && width !== w) {
const outH = Math.max(2, Math.round((h * width) / w / 2) * 2);
filters.push(`scale=${width}:${outH}`);
[w, h] = [width, outH];
}
return decode(clip.input, filters, w, h, gray, limit);
}
/** The frame shown at `t`, picked by its index: seeking by time can land a frame early or late. */
export function readFrameAt(
info: VideoInfo,
t: number,
opts: Omit<ReadOptions, "start" | "seconds" | "fps"> = {},
): Frames {
const k = Math.min(info.frames - 1, frameAt(t, info.fps));
return readClip(
{ ...videoClip(info.path, 0, undefined, info), filters: [`select=eq(n\\,${k})`] },
{ ...opts, limit: 1 },
);
}
/** Frames of one ROI of a video file as raw RGB (or grey) bytes. */
export function readFrames(path: string, opts: ReadOptions = {}): Frames {
return readClip(videoClip(path, opts.start, opts.seconds), opts);
}
/** Every frame (or the first `limit`) ffmpeg's `input` arguments yield after `filters`, which must end at `w`x`h`. */
export function decode(
input: string[],
filters: string[],
w: number,
h: number,
gray = false,
limit?: number,
): Frames {
const raw = runFfmpeg([
...input,
"-an",
"-vf",
[...filters, `format=${gray ? "gray" : "rgb24"}`].join(","),
...(limit ? ["-frames:v", String(limit)] : []),
"-f",
"rawvideo",
"-",
]);
const size = w * h * (gray ? 1 : 3);
const frames: Uint8Array[] = [];
for (let i = 0; i + size <= raw.length; i += size) frames.push(raw.subarray(i, i + size));
return { frames, w, h };
}
/** BT.601 luma of RGB bytes. */
export function grey(rgb: Uint8Array): Uint8Array {
const out = new Uint8Array(rgb.length / 3);
for (let i = 0; i < out.length; i++)
out[i] = (77 * rgb[3 * i]! + 150 * rgb[3 * i + 1]! + 29 * rgb[3 * i + 2]!) >> 8;
return out;
}
/** Mean absolute difference of two equal-length byte arrays. */
export function meanAbsDiff(a: Uint8Array, b: Uint8Array): number {
let s = 0;
for (let i = 0; i < a.length; i++) s += Math.abs(a[i]! - b[i]!);
return s / (a.length || 1);
}
/** Writes raw RGB (or grey) bytes as an image; the extension picks the format. */
export function writeImage(path: string, px: Uint8Array, w: number, h: number, gray = false) {
const header = Buffer.from(`${gray ? "P5" : "P6"}\n${w} ${h}\n255\n`);
runFfmpeg(
["-y", "-f", gray ? "pgm_pipe" : "ppm_pipe", "-i", "-", "-frames:v", "1", path],
Buffer.concat([header, Buffer.from(px.buffer, px.byteOffset, px.byteLength)]),
);
return path;
}
export const round = (n: number, d = 3) => Math.round(n * 10 ** d) / 10 ** d;
/** "1,2.5,4" as numbers. */
export function parseNumbers(text: unknown, flag: string): number[] {
const out = String(text ?? "")
.split(",")
.filter((s) => s.trim() !== "")
.map(Number);
if (!out.length || out.some((n) => !Number.isFinite(n)))
throw new Fix(
`${flag} must be numbers separated by commas, got "${String(text)}"`,
`${flag} 1.2,3.4`,
);
return out;
}
/** "start,end" seconds with 0 <= start < end. */
export function parseRange(text: unknown, flag = "--range"): [number, number] {
const v = parseNumbers(text, flag);
if (v.length !== 2 || v[0]! < 0 || v[1]! <= v[0]!)
throw new Fix(
`${flag} must be start,end seconds with start < end, got "${String(text)}"`,
`${flag} 2.1,4.5`,
);
return [v[0]!, v[1]!];
}
/** "x,y,w,h" in pixels, inside a `width`x`height` frame when one is given. */
export function parseRoi(text: unknown, width?: number, height?: number): Roi {
const v = parseNumbers(text, "--roi").map(Math.round);
if (v.length !== 4 || v[2]! <= 0 || v[3]! <= 0)
throw new Fix(
`--roi must be x,y,w,h with a positive size, got "${String(text)}"`,
"--roi 100,200,400,300",
);
const roi: Roi = [v[0]!, v[1]!, v[2]!, v[3]!];
if (
width &&
height &&
(roi[0] < 0 || roi[1] < 0 || roi[0] + roi[2] > width || roi[1] + roi[3] > height)
)
throw new Fix(
`--roi ${roi.join(",")} runs past the ${width}x${height} frame`,
`keep it inside 0,0,${width},${height}`,
);
return roi;
}
/** The folder a tool writes its pictures to, made on demand: --out, or .hyperframes/<tool> in the current folder. */
export function outDir(out: unknown, tool: string): string {
const dir = typeof out === "string" && out ? resolve(out) : resolve(join(".hyperframes", tool));
mkdirSync(dir, { recursive: true });
return dir;
}
/** The frame shown at time `t`, allowing for times printed to the millisecond (38/24 s printed as 1.583). */
export const frameAt = (t: number, fps: number) =>
Math.max(0, Math.floor((t + 0.0005) * fps + 1e-6));
/** JSON with each innermost array or object on one line: readable, and a fraction of the tokens. */
export const formatJson = (value: unknown) =>
JSON.stringify(value, null, 1).replace(/[[{][^[\]{}]*[\]}]/g, (m) => m.replace(/\n\s*/g, " "));
+54
View File
@@ -0,0 +1,54 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { probeVideo } from "./media.js";
import { objects } from "./objects.js";
const ffmpeg = findFFmpeg();
const dir = mkdtempSync(join(tmpdir(), "hf-objects-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
/** White 16 px squares centred on `points` over black. */
function squares(name: string, points: [number, number][]): string {
const out = join(dir, name);
const boxes = points.map(
([x, y]) =>
`drawbox=x=${Math.round(x - 8)}:y=${Math.round(y - 8)}:w=16:h=16:color=white:t=fill`,
);
execFileSync(ffmpeg!, [
...["-nostdin", "-v", "error", "-y", "-f", "lavfi", "-i", "color=black:s=320x180:d=0.5:r=10"],
...["-vf", boxes.join(","), "-pix_fmt", "yuv444p", "-c:v", "libx264", "-crf", "0", out],
]);
return out;
}
const measure = (video: string) => objects(video, probeVideo(video), 0.2, [0, 0, 320, 180]);
describe.skipIf(!ffmpeg || !findFFprobe())("objects", () => {
it("reads six squares on a circle as an evenly spaced ring", () => {
const ring = Array.from({ length: 6 }, (_, i): [number, number] => [
160 + 60 * Math.cos((i * Math.PI) / 3),
90 + 60 * Math.sin((i * Math.PI) / 3),
]);
const r = measure(squares("ring.mp4", ring));
expect(r.count).toBe(6);
expect(r.arrangement).toMatchObject({
kind: "ring",
count: 6,
evenly_spaced: true,
touching: false,
});
});
it("reads a 3x2 layout as a grid", () => {
const grid: [number, number][] = [60, 160, 260].flatMap((x) =>
[50, 130].map((y): [number, number] => [x, y]),
);
const r = measure(squares("grid.mp4", grid));
expect(r.count).toBe(6);
expect(r.arrangement).toMatchObject({ kind: "grid", rows: 2, cols: 3 });
});
});
+198
View File
@@ -0,0 +1,198 @@
// Separate elements at one instant, ported from light-reverse's measure.objects: count, sizes, and whether they
// form a ring, grid, row, column or stack. Elements are 8-connected blobs of pixels far from the background (the
// ROI border's median); positions and sizes are shares of the full frame.
import { blobs, type Frame } from "./box.js";
import { Fix, readFrameAt, round, type Roi, type VideoInfo } from "./media.js";
const MAX_W = 960;
const MIN_SHARE = 0.0003;
interface Found {
cx: number;
cy: number;
w: number;
h: number;
x0: number;
y0: number;
area: number;
}
/** Where working pixels sit in the frame: offset of the ROI and video pixels per working pixel. */
interface Geometry {
W: number;
H: number;
x: number;
y: number;
sx: number;
sy: number;
}
const toShare = (g: Geometry, x: number, y: number) => ({
x: round((g.x + x * g.sx) / g.W, 3),
y: round((g.y + y * g.sy) / g.H, 3),
});
const mean = (v: number[]) => v.reduce((s, x) => s + x, 0) / (v.length || 1);
const std = (v: number[]) => Math.sqrt(mean(v.map((x) => (x - mean(v)) ** 2)));
const median = (v: number[]) => {
const s = [...v].sort((a, b) => a - b);
return s.length % 2 ? s[s.length >> 1]! : (s[s.length / 2 - 1]! + s[s.length / 2]!) / 2;
};
/** How many runs `values` form when gaps over `gap` split them. */
const runs = (values: number[], gap: number) => {
const v = [...values].sort((a, b) => a - b);
return 1 + v.slice(1).filter((x, i) => x - v[i]! > gap).length;
};
/** The arrangement of the largest group of like-sized elements; a stray glyph or crumb joins nothing. */
// fallow-ignore-next-line unused-export complexity
export function arrangement(all: Found[], g: Geometry) {
let found = all;
if (found.length >= 3) {
const med = median(found.map((o) => o.area));
const like = found.filter((o) => o.area >= 0.6 * med && o.area <= 1.6 * med);
if (like.length >= 3) found = like;
}
if (found.length < 3) return { kind: "fewer than 3 elements", count: found.length };
const areas = found.map((o) => o.area);
const similar = std(areas) / mean(areas) < 0.35;
const c = found.map((o) => [o.cx * g.sx, o.cy * g.sy] as const); // video px
const size = median(found.map((o) => Math.max(o.w * g.sx, o.h * g.sy)));
const boxes = found.map((o) => [o.x0, o.y0, o.x0 + o.w, o.y0 + o.h] as const);
const overlaps = boxes
.flatMap((a, i) =>
boxes.slice(i + 1).map((b) => a[0] < b[2] && b[0] < a[2] && a[1] < b[3] && b[1] < a[3]),
)
.filter(Boolean).length;
if (similar) {
const centre = [mean(c.map((p) => p[0])), mean(c.map((p) => p[1]))] as const;
const r = c.map((p) => Math.hypot(p[0] - centre[0], p[1] - centre[1]));
if (mean(r) > size * 0.6 && std(r) / mean(r) < 0.12) {
const ang = c
.map(
(p) =>
((((Math.atan2(p[1] - centre[1], p[0] - centre[0]) * 180) / Math.PI) % 360) + 360) %
360,
)
.sort((a, b) => a - b);
const gaps = [...ang.slice(1), ang[0]! + 360].map((a, i) => a - ang[i]!);
const radius = mean(r);
return {
kind: "ring",
count: found.length,
centre: toShare(g, centre[0] / g.sx, centre[1] / g.sy),
ring_diameter_share_of_width: round((2 * radius) / g.W, 4),
element_to_ring_diameter: round(size / (2 * radius), 2),
evenly_spaced: std(gaps) < 12,
touching: 2 * radius * Math.sin(Math.PI / found.length) <= size * 1.1,
};
}
const rows = runs(
c.map((p) => p[1]),
size * 0.5,
);
const cols = runs(
c.map((p) => p[0]),
size * 0.5,
);
if (rows * cols >= found.length && found.length >= 0.7 * rows * cols && (rows > 1 || cols > 1))
return {
kind: rows > 1 && cols > 1 ? "grid" : rows === 1 ? "row" : "column",
rows,
cols,
element_share_of_width: round(size / g.W, 3),
};
}
if (overlaps >= found.length - 1) {
const order = c.map((_, i) => i).sort((i, j) => c[i]![0] + c[i]![1] - (c[j]![0] + c[j]![1]));
const steps = order
.slice(1)
.map((i, k) => [c[i]![0] - c[order[k]!]![0], c[i]![1] - c[order[k]!]![1]]);
return {
kind: "stack",
count: found.length,
overlapping_pairs: overlaps,
mean_offset_share: {
x: round(mean(steps.map((s) => s[0]!)) / g.W, 3),
y: round(mean(steps.map((s) => s[1]!)) / g.H, 3),
},
};
}
return {
kind: "scattered",
count: found.length,
overlapping_pairs: overlaps,
similar_sizes: similar,
};
}
const hex = (c: number[]) =>
`#${c.map((v) => Math.round(v).toString(16).padStart(2, "0")).join("")}`;
/** The elements standing out from the background at `t` inside `roi`. */
// fallow-ignore-next-line complexity
export function objects(video: string, info: VideoInfo, t: number, roi: Roi, tol = 40) {
const width = roi[2] > MAX_W ? MAX_W : 0;
const { frames, w, h } = readFrameAt(info, t, { roi, width });
const px = frames[0];
if (!px)
throw new Fix(
`no frame at ${t}s`,
`pick a time before ${round(info.duration - 1 / info.fps, 2)}s`,
);
const f: Frame = { px, w, h };
const g: Geometry = {
W: info.width,
H: info.height,
x: roi[0],
y: roi[1],
sx: roi[2] / w,
sy: roi[3] / h,
};
// the background is the median of the ROI's outermost rows and columns
const border: number[][] = [[], [], []];
for (let y = 0; y < h; y++)
for (let x = 0; x < w; x++)
if (x === 0 || y === 0 || x === w - 1 || y === h - 1)
for (let ch = 0; ch < 3; ch++) border[ch]!.push(px[(y * w + x) * 3 + ch]!);
const bg = border.map(median);
const { id, list } = blobs(
f,
(r, gr, b) => Math.max(Math.abs(r - bg[0]!), Math.abs(gr - bg[1]!), Math.abs(b - bg[2]!)) > tol,
);
const sums = list.map(() => [0, 0]);
for (let i = 0; i < id.length; i++)
if (id[i]) {
sums[id[i]!]![0]! += i % w;
sums[id[i]!]![1]! += Math.floor(i / w);
}
const found = list
.flatMap((b, i): Found[] =>
i === 0 || b.n < MIN_SHARE * w * h
? []
: [
{
cx: sums[i]![0]! / b.n,
cy: sums[i]![1]! / b.n,
w: b.x1 - b.x0 + 1,
h: b.y1 - b.y0 + 1,
x0: b.x0,
y0: b.y0,
area: b.n,
},
],
)
.sort((a, b) => b.area - a.area);
return {
t,
background: hex(bg),
count: found.length,
elements: found.slice(0, 24).map((o) => ({
...toShare(g, o.cx, o.cy),
w: round((o.w * g.sx) / g.W, 3),
h: round((o.h * g.sy) / g.H, 3),
})),
arrangement: arrangement(found, g),
};
}
+109
View File
@@ -0,0 +1,109 @@
import { execFileSync } from "node:child_process";
import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir, userInfo } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { faceSummary, faceVerdict } from "./face.js";
import { flowMoves, flowSeries, similarity } from "./flow.js";
import { probeVideo } from "./media.js";
import { OPENCV_DIR } from "./opencv.js";
import { trackTemplate } from "./track.js";
// opencv.js is downloaded on first use. Tests run in a home folder of their own, so they borrow this machine's
// verified cache when it has one and skip otherwise, rather than download 11 MB on every CI box.
const cached = join(userInfo().homedir, ".cache", "hyperframes", "opencv", "4.10.0");
if (existsSync(join(cached, "opencv.js")) && !existsSync(join(OPENCV_DIR, "opencv.js")))
cpSync(cached, OPENCV_DIR, { recursive: true });
const ready = !!findFFmpeg() && !!findFFprobe() && existsSync(join(OPENCV_DIR, "opencv.js"));
const dir = mkdtempSync(join(tmpdir(), "hf-opencv-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
describe("similarity", () => {
it("recovers a shift, scale and turn despite outliers", () => {
const [from, to]: [number[], number[]] = [[], []];
const [s, a, tx, ty] = [1.1, (5 * Math.PI) / 180, 7, -3];
for (let i = 0; i < 60; i++) {
const [x, y] = [(i * 37) % 100, (i * 53) % 80];
from.push(x, y);
const outlier = i % 5 === 0;
to.push(
outlier ? x + 40 : s * (Math.cos(a) * x - Math.sin(a) * y) + tx,
outlier ? y - 25 : s * (Math.sin(a) * x + Math.cos(a) * y) + ty,
);
}
const fit = similarity(from, to)!;
expect(fit.scale).toBeCloseTo(1.1, 3);
expect(fit.rot).toBeCloseTo(5, 2);
expect(fit.tx).toBeCloseTo(7, 2);
});
});
describe("faces", () => {
it("grades the subject's face by its height", () => {
expect(faceVerdict(0.3)).toBe("pass");
expect(faceVerdict(0.1)).toBe("warn");
expect(faceVerdict(0.1, true)).toBe("fail");
expect(faceVerdict(null)).toBe("unmeasured");
const r = faceSummary([
{
frame: 0,
t: 0,
boxes: [
[0.4, 0.2, 0.2, 0.3],
[0.1, 0.1, 0.05, 0.08],
],
},
{ frame: 5, t: 0.2, boxes: [] },
]);
expect(r.face_height).toBe(0.3);
expect(r.verdict).toBe("unmeasured");
});
});
describe.skipIf(!ready)("opencv tools", () => {
// A textured 48 px square slides 400 px right from 0.3 s to 1.3 s.
const video = join(dir, "slide.mp4");
if (ready) {
const ff = findFFmpeg()!;
execFileSync(ff, [
"-v",
"error",
"-y",
"-f",
"lavfi",
"-i",
"testsrc2=s=48x48",
"-frames:v",
"1",
join(dir, "tex.png"),
]);
execFileSync(ff, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=0x202830:s=640x360:r=30:d=2"],
...["-loop", "1", "-i", join(dir, "tex.png"), "-filter_complex"],
"[0][1]overlay=x='40+400*min(max((t-0.3)/1.0,0),1)':y=150:shortest=1",
...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "10", video],
]);
}
it("follows a template and fits its move", async () => {
const r = await trackTemplate({
info: probeVideo(video),
at: 0,
box: [40, 150, 48, 48],
to: 2,
});
expect(r.lost).toBeNull();
const move = r.moves.find((m) => m.prop === "x");
expect(move).toMatchObject({ from: 64, to: 464, start_s: 0.3, duration_s: 1, ease: "none" });
});
it("reverses the slide from optical flow", async () => {
const info = probeVideo(video);
const rows = await flowSeries(info, 0, 2, [0, 100, 640, 160], 320);
const [move] = flowMoves(rows, info.fps, [0, 100, 640, 160], 0);
expect(move).toMatchObject({ kind: "tween", start_s: 0.3, duration_s: 1 });
expect(Number(move!.x)).toBeGreaterThan(360);
expect(Number(move!.x)).toBeLessThan(440);
});
});
+175
View File
@@ -0,0 +1,175 @@
// OpenCV for the measuring tools that need it (template tracking, optical flow, faces), as opencv.js: one WebAssembly
// file that gives every machine the same numbers. It is not shipped in the CLI; the first use downloads it and the
// Haar face models, pinned by size and sha256, into ~/.cache/hyperframes/opencv/<version>/.
import { mkdirSync, readFileSync, renameSync, rmSync } from "node:fs";
import { createRequire } from "node:module";
import { homedir } from "node:os";
import { join } from "node:path";
import { downloadToFile } from "../cloud/download.js";
import { verifies, type ModelFile } from "../whisper/modelFile.js";
import { Fix } from "./media.js";
const VERSION = "4.10.0";
export const OPENCV_DIR = join(homedir(), ".cache", "hyperframes", "opencv", VERSION);
interface Pinned extends ModelFile {
url: string;
}
// opencv.js is @techstark/opencv-js 4.10.0-release.1 (Apache-2.0); the cascades are OpenCV's own (BSD-3-Clause).
const CASCADES = "https://raw.githubusercontent.com/opencv/opencv/4.10.0/data/haarcascades";
const FILES: readonly Pinned[] = [
{
name: "opencv.js",
bytes: 10_378_215,
sha256: "19b46167b59efbef9cc3836264b0b657110833e4b1df84282004b8b3141c048d",
url: "https://cdn.jsdelivr.net/npm/@techstark/opencv-js@4.10.0-release.1/dist/opencv.js",
},
{
name: "haarcascade_frontalface_alt2.xml",
bytes: 540_616,
sha256: "7b0c967d9abbdfbde025eb9c786947d151b6426040d07a8f9562ed8fd90724b4",
url: `${CASCADES}/haarcascade_frontalface_alt2.xml`,
},
{
name: "haarcascade_profileface.xml",
bytes: 828_514,
sha256: "b39a4a3be45539db146a7fc1d3e761a292c196eb88421185e6a615b3055e612d",
url: `${CASCADES}/haarcascade_profileface.xml`,
},
];
/** The parts of the OpenCV API the measuring tools use. */
export interface Mat {
rows: number;
cols: number;
data32F: Float32Array;
data64F: Float64Array;
data: Uint8Array;
delete(): void;
}
interface Size {
width: number;
height: number;
}
export interface Rect {
x: number;
y: number;
width: number;
height: number;
}
export interface RectVector {
size(): number;
get(i: number): Rect;
delete(): void;
}
export interface Cv {
Mat: new () => Mat;
RectVector: new () => RectVector;
Size: new (w: number, h: number) => Size;
CascadeClassifier: new () => {
load(path: string): boolean;
detectMultiScale(
gray: Mat,
out: RectVector,
scale: number,
neighbours: number,
flags: number,
min: Size,
): void;
delete(): void;
};
matFromArray(rows: number, cols: number, type: number, data: ArrayLike<number>): Mat;
matchTemplate(image: Mat, templ: Mat, result: Mat, method: number): void;
minMaxLoc(src: Mat): {
minVal: number;
maxVal: number;
minLoc: { x: number; y: number };
maxLoc: { x: number; y: number };
};
calcOpticalFlowFarneback(
prev: Mat,
next: Mat,
flow: Mat,
pyrScale: number,
levels: number,
winsize: number,
iterations: number,
polyN: number,
polySigma: number,
flags: number,
): void;
estimateAffine2D(from: Mat, to: Mat): Mat;
FS_createDataFile(
parent: string,
name: string,
data: Uint8Array,
read: boolean,
write: boolean,
own: boolean,
): void;
CV_8UC1: number;
CV_32FC2: number;
TM_CCOEFF_NORMED: number;
}
const isCv = (v: unknown): v is Cv =>
typeof v === "object" && v !== null && "Mat" in v && "matFromArray" in v;
/** Every pinned file present and verified; the missing ones fetched. */
// fallow-ignore-next-line complexity
async function ensureOpenCv(onStatus?: (message: string) => void): Promise<void> {
const missing: Pinned[] = [];
for (const f of FILES) if (!(await verifies(join(OPENCV_DIR, f.name), f))) missing.push(f);
if (!missing.length) return;
mkdirSync(OPENCV_DIR, { recursive: true });
for (const f of missing) {
onStatus?.(`Downloading ${f.name} (${Math.round(f.bytes / 1e5) / 10} MB, once)...`);
const temp = join(OPENCV_DIR, `${f.name}.part-${process.pid}`);
try {
await downloadToFile(f.url, temp);
if (!(await verifies(temp, f)))
throw new Fix(
`${f.name} did not match its pinned size and sha256`,
"run the command again to retry the download",
);
renameSync(temp, join(OPENCV_DIR, f.name));
} catch (err) {
if (err instanceof Fix) throw err;
throw new Fix(
`could not download ${f.name} (${err instanceof Error ? err.message : String(err)})`,
"check the network and run the command again",
);
} finally {
rmSync(temp, { force: true });
}
}
}
let loaded: Promise<Cv> | null = null;
/** opencv.js, ready to call, with the face models in its file system. Loaded once per process. */
export function loadOpenCv(onStatus?: (message: string) => void): Promise<Cv> {
loaded ??= (async () => {
await ensureOpenCv(onStatus);
const mod: unknown = createRequire(import.meta.url)(join(OPENCV_DIR, "opencv.js"));
if (typeof mod !== "object" || mod === null) throw new Error("opencv.js did not load");
// Wait on its init hook, never on the module: it is a thenable that does not settle under await.
await new Promise<void>((resolve) => {
if ("Mat" in mod) resolve();
else Object.assign(mod, { onRuntimeInitialized: () => resolve() });
});
// Ready now; without its `then`, promises can carry it like any other object.
Reflect.deleteProperty(mod, "then");
if (!isCv(mod)) throw new Error("opencv.js did not initialize");
const cv = mod;
for (const f of FILES.filter((x) => x.name.endsWith(".xml")))
cv.FS_createDataFile("/", f.name, readFileSync(join(OPENCV_DIR, f.name)), true, false, false);
return cv;
})();
return loaded;
}
/** Grey bytes as an 8-bit single-channel Mat. */
export const greyMat = (cv: Cv, px: Uint8Array, w: number, h: number) =>
cv.matFromArray(h, w, cv.CV_8UC1, px);
+32
View File
@@ -0,0 +1,32 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { rhythm } from "./rhythm.js";
const ffmpeg = findFFmpeg();
const dir = mkdtempSync(join(tmpdir(), "hf-rhythm-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
const run = (args: string[]) => execFileSync(ffmpeg!, ["-nostdin", "-v", "error", "-y", ...args]);
describe.skipIf(!ffmpeg || !findFFprobe())("rhythm", () => {
it("reads a held picture as a static camera and a centred zoom as a push in", () => {
const still = join(dir, "still.png");
run(["-f", "lavfi", "-i", "testsrc=s=320x180:d=1", "-frames:v", "1", still]);
const held = join(dir, "held.mp4");
run(["-loop", "1", "-i", still, "-t", "2", "-r", "30", "-pix_fmt", "yuv420p", held]);
const zoom = join(dir, "zoom.mp4");
run([
...["-i", still, "-vf"],
"zoompan=z='1+0.003*on':x='iw/2-iw/zoom/2':y='ih/2-ih/zoom/2':d=60:s=320x180:fps=30",
...["-pix_fmt", "yuv420p", zoom],
]);
expect(rhythm(held).shots.map((s) => s.camera)).toEqual(["static"]);
const pushed = rhythm(zoom).shots;
expect(pushed).toHaveLength(1);
expect(pushed[0]!.camera).toBe("push in");
expect(pushed[0]!.zoom).toBeGreaterThan(1.03);
});
});
+219
View File
@@ -0,0 +1,219 @@
// How a film moves over time, ported from the Studio's motion-rhythm.mjs: the share of frames where the picture
// changes, the still holds, the rate the animation updates at (on ones, twos, threes) and the camera's zoom and pan
// inside each shot. Zoom and pan are the dominant whole-frame transform every 0.5 s, so a large moving subject can
// read as camera; confirm a move on frames.
import { findCuts } from "./cuts.js";
import { meanAbsDiff, probeVideo, readFrames, round, type Roi } from "./media.js";
const CUT = 25;
const W = 160;
function bilinear(img: Uint8Array, w: number, h: number, x: number, y: number) {
const [x0, y0] = [Math.floor(x), Math.floor(y)];
if (x0 < 0 || y0 < 0 || x0 + 1 >= w || y0 + 1 >= h) return -1;
const [fx, fy, i] = [x - x0, y - y0, y0 * w + x0];
return (
(img[i]! * (1 - fx) + img[i + 1]! * fx) * (1 - fy) +
(img[i + w]! * (1 - fx) + img[i + w + 1]! * fx) * fy
);
}
interface CameraStep {
s: number;
dx: number;
dy: number;
residual: number;
still: number;
}
/** Best s, dx, dy with B(x) ≈ A(c + (x − c)/s + d): s > 1 is a push in. */
function cameraStep(a: Uint8Array, b: Uint8Array, w: number, h: number): CameraStep {
const [cx, cy, m] = [w / 2, h / 2, Math.round(w * 0.1)];
const cost = (s: number, dx: number, dy: number) => {
let [sum, n] = [0, 0];
for (let y = m; y < h - m; y += 2)
for (let x = m; x < w - m; x += 2) {
const v = bilinear(a, w, h, cx + (x - cx) / s + dx, cy + (y - cy) / s + dy);
if (v >= 0) {
sum += Math.abs(v - b[y * w + x]!);
n++;
}
}
return n ? sum / n : 1e9;
};
let best = [cost(1, 0, 0), 1, 0, 0];
const search = (ss: number[], dxs: number[], dys: number[]) => {
for (const s of ss)
for (const dx of dxs)
for (const dy of dys) {
const c = cost(s, dx, dy);
if (c < best[0]!) best = [c, s, dx, dy];
}
};
const range = (c: number, step: number, n: number) =>
Array.from({ length: 2 * n + 1 }, (_, i) => c + (i - n) * step);
// coarse first, wide enough for a handheld swing (±10% zoom, ±10% of the width), then fine around the best
search(range(1, 0.02, 5), range(0, 4, 4), range(0, 4, 4));
search(range(best[1]!, 0.005, 2), range(best[2]!, 1, 2), range(best[3]!, 1, 2));
search(range(best[1]!, 0.0025, 2), range(best[2]!, 0.5, 2), range(best[3]!, 0.5, 2));
return { s: best[1]!, dx: best[2]!, dy: best[3]!, residual: best[0]!, still: cost(1, 0, 0) };
}
/** True when the whole frame moved: the best zoom and pan leave well under the change a still camera leaves. A
* presenter swaying in front of a locked camera changes a small area, and moving it barely helps. */
const cameraMoved = (step: CameraStep) => step.residual < 0.7 * step.still;
interface Shot {
start: number;
end?: number;
zoom: number;
pan_x: number;
pan_y: number;
path: number;
residuals: number[];
}
export interface Rhythm {
start: number;
seconds: number;
fps: number;
moving_share: number;
longest_hold_s: number;
median_hold_s: number;
holds_over_1s_per_min: number;
update_fps: number;
still_threshold: number;
cuts_per_min: number;
camera_moves_per_min: number;
shots: {
start: number;
end: number;
camera: string;
zoom: number;
pan_x: number;
pan_y: number;
path: number;
}[];
}
/** `still` is the reference's still threshold, so yours is measured at its level. */
// fallow-ignore-next-line complexity
export function rhythm(path: string, start = 0, seconds = 0, roi?: Roi, still?: number): Rhythm {
const p = probeVideo(path);
const secs = seconds || p.duration - start;
const { frames: fs, h } = readFrames(path, { start, seconds: secs, roi, width: W, gray: true });
const d = fs.slice(1).map((f, i) => meanAbsDiff(f, fs[i]!));
// mean |Δluma| on a 160 px frame below which nothing visibly changed; grain and compression raise the floor
// (capped, so a window that never stops moving is not read as noise)
const STILL =
still ??
Math.min(
1.5,
Math.max(0.5, 2 * ([...d].sort((x, y) => x - y)[Math.floor(d.length * 0.1)] ?? 0)),
);
// Shots split where measure cuts and diff find cuts; CUT stays the ceiling of what counts as motion.
const cuts = findCuts(fs).map((c) => c.frame);
// a frame is moving when it or a neighbour changes, so animation on twos or threes still counts as motion
const moving = d.map((_, i) =>
[d[i - 1], d[i], d[i + 1]].some((v) => v !== undefined && v > STILL && v <= CUT),
);
const holds: number[] = [];
for (let i = 0, run = 0; i <= moving.length; i++) {
if (i < moving.length && !moving[i] && d[i]! <= CUT) run++;
else {
if (run) holds.push(run / p.fps);
run = 0;
}
}
// inside stretches that are moving, a repeated frame means the animation updates on twos or threes
let [inMotion, repeats] = [0, 0];
for (let i = 2; i < d.length - 2; i++) {
const near = Math.max(d[i - 2]!, d[i - 1]!, d[i + 1]!, d[i + 2]!);
if (near > STILL && near <= CUT) {
inMotion++;
if (d[i]! < 0.2 * near) repeats++;
}
}
const step = Math.max(1, Math.round(p.fps / 2));
const shots: Shot[] = [];
const fresh = (t: number): Shot => ({
start: t,
zoom: 1,
pan_x: 0,
pan_y: 0,
path: 0,
residuals: [],
});
let shot = fresh(0);
for (let i = 0; i + step < fs.length; i += step) {
const cut = cuts.find((c) => c > i && c <= i + step);
if (cut) {
shots.push({ ...shot, end: cut / p.fps });
shot = fresh(cut / p.fps);
continue;
}
const c = cameraStep(fs[i]!, fs[i + step]!, W, h);
shot.zoom *= c.s;
shot.pan_x += c.dx / W;
shot.pan_y += c.dy / h;
if (cameraMoved(c)) shot.path += Math.hypot(c.dx / W, c.dy / h) + Math.abs(Math.log(c.s));
shot.residuals.push(c.residual);
}
shots.push({ ...shot, end: fs.length / p.fps });
// what is left after the best whole-frame transform; far above the film's usual level, the frame changed in a way
// no zoom or pan explains (a swing past the search, a rotation, a large subject crossing the frame)
const all = shots.flatMap((s) => s.residuals).sort((x, y) => x - y);
const UNEXPLAINED = Math.max(12, 3 * (all[Math.floor(all.length / 2)] ?? 0));
const kind = (s: Shot) => {
if ((s.end ?? 0) - s.start < 0.6) return "short";
const z = s.zoom - 1;
if (Math.abs(z) > 0.03) return z > 0 ? "push in" : "pull out";
if (Math.hypot(s.pan_x, s.pan_y) > 0.03) return "pan/drift";
if (s.path > 0.08) return "handheld";
if (s.residuals.some((r) => r > UNEXPLAINED)) return "moving, unexplained (confirm on frames)";
return "static";
};
const sorted = [...holds].sort((x, y) => x - y);
return {
start,
seconds: round(secs, 2),
fps: round(p.fps, 2),
moving_share: round(moving.filter(Boolean).length / Math.max(1, d.length), 2),
longest_hold_s: round(sorted.at(-1) ?? 0, 2),
median_hold_s: round(sorted[Math.floor(sorted.length / 2)] ?? 0, 2),
holds_over_1s_per_min: round((holds.filter((x) => x >= 1).length / secs) * 60, 1),
update_fps: round(p.fps * (1 - repeats / Math.max(1, inMotion)), 1),
still_threshold: round(STILL, 2),
cuts_per_min: round((cuts.length / secs) * 60, 1),
camera_moves_per_min: round(
(shots.filter((s) => /push|pull|pan|handheld/.test(kind(s))).length / secs) * 60,
1,
),
shots: shots.map((s) => ({
start: round(start + s.start, 2),
end: round(start + (s.end ?? 0), 2),
camera: kind(s),
zoom: round(s.zoom, 3),
pan_x: round(s.pan_x, 3),
pan_y: round(s.pan_y, 3),
path: round(s.path, 3),
})),
};
}
const RATIO_KEYS = [
"moving_share",
"longest_hold_s",
"median_hold_s",
"holds_over_1s_per_min",
"update_fps",
"cuts_per_min",
"camera_moves_per_min",
] as const;
/** Yours over the reference's, key by key; null where the reference's is 0. */
export function compareRhythm(ref: Rhythm, yours: Rhythm) {
return Object.fromEntries(
RATIO_KEYS.map((k) => [k, ref[k] ? round(yours[k] / ref[k], 2) : null]),
);
}
+48
View File
@@ -0,0 +1,48 @@
// The segments of a REVERSE.md: `## S01 · 0.00–2.40 s · title` headings, each with an optional
// `Pixels: rebuilt | reused | regenerated` line saying where that window's picture comes from.
import { existsSync, readFileSync } from "node:fs";
import { Fix } from "./media.js";
/** rebuilt: drawn in HTML; reused: cut from the original; regenerated: new media made for it. */
export type PixelSource = "rebuilt" | "reused" | "regenerated";
export interface Segment {
id: string;
start: number;
end: number;
title: string;
pixels: PixelSource;
}
const HEADING =
/^##\s+(S\d+)\s*[·|-]\s*(\d+(?:\.\d+)?)\s*[–—-]\s*(\d+(?:\.\d+)?)\s*s\b\s*(?:[·|-]\s*(.*))?$/gm;
const SOURCES: readonly PixelSource[] = ["rebuilt", "reused", "regenerated"];
const PIXELS = /^Pixels:\s*(rebuilt|reused|regenerated)\b/im;
export function parseSegments(markdown: string): Segment[] {
const heads = [...markdown.matchAll(HEADING)];
return heads.map((m, i) => {
const body = markdown.slice(m.index! + m[0].length, heads[i + 1]?.index ?? markdown.length);
const found = PIXELS.exec(body)?.[1]?.toLowerCase();
const pixels = SOURCES.find((source) => source === found) ?? "rebuilt";
return {
id: m[1]!,
start: Number(m[2]),
end: Number(m[3]),
title: (m[4] ?? "").trim(),
pixels,
};
});
}
export function readSegments(path: string): Segment[] {
if (!existsSync(path))
throw new Fix(`no segments file at ${path}`, "pass the project's REVERSE.md");
const segments = parseSegments(readFileSync(path, "utf8"));
if (!segments.length)
throw new Fix(
`${path} has no "## S01 · 0.00–2.40 s · title" segment headings`,
"write one heading per segment, as the reverse-any-video spec shows",
);
return segments;
}
+14
View File
@@ -0,0 +1,14 @@
/** Pearson correlation of the first min(len) values, or null under `minN` values or with no variance. */
export function pearson(a: number[], b: number[], minN = 2): number | null {
const n = Math.min(a.length, b.length);
if (n < minN) return null;
const ma = a.slice(0, n).reduce((s, v) => s + v, 0) / n;
const mb = b.slice(0, n).reduce((s, v) => s + v, 0) / n;
let [num, da, db] = [0, 0, 0];
for (let i = 0; i < n; i++) {
num += (a[i]! - ma) * (b[i]! - mb);
da += (a[i]! - ma) ** 2;
db += (b[i]! - mb) ** 2;
}
return da && db ? num / Math.sqrt(da * db) : null;
}
+42
View File
@@ -0,0 +1,42 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { probeVideo } from "./media.js";
import { avSync } from "./sync.js";
const ffmpeg = findFFmpeg();
const dir = mkdtempSync(join(tmpdir(), "hf-sync-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
// Uneven bursts so only one lag lines them up.
const BURSTS = [
[0.3, 0.5],
[0.9, 1.0],
[1.4, 1.7],
[2.1, 2.2],
[2.5, 2.8],
];
const during = (shift: number) =>
BURSTS.map(([a, b]) => `between(t,${a! + shift},${b! + shift})`).join("+");
describe.skipIf(!ffmpeg || !findFFprobe())("avSync", () => {
it("finds a picture two frames late", () => {
const out = join(dir, "late.mp4");
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=0x404040:s=96x96:r=30:d=3"],
...["-f", "lavfi", "-i", "sine=f=220:d=3"],
"-filter_complex",
`[0]drawbox=x=24:y=40:w=48:h=16:color=white:t=fill:enable='${during(2 / 30)}'[v];` +
`[1]volume='if(${during(0)},1,0)':eval=frame[a]`,
...["-map", "[v]", "-map", "[a]", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-qp", "0", out],
]);
const r = avSync(probeVideo(out), [24, 40, 48, 16], 0, 3);
expect(r.best_lag_frames).toBeGreaterThan(1.5);
expect(r.best_lag_frames).toBeLessThan(2.5);
expect(r.best_lag_ms).toBeGreaterThan(0);
expect(r.confidence).not.toBe("none");
});
});
+130
View File
@@ -0,0 +1,130 @@
// Picture against sound for a person speaking, ported from the Studio's hf-media av_sync: per frame the mouth
// region's mean luma change, per frame the change of the sound's loudness, and the lag where the two correlate best.
// A motion and energy proxy, not phoneme sync: head motion, gestures, cuts and music in the ROI or mix move it.
import { Fix, readClip, round, runFfmpeg, videoClip, type Roi, type VideoInfo } from "./media.js";
import { pearson } from "./stats.js";
const RATE = 16000;
/** Mono samples of the window at RATE Hz. */
function samples(info: VideoInfo, start: number, seconds: number): Float32Array {
const raw = runFfmpeg([
...["-ss", String(start), "-i", info.path, "-t", String(seconds), "-vn"],
...["-ac", "1", "-ar", String(RATE), "-f", "f32le", "-"],
]);
return new Float32Array(raw.buffer.slice(raw.byteOffset, raw.byteOffset + (raw.byteLength & ~3)));
}
export interface SyncResult {
best_lag_frames: number | null;
best_lag_ms: number | null;
peak_correlation: number | null;
zero_lag_correlation: number | null;
margin_over_other_lags: number | null;
confidence: "high" | "medium" | "low" | "none";
speech_active_fraction: number;
mouth_motion_mean: number;
note: string | null;
}
// fallow-ignore-next-line complexity
export function avSync(
info: VideoInfo,
mouth: Roi,
start: number,
end: number,
maxLagMs = 400,
): SyncResult {
if (!info.audio)
throw new Fix(
`${info.path} has no sound`,
"measure sync needs the picture and its sound in one file",
);
if (!(maxLagMs > 0)) throw new Fix("--max-lag-ms must be positive", "--max-lag-ms 400");
const { frames, w, h } = readClip(videoClip(info.path, start, end - start, info), {
roi: mouth,
width: 160,
gray: true,
});
const motion = frames.map((f, i) => {
if (!i) return 0;
let s = 0;
for (let k = 0; k < f.length; k++) s += Math.abs(f[k]! - frames[i - 1]![k]!);
return s / (w * h);
});
const audio = samples(info, start, end - start);
const frame = 1 / info.fps;
const envelope = frames.map((_, i) => {
const [a, b] = [Math.floor(i * frame * RATE), Math.floor((i + 1) * frame * RATE)];
const chunk = audio.subarray(a, Math.max(b, a + 1));
let s = 0;
for (const v of chunk) s += v * v;
return chunk.length ? Math.sqrt(s / chunk.length) : 0;
});
const speech = envelope.slice(1).map((v, i) => Math.abs(v - envelope[i]!));
const mouthMotion = motion.slice(1);
const reach = Math.max(1, Math.round(maxLagMs / 1000 / frame));
// Positive lag: the mouth moves after the sound, the picture is late.
const curve = Array.from({ length: 2 * reach + 1 }, (_, i) => {
const lag = i - reach;
const [a, b] =
lag >= 0
? [speech.slice(0, speech.length - lag), mouthMotion.slice(lag)]
: [speech.slice(-lag), mouthMotion.slice(0, lag)];
return { lag, r: pearson(a, b, 10) };
});
const top = Math.max(...envelope);
const active = top > 0 ? envelope.filter((v) => v > 0.1 * top).length / envelope.length : 0;
const base = {
speech_active_fraction: round(active, 3),
mouth_motion_mean: round(mouthMotion.reduce((s, v) => s + v, 0) / (mouthMotion.length || 1), 3),
};
const valid = curve.filter((c) => c.r !== null);
if (!valid.length)
return {
best_lag_frames: null,
best_lag_ms: null,
peak_correlation: null,
zero_lag_correlation: null,
margin_over_other_lags: null,
confidence: "none",
...base,
note: "no mouth motion or no change in the sound in this window; check the ROI and the window",
};
const best = curve.reduce(
(b, c, i) => ((c.r ?? -Infinity) > (curve[b]!.r ?? -Infinity) ? i : b),
0,
);
const peak = curve[best]!.r!;
// A parabola through the peak and its neighbours places it between frames.
let shift = 0;
const [left, right] = [curve[best - 1]?.r, curve[best + 1]?.r];
if (typeof left === "number" && typeof right === "number") {
const bend = left - 2 * peak + right;
if (bend < 0) shift = (0.5 * (left - right)) / bend;
}
const lag = curve[best]!.lag;
const rest = curve.filter((c) => Math.abs(c.lag - lag) > 2 && c.r !== null).map((c) => c.r!);
const margin = rest.length ? peak - Math.max(...rest) : null;
const edge = best === 0 || best === curve.length - 1;
const confidence =
peak >= 0.5 && margin !== null && margin >= 0.1 && !edge && active >= 0.1
? "high"
: peak >= 0.3 && !edge
? "medium"
: "low";
const notes = [
...(edge ? ["the peak is at the search edge; raise --max-lag-ms or check the window"] : []),
...(active < 0.1 ? ["little speech in this window; choose a span with continuous speech"] : []),
];
return {
best_lag_frames: round(lag + shift, 2),
best_lag_ms: round((lag + shift) * frame * 1000, 1),
peak_correlation: round(peak, 3),
zero_lag_correlation: round(curve.find((c) => c.lag === 0)?.r ?? 0, 3),
margin_over_other_lags: margin === null ? null : round(margin, 3),
confidence,
...base,
note: notes.join("; ") || null,
};
}
+123
View File
@@ -0,0 +1,123 @@
// One textured thing followed frame by frame (a cursor, a hand, a product), ported from hf-media's track: the box at
// one moment is the template, each next frame is searched near the last match by normalized cross-correlation, and
// the track ends at a cut, a weak match or an ambiguous one rather than guessing. Moves come out as eases.
import { findCuts } from "./cuts.js";
import { fitMoves, printable, type Row } from "./curve.js";
import { readClip, round, videoClip, type Roi, type VideoInfo } from "./media.js";
import { greyMat, loadOpenCv } from "./opencv.js";
const WORK_W = 960;
export interface TrackOptions {
info: VideoInfo;
/** The moment the template is cut from, and the box it is cut from (source pixels). */
at: number;
box: Roi;
to: number;
/** How far it may move between two frames, in source pixels. */
radius?: number;
minScore?: number;
/** How far the best match must beat the runner-up elsewhere. */
minMargin?: number;
}
export interface Tracked {
rows: Row[];
scores: number[];
lost: { t: number; reason: "cut" | "low_score" | "ambiguous"; score?: number } | null;
moves: ReturnType<typeof printable>;
/** Working size the search ran at, for drawing. */
scale: number;
}
// fallow-ignore-next-line complexity
export async function trackTemplate(o: TrackOptions): Promise<Tracked> {
const cv = await loadOpenCv((m) => console.error(m));
const k = Math.min(1, WORK_W / o.info.width);
const { frames, w, h } = readClip(videoClip(o.info.path, o.at, o.to - o.at, o.info), {
width: Math.round(o.info.width * k),
gray: true,
});
const cuts = new Set(findCuts(frames).map((c) => c.frame));
const [bx, by, bw, bh] = o.box.map((v) => Math.round(v * k));
const tw = Math.max(4, Math.min(bw!, w - bx!));
const th = Math.max(4, Math.min(bh!, h - by!));
const templ = new Uint8Array(tw * th);
for (let y = 0; y < th; y++)
templ.set(frames[0]!.subarray((by! + y) * w + bx!, (by! + y) * w + bx! + tw), y * tw);
const tMat = greyMat(cv, templ, tw, th);
const radius = Math.max(2, Math.round((o.radius ?? 48) * k));
const fps = o.info.fps;
const centre = (x: number, y: number): Row => ({
t: 0,
x: (x + tw / 2) / k,
y: (y + th / 2) / k,
w: tw / k,
h: th / k,
ink: 1,
});
const rows: Row[] = [{ ...centre(bx!, by!), t: round(o.at, 3) }];
const scores = [1];
let [px, py] = [bx!, by!];
let lost: Tracked["lost"] = null;
try {
for (let i = 1; i < frames.length; i++) {
const t = round(o.at + i / fps, 3);
if (cuts.has(i)) {
lost = { t, reason: "cut" };
break;
}
const [x0, y0] = [Math.max(0, px - radius), Math.max(0, py - radius)];
const [x1, y1] = [Math.min(w, px + tw + radius), Math.min(h, py + th + radius)];
const sw = x1 - x0;
const sh = y1 - y0;
const area = new Uint8Array(sw * sh);
for (let y = 0; y < sh; y++)
area.set(frames[i]!.subarray((y0 + y) * w + x0, (y0 + y) * w + x1), y * sw);
const aMat = greyMat(cv, area, sw, sh);
const result = new cv.Mat();
try {
cv.matchTemplate(aMat, tMat, result, cv.TM_CCOEFF_NORMED);
const { maxVal, maxLoc } = cv.minMaxLoc(result);
// The runner-up away from the best match's own correlation lobe: a second cursor, a repeated icon.
let second = -Infinity;
const [rw, dx, dy] = [result.cols, Math.max(1, tw >> 1), Math.max(1, th >> 1)];
result.data32F.forEach((v, j) => {
const [cx, cy] = [j % rw, Math.floor(j / rw)];
if ((Math.abs(cx - maxLoc.x) > dx || Math.abs(cy - maxLoc.y) > dy) && v > second)
second = v;
});
const margin = Number.isFinite(second) ? maxVal - second : 1;
if (maxVal < (o.minScore ?? 0.6) || margin < (o.minMargin ?? 0.05)) {
lost = {
t,
reason: maxVal < (o.minScore ?? 0.6) ? "low_score" : "ambiguous",
score: round(maxVal, 3),
};
break;
}
[px, py] = [x0 + maxLoc.x, y0 + maxLoc.y];
rows.push({ ...centre(px, py), t });
scores.push(maxVal);
} finally {
aMat.delete();
result.delete();
}
}
} finally {
tMat.delete();
}
const quant = 1 / k;
const moves = (["x", "y"] as const)
.flatMap((p) =>
fitMoves(
rows,
p,
fps,
Math.max(3 * quant, 0.01 * Math.max(o.info.width, o.info.height)),
quant,
),
)
.sort((a, b) => a.start_s - b.start_s);
return { rows, scores, lost, moves: printable(moves), scale: k };
}
@@ -0,0 +1 @@
export function regenerateIndex(projectDir: string): void;
@@ -18,3 +18,24 @@ export function withReservedFileSync<T>(
ext: string,
populate: (reservation: Reservation) => T,
): T;
export interface MediaRecord {
id: string;
type: string;
path: string;
source: string;
description: string;
duration?: number;
provenance?: Record<string, unknown>;
}
export function recordInPlace(
projectDir: string,
record: {
type: string;
path: string;
source: string;
description?: string;
duration?: number;
provenance?: Record<string, unknown>;
},
): MediaRecord;