ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-10-30 09:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(skills): reverse-any-video moves to the Lean version, keeping the sign-in gate

Browse Source 
The skill is the 10-24 Lean round's text: probe, cuts, the original's sound, frames at
every change, components in HTML, sub-assets cut, people re-shot with HeyGen
(references/people.md, match.md), one diff at the end. check, layers, spec and watch go.
The measure, diff, analyze and heygen CLI it calls take that run's versions.

Kept from the gate: sign in before starting and open with references/start.md, which now
promises what Lean does; no price talk around HeyGen re-shoots; the refusal wording in
analyze and heygen/v3; billing in cli and help; media-use's sign-in and credits rules,
with Lean's one-retry-per-named-fault rule added to references/heygen.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Miao Yang
2026-10-29 18:22:53 +08:00
co-authored by Claude Opus 5.5
parent 169c5ddae6
commit a0f6ca7469
51 changed files with 3815 additions and 569 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli/src/commands/analyze.test.ts
+32
View File
@@ -0,0 +1,32 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg } from "../browser/ffmpeg.js";
import { probeVideo } from "../measure/media.js";
import { prepare } from "./analyze.js";
const ffmpeg = findFFmpeg();
const dir = mkdtempSync(join(tmpdir(), "hf-analyze-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
describe("analyze", () => {
it.skipIf(!ffmpeg)("sends the whole part when it slows it down", async () => {
const src = join(dir, "src.mp4");
execFileSync(ffmpeg!, [
"-v",
"error",
"-y",
"-f",
"lavfi",
"-i",
"testsrc2=d=4:s=320x240:r=30",
src,
]);
const info = await probeVideo(src);
const out = join(dir, "part.mp4");
writeFileSync(out, prepare(info, [1, 3], 3, dir));
expect((await probeVideo(out)).duration).toBeCloseTo(6, 0);
});
});
packages/cli/src/commands/analyze.ts
+92 -54
View File
@@ -12,10 +12,10 @@ import {
} from "node:fs";
import { tmpdir } from "node:os";
import { basename, dirname, extname, join, resolve } from "node:path";
import { defineCommand } from "../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "./_examples.js";
import { tryResolveCredential } from "../auth/resolver.js";
import { idempotencyKey, runJob } from "../heygen/v3.js";
import { idempotencyKey, runJob, type Job } from "../heygen/v3.js";
import {
DESCRIBE_SCHEMA,
mergeDrafts,
@@ -39,8 +39,8 @@ export const examples: Example[] = [
];
const ROUTE = "/v3/video-analyses";
const MODEL = "gemini-4-argon";
const HEYGEN_FPS = 2;
export const MODEL = "gemini-4-argon";
export const HEYGEN_FPS = 2;
const HEYGEN_MAX_S = 90;
const PART_S = 30;
const INLINE_BYTES = 16 * 1024 * 1024;
@@ -65,25 +65,92 @@ export function partsOf(duration: number, cuts: number[]): [number, number][] {
export const slowBy = (fps: number, seconds: number) =>
Math.max(1, Math.min(fps / HEYGEN_FPS, HEYGEN_MAX_S / seconds));
/** One part as HeyGen reads it: slowed, 720 px on its long side, silent when slowed, under the inline cap. */
function prepare(info: VideoInfo, [a, b]: [number, number], slow: number, dir: string): Buffer {
const fit = info.width >= info.height ? "scale=720:-2" : "scale=-2:720";
/** The first of a few qualities that fits HeyGen's inline cap; `args` are ffmpeg's for that quality and output. */
export function underInlineCap(
dir: string,
name: string,
args: (crf: number, out: string) => string[],
seconds: number,
): Buffer {
for (const crf of [23, 30, 36]) {
const out = join(dir, `part-${a}-${crf}.mp4`);
runFfmpeg([
...["-y", "-ss", String(a), "-i", info.path, "-t", String(b - a)],
...["-vf", `${fit},setsar=1,setpts=${slow}*PTS`, "-r", String(HEYGEN_FPS * 2)],
...(slow === 1 && info.audio ? ["-c:a", "aac", "-b:a", "96k"] : ["-an"]),
...["-c:v", "libx264", "-crf", String(crf), "-pix_fmt", "yuv420p", out],
]);
const out = join(dir, `${name}-${crf}.mp4`);
runFfmpeg(args(crf, out));
if (statSync(out).size <= INLINE_BYTES) return readFileSync(out);
}
throw new Fix(
`a ${round(b - a, 1)} s part stays over HeyGen's 16 MB inline cap`,
`a ${round(seconds, 1)} s part stays over HeyGen's 16 MB inline cap`,
"analyze a shorter window with --start/--end",
);
}
/** One part as HeyGen reads it: slowed, 720 px on its long side, silent when slowed, under the inline cap. */
export function prepare(
info: VideoInfo,
[a, b]: [number, number],
slow: number,
dir: string,
): Buffer {
const fit = info.width >= info.height ? "scale=720:-2" : "scale=-2:720";
return underInlineCap(
dir,
`part-${a}`,
(crf, out) => [
// -t bounds the slowed output: the part times the slow-down, or the model sees only the first 1/slow of it.
...["-y", "-ss", String(a), "-i", info.path, "-t", String((b - a) * slow)],
...["-vf", `${fit},setsar=1,setpts=${slow}*PTS`, "-r", String(HEYGEN_FPS * 2)],
...(slow === 1 && info.audio ? ["-c:a", "aac", "-b:a", "96k"] : ["-an"]),
...["-c:v", "libx264", "-crf", String(crf), "-pix_fmt", "yuv420p", out],
],
b - a,
);
}
export interface Asked {
answer: unknown;
job: string;
usage: unknown;
}
/** One clip and one question to HeyGen's video model; the same request picks up the same job. */
export async function askVideo(
video: Buffer,
prompt: string,
schema: unknown,
opts: {
model?: string;
oauth?: boolean;
what: string;
project?: string;
onWait?: (job: Job) => void;
},
): Promise<Asked> {
const body = {
model: opts.model ?? MODEL,
video: { type: "base64", media_type: "video/mp4", data: video.toString("base64") },
prompt,
response_schema: schema,
max_output_tokens: 32768,
};
const job = await runJob(
ROUTE,
body,
idempotencyKey(ROUTE, [video, prompt, JSON.stringify(schema), body.model]),
(id) => `${ROUTE}/${encodeURIComponent(id)}`,
{ log: { projectDir: opts.project, what: opts.what }, onWait: opts.onWait },
).catch((err: unknown) => {
if (err instanceof Fix && /refused the credential/.test(err.message) && opts.oauth)
throw new Fix(
"HeyGen's video analysis is not available with this sign-in yet",
"continue without analyze: measure the video and look at its frames",
);
throw err;
});
const output = job.output;
const answer =
typeof output === "object" && output !== null && "json" in output ? output.json : output;
return { answer, job: job.id, usage: job.usage ?? null };
}
function defaultOut(video: string): string {
const stem = basename(video, extname(video));
return existsSync(".reverse")
@@ -92,7 +159,7 @@ function defaultOut(video: string): string {
}
/** The project a job log belongs to: the nearest folder up from here with index.html. */
function projectHere(): string | undefined {
export function projectHere(): string | undefined {
for (let dir = resolve("."); ; dir = dirname(dir)) {
if (existsSync(join(dir, "index.html"))) return dir;
if (dirname(dir) === dir) return undefined;
@@ -172,52 +239,23 @@ export default defineCommand({
`Video: ${info.width}x${info.height}.`,
];
const prompt = [question, ...notes].join("\n\n");
const body = {
model: String(args.model ?? MODEL),
video: { type: "base64", media_type: "video/mp4", data: video.toString("base64") },
prompt,
response_schema: schema,
max_output_tokens: 32768,
};
console.error(
`analyze: part ${i + 1}/${parts.length}, ${round(part[0], 2)}–${round(part[1], 2)} s, slowed ${round(slow, 2)}x`,
);
const job = await runJob(
ROUTE,
body,
idempotencyKey(ROUTE, [video, prompt, JSON.stringify(schema), body.model]),
(id) => `${ROUTE}/${encodeURIComponent(id)}`,
{
log: {
projectDir: project,
what: `analyze ${basename(info.path)} ${round(part[0], 2)}-${round(part[1], 2)}s`,
},
onWait: (j) => console.error(`analyze: part ${i + 1} ${j.status}…`),
},
).catch((err: unknown) => {
if (
err instanceof Fix &&
/refused the credential/.test(err.message) &&
credential.type === "oauth"
)
throw new Fix(
"HeyGen's video analysis is not available with this sign-in yet",
"continue without analyze: measure the video and look at its frames",
);
throw err;
const asked = await askVideo(video, prompt, schema, {
model: args.model === undefined ? undefined : String(args.model),
oauth: credential.type === "oauth",
project,
what: `analyze ${basename(info.path)} ${round(part[0], 2)}-${round(part[1], 2)}s`,
onWait: (j) => console.error(`analyze: part ${i + 1} ${j.status}…`),
});
const output = job.output;
const answer =
typeof output === "object" && output !== null && "json" in output
? output.json
: output;
answers.push(toFilmTime(answer, slow, part[0]));
answers.push(toFilmTime(asked.answer, slow, part[0]));
done.push({
start: round(part[0], 3),
end: round(part[1], 3),
slow: round(slow, 2),
job: job.id,
usage: job.usage ?? null,
job: asked.job,
usage: asked.usage,
});
}
} finally {
packages/cli/src/commands/diff.ts
+162 -11
View File
@@ -1,5 +1,6 @@
import { statSync } from "node:fs";
import { basename, extname, join, resolve } from "node:path";
import { defineCommand } from "../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "./_examples.js";
import { c } from "../ui/colors.js";
import { failCommand, setCommandExitCode } from "../utils/commandResult.js";
@@ -10,10 +11,16 @@ import {
DEFAULT_MIN_MEAN,
DEFAULT_MIN_P05,
diffVideos,
findLedger,
type DiffResult,
} from "../measure/diff.js";
import { Fix, formatJson, outDir, parseNumbers, parseRange } from "../measure/media.js";
import { Fix, formatJson, outDir, parseNumbers, parseRange, probeVideo } from "../measure/media.js";
import { readSegments } from "../measure/segments.js";
import { tryResolveCredential } from "../auth/resolver.js";
import { attachDiagnoses, explainTargets, targetsOf } from "../heygen/explain.js";
import { projectHere } from "./analyze.js";
import { repeated } from "./media-use/generate.js";
import { parseMask } from "../heygen/generate.js";
export const examples: Example[] = [
[
@@ -28,10 +35,18 @@ export const examples: Example[] = [
"Score chosen moments of a project without rendering it",
"hyperframes diff .reverse/source.mp4 . --at 0.5,2.4,3.1",
],
[
"Score one drawn element (a title's box) by itself at chosen moments of a project",
"hyperframes diff .reverse/source.mp4 . --at 1,1.5,2 --drawn 120,670,485,115",
],
[
"Judge each REVERSE.md segment of the final render and write a side-by-side",
"hyperframes diff .reverse/source.mp4 renders/final.mp4 --segments REVERSE.md --sbs",
],
[
"Also ask HeyGen's video model what differs in each failing drawn element (a lead, not a verdict)",
"hyperframes diff .reverse/source.mp4 renders/final.mp4 --segments REVERSE.md --explain",
],
];
function number(value: unknown, flag: string): number | undefined {
@@ -40,6 +55,31 @@ function number(value: unknown, flag: string): number | undefined {
return n;
}
/** `x,y,w,h` or `x,y,w,h,a-b` in the original's pixels and seconds. */
function parseDrawn(text: string): ReturnType<typeof parseMask> {
try {
return parseMask(text);
} catch {
throw new Fix(`--drawn ${text} is not x,y,w,h[,a-b]`, "--drawn 120,670,485,115,0-2.77");
}
}
function personLine(p: NonNullable<DiffResult["segments"]>[number]["person"] & object): string {
const off = [
...(p.framing ?? [])
.filter((f) => !f.pass)
.map(
(f) => `framing off ${f.window[0]}–${f.window[1]} s (shift ${f.shift}, size ×${f.scale})`,
),
...(p.text ?? []).filter((t) => t.carried).map((t) => `${t.clip} carries ${t.drawn}'s text`),
...(p.sync ?? [])
.filter((x) => x.pass === false)
.map((x) => `lips ${x.lag_ms} ms off at ${x.window[0]} s`),
];
const moved = p.energy ? `, motion ×${p.energy.mean_ratio} (reported)` : "";
return `person ${p.pass === null ? "—" : p.pass ? "✓" : "✗"}${off.length ? ` ${off.join("; ")}` : ""}${moved}`;
}
function printHuman(r: DiffResult) {
const mark = r.pass ? c.success("✓ pass") : c.error("✗ fail");
console.log(
@@ -49,8 +89,16 @@ function printHuman(r: DiffResult) {
console.log(
` missing content in ${Math.round(r.missing_content.share * 100)}% of frames: ${r.missing_content.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}`,
);
if (r.missing_content.texture.share > 0)
console.log(
` missing texture (reported, not failed) in ${Math.round(r.missing_content.texture.share * 100)}% of frames: ${r.missing_content.texture.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}`,
);
for (const m of r.worst)
console.log(` worst ${m.t}s ssim ${m.ssim} cell ${m.cell.roi.join(",")} at ${m.cell.ssim}`);
if (r.whole_frame_reuse?.spans.length)
console.log(
` whole frame of the original in ${Math.round(r.whole_frame_reuse.share * 100)}% of frames${r.whole_frame_reuse.pass ? "" : `, with its drawn things at ${r.whole_frame_reuse.drawn.map(([a, b]) => `${a}–${b} s`).join(", ")} (a copy)`}`,
);
if (r.cuts)
console.log(
` cuts: ${r.cuts.reference.length} in the original${r.cuts.pass ? ", all matched" : `; missing ${r.cuts.missing.join(", ") || "none"}, extra ${r.cuts.extra.join(", ") || "none"}`}`,
@@ -63,16 +111,54 @@ function printHuman(r: DiffResult) {
console.log(
` sound: ${"offset_ms" in r.sound ? (r.sound.offset_ms === null ? (r.sound.note ?? "not aligned") : `${r.sound.offset_ms} ms off`) : `original ${r.sound.original ? "has" : "has no"} sound, yours ${r.sound.yours ? "has" : "has no"}`}${r.sound.pass ? "" : " (fails)"}`,
);
if (r.components)
console.log(
` components: ${r.components.passed} of ${r.components.total} pass${r.components.mean === null ? "" : `, mean ${r.components.mean}`}`,
);
if (r.person) {
const g = r.person.gates;
const mark = (v: boolean | null, failed: string[]) =>
v === null ? "unmeasured" : v ? "ok" : `fails in ${failed.join(", ")}`;
console.log(
` person: framing ${mark(g.framing, r.person.failed.framing)}, text in the clip ${mark(g.text, r.person.failed.text)}, lips ${mark(g.sync, r.person.failed.sync)}; reported: motion ×${r.person.energy ?? "?"}, ${r.person.shots_at_cuts.at_cuts} of ${r.person.shots_at_cuts.total} clips start at a cut${r.person.shots_at_cuts.cuts_without_shot.length ? `, no new clip at ${r.person.shots_at_cuts.cuts_without_shot.join(", ")} s` : ""}`,
);
}
console.log(` every frame: ${r.frames}`);
for (const s of r.segments ?? []) {
const state = s.pass === null ? c.dim("—") : s.pass ? c.success("✓") : c.error("✗");
const detail = s.ssim
? `ssim ${s.ssim.mean} / p05 ${s.ssim.p05} / cell ${s.ssim.cell_p05}`
: s.energy
? `motion ×${s.energy.mean_ratio}, luma ×${s.look?.luma_ratio}`
: (s.note ?? "");
console.log(` ${state} ${s.id} ${s.window[0]}–${s.window[1]} s ${s.pixels}: ${detail}`);
const detail =
[
s.ssim && `ssim ${s.ssim.mean} / p05 ${s.ssim.p05} / cell ${s.ssim.cell_p05}`,
s.energy && `motion ×${s.energy.mean_ratio}, luma ×${s.look?.luma_ratio}`,
s.person && personLine(s.person),
]
.filter(Boolean)
.join("; ") ||
(s.note ?? "");
const region = s.regenerated ? `; person in ${s.regenerated.roi.join(",")}` : "";
console.log(
` ${state} ${s.id} ${s.window[0]}–${s.window[1]} s ${s.pixels}${s.regenerated ? " outside the region" : ""}: ${detail}${region}`,
);
}
for (const d of r.drawn ?? []) {
const state = d.pass === null ? c.dim("—") : d.pass ? c.success("✓") : c.error("✗");
const detail =
d.pass === null
? "no ink found in the box (crop it tight around the letters)"
: `score ${d.score} / p05 ${d.p05}, off by ${d.offset!.join(",")} px${d.ink === undefined ? "" : `, ink ×${d.ink}`}`;
console.log(
` ${state} ${d.id} ${d.box.join(",")} ${d.window[0]}–${d.window[1]} s: ${detail}`,
);
}
for (const s of r.segments ?? [])
for (const [id, d] of [
...(s.drawn ?? []).map((e) => [e.id, e.diagnosis] as const),
[s.id, s.worst_cell?.diagnosis] as const,
])
if (d)
console.log(
` ${c.dim("?")} ${id} ${d.window.map(([a, b]) => `${a}–${b} s`).join(", ")} (unverified): ${d.matches ? "matches" : d.differences.map((x) => `${x.aspect}: original ${x.original}; yours ${x.yours}`).join(" · ") || "no answer"}`,
);
for (const w of r.warnings) console.log(` ${c.warn("⚠")} ${w}`);
if (r.sheet) console.log(` sheet ${r.sheet}`);
if (r.sbs) console.log(` side by side ${r.sbs}`);
@@ -99,6 +185,16 @@ export default defineCommand({
description:
"REVERSE.md: a verdict per `## S01 · a–b s` segment (Pixels: regenerated compares motion and look)",
},
ledger: {
type: "string",
description:
"The project's media ledger, to find whole-frame cuts of the original (default: .media/manifest.jsonl beside the project or REVERSE.md)",
},
project: {
type: "string",
description:
"The project folder, to find the ledger's generated clips and check them for the original's text (default: the folder holding .media)",
},
"min-mean": { type: "string", description: `Mean SSIM to pass (default ${DEFAULT_MIN_MEAN})` },
"min-p05": {
type: "string",
@@ -122,22 +218,63 @@ export default defineCommand({
type: "string",
description: "Render-ready timeout in ms for a project folder (default 5000)",
},
explain: {
type: "boolean",
description:
"Ask HeyGen's video model what differs in each failing drawn element (and failing segment's worst cell); experimental, needs a HeyGen API key; an unverified lead",
default: false,
},
"explain-only": {
type: "string",
description:
"Ask about these element or segment ids instead, passing or not, e.g. S02.D6,S04",
},
drawn: {
type: "string",
description:
"A drawn element's box x,y,w,h in the original's pixels, optionally its seconds a-b (repeatable): scored by itself whatever is behind it, with its offset and ink (yours over the original's: under 1 is lighter, thinner or smaller)",
},
json: { type: "boolean", description: "Print one JSON result", default: false },
},
async run({ args }) {
async run({ args, rawArgs }) {
const json = args.json === true;
try {
if (args.at !== undefined && args.range !== undefined)
throw new Fix("--at and --range both given", "pass one of them");
const yours = resolve(String(args.yours));
const stem = basename(yours, extname(yours)) || "project";
const segmentsFile = args.segments === undefined ? undefined : resolve(String(args.segments));
const only =
args["explain-only"] === undefined
? undefined
: String(args["explain-only"])
.split(",")
.map((id) => id.trim())
.filter(Boolean);
const explain = args.explain === true || only !== undefined;
const credential = explain ? await tryResolveCredential() : undefined;
if (explain && !credential)
throw new Fix(
"--explain needs a HeyGen sign-in",
"set HEYGEN_API_KEY, or diff without --explain and look at the failing boxes' frames",
);
if (explain && (segmentsFile === undefined || statSync(yours).isDirectory()))
throw new Fix(
"--explain reads a render's failing elements",
"diff a rendered video with --segments REVERSE.md --explain",
);
const result = await diffVideos({
reference: resolve(String(args.reference)),
yours,
at: args.at === undefined ? undefined : parseNumbers(args.at, "--at"),
range: args.range === undefined ? undefined : parseRange(args.range),
segments:
args.segments === undefined ? undefined : readSegments(resolve(String(args.segments))),
segments: segmentsFile === undefined ? undefined : readSegments(segmentsFile),
ledger:
args.ledger === undefined
? findLedger(yours, segmentsFile)
: resolve(String(args.ledger)),
project: args.project === undefined ? undefined : resolve(String(args.project)),
drawn: repeated(rawArgs, "--drawn").map(parseDrawn),
thresholds: {
mean: number(args["min-mean"], "--min-mean"),
p05: number(args["min-p05"], "--min-p05"),
@@ -149,6 +286,20 @@ export default defineCommand({
timeoutMs: number(args.timeout, "--timeout"),
log: (message) => console.error(message),
});
if (explain) {
const targets = targetsOf(result, only);
console.error(`diff: asking HeyGen about ${targets.length} elements`);
attachDiagnoses(
result,
await explainTargets(targets, {
reference: probeVideo(result.reference),
yours: probeVideo(result.yours),
oauth: credential!.type === "oauth",
project: projectHere(),
log: (message) => console.error(message),
}),
);
}
if (json) console.log(formatJson(withMeta({ ok: true, ...result })));
else printHuman(result);
if (!result.pass) setCommandExitCode(1);
packages/cli/src/commands/measure.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "./_examples.js";
export const examples: Example[] = [
packages/cli/src/commands/measure/box.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
num,
packages/cli/src/commands/measure/continuity.ts
+1 -1
View File
@@ -1,5 +1,5 @@
import { resolve } from "node:path";
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, outArg, runMeasure } from "../../measure/command.js";
import { continuity } from "../../measure/continuity.js";
packages/cli/src/commands/measure/curve.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
packages/cli/src/commands/measure/cuts.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, runMeasure, videoArg, window } from "../../measure/command.js";
import { findCuts, signatures } from "../../measure/cuts.js";
packages/cli/src/commands/measure/energy.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, runMeasure } from "../../measure/command.js";
import { compareEnergy, energyCurve } from "../../measure/energy.js";
packages/cli/src/commands/measure/events.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import { writeFileSync } from "node:fs";
import { join } from "node:path";
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
packages/cli/src/commands/measure/extract.ts
+1 -1
View File
@@ -4,7 +4,7 @@
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, relative, resolve } from "node:path";
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, roiArg, runMeasure, videoAndRoi, videoArg } from "../../measure/command.js";
import { Fix, frameAt, round, runFfmpeg, type Roi, type VideoInfo } from "../../measure/media.js";
packages/cli/src/commands/measure/face.ts
+67 -9
View File
@@ -1,14 +1,24 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, runMeasure, videoArg, window } from "../../measure/command.js";
import { faceBoxes, faceSummary } from "../../measure/face.js";
import { probeVideo } from "../../measure/media.js";
import {
FIT_MIN_FACES,
againstStart,
faceBoxes,
faceFit,
faceSummary,
} from "../../measure/face.js";
import { Fix, probeVideo, round } from "../../measure/media.js";
export const examples: Example[] = [
[
"How big and where the presenter's face is in a shot",
"hyperframes measure face ref.mp4 --start 4 --end 9.5",
],
[
"The scale and x, y that put a generated clip's face on the original's face over 2-6 s",
"hyperframes measure face presenter.mp4 --against ref.mp4 --from 2 --to 6",
],
];
export default defineCommand({
@@ -19,29 +29,77 @@ export default defineCommand({
},
args: {
video: videoArg,
start: { type: "string", description: "First second (default 0)" },
end: { type: "string", description: "Last second (default: the end)" },
start: {
type: "string",
description:
"First second (default 0; with --against, 0 for a shot of its own and --from for a render as long as the original)",
},
end: {
type: "string",
description:
"Last second (default: the end; with --against, --start plus the length of --from/--to)",
},
samples: {
type: "string",
description: "Frames sampled, evenly, the last one included (default 12)",
description: "Frames sampled, evenly, the last one included (default 8)",
},
cast: {
type: "boolean",
description: "The person is one face in a crowd: grade the smallest face",
description:
"The person is one face in a crowd: grade the smallest face (and look for faces down to 20 px)",
default: false,
},
against: {
type: "string",
description:
"The original video: print the GSAP scale and x, y (canvas pixels) that put this clip's face on the original's over --from/--to",
},
from: { type: "string", description: "With --against: the original's first second" },
to: { type: "string", description: "With --against: the original's last second" },
json: jsonArg,
},
async run({ args }) {
await runMeasure(async () => {
const info = probeVideo(String(args.video));
const cast = args.cast === true;
const samples = Math.max(1, num(args.samples, "--samples", 8));
const opts = cast ? { minFace: 0 } : {};
if (args.against !== undefined) {
const ref = probeVideo(String(args.against));
const [a, b] = window(ref, args.from, args.to, ["--from", "--to"]);
const s = num(args.start, "--start", againstStart(info.duration, ref.duration, a));
const e = Math.min(info.duration, s + (b - a));
if (!(e > s))
throw new Fix(`--start ${s} is past this clip's end`, `--start 0 (or leave it out)`);
const [fo, fc] = [
await faceBoxes(ref, a, b, samples, opts),
await faceBoxes(info, s, e, samples, opts),
];
const rel = (f: { t: number; boxes: number[][] }[], t0: number) =>
f.map((x) => ({ t: x.t - t0, boxes: x.boxes }));
const fit = faceFit(rel(fo, a), rel(fc, s), ref, info, cast);
return {
video: info.path,
against: ref.path,
window: [round(s, 3), round(e, 3)],
against_window: [a, b],
canvas: [ref.width, ref.height],
...fit,
...(fit.transform === null
? {
note: `fewer than ${FIT_MIN_FACES} moments with a face in both: try a wider window or more --samples`,
}
: {}),
reading:
"gsap.set(clip, { scale, x, y }) on a clip element that fills the canvas (object-fit: cover) with the default transform origin (its centre); x, y and the faces' centre and height are in the canvas's (the original's) pixels, clip_face before the transform; residual is the median face-centre miss left after it, in face heights",
};
}
const [a, b] = window(info, args.start, args.end);
const samples = await faceBoxes(info, a, b, Math.max(1, num(args.samples, "--samples", 12)));
return {
video: info.path,
start: a,
end: b,
...faceSummary(samples, args.cast === true),
...faceSummary(await faceBoxes(info, a, b, samples, opts), cast),
reading:
"face_height is the face's height over the frame's; boxes are x, y, w, h as shares of the frame",
};
packages/cli/src/commands/measure/flow.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
packages/cli/src/commands/measure/frames.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import { mkdtempSync, readdirSync, renameSync, rmSync } from "node:fs";
import { join } from "node:path";
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
packages/cli/src/commands/measure/ink.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, runMeasure, videoAndRoi, videoArg, window } from "../../measure/command.js";
import { scan, summary } from "../../measure/ink.js";
packages/cli/src/commands/measure/look.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
packages/cli/src/commands/measure/objects.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, roiArg, runMeasure, videoAndRoi, videoArg } from "../../measure/command.js";
import { objects } from "../../measure/objects.js";
packages/cli/src/commands/measure/probe.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, runMeasure, videoArg } from "../../measure/command.js";
import {
packages/cli/src/commands/measure/rhythm.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
packages/cli/src/commands/measure/sync.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
packages/cli/src/commands/measure/track.ts
+1 -1
View File
@@ -1,5 +1,5 @@
import { join } from "node:path";
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, outArg, runMeasure, videoArg } from "../../measure/command.js";
import { Fix, outDir, parseRoi, probeVideo, round, runFfmpeg } from "../../measure/media.js";
packages/cli/src/commands/measure/trail.ts
+1 -1
View File
@@ -2,7 +2,7 @@
// onion skin (earliest faintest), the moving part's path with numbered points, and a heat map of where the picture
// changed. One look tells a slide from a scale from a mask reveal; `keyframes --shot` does the same for a project.
import { join } from "node:path";
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import {
jsonArg,
packages/cli/src/commands/media-use.ts
-1
View File
@@ -96,7 +96,6 @@ function subcommand(name: MediaUseVerb, invoke: InvokeMediaUse) {
export function createMediaUseCommand(invoke: InvokeMediaUse = invokeEngine) {
const subCommands = {
...Object.fromEntries(MEDIA_USE_VERBS.map((name) => [name, () => subcommand(name, invoke)])),
// Made on HeyGen through the hyperframes sign-in, not the resolve engine.
generate: () => import("./media-use/generate.js").then((m) => m.default),
};
return defineCommand({
packages/cli/src/commands/media-use/generate.ts
+25 -4
View File
@@ -1,10 +1,12 @@
import { defineCommand } from "../../utils/defineCommand.js";
import { defineCommand } from "citty";
import type { Example } from "../_examples.js";
import { jsonArg, num, runMeasure } from "../../measure/command.js";
import { Fix } from "../../measure/media.js";
import {
GENERATE_TYPES,
checkGuide,
generate,
parseMask,
type GenerateRequest,
type GenerateType,
} from "../../heygen/generate.js";
@@ -21,7 +23,7 @@ export const examples: Example[] = [
],
[
"The original's presenter re-shot clean, saying an 8 s stretch of the original voice",
'hyperframes media-use generate --type video --reference assets/presenter-clean.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --prompt "the woman talks to camera, seated, soft daylight" --seconds 8 --aspect 9:16 --out assets/s02-presenter.mp4',
'hyperframes media-use generate --type video --reference assets/presenter-clean.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --mask 110,615,390,165 --prompt "the woman talks to camera, seated, soft daylight" --seconds 8 --aspect 9:16 --out assets/s02-presenter.mp4',
],
[
"The person in a photo saying a script in a chosen voice",
@@ -31,6 +33,17 @@ export const examples: Example[] = [
const isType = (v: string): v is GenerateType => GENERATE_TYPES.some((t) => t === v);
/** Every value of a flag given more than once (citty keeps only the last). */
export function repeated(raw: string[], flag: string): string[] {
return raw.flatMap((a, i) =>
a === flag && raw[i + 1] !== undefined
? [raw[i + 1]!]
: a.startsWith(`${flag}=`)
? [a.slice(flag.length + 1)]
: [],
);
}
export default defineCommand({
meta: {
name: "generate",
@@ -62,6 +75,11 @@ export default defineCommand({
type: "string",
description: "video: a clip whose camera and body motion the shot follows (up to 15 s)",
},
mask: {
type: "string",
description:
"video: x,y,w,h[,a-b] of the --motion clip to blank before sending (its drawn text; a-b in the clip's seconds); repeat for more",
},
"first-frame": { type: "string", description: "video: the picture the shot starts from" },
photo: { type: "string", description: "avatar: the person's photo" },
script: { type: "string", description: "avatar, voice: the words spoken" },
@@ -88,7 +106,7 @@ export default defineCommand({
},
json: jsonArg,
},
async run({ args }) {
async run({ args, rawArgs }) {
await runMeasure(async () => {
const type = String(args.type);
if (!isType(type))
@@ -104,6 +122,7 @@ export default defineCommand({
firstFrame: args["first-frame"],
audio: args.audio,
motion: args.motion,
masks: repeated(rawArgs, "--mask").map(parseMask),
photo: args.photo,
script: args.script,
voice: args.voice,
@@ -114,7 +133,8 @@ export default defineCommand({
instrumental: !args.vocals,
};
const price = priceOf(request);
if (args.plan)
if (args.plan) {
checkGuide(request);
return {
plan: true,
type,
@@ -122,6 +142,7 @@ export default defineCommand({
price,
next: "after the person agrees to this price, run the same command without --plan",
};
}
const made = await generate(request, (status) => console.error(`generate: ${status}…`));
return { ...made, price };
});
packages/cli/src/commands/snapshot.ts
-1
View File
@@ -706,7 +706,6 @@ export async function captureSnapshots(
review?: boolean;
/** Progress lines; `--json` sends them to stderr so stdout stays one envelope. */
log?: (message: string) => void;
/** False for working frames another command reads and discards: nothing enters project history. */
record?: boolean;
},
): Promise<CapturedSnapshots> {
packages/cli/src/heygen/explain.test.ts
+140
View File
@@ -0,0 +1,140 @@
import { execFileSync } from "node:child_process";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
vi.mock("../cloud/auth.js", () => ({
resolveCloudAuthHeaders: async () => ({ "x-api-key": "k" }),
resolveCloudBaseUrl: () => "https://api.test",
}));
const { attachDiagnoses, explainQuestion, explainTargets, layoutOf, piecesOf, targetsOf } =
await import("./explain.js");
const { diffVideos } = await import("../measure/diff.js");
const { parseSegments } = await import("../measure/segments.js");
const { probeVideo } = await import("../measure/media.js");
const ffmpeg = findFFmpeg();
const noTools = !ffmpeg || !findFFprobe();
const dir = mkdtempSync(join(tmpdir(), "hf-explain-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
afterEach(() => vi.unstubAllGlobals());
const reply = (body: unknown) =>
new Response(JSON.stringify(body), {
status: 200,
headers: { "content-type": "application/json" },
});
describe("explain layout", () => {
it("stacks a wide box, puts a tall one side by side, and keeps the picture near-square and large", () => {
const wide = layoutOf([147, 590, 786, 150], 1080, 1920);
expect(wide.layout).toBe("original above yours");
expect(wide.crop).toEqual([140, 584, 798, 162]);
expect(Math.max(...wide.size)).toBe(1200);
expect(Math.max(...wide.size) / Math.min(...wide.size)).toBeLessThanOrEqual(1.26);
// The element spans the picture's width.
expect(wide.size[0]).toBe(1200);
const tall = layoutOf([108, 174, 60, 118], 720, 1280);
expect(tall.layout).toBe("original left of yours");
expect(tall.filter).toContain("hstack");
// A small box is scaled up, never down to a thumbnail.
expect(Math.min(...tall.size)).toBeGreaterThan(600);
});
it("widens a short window a little and asks a long one at its two ends", () => {
expect(piecesOf([2.64, 3.44], 36)).toEqual([[2.39, 3.69]]);
expect(piecesOf([11.14, 19.69], 36)).toEqual([
[10.89, 12.89],
[17.94, 19.94],
]);
expect(piecesOf([0, 1], 1)).toEqual([[0, 1]]);
});
});
describe.skipIf(noTools)("diff --explain", () => {
it("asks the fixed question about each failing drawn element at native size and files the answer under it", async () => {
// Three white strokes over a plain blue picture; yours draws them 24 px to the right.
const strokes = (x0: number) =>
[0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(",");
const make = (name: string, x0: number) => {
const out = join(dir, name);
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=0x2050a0:s=320x180:r=30:d=3"],
...["-vf", strokes(x0), "-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "0", out],
]);
return out;
};
const [ref, yours] = [make("ref.mp4", 120), make("off.mp4", 144)];
const md =
"## S01 · 0–3 s · Label over a presenter\nPixels: regenerated\nRegenerated: 0, 0, 320, 180\nDrawn: 118, 108, 42, 28, 1.0–2.0 s\nDrawn: 10, 10, 40, 20, 0–1 s\n";
const result = await diffVideos({
reference: ref,
yours,
segments: parseSegments(md),
outDir: join(dir, "diff"),
});
const targets = targetsOf(result);
// The empty second box passes, so only the label is asked about.
expect(targets).toEqual([{ id: "S01.D1", box: [118, 108, 42, 28], window: [1, 2] }]);
expect(targetsOf(result, ["S01.D2"]).map((t) => t.id)).toEqual(["S01.D2"]);
const sent: { url: string; body?: Record<string, unknown> }[] = [];
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
sent.push({ url, ...(init?.body ? { body: JSON.parse(String(init.body)) } : {}) });
// The clip is slowed 6x, so 4.5 s in it is 0.75 s after the window starts.
return reply({
data: {
id: "a1",
status: "completed",
usage: { input_tokens: 812, output_tokens: 90 },
output: {
json: {
matches: false,
differences: [
{ aspect: "position", original: "left", yours: "right", start: 4.5, end: 9 },
],
},
},
},
});
});
const found = await explainTargets(targets, {
reference: probeVideo(ref),
yours: probeVideo(yours),
});
expect(sent).toHaveLength(1);
expect(sent[0]!.url).toBe("https://api.test/v3/video-analyses");
const body = sent[0]!.body!;
const prompt = String(body.prompt);
expect(prompt.startsWith(explainQuestion("original above yours"))).toBe(true);
// Neither the score nor the verdict reaches the model.
expect(prompt).not.toMatch(/score|ssim|fail|pass|0\.\d/i);
// The video is the two crops, upscaled near-square, slowed for 2 fps.
const video = join(dir, "sent.mp4");
const sentVideo = body.video;
if (typeof sentVideo !== "object" || sentVideo === null || !("data" in sentVideo))
throw new Error("no video sent");
writeFileSync(video, Buffer.from(String(sentVideo.data), "base64"));
const info = probeVideo(video);
expect([info.width, info.height]).toEqual(layoutOf([118, 108, 42, 28], 320, 180).size);
expect(info.duration).toBeGreaterThan(8);
expect(info.audio).toBe(false);
attachDiagnoses(result, found);
const d = result.segments![0]!.drawn![0]!;
expect(d.diagnosis).toMatchObject({
unverified: true,
window: [[0.75, 2.25]],
matches: false,
differences: [
{ aspect: "position", original: "left", yours: "right", start: 1.5, end: 2.25 },
],
layout: "original above yours",
input_tokens: 812,
jobs: ["a1"],
});
expect(result.segments![0]!.drawn![1]!.diagnosis).toBeUndefined();
});
});
packages/cli/src/heygen/explain.ts
+270
View File
@@ -0,0 +1,270 @@
// `diff --explain`: pixels find which drawn element fails, HeyGen's video model says what differs in it. Its route
// samples at most 2 fps at low resolution, so each frame is a thumbnail: the element's box is cut from both videos at
// native resolution, the original above yours (left of yours for a tall box), padded to near-square and scaled until
// the element is about 600 px, over a short window slowed so 2 fps sees about 12 fps. The question is fixed and never
// carries the score, the verdict or a guess at the cause, which would prime the answer. An answer is a lead: unverified
// until the frames and the pixels confirm it.
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import { askVideo, HEYGEN_FPS, slowBy, underInlineCap } from "../commands/analyze.js";
import type { DiffResult } from "../measure/diff.js";
import { round, type Roi, type VideoInfo } from "../measure/media.js";
import { toFilmTime } from "./reverseDraft.js";
// Pixels kept around the box, the grey line between the two sides, the widest picture sent, and its long side.
const MARGIN = 6;
const GAP = 8;
const MAX_ASPECT = 1.25;
const LONG_SIDE = 1200;
// The window is widened this much each side to see the element arrive and leave; longer than MAX_S, its first and
// last PIECE_S seconds are asked apart. Frames a second the model should see.
const LEAD_S = 0.25;
const MAX_S = 4;
const PIECE_S = 2;
const EXPLAIN_FPS = 12;
const PARALLEL = 4;
/** What a difference is about: how the element looks, or how it moves. */
export const APPEARANCE = [
"words",
"font",
"weight",
"size",
"position",
"colour",
"material",
"doubled",
];
export const MOTION = ["appears", "changes", "leaves", "timing"];
const str = { type: "string" };
const num = { type: "number" };
export const EXPLAIN_SCHEMA = {
type: "object",
properties: {
matches: { type: "boolean" },
differences: {
type: "array",
items: {
type: "object",
properties: {
aspect: { type: "string", enum: [...APPEARANCE, ...MOTION, "other"] },
original: str,
yours: str,
start: num,
end: num,
},
required: ["aspect", "original", "yours", "start", "end"],
},
},
},
required: ["matches", "differences"],
};
export type Layout = "original above yours" | "original left of yours";
/** The one question, worded for where each side sits. */
export function explainQuestion(layout: Layout): string {
const [first, second] = layout === "original above yours" ? ["TOP", "BOTTOM"] : ["LEFT", "RIGHT"];
return `${first} is the original video, ${second} is a recreation of the same moment. Both are cut to the same box around one drawn element: text, a number, a badge or a card. The person, the scenery and anything behind the element were remade on purpose: ignore them.
Compare only the drawn element. List each way the ${second} one differs from the ${first} one: the words or letters, font, weight or italic, size, position, colour, material (solid, gradient, glass, chrome, outline, glow, shadow), doubled or ghosted letters, how it appears, how it changes while on screen, how it leaves, and timing. Say what each side shows. If the drawn element matches, set matches to true and list nothing. Report only what you can see in the frames.`;
}
/** The box with a margin, on even pixels inside the frame. */
function cropOf([x, y, w, h]: Roi, fw: number, fh: number): Roi {
const even = (v: number) => Math.floor(v / 2) * 2;
const x0 = even(Math.max(0, x - MARGIN));
const y0 = even(Math.max(0, y - MARGIN));
return [
x0,
y0,
Math.max(2, even(Math.min(fw, x + w + MARGIN) - x0)),
Math.max(2, even(Math.min(fh, y + h + MARGIN) - y0)),
];
}
/** How a box is laid out for the model: both sides, padded to near-square, scaled to LONG_SIDE. */
export function layoutOf(
box: Roi,
fw: number,
fh: number,
): { crop: Roi; layout: Layout; size: [number, number]; filter: string } {
const crop = cropOf(box, fw, fh);
const [, , w, h] = crop;
const above = w >= h;
const [sw, sh] = above ? [w, 2 * h + GAP] : [2 * w + GAP, h];
const even = (v: number) => Math.ceil(v / 2) * 2;
const [pw, ph] = [even(Math.max(sw, sh / MAX_ASPECT)), even(Math.max(sh, sw / MAX_ASPECT))];
const k = LONG_SIDE / Math.max(pw, ph);
const size: [number, number] = [even(pw * k), even(ph * k)];
const cut = `crop=${w}:${h}:${crop[0]}:${crop[1]},setsar=1`;
const filter = [
`[0:v]${cut},pad=${above ? `${w}:${h + GAP}` : `${w + GAP}:${h}`}:0:0:color=gray[a]`,
`[1:v]scale=${fw}:${fh},${cut}[b]`,
`[a][b]${above ? "vstack" : "hstack"},pad=${pw}:${ph}:(ow-iw)/2:(oh-ih)/2:color=black,scale=${size[0]}:${size[1]}:flags=lanczos`,
].join(";");
return { crop, layout: above ? "original above yours" : "original left of yours", size, filter };
}
/** The film seconds asked about: the window widened a little, or its two ends when it is long. */
export function piecesOf([a, b]: [number, number], duration: number): [number, number][] {
const [s, e] = [Math.max(0, a - LEAD_S), Math.min(duration, b + LEAD_S)];
if (e - s <= MAX_S) return [[round(s, 3), round(e, 3)]];
return [
[round(s, 3), round(s + PIECE_S, 3)],
[round(e - PIECE_S, 3), round(e, 3)],
];
}
export interface Difference {
aspect: string;
original: string;
yours: string;
start?: number;
end?: number;
}
export interface Diagnosis {
/** A lead from a video model, never a verdict: look at the box's frames and confirm with pixels before fixing. */
unverified: true;
/** The film seconds the model watched (one or two pieces). */
window: [number, number][];
matches: boolean | null;
differences: Difference[];
layout: Layout;
/** The picture sent, in pixels, and the input tokens the route counted (low ones mean it saw thumbnails). */
size: [number, number];
input_tokens: number | null;
jobs: string[];
}
export interface Target {
id: string;
box: Roi;
window: [number, number];
}
/** What `--explain` asks about: every failing drawn element and failing SSIM segment's worst cell, or those named. */
export function targetsOf(result: DiffResult, only?: string[]): Target[] {
const wanted = (id: string, pass: boolean | null) => (only ? only.includes(id) : pass === false);
return (result.segments ?? []).flatMap((s) => [
...(s.drawn ?? [])
.filter((d) => wanted(d.id, d.pass))
.map((d) => ({ id: d.id, box: d.box, window: d.window })),
...(s.worst_cell && wanted(s.id, s.pass)
? [
{
id: s.id,
box: s.worst_cell.roi,
window: [
Math.max(s.window[0], s.worst_cell.t - PIECE_S / 2),
Math.min(s.window[1], s.worst_cell.t + PIECE_S / 2),
] as [number, number],
},
]
: []),
]);
}
const isRecord = (v: unknown): v is Record<string, unknown> =>
typeof v === "object" && v !== null && !Array.isArray(v);
export interface ExplainOptions {
reference: VideoInfo;
yours: VideoInfo;
model?: string;
oauth?: boolean;
project?: string;
log?: (message: string) => void;
}
/** One target asked piece by piece. */
async function explainOne(t: Target, o: ExplainOptions, dir: string): Promise<Diagnosis> {
const { layout, size, filter } = layoutOf(t.box, o.reference.width, o.reference.height);
const pieces = piecesOf(t.window, Math.min(o.reference.duration, o.yours.duration));
const differences: Difference[] = [];
const matches: boolean[] = [];
const jobs: string[] = [];
let tokens: number | null = null;
for (const [n, [a, b]] of pieces.entries()) {
const slow = slowBy(EXPLAIN_FPS, b - a);
const video = underInlineCap(
dir,
`${t.id}-${n}`,
(crf, out) => [
...["-y", "-ss", String(a), "-t", String(b - a), "-i", o.reference.path],
...["-ss", String(a), "-t", String(b - a), "-i", o.yours.path],
...["-filter_complex", `${filter},setpts=${slow}*PTS`, "-r", String(HEYGEN_FPS * 2)],
...["-an", "-c:v", "libx264", "-crf", String(crf), "-pix_fmt", "yuv420p", out],
],
b - a,
);
const prompt = [
explainQuestion(layout),
`This clip is ${round((b - a) * slow, 2)} s long${slow > 1 ? `: it plays at 1/${round(slow, 2)} speed and has no sound; judge motion as if at normal speed` : ""}. Give times in seconds of this clip.`,
].join("\n\n");
o.log?.(`explain: ${t.id} ${a}–${b} s`);
const asked = await askVideo(video, prompt, EXPLAIN_SCHEMA, {
model: o.model,
oauth: o.oauth,
project: o.project,
what: `diff --explain ${basename(o.yours.path)} ${t.id} ${a}-${b}s`,
});
jobs.push(asked.job);
const used = isRecord(asked.usage) ? asked.usage.input_tokens : undefined;
if (typeof used === "number") tokens = (tokens ?? 0) + used;
const answer = toFilmTime(asked.answer, slow, a);
if (!isRecord(answer)) continue;
if (typeof answer.matches === "boolean") matches.push(answer.matches);
for (const d of Array.isArray(answer.differences) ? answer.differences : [])
if (isRecord(d))
differences.push({
aspect: String(d.aspect ?? "other"),
original: String(d.original ?? ""),
yours: String(d.yours ?? ""),
...(typeof d.start === "number" ? { start: d.start } : {}),
...(typeof d.end === "number" ? { end: d.end } : {}),
});
}
return {
unverified: true,
window: pieces,
matches: matches.length ? matches.every(Boolean) && !differences.length : null,
differences,
layout,
size,
input_tokens: tokens,
jobs,
};
}
/** Each target's diagnosis, a few at a time, by id. */
export async function explainTargets(
targets: Target[],
o: ExplainOptions,
): Promise<Map<string, Diagnosis>> {
const out = new Map<string, Diagnosis>();
const dir = mkdtempSync(join(tmpdir(), "hf-explain-"));
try {
const queue = [...targets];
await Promise.all(
Array.from({ length: Math.min(PARALLEL, queue.length) }, async () => {
for (let t = queue.shift(); t; t = queue.shift())
out.set(t.id, await explainOne(t, o, dir));
}),
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
return out;
}
/** Each diagnosis written under its box in the result. */
export function attachDiagnoses(result: DiffResult, found: Map<string, Diagnosis>): void {
for (const s of result.segments ?? []) {
for (const d of s.drawn ?? []) d.diagnosis = found.get(d.id) ?? d.diagnosis;
if (s.worst_cell) s.worst_cell.diagnosis = found.get(s.id) ?? s.worst_cell.diagnosis;
}
}
packages/cli/src/heygen/generate.test.ts
+152 -1
View File
@@ -10,7 +10,7 @@ vi.mock("../cloud/auth.js", () => ({
resolveCloudBaseUrl: () => "https://api.test",
}));
const { generate, urlOf } = await import("./generate.js");
const { generate, parseMask, urlOf } = await import("./generate.js");
const { priceOf } = await import("./pricing.js");
const ffmpeg = findFFmpeg();
const project = mkdtempSync(join(tmpdir(), "hf-generate-test-"));
@@ -20,6 +20,17 @@ afterEach(() => vi.unstubAllGlobals());
const json = (body: unknown) => new Response(JSON.stringify(body), { status: 200 });
/** A 30 fps clip of these lavfi sources one after another: each new source is a cut. */
function shots(name: string, sources: string[], size = "180x320"): string {
const path = join(project, name);
execFileSync(ffmpeg!, [
...["-v", "error", "-y"],
...sources.flatMap((src) => ["-f", "lavfi", "-i", `${src}:s=${size}:r=30`]),
...["-filter_complex", `concat=n=${sources.length}:v=1`, "-pix_fmt", "yuv420p", path],
]);
return path;
}
describe("generate", () => {
it("finds the file URL wherever the route keeps it", () => {
expect(
@@ -135,6 +146,146 @@ describe("generate", () => {
).rejects.toThrow(/--first-frame cannot be used/);
});
it.skipIf(!ffmpeg)("blanks the motion clip's masked regions before sending it", async () => {
// A guide with white letters at 40,100 over a moving picture; the mask covers them for its first 0.5 s.
const guide = join(project, "titled.mp4");
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=d=1:s=180x320", "-vf"],
"drawbox=x=40:y=100:w=100:h=10:color=white:t=fill,drawbox=x=40:y=120:w=100:h=10:color=white:t=fill",
guide,
]);
const face = join(project, "face2.png");
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=gray:s=36x64", "-frames:v", "1"],
face,
]);
const sent: Record<string, unknown>[] = [];
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
if (init?.body) sent.push(JSON.parse(String(init.body)));
if (url === "https://files.test/m.mp4") return new Response(new Uint8Array([7]));
return json({
data: { id: "v3", status: "completed", video_url: "https://files.test/m.mp4" },
});
});
await generate({
type: "video",
out: join(project, "assets/masked.mp4"),
prompt: "walks",
references: [face],
motion: guide,
masks: [parseMask("30,90,120,50,0-0.5")],
});
const [video] = sent[0]!.reference_videos as { data: string }[];
const sentClip = join(project, "sent.mp4");
writeFileSync(sentClip, Buffer.from(video!.data, "base64"));
// The letters are two white rows of bars: the largest step between neighbouring rows' brightness is their edge.
// Filled from around, no such step is left; after 0.5 s they are back.
const edge = (t: number) => {
const px = execFileSync(ffmpeg!, [
...["-v", "error", "-ss", String(t), "-i", sentClip, "-frames:v", "1"],
...["-vf", "crop=100:40:40:95,format=gray", "-f", "rawvideo", "-"],
]);
const rows = Array.from({ length: 40 }, (_, y) => px.subarray(y * 100, y * 100 + 100));
const mean = rows.map((r) => r.reduce((a, v) => a + v, 0) / r.length);
return Math.max(...mean.slice(1).map((m, y) => Math.abs(m - mean[y]!)));
};
expect(edge(0.2)).toBeLessThan(30);
expect(edge(0.8)).toBeGreaterThan(60);
expect(readFileSync(join(project, ".media/manifest.jsonl"), "utf8")).toContain(
'"motion_masks":[[30,90,120,50,0,0.5]]',
);
expect(() => parseMask("30,90,120")).toThrow(/x,y,w,h/);
const { repeated } = await import("../commands/media-use/generate.js");
expect(
repeated(["--mask", "1,1,2,2", "--out", "a.mp4", "--mask=3,3,4,4,0-1"], "--mask"),
).toEqual(["1,1,2,2", "3,3,4,4,0-1"]);
await expect(
generate({ type: "video", out: "x.mp4", prompt: "p", masks: [parseMask("1,1,2,2")] }),
).rejects.toThrow(/--mask blanks regions of the --motion clip/);
});
it.skipIf(!ffmpeg)("refuses a guide with a cut inside before sending anything", async () => {
const guide = shots("two-shots.mp4", ["color=c=red:d=1", "testsrc2=d=1"]);
const face = join(project, "face3.png");
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=gray:s=36x64", "-frames:v", "1"],
face,
]);
const fetched = vi.fn();
vi.stubGlobal("fetch", fetched);
const request = {
type: "video" as const,
out: join(project, "assets/refused.mp4"),
prompt: "talks",
references: [face],
motion: guide,
};
await expect(generate(request)).rejects.toMatchObject({
message: `--motion ${guide} has a cut at 1.00 s`,
fix: "guide has a cut at 1.00 s: generate 0–1.00 s and 1.00–2.00 s as two shots, each with its own --motion clip",
});
expect(fetched).not.toHaveBeenCalled();
expect(existsSync(request.out)).toBe(false);
});
it.skipIf(!ffmpeg)("reports where HeyGen cut inside the returned shot", async () => {
// The guide's last frame is another picture: a cut within a frame of the end is no second shot, so it is sent.
const guide = shots("stray-frame.mp4", ["testsrc2=d=1", "color=c=blue:d=0.02"]);
const made = shots("heygen-cut.mp4", ["testsrc2=d=1", "color=c=red:d=1"]);
const clean = shots("heygen-clean.mp4", ["testsrc2=d=2"]);
const face = join(project, "face4.png");
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=gray:s=36x64", "-frames:v", "1"],
face,
]);
let file = made;
const sent: Record<string, unknown>[] = [];
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
if (init?.body) sent.push(JSON.parse(String(init.body)));
if (url === "https://files.test/c.mp4") return new Response(readFileSync(file));
return json({
data: { id: `c${sent.length}`, status: "completed", video_url: "https://files.test/c.mp4" },
});
});
const request = { type: "video" as const, prompt: "walks", references: [face], motion: guide };
const cut = await generate({ ...request, out: join(project, "assets/cut.mp4") });
expect(sent).toHaveLength(1);
expect(cut.cuts_inside).toEqual([1]);
expect(readFileSync(join(project, ".media/manifest.jsonl"), "utf8")).toContain(
'"cuts_inside":[1]',
);
file = clean;
const one = await generate({ ...request, out: join(project, "assets/one.mp4"), seconds: 6 });
expect(one.cuts_inside).toEqual([]);
});
it.skipIf(!ffmpeg)(
"asks for 768p whatever the source's size, as 1080p bills about twice",
async () => {
const guide = shots("tall.mp4", ["testsrc2=d=1"], "1080x1920");
const sent: Record<string, unknown>[] = [];
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
if (init?.body) sent.push(JSON.parse(String(init.body)));
if (url === "https://files.test/t.mp4") return new Response(new Uint8Array([7]));
return json({
data: { id: "t1", status: "completed", video_url: "https://files.test/t.mp4" },
});
});
const r = await generate({
type: "video",
out: join(project, "assets/tall.mp4"),
prompt: "walks",
motion: guide,
aspect: "9:16",
});
expect(sent[0]).toMatchObject({ resolution: "768p", aspect_ratio: "9:16" });
// An unreadable download is kept and its cuts reported as unknown.
expect(r.cuts_inside).toBeNull();
// A reference shot bills its seconds and the guide's at $0.05: 5 s and the 1 s guide.
expect(priceOf({ type: "video", out: "a.mp4", motion: guide }).usd).toBe(0.3);
},
);
it("waits for a queued shot and downloads it", async () => {
const answers = [
json({ data: { video_id: "v1", status: "pending" } }),
packages/cli/src/heygen/generate.ts
+117 -9
View File
@@ -4,7 +4,8 @@
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, extname, join, relative, resolve } from "node:path";
import { Fix, runFfmpeg } from "../measure/media.js";
import { findCuts, signatures } from "../measure/cuts.js";
import { Fix, probeVideo, round, runFfmpeg, videoClip } from "../measure/media.js";
import { idempotencyKey, runJob, type Job } from "./v3.js";
export type GenerateType = "image" | "video" | "avatar" | "voice" | "music";
@@ -26,6 +27,8 @@ export interface GenerateRequest {
audio?: string;
/** video: a clip whose camera and body motion the shot follows (the original's same stretch, up to 15 s). */
motion?: string;
/** video: regions of the motion clip blanked before it is sent (its drawn text), so the shot does not copy them. */
masks?: Mask[];
/** video: the shot starts from this picture. */
firstFrame?: string;
/** avatar: the photo of the person who speaks. */
@@ -40,6 +43,38 @@ export interface GenerateRequest {
bpm?: number;
}
/** A region of the motion clip in its pixels, and optionally the seconds of the clip it covers. */
export interface Mask {
box: [number, number, number, number];
window?: [number, number];
}
/** `x,y,w,h` or `x,y,w,h,a-b` (seconds of the motion clip). */
export function parseMask(text: string): Mask {
const m =
/^\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:,\s*(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)\s*)?$/.exec(
text,
);
if (!m || Number(m[3]) <= 0 || Number(m[4]) <= 0)
throw new Fix(`--mask ${text} is not x,y,w,h[,a-b]`, "--mask 190,405,700,175,1.0-2.7");
const box: Mask["box"] = [Number(m[1]), Number(m[2]), Number(m[3]), Number(m[4])];
return m[5] ? { box, window: [Number(m[5]), Number(m[6])] } : { box };
}
/**
* Filters that fill each mask's region from the pixels around it (ffmpeg delogo), so no letter survives and the
* motion around it stays. delogo needs a pixel of picture on every side, so boxes are kept 1 px inside the frame.
*/
export function maskFilters(masks: Mask[], width: number, height: number): string[] {
return masks.flatMap(({ box: [x, y, w, h], window }) => {
const [x0, y0] = [Math.max(1, x), Math.max(1, y)];
const [x1, y1] = [Math.min(width - 2, x + w), Math.min(height - 2, y + h)];
if (x1 - x0 < 1 || y1 - y0 < 1) return [];
const when = window ? `:enable='between(t,${window[0]},${window[1]})'` : "";
return [`delogo=x=${x0}:y=${y0}:w=${x1 - x0}:h=${y1 - y0}${when}`];
});
}
interface Route {
path: string;
poll: (id: string) => string;
@@ -56,6 +91,9 @@ type Media = (
) => { type: "base64"; media_type: string; data: string };
const DEFAULTS = { image: "nano-banana-2", video: "heygen-video-1", music: "lyria-3-pro" } as const;
export const VIDEO_RESOLUTION = "768p";
/** HeyGen follows at most this many seconds of a guide clip; the rest is not sent. */
export const GUIDE_MAX_S = 15;
const MUSIC_LENGTHS = [30, 60, 120, 180];
const need = <T>(value: T | undefined, flag: string, example: string): T => {
@@ -97,7 +135,9 @@ export const ROUTES: Record<GenerateType, Route> = {
mode: spoken ? "reference_to_video" : r.firstFrame ? "image_to_video" : "text_to_video",
prompt: need(r.prompt, "--prompt", '--prompt "slow push in on the desk"'),
duration: Math.min(15, Math.max(5, Math.round(r.seconds ?? 5))),
resolution: "768p",
// 1080p bills about twice 768p (2026-10-23, 5 s 9:16 jobs: text $0.10 vs $0.20, a reference picture $0.25
// vs $0.45, a 1 s guide clip $0.30 vs $0.55), so every shot stays at 768p whatever the source's size.
resolution: VIDEO_RESOLUTION,
...(r.firstFrame ? { image: media(r.firstFrame, "image") } : {}),
...(r.firstFrame ? {} : { aspect_ratio: r.aspect ?? "16:9" }),
...(r.references?.length
@@ -172,8 +212,11 @@ export function urlOf(job: Job, paths: string[]): string | null {
return null;
}
/** A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small. */
function inline(work: string): Media {
/**
* A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion
* clip with its masks blanked.
*/
function inline(work: string, masks: Mask[] = []): Media {
return (path, kind) => {
if (!existsSync(path))
throw new Fix(`no file at ${path}`, "pass a picture or sound from the project");
@@ -197,12 +240,17 @@ function inline(work: string): Media {
else if (kind === "audio")
runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]);
// A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s.
else
runFfmpeg([
...["-y", "-i", path, "-t", "15", "-an", "-vf"],
else {
const { width, height } = masks.length ? probeVideo(path) : { width: 0, height: 0 };
const filters = [
...maskFilters(masks, width, height),
"scale='if(gt(iw,ih),min(720,iw),-2)':'if(gt(iw,ih),-2,min(720,ih))'",
];
runFfmpeg([
...["-y", "-i", path, "-t", "15", "-an", "-vf", filters.join(",")],
...["-c:v", "libx264", "-crf", "28", "-preset", "veryfast", "-pix_fmt", "yuv420p", out],
]);
}
return {
type: "base64",
media_type: { image: "image/jpeg", audio: "audio/mpeg", video: "video/mp4" }[kind],
@@ -211,6 +259,42 @@ function inline(work: string): Media {
};
}
/**
* The seconds where a new shot starts in a video's first `seconds`, and the length read. A cut within a frame of
* either end leaves a stray frame, not a second shot, and does not count.
*/
export function cutsInside(path: string, seconds?: number): { cuts: number[]; length: number } {
const info = probeVideo(path);
const clip = videoClip(info.path, 0, seconds, info);
const { frames } = signatures(clip.input, clip.filters);
const cuts = findCuts(frames)
.filter((c) => c.frame > 1 && c.frame < frames.length - 1)
.map((c) => round(c.frame / info.fps, 3));
return { cuts, length: round(frames.length / info.fps, 3) };
}
const COUNT = ["", "one", "two", "three", "four", "five", "six"];
/**
* Refuses a --motion clip holding more than one shot before anything is sent: HeyGen follows the guide's cut into
* a cut of its own, so each shot is its own generation.
*/
export function checkGuide(r: Pick<GenerateRequest, "motion">): void {
if (!r.motion) return;
const { cuts, length } = cutsInside(r.motion, GUIDE_MAX_S);
if (!cuts.length) return;
const s = (t: number) => t.toFixed(2);
const and = (items: string[]) => `${items.slice(0, -1).join(", ")} and ${items.at(-1)}`;
const at =
cuts.length > 1 ? `cuts at ${and(cuts.map((t) => `${s(t)} s`))}` : `a cut at ${s(cuts[0]!)} s`;
const edges = [0, ...cuts, length];
const shots = edges.slice(1).map((e, i) => `${i ? s(edges[i]!) : "0"}–${s(e)} s`);
throw new Fix(
`--motion ${r.motion} has ${at}`,
`guide has ${at}: generate ${and(shots)} as ${COUNT[shots.length] ?? shots.length} shots, each with its own --motion clip`,
);
}
/** The nearest folder at or above `from` holding index.html. */
function projectOf(from: string): string | null {
for (let dir = resolve(from); ; dir = dirname(dir)) {
@@ -225,6 +309,8 @@ export interface Generated {
job: string;
route: string;
ledger?: string;
/** video: seconds where HeyGen cut inside the returned shot; null when the file could not be read. */
cuts_inside?: number[] | null;
}
export async function generate(
@@ -239,7 +325,10 @@ export async function generate(
const project = projectOf(dirname(out));
const work = mkdtempSync(join(tmpdir(), "hf-generate-"));
try {
const body = route.body(r, inline(work));
if (r.masks?.length && !r.motion)
throw new Fix("--mask blanks regions of the --motion clip", "pass --motion with --mask");
checkGuide(r);
const body = route.body(r, inline(work, r.masks));
const key = idempotencyKey(route.path, [JSON.stringify(body)]);
const job = await runJob(route.path, body, key, (id) => route.poll(encodeURIComponent(id)), {
log: {
@@ -262,6 +351,14 @@ export async function generate(
"run the same command again; it re-reads the job",
);
writeFileSync(out, Buffer.from(await res.arrayBuffer()));
// HeyGen sometimes cuts inside a shot on its own; the shot is paid for and kept, the cuts are reported.
let cuts: number[] | null | undefined;
if (r.type === "video")
try {
cuts = cutsInside(out).cuts;
} catch {
cuts = null;
}
let ledger: string | undefined;
if (project) {
const { recordInPlace } = await import("../media-use/lib/manifest.mjs");
@@ -284,12 +381,23 @@ export async function generate(
...(r.photo ? { photo: describe(r.photo) } : {}),
...(r.audio ? { audio: describe(r.audio) } : {}),
...(r.motion ? { motion: describe(r.motion) } : {}),
...(r.masks?.length
? { motion_masks: r.masks.map((m) => [...m.box, ...(m.window ?? [])]) }
: {}),
...(r.voice ? { voice: r.voice } : {}),
...(cuts !== undefined ? { cuts_inside: cuts } : {}),
},
}).id;
regenerateIndex(project);
}
return { out, type: r.type, job: job.id, route: route.path, ...(ledger ? { ledger } : {}) };
return {
out,
type: r.type,
job: job.id,
route: route.path,
...(ledger ? { ledger } : {}),
...(cuts !== undefined ? { cuts_inside: cuts } : {}),
};
} finally {
rmSync(work, { recursive: true, force: true });
}
packages/cli/src/heygen/pricing.ts
+12 -1
View File
@@ -1,7 +1,8 @@
// Prices before a paid call, from the HyperFrames app's creation pricing (hyperframes-internal
// packages/desktop/shared/creationPricing.mjs, 2026-10-22). HeyGen charges in units of $1/60, rounded up; the label
// rounds up to the cent. Routes HeyGen does not price ahead say so instead of guessing.
import type { GenerateRequest } from "./generate.js";
import { probeVideo, round } from "../measure/media.js";
import { GUIDE_MAX_S, type GenerateRequest } from "./generate.js";
export interface Price {
usd: number | null;
@@ -21,6 +22,9 @@ const GPT_IMAGE: Record<string, number> = {
};
const NANO_IMAGE = 0.067;
const VIDEO_768P_PER_S = 0.018;
// A reference shot (pictures, voice or guide clip) bills more, and its guide clip's seconds bill too: measured
// 2026-10-23 from the wallet, 5 s with a picture $0.25, 10 s $0.50, 5 s with a 1 s guide $0.30, with a 3 s guide $0.40.
const VIDEO_REF_768P_PER_S = 0.05;
const MUSIC_PER_TRACK = 0.08;
const AVATAR_IV_PER_MIN = 4.83;
// An avatar's length is estimated from its script, as the app does.
@@ -40,6 +44,13 @@ export function priceOf(r: GenerateRequest): Price {
}
case "video": {
const seconds = Math.min(15, Math.max(5, Math.round(r.seconds ?? 5)));
if (r.audio || r.references?.length || r.motion) {
const guide = r.motion ? Math.min(GUIDE_MAX_S, probeVideo(r.motion).pictureDuration) : 0;
return {
usd: cents((seconds + guide) * VIDEO_REF_768P_PER_S),
basis: `heygen-video-1 reference shot at 768p, ${seconds} s${guide ? ` and the ${round(guide, 2)} s guide` : ""} at $0.05 a second`,
};
}
return {
usd: cents(seconds * VIDEO_768P_PER_S),
basis: `heygen-video-1 at 768p, ${seconds} s at $0.018 a second`,
packages/cli/src/measure/cuts.test.ts
+52
View File
@@ -0,0 +1,52 @@
import { describe, expect, it } from "vitest";
import { confirmsCut, cutsIn, findCuts, SIG_H, SIG_W } from "./cuts.js";
/** Frame changes: `base` everywhere, with the given changes at their frames. */
const changes = (n: number, base: number, at: Record<number, number>) =>
Array.from({ length: n }, (_, i) => (i === 0 ? 0 : (at[i] ?? base)));
describe("cutsIn", () => {
it("finds a hard cut and a two-frame whip as one cut", () => {
expect(cutsIn(changes(60, 1, { 20: 40, 40: 30, 41: 25 }))).toEqual([
{ frame: 20, frames: 1, change: 40 },
{ frame: 40, frames: 2, change: 30 },
]);
});
it("finds a soft same-angle punch-in that nothing near it rivals", () => {
// dan 9.743 s: 11.3 grey levels against neighbours of about 1.
expect(cutsIn(changes(60, 1, { 30: 11.3 }))).toEqual([
{ frame: 30, frames: 1, change: 11.3, soft: true },
]);
});
it("leaves a smaller change among moving frames or a stepped animation alone", () => {
// Hands moving: neighbours up to 5, so 11 is not alone.
expect(cutsIn(changes(60, 1, { 25: 5, 30: 11 }))).toEqual([]);
// A 15 fps animation in a 30 fps file changes every other frame by about as much.
const stepped = Array.from({ length: 60 }, (_, i) => (i % 2 ? 0 : 10.5));
expect(cutsIn(stepped)).toEqual([]);
expect(cutsIn(changes(60, 1, { 30: 9 }))).toEqual([]);
});
});
describe("confirmsCut", () => {
it("matches a soft cut with a clear change within a frame on the other side", () => {
const cut = { frame: 30, frames: 1, change: 11.3, soft: true as const };
expect(confirmsCut(changes(60, 1, { 31: 7 }), cut)).toBe(true);
expect(confirmsCut(changes(60, 1, { 32: 7 }), cut)).toBe(false);
expect(confirmsCut(changes(60, 1, { 30: 4 }), cut)).toBe(false);
expect(confirmsCut(changes(60, 2, { 30: 7 }), cut)).toBe(false);
});
});
describe("findCuts", () => {
it("reads changes from grey frames", () => {
const frame = (v: number) => new Uint8Array(SIG_W * SIG_H).fill(v);
const frames = [
...Array.from({ length: 20 }, () => frame(100)),
...Array.from({ length: 20 }, () => frame(140)),
];
expect(findCuts(frames)).toEqual([{ frame: 20, frames: 1, change: 40 }]);
});
});
packages/cli/src/measure/cuts.ts
+64 -16
View File
@@ -1,6 +1,7 @@
// Cuts from the frame-to-frame difference itself, ported from the Studio's videoShots.mjs: ffmpeg's scene score
// missed two of the James Blake source's cuts at 0.25, so a cut is a frame far unlike the one before and several
// times unlike its neighbours' changes.
// times unlike its neighbours' changes. A punch-in at the same angle changes less (dan's 9.743 s: 11 grey levels),
// so a smaller change also counts when nothing else near it changes anywhere near as much: a soft cut.
import { decode, meanAbsDiff, type Frames } from "./media.js";
// Small enough to read a minute quickly, large enough to tell a cut from motion.
@@ -8,6 +9,25 @@ export const SIG_W = 64;
export const SIG_H = 36;
const CUT_LEVEL = 12;
const CUT_RATIO = 4;
// A soft cut: at least SOFT_LEVEL, SOFT_RATIO times its neighbours' median and SOFT_ALONE times the largest of them.
const SOFT_LEVEL = 10;
const SOFT_RATIO = 8;
const SOFT_ALONE = 4;
// The other side confirms a soft cut with a change of CONFIRM_LEVEL, CUT_RATIO times its neighbours' median and
// CONFIRM_SHARE of the cut's own change, within a frame of it.
const CONFIRM_LEVEL = 6;
const CONFIRM_SHARE = 0.5;
const NEIGHBOURS = 12;
export interface Cut {
frame: number;
/** A whip or a two-frame dissolve changes on consecutive frames: one cut this many frames long. */
frames: number;
/** Its largest change, in mean grey levels. */
change: number;
/** Found only by the soft rule. */
soft?: true;
}
/** Every frame of ffmpeg's `input` (after `filters`) as SIG_W x SIG_H grey levels. */
export function signatures(input: string[], filters: string[] = []): Frames {
@@ -19,26 +39,54 @@ const median = (values: number[]) => {
return sorted[Math.floor(sorted.length / 2)] ?? 0;
};
/**
* The frame indexes where a new shot starts, each with its change (mean grey levels) and length: a whip or a
* two-frame dissolve changes on consecutive frames, which is one cut `frames` long, not a shot of one frame.
*/
export function findCuts(
frames: Uint8Array[],
): { frame: number; frames: number; change: number }[] {
const diffs = frames.map((f, i) => (i ? meanAbsDiff(frames[i - 1]!, f) : 0));
const hits = diffs.flatMap((d, i) => {
if (i === 0 || d < CUT_LEVEL) return [];
const around = diffs.slice(Math.max(1, i - 12), i).concat(diffs.slice(i + 1, i + 13));
return d >= CUT_RATIO * Math.max(1, median(around)) ? [{ frame: i, change: d }] : [];
/** Each frame's mean grey difference from the one before (0 for the first). */
export function frameChanges(frames: Uint8Array[]): number[] {
return frames.map((f, i) => (i ? meanAbsDiff(frames[i - 1]!, f) : 0));
}
const neighbours = (diffs: number[], i: number) =>
diffs.slice(Math.max(1, i - NEIGHBOURS), i).concat(diffs.slice(i + 1, i + NEIGHBOURS + 1));
/** The frame indexes where a new shot starts, from each frame's change (`frameChanges`). */
export function cutsIn(diffs: number[]): Cut[] {
const hits = diffs.flatMap((d, i): { frame: number; change: number; soft?: true }[] => {
if (i === 0) return [];
const around = neighbours(diffs, i);
const typical = Math.max(1, median(around));
if (d >= CUT_LEVEL && d >= CUT_RATIO * typical) return [{ frame: i, change: d }];
const alone = d >= SOFT_ALONE * Math.max(0, ...around);
return d >= SOFT_LEVEL && d >= SOFT_RATIO * typical && alone
? [{ frame: i, change: d, soft: true }]
: [];
});
const out: { frame: number; frames: number; change: number }[] = [];
for (const h of hits) {
const out: Cut[] = [];
for (const { soft, ...h } of hits) {
const last = out.at(-1);
if (last && h.frame === last.frame + last.frames) {
last.frames++;
last.change = Math.max(last.change, h.change);
} else out.push({ ...h, frames: 1 });
if (!soft) delete last.soft;
} else out.push({ ...h, frames: 1, ...(soft ? { soft } : {}) });
}
return out;
}
/** The frame indexes where a new shot starts, each with its change and length. */
export function findCuts(frames: Uint8Array[]): Cut[] {
return cutsIn(frameChanges(frames));
}
/**
* Whether these changes show a clear change within a frame of `frame`, though maybe under a cut's thresholds: a
* soft cut found on the other side counts as matched by it, as a soft cut is a change near the line.
*/
export function confirmsCut(diffs: number[], cut: Cut): boolean {
return [cut.frame - 1, cut.frame, cut.frame + 1].some((i) => {
const d = diffs[i];
if (i < 1 || d === undefined) return false;
return (
d >= Math.max(CONFIRM_LEVEL, CONFIRM_SHARE * cut.change) &&
d >= CUT_RATIO * Math.max(1, median(neighbours(diffs, i)))
);
});
}
packages/cli/src/measure/diff.test.ts
+546 -2
View File
@@ -1,10 +1,21 @@
import { execFileSync } from "node:child_process";
import { existsSync, mkdtempSync, rmSync } from "node:fs";
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { diffVideos, pickedFrames, selectFilter } from "./diff.js";
import {
cellsOutside,
diffVideos,
longRuns,
pickedFrames,
regionRoi,
selectFilter,
soundOffset,
wholeFrameCuts,
} from "./diff.js";
import { probeVideo } from "./media.js";
import { carriedFrame, shotFraming, syncPass } from "./person.js";
import { parseSegments } from "./segments.js";
const ffmpeg = findFFmpeg();
@@ -42,6 +53,145 @@ describe("parseSegments", () => {
{ id: "S04", start: 6.5, end: 8, title: "", pixels: "rebuilt" },
]);
});
it("reads a regenerated region inside a rebuilt segment, in pixels or shares", () => {
const md = [
"## S01 · 0–2 s · Board under the presenter",
"Pixels: rebuilt",
"Regenerated: 0, 95, 720, 600 (the re-shot presenter)",
"## S02 · 2–4 s",
"Regenerated: 0 0.5 1 0.5",
"## S03 · 4–6 s",
"Pixels: regenerated",
"Regenerated: 0, 0, 10, 10",
].join("\n");
const [a, b, c] = parseSegments(md);
expect(a!.regenerated).toEqual([0, 95, 720, 600]);
expect(b!.regenerated).toEqual([0, 0.5, 1, 0.5]);
expect(c!.regenerated).toBeUndefined();
});
it("reads drawn elements over a regenerated person, and whether a segment quotes copy", () => {
const md = [
"## S01 · 0–2 s · Title over the presenter",
"Pixels: rebuilt",
"Regenerated: 0, 0, 608, 1080",
'Copy: "IS NO" / "LONGER"',
"Drawn: 110, 615, 390, 165 (the title)",
"Drawn: 40 40 120 120, logo 0.5–1.5 s",
"## S02 · 2–4 s",
"Copy: none, footage only",
].join("\n");
const [a, b] = parseSegments(md);
expect(a!.drawn).toEqual([
{ box: [110, 615, 390, 165] },
{ box: [40, 40, 120, 120], window: [0.5, 1.5] },
]);
expect(a!.copy).toBe(true);
expect(b!.drawn).toBeUndefined();
expect(b!.copy).toBeUndefined();
});
});
describe("whole-frame cuts", () => {
it("keeps the ledger's cuts and stills from the original that cover the whole frame", () => {
const ledger = [
{
type: "video",
provenance: { provider: "original", window: [0, 12.267], roi: [0, 0, 720, 1280] },
},
{ type: "video", provenance: { provider: "original", window: [1, 2] } },
{
type: "video",
provenance: { provider: "original", window: [3, 4], roi: [0, 0, 720, 704] },
},
{ type: "image", provenance: { provider: "original", at: 3, roi: [0, 0, 720, 1280] } },
{ type: "image", provenance: { provider: "original", at: 5, roi: [60, 260, 480, 560] } },
{ type: "video", provenance: { provider: "heygen", window: [0, 5] } },
]
.map((e) => JSON.stringify(e))
.join("\n");
expect(wholeFrameCuts(`${ledger}\nnot json`, 720, 1280)).toEqual({
windows: [
[0, 12.267],
[1, 2],
],
stills: [3],
});
});
it("drops runs of frames shorter than the minimum", () => {
const on = [true, true, false, true, true, true, true, true];
expect(longRuns(on, [0, 1, 2, 3, 4, 5, 9, 10], 3)).toEqual([
false,
false,
false,
true,
true,
true,
false,
false,
]);
});
});
describe("regenerated regions", () => {
it("scales shares to pixels, clips to the frame and leaves the covered cells to the region", () => {
expect(regionRoi([0, 0.5, 1, 0.5], 320, 180)).toEqual([0, 90, 320, 90]);
expect(regionRoi([600, 1200, 400, 200], 720, 1280)).toEqual([600, 1200, 120, 80]);
// 80x44 cells: the bottom half covers rows 2 and 3 (row 2 from y 88, all but 2 px of it).
expect(cellsOutside([0, 90, 320, 90], 320, 180)).toEqual([0, 1, 2, 3, 4, 5, 6, 7]);
});
});
describe("person gates", () => {
const at = (t: number, ...boxes: number[][]) => ({ t, boxes });
const face = [0.4, 0.3, 0.2, 0.12];
const original = [0, 0.4, 0.8, 1.2].map((t) => at(t, face));
it("fails a shot's framing only when the face sits clearly off the original's", () => {
const moved = (dy: number, grow = 1) =>
[0, 0.4, 0.8, 1.2].map((t) => at(t, [0.4, 0.3 + dy, 0.2 * grow, 0.12 * grow]));
expect(shotFraming([0, 1.5], original, moved(0.01))).toMatchObject({ pass: true });
// Half a face height lower, or 40% bigger.
expect(shotFraming([0, 1.5], original, moved(0.06))).toMatchObject({ pass: false });
expect(shotFraming([0, 1.5], original, moved(0, 1.4))).toMatchObject({ pass: false });
// One stray moment off (a hand over the face, a photo) is not a framing.
const stray = original.map((m, i) => (i === 1 ? at(m.t, [0.4, 0.6, 0.2, 0.12]) : m));
expect(shotFraming([0, 1.5], original, stray)).toMatchObject({ pass: true });
// Another face in yours (a picture on the wall) does not stand in for the person.
const crowd = [0, 0.4, 0.8, 1.2].map((t) => at(t, [0.05, 0.05, 0.3, 0.2], face));
expect(shotFraming([0, 1.5], original, crowd)).toMatchObject({
shift: 0,
pass: true,
});
// Too few faces to judge.
expect(shotFraming([0, 0.5], original, original)).toBeUndefined();
});
it("judges lips only on a confident reading, inside +45 / −125 ms", () => {
expect(syncPass({ best_lag_ms: 30, confidence: "high" })).toBe(true);
expect(syncPass({ best_lag_ms: -120, confidence: "high" })).toBe(true);
expect(syncPass({ best_lag_ms: 80, confidence: "high" })).toBe(false);
expect(syncPass({ best_lag_ms: -200, confidence: "high" })).toBe(false);
expect(syncPass({ best_lag_ms: -300, confidence: "medium" })).toBeNull();
});
it("finds the original's letters in a generated clip, not the place they share", () => {
// A textured place (the same in the original, the clean still and the clip) and white letters over it.
const [w, h] = [60, 30];
const place = new Uint8Array(w * h * 3).map((_, i) => 60 + ((i * 37) % 50));
const lettered = (px: Uint8Array) => {
const out = new Uint8Array(px);
for (let y = 8; y < 22; y++)
for (const x0 of [10, 25, 40])
for (let x = x0; x < x0 + 4; x++) out.set([255, 255, 255], (y * w + x) * 3);
return out;
};
const original = lettered(place);
expect(carriedFrame(original, lettered(place), place, w, h, undefined)).toBeGreaterThan(0.9);
expect(carriedFrame(original, place, place, w, h, undefined)).toBeLessThan(0.1);
});
});
describe("frame picks", () => {
@@ -119,6 +269,32 @@ describe.skipIf(noTools)("diffVideos", () => {
expect(r.pass).toBe(false);
const same = await diffVideos({ reference: page, yours: page, outDir: join(dir, "page") });
expect(same.missing_content).toMatchObject({ share: 0, pass: true });
// A regenerated segment's picture is its own: what it leaves out is not missing.
const regen = await diffVideos({
reference: page,
yours,
segments: parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n"),
outDir: join(dir, "emptied-regen"),
});
expect(regen.missing_content).toMatchObject({ share: 0, pass: true });
});
it("reports missing texture apart from missing shapes, without failing it", async () => {
// A fine 4 px checker in the bottom-left cell: flat at an eighth of the width, so texture, not structure.
const checker =
"if(lt(X\\,80)*gte(Y\\,135)\\,if(mod(floor(X/4)+floor(Y/4)\\,2)\\,60\\,200)\\,235)";
const grain = clip("grain.mp4", [
...["-f", "lavfi", "-i", "color=c=white:s=320x180:r=30:d=2", "-vf"],
`format=gray,geq=lum='${checker}',format=yuv420p`,
]);
const plain = clip("plain.mp4", [
...["-f", "lavfi", "-i", "color=c=white:s=320x180:r=30:d=2", "-vf"],
"format=gray,geq=lum=235,format=yuv420p",
]);
const r = await diffVideos({ reference: grain, yours: plain, outDir: join(dir, "grain") });
expect(r.missing_content).toMatchObject({ share: 0, pass: true });
expect(r.missing_content.texture.share).toBeGreaterThan(0.9);
expect(r.warnings.some((w) => w.includes("reported, not failed"))).toBe(true);
});
it("scores only a range, on the original's frame numbers", async () => {
@@ -155,6 +331,374 @@ describe.skipIf(noTools)("diffVideos", () => {
expect(r.pass).toBe(false);
});
it("matches a soft cut in the original with a hard one in yours", async () => {
// Grey 0x80 to 0x8b changes the signature by 10 levels: a soft cut. Yours jumps further on the same frame.
const step = (name: string, to: string, at: number) =>
clip(name, [
...["-f", "lavfi", "-i", `color=c=0x808080:s=320x180:r=30:d=${at}`],
...["-f", "lavfi", "-i", `color=c=${to}:s=320x180:r=30:d=${3 - at}`],
...["-filter_complex", "[0][1]concat=n=2:v=1[v]", "-map", "[v]"],
]);
const soft = step("soft-ref.mp4", "0x8b8b8b", 1);
const r = await diffVideos({
reference: soft,
yours: step("soft-hard.mp4", "0xa0a0a0", 1),
outDir: join(dir, "soft"),
});
expect(r.cuts).toMatchObject({ reference: [1], missing: [], extra: [], pass: true });
const late = await diffVideos({
reference: soft,
yours: step("soft-late.mp4", "0xa0a0a0", 1.2),
outDir: join(dir, "soft-late"),
});
expect(late.cuts).toMatchObject({ missing: [1], extra: [1.2], pass: false });
});
it("judges a rebuilt segment's regenerated region by motion and look, the rest by SSIM", async () => {
// The bottom half mirrored, as a re-shot presenter differs in pixels but moves the same.
const mixed = clip("mixed.mp4", [
...["-i", ref, "-filter_complex"],
"[0]split[a][b];[b]crop=320:90:0:90,hflip[f];[a][f]overlay=0:90",
]);
const md = "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\n";
const plain = await diffVideos({
reference: ref,
yours: mixed,
segments: parseSegments(md),
outDir: join(dir, "mixed-plain"),
});
expect(plain.segments?.[0]?.pass).toBe(false);
const r = await diffVideos({
reference: ref,
yours: mixed,
segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`),
outDir: join(dir, "mixed"),
});
const [s] = r.segments!;
expect(s!.ssim!.mean).toBeGreaterThan(0.99);
expect(s!.regenerated?.roi).toEqual([0, 90, 320, 90]);
expect(s!.person?.energy?.mean_ratio).toBeCloseTo(1, 1);
expect(s!.pass).toBe(true);
// A wrong box outside the region still fails the segment.
const boxed = clip("mixed-box.mp4", [
...["-i", mixed, "-vf"],
"drawbox=x=20:y=10:w=60:h=40:color=red:t=fill",
]);
const bad = await diffVideos({
reference: ref,
yours: boxed,
segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`),
outDir: join(dir, "mixed-box"),
});
expect(bad.segments?.[0]?.pass).toBe(false);
});
it("judges each drawn element inside a regenerated region by itself, not by what is behind it", async () => {
// A label (three white strokes) over a soft bottom half, a regenerated person (here 5 frames late); yours
// draws it 24 px off.
const strokes = (x0: number) =>
[0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(",");
const label = strokes(120);
const soft = (delay: number) =>
`[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`;
const base = clip("label-ref.mp4", [...["-i", ref, "-filter_complex"], `${soft(0)},${label}`]);
const right = clip("label-right.mp4", [
...["-i", ref, "-filter_complex"],
`${soft(5)},${label}`,
]);
const off = clip("label-off.mp4", [
...["-i", ref, "-filter_complex"],
`${soft(5)},${strokes(144)}`,
]);
const md =
"## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\n";
const judge = (yours: string, text: string, name: string) =>
diffVideos({
reference: base,
yours,
segments: parseSegments(text),
outDir: join(dir, name),
});
// Without a Drawn line the label hides in the person's box.
expect((await judge(off, md, "drawn-hidden")).segments?.[0]?.pass).toBe(true);
const drawn = `${md}Drawn: 118, 108, 42, 28\n`;
const good = (await judge(right, drawn, "drawn-right")).segments![0]!;
expect(good.pass).toBe(true);
expect(good.drawn?.[0]).toMatchObject({ id: "S01.D1", box: [118, 108, 42, 28], pass: true });
expect(good.scored).toContain("1 drawn elements, each scored by itself");
const bad = await judge(off, drawn, "drawn-off");
expect(bad.segments?.[0]?.pass).toBe(false);
expect(bad.segments?.[0]?.drawn?.[0]?.pass).toBe(false);
// A box over the whole frame is flagged.
const whole = await judge(right, md.replace("0, 90, 320, 90", "0, 0, 320, 180"), "drawn-whole");
expect(whole.warnings.some((w) => w.includes("covers the whole frame"))).toBe(true);
});
it("scores a drawn element by itself without segments, and its ink says when yours is thinner", async () => {
// The same three-stroke label over two different flat backdrops; yours draws it at half the stroke width.
const strokes = (w: number) =>
[0, 14, 28].map((x) => `drawbox=x=${120 + x}:y=110:w=${w}:h=24:color=white:t=fill`).join(",");
const backdrop = (colour: string) => [
"-f",
"lavfi",
"-i",
`color=c=${colour}:s=320x180:d=3:r=30`,
];
const base = clip("solo-ref.mp4", [...backdrop("0x305070"), "-vf", strokes(6)]);
const right = clip("solo-right.mp4", [...backdrop("0x604030"), "-vf", strokes(6)]);
const thin = clip("solo-thin.mp4", [...backdrop("0x604030"), "-vf", strokes(3)]);
const box: [number, number, number, number] = [116, 106, 44, 32];
const good = await diffVideos({
reference: base,
yours: right,
at: [0.5, 1, 1.5, 2],
drawn: [{ box }, { box, window: [1, 3] }],
outDir: join(dir, "solo-right"),
});
expect(good.segments).toBeUndefined();
expect(good.drawn).toHaveLength(2);
expect(good.drawn?.[0]).toMatchObject({ id: "D1", box, window: [0, 3], frames: 4, pass: true });
expect(good.drawn?.[1]).toMatchObject({ id: "D2", window: [1, 3], frames: 3, pass: true });
const bad = await diffVideos({
reference: base,
yours: thin,
at: [0.5, 1, 1.5, 2],
drawn: [{ box }],
outDir: join(dir, "solo-thin"),
});
expect(good.drawn?.[0]).toMatchObject({ score: 1, offset: [0, 0], ink: 1 });
// Half-weight letters in the right place: the score barely drops, the ink says it.
expect(bad.drawn?.[0]?.score).toBeLessThan(0.9);
expect(bad.drawn?.[0]?.ink).toBe(0.5);
});
it("fails the original's whole frame played where copy is meant to be rebuilt", async () => {
const ledger = join(dir, "manifest.jsonl");
writeFileSync(
ledger,
JSON.stringify({
type: "video",
path: "assets/all.mp4",
provenance: { provider: "original", window: [0, 3], roi: [0, 0, 320, 180] },
}) + "\n",
);
const titled = '## S01 · 0–3 s · Title\nPixels: rebuilt\nCopy: "TITLE"\n';
const copy = await diffVideos({
reference: ref,
yours: ref,
segments: parseSegments(titled),
ledger,
outDir: join(dir, "copy"),
});
expect(copy.whole_frame_reuse).toMatchObject({ spans: [[0, 3]], drawn: [[0, 3]], pass: false });
expect(copy.segments?.[0]?.pass).toBe(false);
expect(copy.pass).toBe(false);
// Footage reused with its burned-in title is a copy too (Miao, 2026-10-23: no whole-frame cut with drawn things).
const burned = await diffVideos({
reference: ref,
yours: ref,
segments: parseSegments(
'## S01 · 0–3 s · Title over footage\nPixels: reused\nCopy: "TITLE"\n',
),
ledger,
outDir: join(dir, "burned"),
});
expect(burned.whole_frame_reuse).toMatchObject({ share: 1, drawn: [[0, 3]], pass: false });
// Footage with nothing drawn on it may be cut whole.
const footage = await diffVideos({
reference: ref,
yours: ref,
segments: parseSegments("## S01 · 0–3 s · Footage\nPixels: reused\n"),
ledger,
outDir: join(dir, "footage"),
});
expect(footage.whole_frame_reuse).toMatchObject({ share: 1, drawn: [], pass: true });
expect(footage.pass).toBe(true);
// A cut that only fed a generation (a motion guide) is not played: yours differs from it.
const flipped = clip("guided.mp4", [...["-i", ref, "-vf"], "hflip"]);
const guided = await diffVideos({
reference: ref,
yours: flipped,
segments: parseSegments(titled),
ledger,
outDir: join(dir, "guided"),
});
expect(guided.whole_frame_reuse).toMatchObject({ spans: [], pass: true });
// A generated shot that starts from the original's first frame matches it for a few frames only: not reuse.
const started = clip("started.mp4", [
...["-i", ref, "-filter_complex"],
"[0]split[a][b];[b]hflip[f];[a][f]overlay=enable='gte(t\\,0.2)'",
]);
const start = await diffVideos({
reference: ref,
yours: started,
segments: parseSegments(titled),
ledger,
outDir: join(dir, "started"),
});
expect(start.whole_frame_reuse).toMatchObject({ spans: [], pass: true });
});
it("finds a whole-frame still of the original shown in yours", async () => {
const ledger = join(dir, "stills.jsonl");
writeFileSync(
ledger,
[
{ type: "image", provenance: { provider: "original", at: 1, roi: [0, 0, 320, 180] } },
{ type: "image", provenance: { provider: "original", at: 2.5 } },
]
.map((e) => JSON.stringify(e))
.join("\n"),
);
// Yours holds the original's frame at 1 s for the whole clip.
const held = clip("held.mp4", [
...["-i", ref, "-vf"],
"select=eq(n\\,30),loop=loop=89:size=1,setpts=N/30/TB",
]);
const r = await diffVideos({
reference: ref,
yours: held,
segments: parseSegments('## S01 · 0–3 s · Collage\nPixels: rebuilt\nCopy: "TITLE"\n'),
ledger,
outDir: join(dir, "held"),
});
expect(r.whole_frame_reuse).toMatchObject({ stills: [1], spans: [[0, 3]], pass: false });
});
it("reads sound that starts late as late, and the picture's length as the duration", async () => {
// Clicks at uneven times, with the sound running 0.4 s past the picture.
const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3.4";
const voiced = (name: string, delay: number) =>
clip(name, [
...["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"],
...["-itsoffset", String(delay), "-f", "lavfi", "-i", clicks],
...["-map", "0:v", "-map", "1:a", "-c:a", "aac"],
]);
const on = voiced("voiced.mp4", 0);
const same = await diffVideos({ reference: on, yours: on, outDir: join(dir, "voiced") });
expect(same.duration).toMatchObject({ reference: 3, pass: true });
expect(same.sound).toMatchObject({ offset_ms: 0, pass: true });
const late = await diffVideos({
reference: on,
yours: voiced("late.mp4", 0.2),
outDir: join(dir, "late"),
});
expect(late.sound).toMatchObject({ offset_ms: 200, pass: false });
});
it("places the sound by its decoded timestamps, not the start time the probe reports", () => {
// ffmpeg 5.1 drops an HE-AAC edit list's skip twice: the first sample it keeps is timestamped 0.1 s in while
// the probe still says the track starts at 0. Here a sound timestamped 0.25 s in, probed as starting at 0,
// against the same sound with 0.25 s of silence baked into its samples: in sync.
const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3";
const picture = ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"];
const stamped = clip("stamped.mp4", [
...picture,
...["-itsoffset", "0.25", "-f", "lavfi", "-i", clicks],
...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"],
]);
const baked = clip("baked.mp4", [
...picture,
...["-f", "lavfi", "-i", clicks, "-af", "adelay=250:all=1"],
...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"],
]);
const probedAtZero = { ...probeVideo(stamped), audioStart: 0 };
expect(soundOffset(stamped, baked, probedAtZero, probeVideo(baked))).toMatchObject({
offset_ms: 0,
pass: true,
});
});
it("passes a re-shot person whose motion is off when the components are right", async () => {
// The bottom half (the person) frozen on its first frame: no motion at all. Pixels above match.
const frozen = clip("frozen.mp4", [
...["-i", ref, "-filter_complex"],
"[0]split[a][b];[b]crop=320:90:0:90,trim=end_frame=1,tpad=stop=89:stop_mode=clone[f];[a][f]overlay=0:90",
]);
const r = await diffVideos({
reference: ref,
yours: frozen,
segments: parseSegments(
"## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\nRegenerated: 0, 0.5, 1, 0.5\n",
),
outDir: join(dir, "frozen"),
});
const [s] = r.segments!;
expect(s!.person?.energy?.mean_ratio).toBeLessThan(0.5);
expect(s!.person?.gates).toEqual({ framing: null, text: null, sync: null });
expect(s!.pass).toBe(true);
expect(r.components).toMatchObject({ passed: 1, total: 1 });
expect(r.person?.energy).toBeLessThan(0.5);
expect(r.pass).toBe(true);
});
it("fails a re-shot person whose generated clip carries the original's text", async () => {
// The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half.
const strokes = (x0: number) =>
[0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(",");
const label = strokes(120);
const soft = (delay: number) =>
`[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`;
const base = clip("carried-ref.mp4", [
...["-i", ref, "-filter_complex"],
`${soft(0)},${label}`,
]);
const project = join(dir, "carried");
mkdirSync(join(project, "assets"), { recursive: true });
// The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the
// label over each; over the first, 8 px off, the letters show doubled.
clip("carried/assets/dirty.mp4", [...["-i", ref, "-filter_complex"], `${soft(5)},${label}`]);
clip("carried/assets/clean.mp4", [...["-i", ref, "-filter_complex"], soft(5)]);
execFileSync(ffmpeg!, [
...["-hide_banner", "-loglevel", "error", "-y", "-i", ref, "-filter_complex"],
`${soft(0)},trim=end_frame=1`,
...["-frames:v", "1", join(project, "assets", "still.png")],
]);
const ledger = (path: string) => {
const file = join(project, `${path}.jsonl`);
writeFileSync(
file,
JSON.stringify({
type: "video",
path: `assets/${path}.mp4`,
provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] },
}) + "\n",
);
return file;
};
const md =
"## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n";
const judge = (path: string, x0: number) =>
diffVideos({
reference: base,
yours: clip(`carried-yours-${path}.mp4`, [
...["-i", join(project, "assets", `${path}.mp4`), "-vf"],
strokes(x0),
]),
segments: parseSegments(md),
ledger: ledger(path),
project,
outDir: join(dir, `carried-${path}`),
});
const dirty = await judge("dirty", 128);
expect(dirty.segments?.[0]?.person?.text?.[0]).toMatchObject({
drawn: "S01.D1",
carried: true,
});
expect(dirty.segments?.[0]?.pass).toBe(false);
expect(dirty.person).toMatchObject({ gates: { text: false }, failed: { text: ["S01"] } });
expect(dirty.pass).toBe(false);
const clean = await judge("clean", 120);
expect(clean.segments?.[0]?.drawn?.[0]?.pass).toBe(true);
expect(clean.segments?.[0]?.person?.gates.text).toBe(true);
expect(clean.segments?.[0]?.pass).toBe(true);
expect(clean.person?.shots_at_cuts).toMatchObject({
at_cuts: 1,
total: 1,
cuts_without_shot: [],
});
});
it("judges a regenerated segment by motion and look, not pixels", async () => {
const shifted = clip("shifted.mp4", [...["-i", ref, "-vf"], "hflip"]);
const segments = parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n");
packages/cli/src/measure/diff.ts
+711 -77
View File
@@ -14,8 +14,15 @@ import {
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { findCuts, signatures } from "./cuts.js";
import { basename, dirname, join, resolve } from "node:path";
import { confirmsCut, cutsIn, frameChanges, signatures } from "./cuts.js";
import {
drawnFrames,
drawnVerdict,
readDrawn,
scoreDrawnFrame,
type DrawnVerdict,
} from "./drawn.js";
import { compareEnergy, energyCurve, type EnergyComparison } from "./energy.js";
import { compareLook, look, sameLook, type LookComparison } from "./look.js";
import {
@@ -28,9 +35,20 @@ import {
videoClip,
writeImage,
type Clip,
type Roi,
type VideoInfo,
} from "./media.js";
import {
AT_CUT_S,
clipUses,
generatedClips,
personVerdict,
placeClips,
type GeneratedClip,
type PersonVerdict,
} from "./person.js";
import type { Segment } from "./segments.js";
import type { Diagnosis } from "../heygen/explain.js";
// To be calibrated on the reverse bench; light-reverse's rebuilt moments scored 0.885–0.958 mean SSIM.
export const DEFAULT_MIN_MEAN = 0.92;
@@ -43,19 +61,37 @@ export const DEFAULT_MIN_CELL = 0.75;
// A regenerated shot moves about as much as the original's.
const ENERGY_RATIO: [number, number] = [0.8, 1.25];
const GRID = 4;
const ALL_CELLS = Array.from({ length: GRID * GRID }, (_, c) => c);
const WORST_GAP_S = 0.5;
const SHEET_W = 960;
const CUT_SLACK = 1;
// A segment's cell is left to its regenerated region when the region covers more than this share of it.
const REGION_CELL_SHARE = 0.1;
// Missing content: cells are read at this width, from at most this many frames. A cell whose luma spread in the
// original reaches MISSING_DETAIL while yours is flat (under MISSING_FLAT and MISSING_KEEP of the original's) is
// empty; over MISSING_MAX_SHARE of frames with an empty cell fails. A weaker blur or a missing small label is not
// empty: SSIM and its cell gate grade those (provisional, calibrated on the reverse bench).
// empty: SSIM and its cell gate grade those (provisional, calibrated on the reverse bench). An empty cell is missing
// structure when the original's cell still spreads MISSING_DETAIL at an eighth of that width (shapes, cards, a
// map's land), and missing texture otherwise (a credit line, sea-floor grain): texture is reported, not failed.
const MISSING_W = 192;
const MISSING_SAMPLES = 1800;
const MISSING_DETAIL = 8;
const MISSING_KEEP = 0.35;
const MISSING_FLAT = 4;
const MISSING_MAX_SHARE = 0.02;
const STRUCTURE_SCALE = 8;
// Whole-frame reuse: a cut from the original covering this share of its frame plays the whole frame, and a frame
// whose median cell scores REUSE_SSIM shows it. Drawn things come along with it (a copy) for longer than
// REUSE_MAX_FRAMES fails. A Regenerated box over this share of the frame judges nothing by pixels. A cut matched for
// less than REUSE_MIN_S is not reuse: a generated shot starts from a clean still of the original's first frame. A whole-
// frame still of the original shows where yours, read in grey at STILL_W wide, is within STILL_DIFF of it in most
// of the 4x4 cells.
const WHOLE_FRAME_SHARE = 0.9;
const REUSE_SSIM = 0.9;
const REUSE_MAX_FRAMES = 2;
const REUSE_MIN_S = 0.5;
const STILL_W = 96;
const STILL_DIFF = 4;
/** Which of the original's frames are compared. */
export type Pick =
@@ -82,6 +118,12 @@ export interface DiffOptions {
outDir: string;
/** Render-ready timeout for a project folder's capture. */
timeoutMs?: number;
/** The project's media ledger (`.media/manifest.jsonl`): its cuts from the original show whole-frame reuse. */
ledger?: string;
/** The project folder, to find the ledger's generated clips (default: yours, or the folder holding `.media`). */
project?: string;
/** Drawn elements to score by themselves, segments or not: boxes in the original's pixels, optional windows. */
drawn?: { box: [number, number, number, number]; window?: [number, number] }[];
log?: (message: string) => void;
}
@@ -108,8 +150,22 @@ export interface SegmentVerdict {
pixels: Segment["pixels"];
pass: boolean | null;
ssim?: Scores;
/** A regenerated segment with no person in it (a product shot): judged by motion and look. */
energy?: EnergyComparison;
look?: LookComparison;
/** What `pass` covers, in words: which pixels SSIM scored, and what was judged otherwise. */
scored?: string;
/** A rebuilt or reused segment's regenerated region; `ssim` then scores the cells outside it. */
regenerated?: { roi: [number, number, number, number] };
/**
* The re-shot person: three gates decide (framing, text carried in the generated clip, lip sync); motion energy
* and look are reported, not judged.
*/
person?: PersonVerdict & { energy?: EnergyComparison; look?: LookComparison };
/** Each `Drawn:` element over the regenerated person, scored by itself (drawn.ts), in the order written. */
drawn?: (DrawnVerdict & { id: string; diagnosis?: Diagnosis })[];
/** A segment scored by SSIM: the 4x4 cell with the lowest mean over it, and the moment it is lowest. */
worst_cell?: { roi: Roi; ssim: number; t: number; diagnosis?: Diagnosis };
note?: string;
}
@@ -123,11 +179,24 @@ export interface CutReport {
}
export interface MissingContent {
/** Share of compared frames with a cell the original fills and yours leaves flat. */
/** Share of compared frames with a cell whose shapes the original has and yours leaves flat. */
share: number;
/** The longest such stretches, in seconds. */
spans: [number, number][];
pass: boolean;
/** Frames where yours leaves flat only the original's fine texture in a cell: reported, not failed. */
texture: { share: number; spans: [number, number][] };
}
export interface WholeFrameReuse {
/** Stretches where yours plays a whole-frame cut of the original, in seconds. */
spans: [number, number][];
/** The times (in the original) of the ledger's whole-frame stills that show in yours. */
stills: number[];
share: number;
/** Those inside segments that quote copy: the original's drawn things played along, a copy. */
drawn: [number, number][];
pass: boolean;
}
export interface DiffResult {
@@ -139,11 +208,18 @@ export interface DiffResult {
thresholds: Thresholds;
ssim: Scores;
missing_content: MissingContent;
whole_frame_reuse?: WholeFrameReuse;
worst: Moment[];
cuts?: CutReport;
duration?: { reference: number; yours: number; pass: boolean };
sound?: ReturnType<typeof soundOffset>;
segments?: SegmentVerdict[];
/** What decides the verdict: every drawn element and each rebuilt segment's area outside the person. */
components?: { passed: number; total: number; mean: number | null };
/** The re-shot person over the film: the gates that decide, and what is only reported. */
person?: PersonSummary;
/** Each `drawn` option's element (D1, D2, …), scored by itself: reported, not judged. */
drawn?: (DrawnVerdict & { id: string })[];
sheet?: string;
sbs?: string;
log: string;
@@ -152,6 +228,26 @@ export interface DiffResult {
warnings: string[];
}
export interface PersonSummary {
/** Each gate over every segment: false when one fails, null when none could be measured. */
gates: { framing: boolean | null; text: boolean | null; sync: boolean | null };
/** Segments failing each gate. */
failed: { framing: string[]; text: string[]; sync: string[] };
/** Median of the segments' motion energy ratios (yours over the original's): reported, not judged. */
energy: number | null;
/**
* Generated clips that start at one of the original's cuts, of those placed on the film, and the original's cuts
* inside the person's segments where no clip starts (a jump cut played as one take): reported.
*/
shots_at_cuts: {
at_cuts: number;
total: number;
off: { clip: string; start: number }[];
cuts_without_shot: number[];
};
clips: GeneratedClip[];
}
interface Series {
/** Whole-frame SSIM per compared frame. */
frame: number[];
@@ -247,42 +343,67 @@ function ssimSeries(ref: Clip, yours: Clip, logPath: string): Series {
/**
* Stretches where a cell the original fills is flat in yours. SSIM scores two near-blank patches alike, so a scene
* left empty on a plain background passes SSIM and its cell gate; this compares each cell's luma spread instead.
* `judged(k)` lists the cells of the original's frame k this applies to: a regenerated person's gesture is theirs.
*/
function missingContent(ref: Clip, yours: Clip, ks: number[], fps: number): MissingContent {
function missingContent(
ref: Clip,
yours: Clip,
ks: number[],
fps: number,
judged: (k: number) => number[] = () => ALL_CELLS,
): MissingContent {
const stride = Math.max(1, Math.ceil(ks.length / MISSING_SAMPLES));
const w = MISSING_W;
const h = Math.max(GRID * 2, Math.round((w * ref.height) / ref.width / GRID) * GRID);
const [cw, ch] = [
w / STRUCTURE_SCALE,
Math.max(GRID, Math.round(h / STRUCTURE_SCALE / GRID) * GRID),
];
const thin =
stride > 1 ? [`select=not(mod(n\\,${stride}))`, `setpts=N/(${fps / stride})/TB`] : [];
const spreads = (clip: Clip) =>
decode(clip.input, [...clip.filters, ...thin, `scale=${w}:${h}`], w, h, true).frames.map((px) =>
cellSpreads(px, w, h),
const spreads = (clip: Clip, sw: number, sh: number) =>
decode(clip.input, [...clip.filters, ...thin, `scale=${sw}:${sh}`], sw, sh, true).frames.map(
(px) => cellSpreads(px, sw, sh),
);
const [a, b] = [spreads(ref), spreads(yours)];
const [a, b, coarse] = [spreads(ref, w, h), spreads(yours, w, h), spreads(ref, cw, ch)];
// The thinning select can pass one frame past the pick; times come from ks, so stop with it.
const n = Math.min(a.length, b.length, Math.ceil(ks.length / stride));
const empty = Array.from({ length: n }, (_, j) =>
a[j]!.some(
(s, c) => s >= MISSING_DETAIL && b[j]![c]! < Math.min(MISSING_FLAT, MISSING_KEEP * s),
),
);
const spans: [number, number][] = [];
empty.forEach((e, j) => {
if (!e) return;
const t = round(ks[j * stride]! / fps, 3);
const last = spans.at(-1);
if (last && empty[j - 1])
last[1] = round((ks[Math.min(ks.length - 1, (j + 1) * stride - 1)]! + 1) / fps, 3);
else spans.push([t, round((ks[Math.min(ks.length - 1, (j + 1) * stride - 1)]! + 1) / fps, 3)]);
const n = Math.min(a.length, b.length, coarse.length, Math.ceil(ks.length / stride));
const kinds = Array.from({ length: n }, (_, j) => {
const empty = judged(ks[j * stride]!).filter((c) => {
const s = a[j]![c]!;
return s >= MISSING_DETAIL && b[j]![c]! < Math.min(MISSING_FLAT, MISSING_KEEP * s);
});
return {
structure: empty.some((c) => coarse[j]![c]! >= MISSING_DETAIL),
texture: empty.length > 0,
};
});
const share = n ? empty.filter(Boolean).length / n : 0;
return {
share: round(share, 4),
spans: spans
const spansOf = (on: boolean[]) => {
const spans: [number, number][] = [];
on.forEach((e, j) => {
if (!e) return;
const t = round(ks[j * stride]! / fps, 3);
const end = round((ks[Math.min(ks.length - 1, (j + 1) * stride - 1)]! + 1) / fps, 3);
const last = spans.at(-1);
if (last && on[j - 1]) last[1] = end;
else spans.push([t, end]);
});
return spans
.sort((x, y) => y[1] - y[0] - (x[1] - x[0]))
.slice(0, 8)
.sort((x, y) => x[0] - y[0]),
.sort((x, y) => x[0] - y[0]);
};
const structure = kinds.map((k) => k.structure);
const texture = kinds.map((k) => k.texture && !k.structure);
const share = n ? structure.filter(Boolean).length / n : 0;
return {
share: round(share, 4),
spans: spansOf(structure),
pass: share <= MISSING_MAX_SHARE,
texture: {
share: round(n ? texture.filter(Boolean).length / n : 0, 4),
spans: spansOf(texture),
},
};
}
@@ -342,22 +463,24 @@ function outline(px: Uint8Array, w: number, [x, y, bw, bh]: number[], t = 3) {
}
function cutReport(ref: Clip, yours: Clip, k0: number, fps: number): CutReport {
const a = findCuts(signatures(ref.input, ref.filters).frames).map((c) => c.frame);
const b = findCuts(signatures(yours.input, yours.filters).frames).map((c) => c.frame);
const [da, db] = [ref, yours].map((c) => frameChanges(signatures(c.input, c.filters).frames));
const [a, b] = [cutsIn(da!), cutsIn(db!)];
const used = new Set<number>();
const missing: number[] = [];
for (const f of a) {
const hit = b.find((g) => !used.has(g) && Math.abs(g - f) <= CUT_SLACK);
if (hit === undefined) missing.push(f);
else used.add(hit);
for (const cut of a) {
const hit = b.find((g) => !used.has(g.frame) && Math.abs(g.frame - cut.frame) <= CUT_SLACK);
// A soft cut is near the line: a clear change on the other side matches it.
if (hit) used.add(hit.frame);
else if (!(cut.soft && confirmsCut(db!, cut))) missing.push(cut.frame);
}
const extra = b.filter((g) => !used.has(g.frame) && !(g.soft && confirmsCut(da!, g)));
const t = (f: number) => round((k0 + f) / fps, 3);
return {
reference: a.map(t),
yours: b.map(t),
reference: a.map((c) => t(c.frame)),
yours: b.map((c) => t(c.frame)),
missing: missing.map(t),
extra: b.filter((g) => !used.has(g)).map(t),
pass: missing.length === 0 && b.length === used.size,
extra: extra.map((c) => t(c.frame)),
pass: missing.length === 0 && extra.length === 0,
};
}
@@ -406,47 +529,469 @@ async function captureProject(
};
}
function regeneratedVerdict(s: Segment, reference: string, yours: string, ref: VideoInfo) {
function regeneratedVerdict(
s: Segment,
reference: string,
yours: string,
ref: VideoInfo,
roi?: Roi,
) {
const seconds = s.end - s.start;
const [a, b] = [videoClip(reference, s.start, seconds, ref), videoClip(yours, s.start, seconds)];
const energy = compareEnergy(energyCurve(a), energyCurve(b));
const looks = compareLook(look(a), look(b));
// Yours may be another size: the region is the same share of its frame.
const mine = roi && scaleRoi(roi, b.width / ref.width, b.height / ref.height);
const energy = compareEnergy(energyCurve(a, roi), energyCurve(b, mine));
const looks = compareLook(look(a, 2, roi), look(b, 2, mine));
const moves = energy.mean_ratio >= ENERGY_RATIO[0] && energy.mean_ratio <= ENERGY_RATIO[1];
return { pass: moves && sameLook(looks), energy, look: looks };
}
const scaleRoi = ([x, y, w, h]: Roi, sx: number, sy: number): Roi => {
const [x0, y0] = [Math.round((x * sx) / 2) * 2, Math.round((y * sy) / 2) * 2];
return [
x0,
y0,
Math.max(2, Math.round((w * sx) / 2) * 2),
Math.max(2, Math.round((h * sy) / 2) * 2),
];
};
/** A segment's regenerated region in the original's pixels, clipped to the frame; shares when every value is ≤ 1. */
export function regionRoi(
box: [number, number, number, number],
width: number,
height: number,
): Roi {
const [sx, sy] = box.every((v) => v <= 1) ? [width, height] : [1, 1];
const roi = scaleRoi(box, sx, sy);
const [x, y] = [Math.min(roi[0], width - 2), Math.min(roi[1], height - 2)];
return [x, y, Math.min(roi[2], width - x), Math.min(roi[3], height - y)];
}
/** The grid cells (row by row) a region covers no more than REGION_CELL_SHARE of. */
export function cellsOutside(roi: Roi, width: number, height: number): number[] {
const [cw, ch] = cellSize(width, height);
return ALL_CELLS.filter((c) => {
const [x, y] = [(c % GRID) * cw, Math.floor(c / GRID) * ch];
const ox = Math.max(0, Math.min(x + cw, roi[0] + roi[2]) - Math.max(x, roi[0]));
const oy = Math.max(0, Math.min(y + ch, roi[1] + roi[3]) - Math.max(y, roi[1]));
return ox * oy <= REGION_CELL_SHARE * cw * ch;
});
}
interface VerdictContext {
reference: string;
yours: string;
ref: VideoInfo;
mode: DiffResult["mode"];
t: Thresholds;
/** A segment's drawn elements, scored by themselves. */
drawn: Map<Segment, (DrawnVerdict & { id: string })[]>;
/** A segment's re-shot person, by the three gates. */
person: Map<Segment, PersonVerdict>;
}
function segmentVerdicts(
segments: Segment[],
series: Series,
worstCell: number[],
ks: number[],
ctx: {
reference: string;
yours: string;
ref: VideoInfo;
mode: DiffResult["mode"];
t: Thresholds;
},
ctx: VerdictContext,
): SegmentVerdict[] {
return segments.map((s) => {
const head = { id: s.id, window: [s.start, s.end] as [number, number], pixels: s.pixels };
if (s.pixels === "regenerated")
return ctx.mode === "video"
? { ...head, ...regeneratedVerdict(s, ctx.reference, ctx.yours, ctx.ref) }
: {
...head,
pass: null,
note: "regenerated pixels are judged by motion and look: diff a render",
};
if (s.pixels === "regenerated" && !s.drawn) {
if (ctx.mode !== "video")
return {
...head,
pass: null,
note: "regenerated pixels are judged on a render: diff a render",
};
const moved = regeneratedVerdict(s, ctx.reference, ctx.yours, ctx.ref);
const person = ctx.person.get(s);
// No face to judge (a product shot): motion and look decide.
if (!person || person.pass === null) return { ...head, ...moved };
return {
...head,
pass: person.pass,
scored: "the person: framing, text carried in the generated clip, lip sync",
person: { ...person, energy: moved.energy, look: moved.look },
};
}
const [a, b] = [frameAt(s.start, ctx.ref.fps), frameAt(s.end, ctx.ref.fps)];
const mine = ks.flatMap((k, i) => (k >= a && k < b && i < series.frame.length ? [i] : []));
if (!mine.length)
return { ...head, pass: null, note: "no compared frame falls in this segment" };
if (s.regenerated || s.pixels === "regenerated")
return mixedVerdict(s, head, mine, series, ctx);
const st = scores(
mine.map((i) => series.frame[i]!),
mine.map((i) => worstCell[i]!),
);
return { ...head, pass: passes(st, ctx.t), ssim: st };
return {
...head,
pass: passes(st, ctx.t),
ssim: st,
worst_cell: lowestCell(mine, series, ks, ctx.ref),
};
});
}
/** Of these frames, the cell with the lowest mean SSIM, and the time it is lowest. */
export function lowestCell(
mine: number[],
series: Series,
ks: number[],
ref: { width: number; height: number; fps: number },
): { roi: Roi; ssim: number; t: number } {
const means = series.cells[mine[0]!]!.map(
(_, c) => mine.reduce((sum, i) => sum + series.cells[i]![c]!, 0) / mine.length,
);
const c = means.indexOf(Math.min(...means));
const at = mine.reduce((lo, i) => (series.cells[i]![c]! < series.cells[lo]![c]! ? i : lo));
const [cw, ch] = cellSize(ref.width, ref.height);
return {
roi: [(c % GRID) * cw, Math.floor(c / GRID) * ch, cw, ch],
ssim: round(means[c]!, 4),
t: round(ks[at]! / ref.fps, 3),
};
}
/**
* A segment with a regenerated person (its box, or the whole frame): the person by motion and look, the cells
* outside the box by SSIM, and each drawn element over the person by itself.
*/
function mixedVerdict(
s: Segment,
head: { id: string; window: [number, number]; pixels: Segment["pixels"] },
mine: number[],
series: Series,
ctx: VerdictContext,
): SegmentVerdict {
const whole = s.pixels === "regenerated";
const roi: Roi = whole
? [0, 0, ctx.ref.width, ctx.ref.height]
: regionRoi(s.regenerated!, ctx.ref.width, ctx.ref.height);
const rest = whole ? [] : cellsOutside(roi, ctx.ref.width, ctx.ref.height);
const moved =
ctx.mode === "video"
? regeneratedVerdict(s, ctx.reference, ctx.yours, ctx.ref, whole ? undefined : roi)
: undefined;
const gates = ctx.person.get(s);
const person =
gates || moved
? {
...(gates ?? { gates: { framing: null, text: null, sync: null }, pass: null }),
energy: moved?.energy,
look: moved?.look,
}
: undefined;
const st = rest.length
? scores(
mine.map((i) => rest.reduce((sum, c) => sum + series.cells[i]![c]!, 0) / rest.length),
mine.map((i) => Math.min(...rest.map((c) => series.cells[i]![c]!))),
)
: undefined;
const drawn = ctx.drawn.get(s) ?? [];
const verdicts = [
st ? passes(st, ctx.t) : null,
person?.pass ?? null,
...drawn.map((d) => d.pass),
];
const pass = verdicts.includes(false) ? false : verdicts.every((p) => p === null) ? null : true;
const scored = [
st
? `SSIM: the ${rest.length} cells outside the regenerated box`
: drawn.length
? "drawn elements only (no cell is outside the regenerated box)"
: "no pixels (the regenerated box covers every cell)",
"the person: framing, text carried in the generated clip, lip sync (motion and look reported)",
...(drawn.length ? [`${drawn.length} drawn elements, each scored by itself`] : []),
].join("; ");
const note =
ctx.mode === "project"
? { note: `the regenerated ${whole ? "picture" : "region"} is judged on a render` }
: {};
const parts = {
pass,
scored,
...(st ? { ssim: st } : {}),
...(drawn.length ? { drawn } : {}),
...(person ? { person } : {}),
};
return { ...head, ...parts, ...(whole ? {} : { regenerated: { roi } }), ...note };
}
/**
* The drawn elements over a regenerated person, each scored by itself on its window inside its segment (drawn.ts),
* from one decode of each side.
*/
function scoreDrawn(
segments: Segment[],
refClip: Clip,
yours: Clip,
ks: number[],
ref: VideoInfo,
available: number,
): Map<Segment, (DrawnVerdict & { id: string })[]> {
const jobs = segments.flatMap((s) => drawnOf(s, ref).map((d) => ({ s, ...d })));
const verdicts = scoreElements(jobs, refClip, yours, ks, ref, available);
const out = new Map<Segment, (DrawnVerdict & { id: string })[]>();
jobs.forEach((j, n) => out.set(j.s, [...(out.get(j.s) ?? []), verdicts[n]!]));
return out;
}
/** Drawn elements scored by themselves on their windows, in order, from one decode of each side. */
function scoreElements(
elements: { id: string; box: Roi; window: [number, number] }[],
refClip: Clip,
yours: Clip,
ks: number[],
ref: VideoInfo,
available: number,
): (DrawnVerdict & { id: string })[] {
const jobs = elements.map((d) => ({
...d,
frames: drawnFrames(ks, ref.fps, ...d.window, available),
}));
const live = jobs.filter((j) => j.frames.length);
const [a, b] = [refClip, yours].map((clip) => readDrawn(clip, live, ref.width, ref.height));
return jobs.map((j) => {
const n = live.indexOf(j);
const pairs =
n < 0 ? [] : a![n]!.map((fa, i) => [fa, b![n]![i]] as const).filter(([, fb]) => fb);
const frames = pairs.map(([fa, fb]) => scoreDrawnFrame(fa, fb!, j.box[2], j.box[3]));
return { id: j.id, ...drawnVerdict(j.box, j.window, frames) };
});
}
/** A segment's drawn elements over a regenerated person: id, box in the original's pixels, window. */
function drawnOf(s: Segment, ref: VideoInfo): { id: string; box: Roi; window: [number, number] }[] {
if (!(s.regenerated || s.pixels === "regenerated")) return [];
return (s.drawn ?? []).map((d, n) => ({
id: `${s.id}.D${n + 1}`,
box: regionRoi(d.box, ref.width, ref.height),
window: [Math.max(s.start, d.window?.[0] ?? s.start), Math.min(s.end, d.window?.[1] ?? s.end)],
}));
}
/** The film's headline: the components that decide it, and the person's gates and what is reported. */
export function headline(
segments: SegmentVerdict[],
clips: GeneratedClip[],
t: Thresholds,
cuts: number[] = [],
): { components: NonNullable<DiffResult["components"]>; person: PersonSummary } {
const parts = segments.flatMap((v) => [
...(v.ssim ? [{ pass: passes(v.ssim, t), score: v.ssim.mean }] : []),
...(v.drawn ?? []).flatMap((d) =>
d.pass === null ? [] : [{ pass: d.pass, score: d.score ?? 0 }],
),
]);
const people = segments.flatMap((v) =>
v.person ? [{ id: v.id, window: v.window, ...v.person }] : [],
);
const gate = (key: "framing" | "text" | "sync") => {
const g = people.map((p) => p.gates[key]);
return g.includes(false) ? false : g.some((x) => x === true) ? true : null;
};
const ratios = people.flatMap((p) => (p.energy ? [p.energy.mean_ratio] : []));
const placed = clips.filter((c) => c.at_cut !== undefined);
return {
components: {
passed: parts.filter((p) => p.pass).length,
total: parts.length,
mean: parts.length
? round(parts.reduce((sum, p) => sum + p.score, 0) / parts.length, 4)
: null,
},
person: {
gates: { framing: gate("framing"), text: gate("text"), sync: gate("sync") },
failed: {
framing: people.filter((p) => p.gates.framing === false).map((p) => p.id),
text: people.filter((p) => p.gates.text === false).map((p) => p.id),
sync: people.filter((p) => p.gates.sync === false).map((p) => p.id),
},
energy: ratios.length
? round([...ratios].sort((a, b) => a - b)[ratios.length >> 1]!, 3)
: null,
shots_at_cuts: {
at_cuts: placed.filter((c) => c.at_cut).length,
total: placed.length,
off: placed.flatMap((c) => (c.at_cut ? [] : [{ clip: c.path, start: c.offset! }])),
cuts_without_shot: placed.length
? cuts.filter(
(k) =>
people.some((p) => k > p.window[0] && k < p.window[1]) &&
!placed.some((c) => Math.abs(c.offset! - k) <= AT_CUT_S),
)
: [],
},
clips,
},
};
}
/** A media ledger's cuts from the original that show its whole frame: video windows and the times of stills. */
export function wholeFrameCuts(
ledger: string,
width: number,
height: number,
): { windows: [number, number][]; stills: number[] } {
const isRecord = (v: unknown): v is Record<string, unknown> =>
typeof v === "object" && v !== null;
const numbers = (v: unknown) =>
Array.isArray(v) ? v.filter((x): x is number => typeof x === "number") : [];
type Cut = { window: [number, number] } | { at: number };
const cuts = ledger.split("\n").flatMap((line): Cut[] => {
let entry: unknown;
try {
entry = JSON.parse(line);
} catch {
return [];
}
if (!isRecord(entry) || !isRecord(entry.provenance) || entry.provenance.provider !== "original")
return [];
const p = entry.provenance;
const [window, roi] = [numbers(p.window), numbers(p.roi)];
// A cut with no region is the whole frame.
const area = roi.length === 4 ? roi[2]! * roi[3]! : width * height;
if (area < WHOLE_FRAME_SHARE * width * height) return [];
if (entry.type === "video" && window.length === 2)
return [{ window: [window[0]!, window[1]!] }];
return entry.type === "image" && typeof p.at === "number" ? [{ at: p.at }] : [];
});
return {
windows: cuts.flatMap((c) => ("window" in c ? [c.window] : [])),
stills: [...new Set(cuts.flatMap((c) => ("at" in c ? [c.at] : [])))].sort((a, b) => a - b),
};
}
/** For each still (a time in the original), which compared frames of yours show it. */
function stillsShown(ref: Clip, yours: Clip, stills: number[], fps: number): boolean[][] {
if (!stills.length) return [];
const h = Math.max(GRID, Math.round((STILL_W * ref.height) / ref.width / GRID) * GRID);
const read = (clip: Clip, extra: string[] = [], limit?: number) =>
decode(
clip.input,
[...clip.filters, ...extra, `scale=${STILL_W}:${h}`],
STILL_W,
h,
true,
limit,
).frames;
const mine = read(yours);
return stills.map((at) => {
const still = read(ref, [`select=eq(n\\,${frameAt(at, fps)})`], 1)[0];
if (!still) return mine.map(() => false);
return mine.map((px) => {
const cells = cellDiffs(px, still, STILL_W, h).sort((a, b) => a - b);
return pct(cells, 0.5) <= STILL_DIFF;
});
});
}
/** The mean absolute difference of two grey pictures in each 4x4 cell. */
function cellDiffs(a: Uint8Array, b: Uint8Array, w: number, h: number): number[] {
const [cw, ch] = [Math.floor(w / GRID), Math.floor(h / GRID)];
return ALL_CELLS.map((c) => {
const [x0, y0] = [(c % GRID) * cw, Math.floor(c / GRID) * ch];
let sum = 0;
for (let y = y0; y < y0 + ch; y++)
for (let x = x0; x < x0 + cw; x++) sum += Math.abs(a[y * w + x]! - b[y * w + x]!);
return sum / (cw * ch);
});
}
/** The ledger beside a project or its REVERSE.md (the grade keeps a copy as media-manifest.jsonl). */
export function findLedger(yours: string, segmentsFile?: string): string | undefined {
const dirs = [
...(existsSync(yours) && statSync(yours).isDirectory() ? [yours] : []),
...(segmentsFile ? [dirname(segmentsFile)] : []),
];
return dirs
.flatMap((d) => [join(d, ".media", "manifest.jsonl"), join(d, "media-manifest.jsonl")])
.find((f) => existsSync(f));
}
/** Runs of compared frames that are on, as [start, end) seconds. */
function frameSpans(on: boolean[], ks: number[], fps: number): [number, number][] {
const spans: [number, number][] = [];
on.forEach((v, i) => {
if (!v) return;
const last = spans.at(-1);
if (last && on[i - 1] && ks[i] === ks[i - 1]! + 1) last[1] = round((ks[i]! + 1) / fps, 3);
else spans.push([round(ks[i]! / fps, 3), round((ks[i]! + 1) / fps, 3)]);
});
return spans;
}
/**
* Where yours plays one of the original's whole-frame cuts (inside its window, most cells near the original's own
* pixels, for REUSE_MIN_S or longer) or shows one of its whole-frame stills, and where that frame's segment draws
* copy: the original's drawn things came along with the footage.
*/
function wholeFrameReuse(
cuts: { windows: [number, number][]; stills: number[] },
stills: boolean[][],
series: Series,
ks: number[],
fps: number,
segments?: Segment[],
): WholeFrameReuse {
const inside = (k: number, a: number, b: number) => k >= frameAt(a, fps) && k < frameAt(b, fps);
const played = series.frame.map(
(_, i) =>
cuts.windows.some(([a, b]) => inside(ks[i]!, a, b)) &&
pct(
[...series.cells[i]!].sort((x, y) => x - y),
0.5,
) >= REUSE_SSIM,
);
const long = longRuns(played, ks, Math.ceil(REUSE_MIN_S * fps));
const shown = long.map((v, i) => v || stills.some((on) => on[i]));
const drawn = shown.map(
(v, i) => v && !!segments?.some((s) => s.copy && inside(ks[i]!, s.start, s.end)),
);
return {
spans: frameSpans(shown, ks, fps),
stills: cuts.stills.filter((_, j) => stills[j]?.some(Boolean)),
share: round(shown.filter(Boolean).length / Math.max(1, shown.length), 4),
drawn: frameSpans(drawn, ks, fps),
pass: drawn.filter(Boolean).length <= REUSE_MAX_FRAMES,
};
}
/** `on` with its runs of consecutive frames shorter than `min` turned off. */
export function longRuns(on: boolean[], ks: number[], min: number): boolean[] {
const out = [...on];
for (let i = 0; i < on.length; ) {
if (!on[i]) {
i++;
continue;
}
let j = i + 1;
while (j < on.length && on[j] && ks[j] === ks[j - 1]! + 1) j++;
if (j - i < min) out.fill(false, i, j);
i = j;
}
return out;
}
/** Regenerated boxes that hide what should be judged by pixels. */
function regionWarnings(segments: Segment[], ref: VideoInfo): string[] {
return segments.flatMap((s) => {
const box = s.regenerated && regionRoi(s.regenerated, ref.width, ref.height);
if (s.drawn || !(box || s.pixels === "regenerated")) return [];
if (box && box[2] * box[3] >= WHOLE_FRAME_SHARE * ref.width * ref.height)
return [
`${s.id}: its Regenerated box covers the whole frame, so nothing in it is judged by pixels: box the person, and give each drawn element over them a Drawn: x, y, w, h line`,
];
return s.copy
? [
`${s.id} draws copy over a regenerated person: a drawn element inside the person's box is judged by pixels only with its own Drawn: x, y, w, h line`,
]
: [];
});
}
@@ -517,22 +1062,11 @@ async function worstMoments(
/** Loudness every 10 ms of the first 20 s, as a rising-edge curve: where sounds start. */
function onsetCurve(path: string): number[] {
// Padded from time 0, so a track that starts late reads late.
// On the sound's own timestamps (first_pts=0 pads up to its first decoded sample), not the probe's start_time:
// ffmpeg 5.1 drops an HE-AAC edit list's skip twice, keeps the rest timestamped right, and still probes 0.
const raw = runFfmpeg([
"-i",
path,
"-vn",
"-af",
"aresample=async=1:first_pts=0",
"-t",
"20",
"-ac",
"1",
"-ar",
"8000",
"-f",
"f32le",
"-",
...["-i", path, "-vn", "-af", "aresample=async=1:first_pts=0"],
...["-t", "20", "-ac", "1", "-ar", "8000", "-f", "f32le", "-"],
]);
const samples = new Float32Array(raw.buffer, raw.byteOffset, Math.floor(raw.byteLength / 4));
const rms: number[] = [];
@@ -545,7 +1079,7 @@ function onsetCurve(path: string): number[] {
}
/** How far yours' sound sits from the original's (positive: yours is late), from where sounds start. */
function soundOffset(reference: string, yours: string, ref: VideoInfo, mine: VideoInfo) {
export function soundOffset(reference: string, yours: string, ref: VideoInfo, mine: VideoInfo) {
if (!ref.audio || !mine.audio)
return { original: ref.audio, yours: mine.audio, pass: ref.audio === mine.audio };
const [a, b] = [onsetCurve(reference), onsetCurve(yours)];
@@ -690,11 +1224,23 @@ export async function diffVideos(options: DiffOptions): Promise<DiffResult> {
cell: opts.thresholds?.cell ?? DEFAULT_MIN_CELL,
};
const overall = scores(series.frame, worstCell);
const missing = missingContent(refClip, yours, ks, ref.fps);
const missing = missingContent(refClip, yours, ks, ref.fps, (k) => {
const s = opts.segments?.find(
(seg) => k >= frameAt(seg.start, ref.fps) && k < frameAt(seg.end, ref.fps),
);
if (s?.pixels === "regenerated") return [];
return s?.regenerated
? cellsOutside(regionRoi(s.regenerated, ref.width, ref.height), ref.width, ref.height)
: ALL_CELLS;
});
if (!missing.pass)
warnings.push(
`${Math.round(missing.share * 100)}% of frames leave empty what the original fills (${missing.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}): build what is missing there`,
);
if (missing.texture.share > MISSING_MAX_SHARE)
warnings.push(
`${Math.round(missing.texture.share * 100)}% of frames lack fine texture or small detail the original has (${missing.texture.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}): reported, not failed`,
);
const picks = worstIndexes(worstCell, ks, ref.fps, opts.worst ?? 4);
const { worst, sheet } = await worstMoments(picks, {
refClip,
@@ -708,11 +1254,44 @@ export async function diffVideos(options: DiffOptions): Promise<DiffResult> {
const duration =
yoursInfo && pick.kind === "all"
? {
reference: round(ref.duration, 3),
yours: round(yoursInfo.duration, 3),
pass: Math.abs(ref.duration - yoursInfo.duration) <= 1.5 / ref.fps,
reference: round(ref.pictureDuration, 3),
yours: round(yoursInfo.pictureDuration, 3),
pass: Math.abs(ref.pictureDuration - yoursInfo.pictureDuration) <= 1.5 / ref.fps,
}
: undefined;
const ledgerText =
opts.ledger && existsSync(opts.ledger) ? readFileSync(opts.ledger, "utf8") : undefined;
const people = new Map<Segment, PersonVerdict>();
let clips: ReturnType<typeof placeClips> = [];
if (opts.segments && yoursInfo) {
const project =
opts.project ??
(opts.ledger && basename(dirname(opts.ledger)) === ".media"
? dirname(dirname(opts.ledger))
: undefined);
const refCuts = cuts?.reference ?? [];
const uses = project
? await clipUses(project).catch(() => new Map<string, never[]>())
: new Map<string, never[]>();
clips = ledgerText ? placeClips(generatedClips(ledgerText), project, ref, refCuts, uses) : [];
if (clips.length && clips.every((c) => !c.info))
warnings.push(
"the ledger's generated clips are not found, so text carried in them is not checked: pass --project <folder>",
);
for (const s of opts.segments)
if (s.regenerated || s.pixels === "regenerated")
people.set(
s,
await personVerdict({
ref,
yours: yoursInfo,
window: [s.start, s.end],
cuts: refCuts,
clips,
drawn: drawnOf(s, ref),
}),
);
}
const segments = opts.segments
? segmentVerdicts(opts.segments, series, worstCell, ks, {
reference: opts.reference,
@@ -720,8 +1299,51 @@ export async function diffVideos(options: DiffOptions): Promise<DiffResult> {
ref,
mode,
t,
drawn: scoreDrawn(opts.segments, refClip, yours, ks, ref, series.frame.length),
person: people,
})
: undefined;
if (opts.segments) warnings.push(...regionWarnings(opts.segments, ref));
const elements = opts.drawn?.length
? scoreElements(
opts.drawn.map((d, n) => ({
id: `D${n + 1}`,
box: regionRoi(d.box, ref.width, ref.height),
window: d.window ?? [0, round(ref.pictureDuration, 3)],
})),
refClip,
yours,
ks,
ref,
series.frame.length,
)
: undefined;
const ledger = ledgerText ? wholeFrameCuts(ledgerText, ref.width, ref.height) : undefined;
const reuse =
ledger && (ledger.windows.length || ledger.stills.length)
? wholeFrameReuse(
ledger,
stillsShown({ ...refClip, filters: [] }, yours, ledger.stills, ref.fps),
series,
ks,
ref.fps,
opts.segments,
)
: undefined;
if (reuse && !reuse.pass) {
const seconds = reuse.drawn.reduce((sum, [a, b]) => sum + b - a, 0);
warnings.push(
`${round(seconds, 2)} s play the original's whole frame with its drawn things in it (${reuse.drawn.map(([a, b]) => `${a}–${b} s`).join(", ")}): a copy, not a reverse. Cut only the camera image, each piece on its own box, and rebuild what is drawn`,
);
for (const v of segments ?? [])
if (reuse.drawn.some(([a, b]) => a < v.window[1] && b > v.window[0])) {
v.pass = false;
v.note = "plays the original's whole frame with its drawn things: a copy";
}
} else if (reuse?.spans.length && !opts.segments)
warnings.push(
`${Math.round(reuse.share * 100)}% of frames play the original's whole frame: pass --segments REVERSE.md to tell footage from copied drawings`,
);
const sound =
yoursInfo && pick.kind === "all"
? soundOffset(opts.reference, opts.yours, ref, yoursInfo)
@@ -732,12 +1354,21 @@ export async function diffVideos(options: DiffOptions): Promise<DiffResult> {
: undefined;
const judged = segments ? segments.every((s) => s.pass !== false) : passes(overall, t);
const top = segments
? headline(
segments,
clips.map(({ info: _info, file: _file, still: _still, ...c }) => c),
t,
cuts?.reference,
)
: undefined;
const complete = series.frame.length === ks.length;
return {
pass:
judged &&
complete &&
missing.pass &&
(reuse?.pass ?? true) &&
(cuts?.pass ?? true) &&
(duration?.pass ?? true) &&
(sound?.pass ?? true),
@@ -748,11 +1379,14 @@ export async function diffVideos(options: DiffOptions): Promise<DiffResult> {
thresholds: t,
ssim: overall,
missing_content: missing,
...(reuse ? { whole_frame_reuse: reuse } : {}),
worst,
...(cuts ? { cuts } : {}),
...(duration ? { duration } : {}),
...(sound ? { sound } : {}),
...(top ? top : {}),
...(segments ? { segments } : {}),
...(elements ? { drawn: elements } : {}),
...(sheet ? { sheet } : {}),
...(sbs ? { sbs } : {}),
log: logPath,
packages/cli/src/measure/drawn.test.ts
+74
View File
@@ -0,0 +1,74 @@
import { describe, expect, it } from "vitest";
import { DRAWN_MIN_MEAN, drawnVerdict, scoreDrawnFrame } from "./drawn.js";
const [W, H] = [160, 48];
/** A crop: a background, then bars standing in for letters (x, width) in one colour. */
function crop(
background: (x: number, y: number) => [number, number, number],
bars: [number, number][],
ink: [number, number, number] = [255, 255, 255],
dy = 0,
): Uint8Array {
const px = new Uint8Array(W * H * 3);
for (let y = 0; y < H; y++)
for (let x = 0; x < W; x++) {
const on = y >= 10 + dy && y < 38 + dy && bars.some(([a, w]) => x >= a && x < a + w);
px.set(on ? ink : background(x, y), (y * W + x) * 3);
}
return px;
}
// Two different backgrounds behind the same letters: a warm room and a cool, striped one.
const room = (x: number, y: number): [number, number, number] => [90 + (x % 7), 60 + (y % 5), 40];
const street = (x: number, _y: number): [number, number, number] => [
30,
70 + ((x >> 3) % 2) * 25,
110,
];
const word: [number, number][] = [10, 30, 50, 70, 90, 110, 130].map((x) => [x, 8]);
describe("scoreDrawnFrame", () => {
it("scores the same letters over a different background as a match", () => {
const f = scoreDrawnFrame(crop(room, word), crop(street, word), W, H);
expect(f.score).toBeGreaterThan(DRAWN_MIN_MEAN);
expect(f.offset).toEqual([0, 0]);
});
it("forgives a pixel or two of offset and says how far", () => {
const f = scoreDrawnFrame(crop(room, word), crop(street, word, undefined, 2), W, H);
expect(f.score).toBeGreaterThan(DRAWN_MIN_MEAN);
expect(f.offset).toEqual([0, 2]);
});
it("fails lighter weight, wider tracking, a doubled letter and another fill", () => {
const light = word.map(([x]): [number, number] => [x, 4]);
const wide = word.map(([x, w], i): [number, number] => [x + i * 3, w]);
const doubled = [...word, ...word.map(([x, w]): [number, number] => [x + 12, w])];
for (const yours of [
crop(street, light),
crop(street, wide),
crop(street, doubled),
crop(street, word, [250, 210, 0]),
])
expect(scoreDrawnFrame(crop(room, word), yours, W, H).score).toBeLessThan(DRAWN_MIN_MEAN);
});
it("scores a missing element as nothing matched, and nothing on both sides as no frame", () => {
expect(scoreDrawnFrame(crop(room, word), crop(street, []), W, H).score).toBeLessThan(0.5);
const flat = () => [128, 128, 128] as [number, number, number];
expect(scoreDrawnFrame(crop(flat, []), crop(flat, []), W, H).score).toBeNull();
});
});
describe("drawnVerdict", () => {
it("passes on the mean and the 5th percentile of the frames, and reports ink weight", () => {
const good = scoreDrawnFrame(crop(room, word), crop(street, word), W, H);
const gone = scoreDrawnFrame(crop(room, word), crop(street, []), W, H);
const v = drawnVerdict([0, 0, W, H], [0, 1], Array(10).fill(good));
expect(v).toMatchObject({ pass: true, frames: 10, ink: 1 });
const bad = drawnVerdict([0, 0, W, H], [0, 1], [...Array(9).fill(good), gone]);
expect(bad.pass).toBe(false);
expect(drawnVerdict([0, 0, W, H], [0, 1], [])).toMatchObject({ pass: null, frames: 0 });
});
});
packages/cli/src/measure/drawn.ts
+231
View File
@@ -0,0 +1,231 @@
// A drawn element (text, a badge, a card) over a regenerated person, scored by itself: what is behind it differs on
// purpose, so SSIM over its box measures the background. Each frame finds the element's ink on both sides (pixels
// far from the median colour of their neighbourhood, so a letter stands out from any background behind it), aligns
// the two within a few pixels, and asks how much of either side's ink the other matches in colour. Wrong weight,
// tracking, slant, size, fill, a doubled letter or a late onset all leave ink unmatched; a different background
// leaves none. Calibrated on the reverse bench (10-23): correct titles and captions 0.82–0.94, wrong ones 0.40–0.66.
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { round, runFfmpeg, type Clip, type Roi } from "./media.js";
export const DRAWN_MIN_MEAN = 0.8;
export const DRAWN_MIN_P05 = 0.65;
// A pixel is ink when its colour is this far (RGB distance) from its neighbourhood's median.
const INK_CONTRAST = 60;
// Two pixels match within this RGB distance, one pixel apart at most, after aligning the sides this many pixels.
const MATCH_COLOUR = 45;
const ALIGN = 3;
// Fewer ink pixels than this on both sides: nothing drawn in that frame on either side.
const MIN_INK = 30;
// Elements are read at most this often.
const MAX_FPS = 10;
export interface DrawnFrame {
/** Share of both sides' ink the other side matches (F1 of matched ink), or null when neither side has ink. */
score: number | null;
/** Where yours sits against the original, x and y in pixels (within ±3; further lowers the score). */
offset: [number, number];
inkOriginal: number;
inkYours: number;
}
export interface DrawnVerdict {
box: Roi;
window: [number, number];
frames: number;
pass: boolean | null;
/** Mean and 5th percentile of the frames' scores. */
score?: number;
p05?: number;
/** Median of the frames' offsets. */
offset?: [number, number];
/** Yours' ink over the original's, median of frames where both have ink: under 1 is lighter or smaller. */
ink?: number;
}
/** Each pixel's colour distance from the median colour of the blocks around it. */
export function contrast(px: Uint8Array, w: number, h: number): Float32Array {
const bs = Math.max(4, Math.floor(Math.max(8, Math.floor(h / 2)) / 2));
const [gw, gh] = [Math.ceil(w / bs), Math.ceil(h / bs)];
const med = new Float32Array(gw * gh * 3);
const hist = new Uint32Array(256);
for (let by = 0; by < gh; by++)
for (let bx = 0; bx < gw; bx++) {
const [x0, x1] = [Math.max(0, (bx - 1) * bs), Math.min(w, (bx + 2) * bs)];
const [y0, y1] = [Math.max(0, (by - 1) * bs), Math.min(h, (by + 2) * bs)];
const half = ((x1 - x0) * (y1 - y0)) / 2;
for (let c = 0; c < 3; c++) {
hist.fill(0);
for (let y = y0; y < y1; y++)
for (let x = x0; x < x1; x++) hist[px[(y * w + x) * 3 + c]!]!++;
let [v, seen] = [0, 0];
while (v < 255 && seen + hist[v]! < half) seen += hist[v++]!;
med[(by * gw + bx) * 3 + c] = v;
}
}
const out = new Float32Array(w * h);
for (let y = 0; y < h; y++)
for (let x = 0; x < w; x++) {
const [o, m] = [(y * w + x) * 3, (Math.floor(y / bs) * gw + Math.floor(x / bs)) * 3];
out[y * w + x] = Math.hypot(
px[o]! - med[m]!,
px[o + 1]! - med[m + 1]!,
px[o + 2]! - med[m + 2]!,
);
}
return out;
}
const inkOf = (px: Uint8Array, w: number, h: number) => {
const c = contrast(px, w, h);
return Uint8Array.from(c, (v) => (v > INK_CONTRAST ? 1 : 0));
};
/** One frame of an element: the original's and yours' RGB crops of the same box. */
export function scoreDrawnFrame(a: Uint8Array, b: Uint8Array, w: number, h: number): DrawnFrame {
const [ia, ib] = [inkOf(a, w, h), inkOf(b, w, h)];
const [na, nb] = [ia.reduce((s, v) => s + v, 0), ib.reduce((s, v) => s + v, 0)];
if (na + nb < MIN_INK) return { score: null, offset: [0, 0], inkOriginal: na, inkYours: nb };
// The shift of yours whose ink overlaps the original's most.
let best = { iou: -1, dx: 0, dy: 0 };
for (let dy = -ALIGN; dy <= ALIGN; dy++)
for (let dx = -ALIGN; dx <= ALIGN; dx++) {
let [both, either] = [0, 0];
for (let y = 0; y < h; y++) {
const sy = y - dy;
for (let x = 0; x < w; x++) {
const sx = x - dx;
const vb = sy >= 0 && sy < h && sx >= 0 && sx < w ? ib[sy * w + sx]! : 0;
const va = ia[y * w + x]!;
both += va & vb;
either += va | vb;
}
}
const iou = either ? both / either : 0;
if (iou > best.iou) best = { iou, dx, dy };
}
// Ink on either side counts as matched when the other side has a pixel of its colour within one pixel.
const at = (x: number, y: number) =>
(Math.min(h - 1, Math.max(0, y)) * w + Math.min(w - 1, Math.max(0, x))) * 3;
const near = (p: Uint8Array, o: number, q: Uint8Array, x: number, y: number) => {
for (let ry = -1; ry <= 1; ry++)
for (let rx = -1; rx <= 1; rx++) {
const k = at(x + rx, y + ry);
if (Math.hypot(p[o]! - q[k]!, p[o + 1]! - q[k + 1]!, p[o + 2]! - q[k + 2]!) < MATCH_COLOUR)
return true;
}
return false;
};
let [hitA, hitB, inkA, inkB] = [0, 0, 0, 0];
for (let y = 0; y < h; y++)
for (let x = 0; x < w; x++) {
const i = y * w + x;
const [sx, sy] = [x - best.dx, y - best.dy];
const inB = sy >= 0 && sy < h && sx >= 0 && sx < w && ib[sy * w + sx] === 1;
if (ia[i]) {
inkA++;
if (near(a, i * 3, b, sx, sy)) hitA++;
}
if (inB) {
inkB++;
if (near(b, at(sx, sy), a, x, y)) hitB++;
}
}
const [recall, precision] = [inkA ? hitA / inkA : 0, inkB ? hitB / inkB : 0];
return {
score: recall + precision ? (2 * recall * precision) / (recall + precision) : 0,
offset: [0 - best.dx, 0 - best.dy],
inkOriginal: na,
inkYours: nb,
};
}
const median = (v: number[]) => [...v].sort((x, y) => x - y)[v.length >> 1] ?? 0;
/** The verdict of an element from its frames. */
export function drawnVerdict(
box: Roi,
window: [number, number],
frames: DrawnFrame[],
): DrawnVerdict {
const head = { box, window, frames: frames.length };
const scored = frames.flatMap((f) => (f.score === null ? [] : [f]));
if (!scored.length) return { ...head, pass: null };
const sorted = scored.map((f) => f.score!).sort((x, y) => x - y);
const mean = sorted.reduce((s, v) => s + v, 0) / sorted.length;
const p05 = sorted[Math.min(sorted.length - 1, Math.floor(0.05 * sorted.length))]!;
const both = scored.filter((f) => f.inkOriginal >= MIN_INK && f.inkYours >= MIN_INK);
return {
...head,
pass: mean >= DRAWN_MIN_MEAN && p05 >= DRAWN_MIN_P05,
score: round(mean, 3),
p05: round(p05, 3),
offset: [median(scored.map((f) => f.offset[0])), median(scored.map((f) => f.offset[1]))],
...(both.length ? { ink: round(median(both.map((f) => f.inkYours / f.inkOriginal)), 2) } : {}),
};
}
export interface DrawnJob {
box: Roi;
/** Indexes into the compared frames (the clips' frames after their own filters), in order. */
frames: number[];
}
/**
* Each job's crops from a clip scaled to `width`x`height` (the original's size), one decode for all of them:
* `frames[j][n]` is job j's n-th frame, RGB at the box's size.
*/
export function readDrawn(
clip: Clip,
jobs: DrawnJob[],
width: number,
height: number,
): Uint8Array[][] {
if (!jobs.length) return [];
const work = mkdtempSync(join(tmpdir(), "hf-diff-drawn-"));
try {
const tags = jobs.map((_, j) => `[d${j}]`).join("");
const graph = [
`[0:v]${[...clip.filters, "setsar=1", `scale=${width}:${height}:flags=bicubic`].join(",")},split=${jobs.length}${tags}`,
...jobs.map(({ box: [x, y, w, h], frames }, j) => {
const pick = frames.map((k) => `eq(n\\,${k})`).join("+");
return `[d${j}]select=${pick},crop=${w}:${h}:${x}:${y},format=rgb24[e${j}]`;
}),
].join(";");
const outs = jobs.flatMap((_, j) => [
...["-map", `[e${j}]`, "-fps_mode", "passthrough", "-f", "rawvideo", `e${j}.rgb`],
]);
runFfmpeg([...clip.input, "-an", "-filter_complex", graph, ...outs], undefined, work);
return jobs.map(({ box: [, , w, h] }, j) => {
const raw = readFileSync(join(work, `e${j}.rgb`));
const size = w * h * 3;
const frames: Uint8Array[] = [];
for (let i = 0; i + size <= raw.length; i += size) frames.push(raw.subarray(i, i + size));
return frames;
});
} finally {
rmSync(work, { recursive: true, force: true });
}
}
/** The compared frames (indexes into `ks`) inside [a, b) seconds, thinned to MAX_FPS. */
export function drawnFrames(
ks: number[],
fps: number,
a: number,
b: number,
available: number,
): number[] {
const stride = Math.max(1, Math.round(fps / MAX_FPS));
const inside = ks.flatMap((k, i) =>
i < available && k / fps >= a - 1e-6 && k / fps < b - 1e-6 ? [i] : [],
);
// Every stride-th frame of a run of consecutive frames; moments already that far apart (--at) all count.
let last = -Infinity;
return inside.filter((i) => {
if (ks[i]! - last < stride) return false;
last = ks[i]!;
return true;
});
}
packages/cli/src/measure/face.test.ts
+149
View File
@@ -0,0 +1,149 @@
import { execFileSync } from "node:child_process";
import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir, userInfo } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { againstStart, faceBoxes, faceFit, minWindow, readSamples, sampleFrames } from "./face.js";
import { probeVideo, runFfmpeg } from "./media.js";
import { OPENCV_DIR } from "./opencv.js";
// As in opencv.test.ts: borrow this machine's verified opencv.js cache, skip without one.
const cached = join(userInfo().homedir, ".cache", "hyperframes", "opencv", "4.10.0");
if (existsSync(join(cached, "opencv.js")) && !existsSync(join(OPENCV_DIR, "opencv.js")))
cpSync(cached, OPENCV_DIR, { recursive: true });
const media = !!findFFmpeg() && !!findFFprobe();
const opencv = media && existsSync(join(OPENCV_DIR, "opencv.js"));
const dir = mkdtempSync(join(tmpdir(), "hf-face-test-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
describe("sampleFrames", () => {
it("spreads the samples evenly, the last frame included, each once", () => {
expect(sampleFrames(30, 0, 1, 4)).toEqual([0, 10, 19, 29]);
expect(sampleFrames(30, 2, 2.1, 12)).toEqual([60, 61, 62]);
expect(sampleFrames(24, 1, 1.01, 8)).toEqual([24]);
});
});
describe("minWindow", () => {
it("is a step of the cascades' 20 px x 1.1 ladder, no larger than the share", () => {
expect(minWindow(1280, 0)).toBe(20);
expect(minWindow(400, 0.04)).toBe(20);
// 4% of 1280 is 51.2 px: 20 x 1.1^9 = 47.2 is the last step under it.
expect(minWindow(1280, 0.04)).toBe(47);
expect(minWindow(1080, 0.04)).toBe(43);
});
});
describe("againstStart", () => {
it("reads a generated shot from its own start and a full render on film time", () => {
// A 5.2 s HeyGen shot for the original's 9.5-12.27 s: its face is at 0-2.77 s of the clip, not 9.5.
expect(againstStart(5.17, 12.27, 9.5)).toBe(0);
// A shot longer than the window but shorter than the film is still its own clip (it used to be read from --from).
expect(againstStart(7.29, 12.27, 2.77)).toBe(0);
expect(againstStart(12.27, 12.27, 9.5)).toBe(9.5);
expect(againstStart(12.3, 12.27, 9.5)).toBe(9.5);
});
});
describe("faceFit", () => {
const at = (t: number, ...boxes: number[][]) => ({ t, boxes });
const canvas = { width: 720, height: 1280 };
it("gives the scale about the centre and the x, y that land the clip's face on the original's", () => {
// The clip's face, 0.1 high at (0.5, 0.4); the original's twice the size, 10 px right and 20 px up of where
// scaling about the centre puts it: centre y 640 + 2 (512 - 640) - 20 = 364.
const clip = [0, 0.5, 1, 1.5].map((t) => at(t, [0.45, 0.35, 0.1, 0.1]));
const [w, h] = [0.2, 0.2];
const [cx, cy] = [(360 + 10) / 720, 364 / 1280];
const original = [0, 0.5, 1, 1.5].map((t) =>
at(t, [cx - w / 2, cy - h / 2, w, h], [0.05, 0.05, 0.03, 0.03]),
);
const fit = faceFit(original, clip, canvas, canvas);
expect(fit.moments).toBe(4);
expect(fit.transform).toEqual({ scale: 2, x: 10, y: -20 });
expect(fit.residual).toBe(0);
});
it("fits a clip of another size through object-fit: cover, and shrugs off a stray moment", () => {
// 768x1344 under a 720x1280 canvas: cover scales it by 1280/1344 and crops its sides; its face then sits
// where the original's is, at the same height, so nothing moves.
const s0 = 1280 / 1344;
const ox = (720 - 768 * s0) / 2;
const [fx, fy, fh] = [360, 500, 200];
const clipBox = [
(fx - ox - fh / 2) / (768 * s0),
(fy - fh / 2) / 1280,
fh / (768 * s0),
fh / 1280,
];
const origBox = [(fx - fh / 2) / 720, (fy - fh / 2) / 1280, fh / 720, fh / 1280];
const clip = [0, 0.25, 0.5, 0.75, 1].map((t) => at(t, clipBox));
const original = [0, 0.25, 0.5, 0.75, 1].map((t, i) =>
at(t + 0.02, i === 2 ? [0.1, 0.1, 0.4, 0.4] : origBox),
);
const fit = faceFit(original, clip, canvas, { width: 768, height: 1344 });
expect(fit.transform).toEqual({ scale: 1, x: 0, y: 0 });
});
it("gives no transform without faces on both sides at enough moments", () => {
const fit = faceFit(
[at(0, [0.4, 0.3, 0.2, 0.1]), at(1)],
[at(0, [0.4, 0.3, 0.2, 0.1]), at(1, [0.4, 0.3, 0.2, 0.1])],
canvas,
canvas,
);
expect(fit).toEqual({ moments: 1, transform: null });
// Moments farther apart than a tenth of a second do not pair.
expect(
faceFit(
[at(0, [0.4, 0.3, 0.2, 0.1]), at(1, [0.4, 0.3, 0.2, 0.1])],
[at(0.5, [0.4, 0.3, 0.2, 0.1]), at(1.5, [0.4, 0.3, 0.2, 0.1])],
canvas,
canvas,
).transform,
).toBeNull();
});
});
describe.skipIf(!media)("readSamples", () => {
// Every frame different, keyframes 2.5 s apart and B-frames: what a seek has to get right.
const video = join(dir, "frames.mp4");
if (media)
execFileSync(findFFmpeg()!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=160x96:r=30:d=8"],
...["-c:v", "libx264", "-g", "75", "-bf", "2", "-pix_fmt", "yuv420p", video],
]);
it("seeks to the same frames that decoding from the start picks", () => {
const info = probeVideo(video);
// Nearby samples (one pass) and samples seconds apart (a seek each), off and on keyframes.
const ks = [1, 2, 40, 74, 75, 76, 140, 141, 210, 239];
const got = readSamples(info, ks, 80, 48);
const raw = runFfmpeg([
...["-i", video, "-an", "-vf"],
`select=${ks.map((k) => `eq(n\\,${k})`).join("+")},scale=80:48,format=gray`,
...["-fps_mode", "passthrough", "-f", "rawvideo", "-"],
]);
expect(got).toHaveLength(ks.length);
got.forEach((f, i) =>
expect(Buffer.compare(f, raw.subarray(i * 80 * 48, (i + 1) * 80 * 48))).toBe(0),
);
});
});
describe.skipIf(!opencv)("faceBoxes", () => {
const video = join(dir, "blank.mp4");
if (opencv)
execFileSync(findFFmpeg()!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=0x406080:s=320x568:r=30:d=4"],
...["-c:v", "libx264", "-pix_fmt", "yuv420p", video],
]);
it("labels every sampled frame and finds no face in a blank one", async () => {
const r = await faceBoxes(probeVideo(video), 1, 3, 5);
expect(r.map((s) => s.frame)).toEqual([30, 45, 60, 74, 89]);
expect(r.map((s) => s.t)).toEqual([1, 1.5, 2, 2.467, 2.967]);
expect(r.every((s) => s.boxes.length === 0)).toBe(true);
});
});
packages/cli/src/measure/face.ts
+174 -20
View File
@@ -1,13 +1,24 @@
// How big the face is and where, ported from hf-media's face_size: OpenCV's frontal and profile Haar cascades on
// evenly spaced frames, graded by face height over frame height. A regenerated presenter keeps the likeness from
// FACE_HOLDS up; below FACE_FLOOR a video model cannot carry one at all (the harness's face-size eval, 2026-10-08).
import { frameAt, readClip, round, videoClip, type VideoInfo } from "./media.js";
import { frameAt, round, runFfmpeg, type VideoInfo } from "./media.js";
import { greyMat, loadOpenCv } from "./opencv.js";
export const FACE_HOLDS = 0.25;
export const FACE_SLIDES = 0.15;
export const FACE_FLOOR = 0.06;
/**
* The smallest face looked for by default, as a share of the frame's height: well under FACE_FLOOR (a smaller face
* fails either way), and it skips the cascades' smallest, costliest windows. Those start at 20 px and grow by 10% a
* step, so a minimum on one of the steps finds the faces the full search finds from that size up, with the same
* boxes; scaling the frame down instead moves every box by up to a step.
*/
export const FACE_MIN = 0.04;
const WORK_W = 960;
const STEP = 1.1;
const WINDOW_PX = 20;
// Samples closer than this share one decode from the first of them; farther ones get a seek of their own.
const RUN_S = 2;
type Verdict = "pass" | "warn" | "fail" | "unmeasured";
const ORDER: Verdict[] = ["fail", "unmeasured", "warn", "pass"];
@@ -18,24 +29,87 @@ export function faceVerdict(height: number | null, cast = false): Verdict {
return height < (cast ? FACE_SLIDES : FACE_FLOOR) ? "fail" : "warn";
}
/** Each sampled frame's faces as [x, y, w, h] shares of the frame. */
export async function faceBoxes(info: VideoInfo, start: number, end: number, samples: number) {
const cv = await loadOpenCv((m) => console.error(m));
const [k0, k1] = [
frameAt(start, info.fps),
Math.max(frameAt(start, info.fps), Math.ceil(end * info.fps - 1e-6) - 1),
];
/** `samples` frame indices spread evenly over [start, end), the last frame included. */
export function sampleFrames(fps: number, start: number, end: number, samples: number): number[] {
const k0 = frameAt(start, fps);
const k1 = Math.max(k0, Math.ceil(end * fps - 1e-6) - 1);
const n = Math.min(samples, k1 - k0 + 1);
const ks = [
return [
...new Set(
Array.from({ length: n }, (_, i) => k0 + Math.round((i * (k1 - k0)) / Math.max(1, n - 1))),
),
];
const clip = {
...videoClip(info.path, 0, undefined, info),
filters: [`select=${ks.map((k) => `eq(n\\,${k})`).join("+")}`, `setpts=N/(${info.fpsText})/TB`],
};
const { frames, w, h } = readClip(clip, { width: Math.min(WORK_W, info.width), gray: true });
}
/** The largest window on the cascades' 20 px x 1.1^k ladder that is no more than `share` of a frame `h` high. */
export function minWindow(h: number, share: number): number {
const steps = Math.floor(Math.log(Math.max(1, (share * h) / WINDOW_PX)) / Math.log(STEP) + 1e-9);
return Math.round(WINDOW_PX * STEP ** steps);
}
/**
* Frames `ks` (indices from the first frame) as grey bytes at `w` x `h`, which must keep the video's aspect. Each run
* of nearby frames is one ffmpeg pass that seeks to its first frame and decodes only to its last; a variable-rate
* video, whose indices a seek cannot find, is read from its start.
*/
export function readSamples(info: VideoInfo, ks: number[], w: number, h: number): Uint8Array[] {
const runs: number[][] = [];
for (const k of ks) {
const run = runs.at(-1);
if (run && (info.vfr || (k - run.at(-1)!) / info.fps <= RUN_S)) run.push(k);
else runs.push([k]);
}
const size = w * h;
return runs.flatMap((run) => {
// Half a frame early: an input seek drops the frames before it, so the first one left is frame run[0].
const k0 = info.vfr ? 0 : run[0]!;
const seek = k0 > 0 ? ["-ss", ((k0 - 0.5) / info.fps).toFixed(6)] : [];
const filters = [
`select=${run.map((k) => `eq(n\\,${k - k0})`).join("+")}`,
...(w !== info.width || h !== info.height ? [`scale=${w}:${h}`] : []),
"format=gray",
];
const raw = runFfmpeg([
...seek,
"-i",
info.path,
"-an",
"-vf",
filters.join(","),
"-fps_mode",
"passthrough",
"-frames:v",
String(run.length),
"-f",
"rawvideo",
"-",
]);
const frames: Uint8Array[] = [];
for (let i = 0; i + size <= raw.length; i += size) frames.push(raw.subarray(i, i + size));
return frames;
});
}
/**
* Each sampled frame's faces as [x, y, w, h] shares of the frame. `minFace` is the smallest face looked for as a
* share of the frame's height (default FACE_MIN; 0 looks down to the cascades' 20 px, for crowds).
*/
export async function faceBoxes(
info: VideoInfo,
start: number,
end: number,
samples: number,
opts: { minFace?: number } = {},
) {
const cv = await loadOpenCv((m) => console.error(m));
const ks = sampleFrames(info.fps, start, end, samples);
const w = Math.min(WORK_W, info.width);
const h =
w === info.width
? info.height
: Math.max(2, Math.round((info.height * w) / info.width / 2) * 2);
const frames = readSamples(info, ks, w, h);
const min = minWindow(h, opts.minFace ?? FACE_MIN);
const cascades = ["haarcascade_frontalface_alt2.xml", "haarcascade_profileface.xml"].map(
(name) => {
const c = new cv.CascadeClassifier();
@@ -50,7 +124,7 @@ export async function faceBoxes(info: VideoInfo, start: number, end: number, sam
try {
for (const c of cascades) {
const found = new cv.RectVector();
c.detectMultiScale(gray, found, 1.1, 5, 0, new cv.Size(20, 20));
c.detectMultiScale(gray, found, STEP, 5, 0, new cv.Size(min, min));
for (let j = 0; j < found.size(); j++) {
const r = found.get(j);
boxes.push([r.x / w, r.y / h, r.width / w, r.height / h]);
@@ -67,16 +141,20 @@ export async function faceBoxes(info: VideoInfo, start: number, end: number, sam
}
}
/** The subject's face among a frame's: the largest, or the smallest when the person is cast in a crowd. */
export function subjectFace(boxes: number[][], cast = false): number[] | null {
return boxes.reduce<number[] | null>(
(a, b) => (!a || (cast ? b[3]! < a[3]! : b[3]! > a[3]!) ? b : a),
null,
);
}
/** The subject's face (largest, or the smallest when the person is cast in a crowd), its median and last height. */
export function faceSummary(
samples: { frame: number; t: number; boxes: number[][] }[],
cast = false,
) {
const pick = (boxes: number[][]) =>
boxes.reduce<number[] | null>(
(a, b) => (!a || (cast ? b[3]! < a[3]! : b[3]! > a[3]!) ? b : a),
null,
);
const pick = (boxes: number[][]) => subjectFace(boxes, cast);
const seen = samples.flatMap((s) => {
const b = pick(s.boxes);
return b ? [{ t: s.t, box: b.map((v) => round(v, 4)) }] : [];
@@ -101,3 +179,79 @@ export function faceSummary(
: {}),
};
}
const median = (v: number[]) => [...v].sort((x, y) => x - y)[v.length >> 1]!;
/** Fewest moments with a face on both sides that a fit is given from. */
export const FIT_MIN_FACES = 2;
const PAIR_S = 0.1;
/**
* Where --against starts reading the clip: a render as long as the original runs on film time, so at --from; a
* shot of its own (a generated clip) starts at its own 0.
*/
export function againstStart(clipDuration: number, originalDuration: number, from: number): number {
return Math.abs(clipDuration - originalDuration) <= 0.5 ? from : 0;
}
/**
* The GSAP transform that puts a clip's face on the original's: `scale` about the clip element's centre, then `x`, `y`
* in the original's (the canvas's) pixels, for a clip element that fills the canvas with object-fit: cover. Moments
* pair by time (`t` from each side's window start, within PAIR_S); each side's subject face is taken per moment and
* their centres and heights medianed, so one stray detection does not move the fit. `residual` is the median distance
* left between the faces after it, in the original's face heights.
*/
export function faceFit(
original: { t: number; boxes: number[][] }[],
clip: { t: number; boxes: number[][] }[],
canvas: { width: number; height: number },
clipSize: { width: number; height: number },
cast = false,
) {
const pairs = original.flatMap((o) => {
const near = clip.reduce<(typeof clip)[number] | null>(
(p, q) => (!p || Math.abs(q.t - o.t) < Math.abs(p.t - o.t) ? q : p),
null,
);
if (!near || Math.abs(near.t - o.t) > PAIR_S) return [];
const [a, b] = [subjectFace(o.boxes, cast), subjectFace(near.boxes, cast)];
return a && b ? [{ a, b }] : [];
});
if (pairs.length < FIT_MIN_FACES) return { moments: pairs.length, transform: null };
const { width: W, height: H } = canvas;
// object-fit: cover, centred: the clip's pixel (u, v) lands on the canvas at (ox + u s0, oy + v s0).
const s0 = Math.max(W / clipSize.width, H / clipSize.height);
const [cw, ch] = [clipSize.width * s0, clipSize.height * s0];
const [ox, oy] = [(W - cw) / 2, (H - ch) / 2];
const face = (b: number[], w: number, h: number, x0: number, y0: number) => ({
x: x0 + (b[0]! + b[2]! / 2) * w,
y: y0 + (b[1]! + b[3]! / 2) * h,
h: b[3]! * h,
});
const os = pairs.map((p) => face(p.a, W, H, 0, 0));
const cs = pairs.map((p) => face(p.b, cw, ch, ox, oy));
const mid = (v: { x: number; y: number; h: number }[]) => ({
x: median(v.map((f) => f.x)),
y: median(v.map((f) => f.y)),
h: median(v.map((f) => f.h)),
});
const [o, c] = [mid(os), mid(cs)];
const scale = o.h / c.h;
// About the element's centre (GSAP's default transform origin): p -> centre + scale (p - centre) + (x, y).
const x = o.x - (W / 2 + scale * (c.x - W / 2));
const y = o.y - (H / 2 + scale * (c.y - H / 2));
const residual = median(
pairs.map((_, i) => {
const [p, q] = [os[i]!, cs[i]!];
const [mx, my] = [W / 2 + scale * (q.x - W / 2) + x, H / 2 + scale * (q.y - H / 2) + y];
return Math.hypot(mx - p.x, my - p.y) / p.h;
}),
);
return {
moments: pairs.length,
transform: { scale: round(scale, 3), x: round(x, 1), y: round(y, 1) },
original_face: { centre: [round(o.x, 1), round(o.y, 1)], height: round(o.h, 1) },
clip_face: { centre: [round(c.x, 1), round(c.y, 1)], height: round(c.h, 1) },
residual: round(residual, 3),
};
}
packages/cli/src/measure/media.ts
+11 -2
View File
@@ -48,8 +48,12 @@ export interface VideoInfo {
/** The frame rate as ffmpeg writes it, e.g. 30000/1001. */
fpsText: string;
duration: number;
/** The picture's length: the container's can run on with a longer sound track. */
pictureDuration: number;
frames: number;
audio: boolean;
/** Seconds the sound starts after the picture (0 when it starts with it or there is none). */
audioStart: number;
/** Frames over seconds, which a variable-rate video's nominal rate does not tell. */
avgFps: number;
/** The container's frame rate and the stream's average differ: frames are not evenly spaced. */
@@ -67,6 +71,7 @@ interface ProbeStream {
avg_frame_rate?: string;
nb_frames?: string;
duration?: string;
start_time?: string;
color_transfer?: string;
side_data_list?: { rotation?: number }[];
tags?: { rotate?: string };
@@ -108,6 +113,8 @@ export function probeVideo(path: string): VideoInfo {
const avg = rate(video.avg_frame_rate);
const duration = Number(json.format?.duration ?? video.duration ?? 0);
const counted = Number(video.nb_frames ?? 0);
const sound = streams.find((s) => s.codec_type === "audio");
const frames = counted > 0 ? counted : Math.round(duration * fps);
return {
path,
width: (turned ? video.height : video.width) ?? 0,
@@ -115,8 +122,10 @@ export function probeVideo(path: string): VideoInfo {
fps,
fpsText: fpsText.endsWith("/1") ? fpsText.slice(0, -2) : fpsText,
duration,
frames: counted > 0 ? counted : Math.round(duration * fps),
audio: streams.some((s) => s.codec_type === "audio"),
pictureDuration: Number(video.duration ?? 0) || frames / (avg || fps),
frames,
audio: Boolean(sound),
audioStart: Math.max(0, Number(sound?.start_time ?? 0) - Number(video.start_time ?? 0) || 0),
avgFps: avg || fps,
vfr: avg > 0 && Math.abs(avg - fps) / fps > 0.01,
hdr: /smpte2084|arib-std-b67/.test(video.color_transfer ?? ""),
packages/cli/src/measure/person.ts
+576
View File
@@ -0,0 +1,576 @@
// A re-shot person never matches the original by pixels, so pixels do not judge them. Three gates do: the framing
// is clearly wrong (face centre or height off the original's, per shot), the generated clip carries the original's
// words (its motion guide's text came along), or the lips are out of sync. The rest (motion energy, look, whether
// each generated clip starts at one of the original's cuts) is reported, not judged.
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { contrast } from "./drawn.js";
import { faceBoxes } from "./face.js";
import { probeVideo, round, runFfmpeg, type Roi, type VideoInfo } from "./media.js";
import { avSync, type SyncResult } from "./sync.js";
/**
* A shot's framing is clearly wrong when, over the shot, the face centre sits more than FRAMING_SHIFT face heights
* from the original's or its height is outside FRAMING_SCALE of it (medians). Calibrated on the reverse bench (10-23,
* faces at 4 a second, shots of a second or more): goat round 3's 8.0–10.0 s shot is 0.23 off and ×0.82, dan round 2
* at most 0.16 and ×0.93–1.10, cindy round 3 0.02; dan round 3 misses its 35 s close-up at ×0.77 (and is 0.21–0.23
* off at 3.6 s and 30 s, which face boxes cannot tell from goat). Shorter shots have too few faces to judge.
*/
export const FRAMING_SHIFT = 0.3;
export const FRAMING_SCALE: [number, number] = [0.8, 1.25];
const FRAMING_MIN_S = 1;
const FRAMING_MIN_FACES = 3;
const FACE_RATE = 4;
/** Picture against sound (positive: the picture is late), the broadcast tolerance; judged on high confidence only. */
export const SYNC_MS: [number, number] = [-125, 45];
const SYNC_MIN_S = 2;
const SYNC_MIN_SPEECH = 0.3;
/**
* A generated clip carries a drawn element's text when it shows the original's letters inside the element's box:
* of the original's sharp ink (contrast over TEXT_CONTRAST) where it differs from the clean still the clip was made
* from and where yours draws over the clip (each by DRAWN_DIFF: the letters, not the place), the share the clip has
* within two pixels as sharp ink of the same colour (TEXT_COLOUR). The clip shares the person and the place with the
* original, so only carried letters match there. Calibrated on ig03 round 2 (10-23): t12's "Transparency", "a deep
* understanding", "of this market" 0.30, 0.45, 0.56; every clean box 0.24 or less.
*/
export const CARRIED_TEXT = 0.35;
const TEXT_CONTRAST = 140;
const DRAWN_DIFF = 60;
const TEXT_COLOUR = 15;
const TEXT_MIN_INK = 50;
const CARRIED_SAMPLES = 8;
/** A generated clip found by its sound needs this correlation with the original's; it starts at a cut within this. */
const LOCATE_MIN = 0.6;
export const AT_CUT_S = 0.1;
const ENVELOPE_RATE = 100;
export interface GeneratedClip {
/** As the ledger writes it, relative to the project. */
path: string;
/** The still it was made from (a clean frame of the original), relative to the project. */
reference?: string;
/** Film time of the clip's first frame, and how well its sound matched the original's there. */
offset?: number;
match?: number;
window?: [number, number];
/** Where the film shows it: film window, the clip's time at its start, and its playback rate. */
uses?: ClipUse[];
/** Whether it starts within AT_CUT_S of the film's start or one of the original's cuts. */
at_cut?: boolean;
note?: string;
}
export interface ClipUse {
window: [number, number];
from: number;
rate: number;
}
export interface ShotFraming {
window: [number, number];
/** Medians: face centre distance in the original's face heights, and yours' face height over the original's. */
shift: number;
scale: number;
pass: boolean;
}
export interface CarriedText {
drawn: string;
clip: string;
window: [number, number];
/** How well the generated clip's ink inside the box matches the original's text there. */
score: number;
carried: boolean;
}
export interface PersonVerdict {
/** Each gate: false when it fails, null when it could not be measured. */
gates: { framing: boolean | null; text: boolean | null; sync: boolean | null };
pass: boolean | null;
framing?: ShotFraming[];
text?: CarriedText[];
sync?: {
window: [number, number];
lag_ms: number | null;
original_lag_ms: number | null;
confidence: string;
pass: boolean | null;
}[];
}
const isRecord = (v: unknown): v is Record<string, unknown> => typeof v === "object" && v !== null;
/** The ledger's generated videos (HeyGen), with the film window it records when it does. */
export function generatedClips(ledger: string): GeneratedClip[] {
return ledger.split("\n").flatMap((line) => {
let entry: unknown;
try {
entry = JSON.parse(line);
} catch {
return [];
}
if (!isRecord(entry) || entry.type !== "video" || typeof entry.path !== "string") return [];
const p = entry.provenance;
if (!isRecord(p) || p.provider !== "heygen") return [];
const w = Array.isArray(p.window) ? p.window.filter((v) => typeof v === "number") : [];
const refs = Array.isArray(p.references)
? p.references.filter((v) => typeof v === "string")
: [];
return [
{
path: entry.path,
...(refs[0] ? { reference: refs[0] } : {}),
...(w.length === 2 ? { offset: w[0]!, window: [w[0]!, w[1]!] as [number, number] } : {}),
},
];
});
}
/** Loudness every 10 ms of a file's sound. */
function envelope(path: string): number[] {
const raw = runFfmpeg([...["-i", path, "-vn", "-ac", "1", "-ar", "8000", "-f", "f32le", "-"]]);
const s = new Float32Array(
raw.buffer.slice(raw.byteOffset, raw.byteOffset + (raw.byteLength & ~3)),
);
const step = 8000 / ENVELOPE_RATE;
const out: number[] = [];
for (let i = 0; i + step <= s.length; i += step) {
let e = 0;
for (let j = i; j < i + step; j++) e += s[j]! ** 2;
out.push(Math.sqrt(e / step));
}
return out;
}
/**
* Where `b` sits inside `a` (index of a where b[0] lands) and the correlation there, over lags that keep at least
* nine tenths of b inside a.
*/
export function bestLag(a: number[], b: number[]): { lag: number; r: number } {
let best = { lag: 0, r: -2 };
const keep = Math.ceil(0.9 * b.length);
for (let lag = -(b.length - keep); lag <= a.length - keep; lag++) {
const [i0, i1] = [Math.max(0, -lag), Math.min(b.length, a.length - lag)];
const n = i1 - i0;
if (n < Math.max(50, keep)) continue;
let [ma, mb] = [0, 0];
for (let i = i0; i < i1; i++) {
ma += a[i + lag]!;
mb += b[i]!;
}
[ma, mb] = [ma / n, mb / n];
let [sab, sa, sb] = [0, 0, 0];
for (let i = i0; i < i1; i++) {
const [x, y] = [a[i + lag]! - ma, b[i]! - mb];
sab += x * y;
sa += x * x;
sb += y * y;
}
const r = sa && sb ? sab / Math.sqrt(sa * sb) : -2;
if (r > best.r) best = { lag, r };
}
return best;
}
/**
* Each generated clip placed on the film. Its own start (where HeyGen's shot begins) is the ledger's window when it
* records one, else where the clip's sound (the original's voice it was made from) lines up with the original's.
* Where the film shows it comes from the project's timeline (`uses`, by the clip's path), else its whole window.
* Files resolve against the project.
*/
export function placeClips(
clips: GeneratedClip[],
project: string | undefined,
ref: VideoInfo,
cuts: number[],
shown: Map<string, ClipUse[]> = new Map(),
): (GeneratedClip & { file?: string; info?: VideoInfo; still?: string })[] {
let whole: number[] | undefined;
return clips.map((c) => {
const file = project ? join(project, c.path) : undefined;
let info: VideoInfo | undefined;
try {
info = file ? probeVideo(file) : undefined;
} catch {
info = undefined;
}
if (!info || !file) return { ...c, note: "the clip's file is not in the project" };
const uses = shown.get(normalize(c.path));
let placed: GeneratedClip = c;
if (c.offset === undefined && info.audio && ref.audio) {
whole ??= envelope(ref.path);
const { lag, r } = bestLag(whole, envelope(file));
const offset = round(lag / ENVELOPE_RATE + ref.audioStart - info.audioStart, 3);
placed =
r >= LOCATE_MIN
? { ...c, offset, match: round(r, 2) }
: { ...c, match: round(r, 2), note: "its sound does not match the original's" };
}
if (placed.offset === undefined && uses?.length)
placed = { ...placed, offset: round(uses[0]!.window[0] - uses[0]!.from / uses[0]!.rate, 3) };
if (placed.offset === undefined)
return {
...placed,
file,
info,
note: placed.note ?? "no window in the ledger, no sound and no place in the project",
};
const offset = placed.offset;
const end = Math.min(ref.pictureDuration, offset + info.pictureDuration);
return {
...placed,
window: placed.window ?? [round(Math.max(0, offset), 3), round(end, 3)],
uses: uses ?? [
{
window: [round(Math.max(0, offset), 3), round(end, 3)],
from: Math.max(0, -offset),
rate: 1,
},
],
at_cut: [0, ...cuts].some((t) => Math.abs(t - offset) <= AT_CUT_S),
file,
info,
...(c.reference && existsSync(join(project!, c.reference))
? { still: join(project!, c.reference) }
: {}),
};
});
}
/** Where a project's timeline shows each video file, by project-relative path. */
export async function clipUses(project: string): Promise<Map<string, ClipUse[]>> {
const out = new Map<string, ClipUse[]>();
const index = join(project, "index.html");
if (!existsSync(index)) return out;
const [{ ensureDOMParser }, { describeProject }] = await Promise.all([
import("../utils/dom.js"),
import("../timeline/describeProject.js"),
]);
ensureDOMParser();
const timeline = await describeProject(index);
for (const row of timeline.tracks.find((t) => t.kind === "video")?.rows ?? []) {
if (!row.src || row.absEnd <= row.absStart) continue;
const key = normalize(row.src);
const use = {
window: [round(row.absStart, 3), round(row.absEnd, 3)] as [number, number],
from: row.mediaStart ?? 0,
rate: row.playbackRate ?? 1,
};
out.set(key, [...(out.get(key) ?? []), use]);
}
return out;
}
/** A project-relative path written one way. */
export const normalize = (p: string) => p.replace(/^\.\//, "").replace(/\\/g, "/");
/** RGB crops of one box at these times: the picture scaled to cover `w`x`h` (the original's size) first. */
function boxAt(
info: VideoInfo,
times: number[],
box: Roi,
w: number,
h: number,
): (Uint8Array | undefined)[] {
const ks = times.map((t) => Math.max(0, Math.min(info.frames - 1, Math.round(t * info.fps))));
const unique = [...new Set(ks)].sort((x, y) => x - y);
const work = mkdtempSync(join(tmpdir(), "hf-person-"));
try {
const filters = [
`select=${unique.map((k) => `eq(n\\,${k})`).join("+")}`,
`scale=${w}:${h}:force_original_aspect_ratio=increase:flags=bicubic`,
`crop=${w}:${h}`,
`crop=${box[2]}:${box[3]}:${box[0]}:${box[1]}`,
"format=rgb24",
].join(",");
runFfmpeg(
[
...["-i", info.path, "-an", "-vf", filters, "-fps_mode", "passthrough"],
...["-f", "rawvideo", "out.rgb"],
],
undefined,
work,
);
const raw = readFileSync(join(work, "out.rgb"));
const size = box[2] * box[3] * 3;
return ks.map((k) => {
const i = unique.indexOf(k) * size;
return i + size <= raw.length ? raw.subarray(i, i + size) : undefined;
});
} finally {
rmSync(work, { recursive: true, force: true });
}
}
/** Within `r` pixels of (x, y), some pixel passes `test`. */
function nearby(
w: number,
h: number,
x: number,
y: number,
r: number,
test: (k: number) => boolean,
) {
for (let dy = -r; dy <= r; dy++)
for (let dx = -r; dx <= r; dx++) {
const [xx, yy] = [Math.min(w - 1, Math.max(0, x + dx)), Math.min(h - 1, Math.max(0, y + dy))];
if (test(yy * w + xx)) return true;
}
return false;
}
const dist = (p: Uint8Array, i: number, q: Uint8Array, k: number) =>
Math.hypot(p[i * 3]! - q[k * 3]!, p[i * 3 + 1]! - q[k * 3 + 1]!, p[i * 3 + 2]! - q[k * 3 + 2]!);
/** One frame: the share of the original's letters (apart from the still and from what yours draws) the clip shows. */
export function carriedFrame(
original: Uint8Array,
clip: Uint8Array,
still: Uint8Array | undefined,
w: number,
h: number,
yours: Uint8Array | undefined,
): number | null {
const [sharp, inked] = [contrast(original, w, h), contrast(clip, w, h)];
let [n, hit] = [0, 0];
for (let y = 0; y < h; y++)
for (let x = 0; x < w; x++) {
const i = y * w + x;
if (sharp[i]! <= TEXT_CONTRAST) continue;
if (still && !nearby(w, h, x, y, 2, (k) => dist(original, k, still, k) > DRAWN_DIFF))
continue;
if (yours && !nearby(w, h, x, y, 2, (k) => dist(yours, k, clip, k) > DRAWN_DIFF)) continue;
n++;
const same = (k: number) =>
inked[k]! > TEXT_CONTRAST && dist(original, i, clip, k) < TEXT_COLOUR;
if (nearby(w, h, x, y, 2, same)) hit++;
}
return n >= TEXT_MIN_INK ? hit / n : null;
}
/** One box of a still picture, scaled to cover `w`x`h` (the original's size) first. */
function stillBox(path: string, box: Roi, w: number, h: number): Uint8Array | undefined {
try {
const raw = runFfmpeg([
...["-i", path, "-frames:v", "1", "-vf"],
`scale=${w}:${h}:force_original_aspect_ratio=increase:flags=bicubic,crop=${w}:${h},crop=${box[2]}:${box[3]}:${box[0]}:${box[1]},format=rgb24`,
...["-f", "rawvideo", "-"],
]);
return raw.length >= box[2] * box[3] * 3 ? new Uint8Array(raw) : undefined;
} catch {
return undefined;
}
}
/**
* Whether a generated clip carries the original's text inside a drawn element's box over its window: the clip
* alone, against the original there. A clean clip shows the person and the place behind the letters; a clip made
* from a motion guide with words on it shows those words.
*/
export function carriedText(
ref: VideoInfo,
yours: VideoInfo,
clip: { path: string; info: VideoInfo; uses: ClipUse[]; still?: string },
drawn: { id: string; box: Roi; window: [number, number] },
): CarriedText | undefined {
// The film times both show, each with the clip's own time there.
const spans = clip.uses.flatMap((u) => {
const [a, b] = [Math.max(drawn.window[0], u.window[0]), Math.min(drawn.window[1], u.window[1])];
return b - a >= 1 / ref.fps ? [{ a, b, u }] : [];
});
const total = spans.reduce((s, x) => s + x.b - x.a, 0);
if (!total) return undefined;
// Evenly spread samples over the shown spans.
const times = Array.from({ length: CARRIED_SAMPLES }, (_, i) => {
let at = ((i + 0.5) * total) / CARRIED_SAMPLES;
for (const x of spans) {
if (at <= x.b - x.a)
return { t: x.a + at, c: x.u.from + (at + x.a - x.u.window[0]) * x.u.rate };
at -= x.b - x.a;
}
const x = spans.at(-1)!;
return { t: x.b, c: x.u.from + (x.b - x.u.window[0]) * x.u.rate };
});
const orig = boxAt(
ref,
times.map((x) => x.t),
drawn.box,
ref.width,
ref.height,
);
const generated = boxAt(
clip.info,
times.map((x) => x.c),
drawn.box,
ref.width,
ref.height,
);
const mine = boxAt(
yours,
times.map((x) => x.t),
drawn.box,
ref.width,
ref.height,
);
const still = clip.still ? stillBox(clip.still, drawn.box, ref.width, ref.height) : undefined;
const scores = orig.flatMap((o, i) => {
const g = generated[i];
const s = o && g ? carriedFrame(o, g, still, drawn.box[2], drawn.box[3], mine[i]) : null;
return s === null ? [] : [s];
});
if (!scores.length) return undefined;
const score = scores.reduce((s, v) => s + v, 0) / scores.length;
return {
drawn: drawn.id,
clip: clip.path,
window: [round(spans[0]!.a, 3), round(spans.at(-1)!.b, 3)],
score: round(score, 3),
carried: score >= CARRIED_TEXT,
};
}
const median = (v: number[]) => [...v].sort((x, y) => x - y)[v.length >> 1]!;
const centre = (b: number[]) => [b[0]! + b[2]! / 2, b[1]! + b[3]! / 2] as const;
/** The original's shots inside a window: split at its cuts. */
export function shotsIn(window: [number, number], cuts: number[]): [number, number][] {
const edges = [window[0], ...cuts.filter((t) => t > window[0] && t < window[1]), window[1]];
return edges.slice(1).map((e, i) => [edges[i]!, e]);
}
/**
* One shot's framing from both sides' faces, sampled at the same moments (boxes as shares of a frame `aspect` wide
* per unit high): per moment, the original's subject (its largest face) against yours' face nearest it; the shot's
* medians decide, so one stray detection (a photo, a hand) does not.
*/
export function shotFraming(
window: [number, number],
original: { t: number; boxes: number[][] }[],
yours: { t: number; boxes: number[][] }[],
aspect = 1,
): ShotFraming | undefined {
const inside = (s: { t: number }) => s.t >= window[0] && s.t < window[1];
// Across, a share of the frame is `aspect` times longer than down.
const apart = (b: number[], x: number, y: number) =>
Math.hypot((centre(b)[0] - x) * aspect, centre(b)[1] - y);
const [a, b] = [original.filter(inside), yours.filter(inside)];
const moments = a.flatMap((o, i) => {
const y = b[i];
if (!o.boxes.length || !y?.boxes.length) return [];
const subject = o.boxes.reduce((p, q) => (q[3]! > p[3]! ? q : p));
const [ox, oy] = centre(subject);
const near = y.boxes.reduce((p, q) => (apart(q, ox, oy) < apart(p, ox, oy) ? q : p));
return [{ shift: apart(near, ox, oy) / subject[3]!, scale: near[3]! / subject[3]! }];
});
if (moments.length < FRAMING_MIN_FACES) return undefined;
const shift = round(median(moments.map((m) => m.shift)), 3);
const scale = round(median(moments.map((m) => m.scale)), 3);
return {
window: [round(window[0], 3), round(window[1], 3)],
shift,
scale,
pass: shift <= FRAMING_SHIFT && scale >= FRAMING_SCALE[0] && scale <= FRAMING_SCALE[1],
};
}
/** Lips in sync: the lag inside SYNC_MS, judged only when the reading is confident (it is a motion proxy). */
export function syncPass(r: Pick<SyncResult, "best_lag_ms" | "confidence">): boolean | null {
if (r.confidence !== "high" || r.best_lag_ms === null) return null;
return r.best_lag_ms >= SYNC_MS[0] && r.best_lag_ms <= SYNC_MS[1];
}
/** A segment's person: framing per shot, carried text per drawn element, lip sync per long speaking shot. */
export async function personVerdict(opts: {
ref: VideoInfo;
yours: VideoInfo;
window: [number, number];
cuts: number[];
clips: (GeneratedClip & { info?: VideoInfo; still?: string })[];
drawn: { id: string; box: Roi; window: [number, number] }[];
}): Promise<PersonVerdict> {
const { ref, yours, window } = opts;
const shots = shotsIn(window, opts.cuts);
const n = Math.max(2, Math.round((window[1] - window[0]) * FACE_RATE));
// A face inside a drawn element (a photo on a card) is not the person.
const drawnFace = (t: number, b: number[]) =>
opts.drawn.some(({ box: [x, y, w, h], window: [a, z] }) => {
const [cx, cy] = [(b[0]! + b[2]! / 2) * ref.width, (b[1]! + b[3]! / 2) * ref.height];
return t >= a && t < z && cx >= x && cx < x + w && cy >= y && cy < y + h;
});
const people = (samples: Awaited<ReturnType<typeof faceBoxes>>) =>
samples.map((f) => ({ ...f, boxes: f.boxes.filter((b) => !drawnFace(f.t, b)) }));
const [fo, fy] = [
people(await faceBoxes(ref, window[0], window[1], n)),
people(await faceBoxes(yours, window[0], window[1], n)),
];
const framing = shots.flatMap((s) =>
s[1] - s[0] >= FRAMING_MIN_S
? [shotFraming(s, fo, fy, ref.width / ref.height)].filter((f) => f !== undefined)
: [],
);
const text = opts.drawn.flatMap((d) =>
opts.clips.flatMap((c) =>
c.info && c.uses
? [
carriedText(
ref,
yours,
{ path: c.path, info: c.info, uses: c.uses, still: c.still },
d,
),
].filter((t) => t !== undefined)
: [],
),
);
const sync = shots.flatMap((s) => {
if (s[1] - s[0] < SYNC_MIN_S || !yours.audio) return [];
const mouth = (
info: VideoInfo,
samples: { t: number; boxes: number[][] }[],
): Roi | undefined => {
const boxes = samples
.filter((f) => f.t >= s[0] && f.t < s[1] && f.boxes.length)
.map((f) => f.boxes.reduce((x, y) => (y[3]! > x[3]! ? y : x)));
if (boxes.length < FRAMING_MIN_FACES) return undefined;
const [x, y, w, h] = [0, 1, 2, 3].map((i) => median(boxes.map((b) => b[i]!)));
const roi: Roi = [
Math.round(((x! + 0.2 * w!) * info.width) / 2) * 2,
Math.round(((y! + 0.62 * h!) * info.height) / 2) * 2,
Math.max(8, Math.round((0.6 * w! * info.width) / 2) * 2),
Math.max(8, Math.round((0.38 * h! * info.height) / 2) * 2),
];
return roi[0] + roi[2] <= info.width && roi[1] + roi[3] <= info.height ? roi : undefined;
};
const [mo, my] = [mouth(ref, fo), mouth(yours, fy)];
if (!my) return [];
const mine = avSync(yours, my, s[0], s[1]);
if (mine.speech_active_fraction < SYNC_MIN_SPEECH) return [];
const theirs = mo && ref.audio ? avSync(ref, mo, s[0], s[1]) : undefined;
return [
{
window: [round(s[0], 3), round(s[1], 3)] as [number, number],
lag_ms: mine.best_lag_ms,
original_lag_ms: theirs?.best_lag_ms ?? null,
confidence: mine.confidence,
pass: syncPass(mine),
},
];
});
const gate = (v: (boolean | null)[]) =>
v.includes(false) ? false : v.some((p) => p === true) ? true : null;
const gates = {
framing: gate(framing.map((f) => f.pass)),
text: gate(text.map((t) => !t.carried)),
sync: gate(sync.map((s) => s.pass)),
};
const all = Object.values(gates);
return {
gates,
pass: all.includes(false) ? false : all.some((g) => g === true) ? true : null,
...(framing.length ? { framing } : {}),
...(text.length ? { text } : {}),
...(sync.length ? { sync } : {}),
};
}
packages/cli/src/measure/segments.ts
+39 -1
View File
@@ -1,5 +1,9 @@
// The segments of a REVERSE.md: `## S01 · 0.00–2.40 s · title` headings, each with an optional
// `Pixels: rebuilt | reused | regenerated` line saying where that window's picture comes from.
// `Pixels: rebuilt | reused | regenerated` line saying where that window's picture comes from, and an optional
// `Regenerated: x, y, w, h` line for a rebuilt or reused segment with a regenerated region (a re-shot presenter
// under drawn graphics): the region's box in the source's pixels, or as shares of the frame when all are ≤ 1. Each
// `Drawn: x, y, w, h` line is a drawn element over a regenerated person, still judged by pixels, with an optional
// `a–b s` when it shows for part of the segment. A `Copy:` line that quotes a string says the segment draws text.
import { existsSync, readFileSync } from "node:fs";
import { Fix } from "./media.js";
@@ -12,12 +16,27 @@ export interface Segment {
end: number;
title: string;
pixels: PixelSource;
/** A regenerated region inside a rebuilt or reused segment: [x, y, w, h], pixels or shares of the frame. */
regenerated?: [number, number, number, number];
/** Drawn elements over a regenerated person: their boxes, judged by pixels, and when they show. */
drawn?: { box: [number, number, number, number]; window?: [number, number] }[];
/** The segment quotes copy: it has drawn text. */
copy?: true;
}
const HEADING =
/^##\s+(S\d+)\s*[·|-]\s*(\d+(?:\.\d+)?)\s*[–—-]\s*(\d+(?:\.\d+)?)\s*s\b\s*(?:[·|-]\s*(.*))?$/gm;
const SOURCES: readonly PixelSource[] = ["rebuilt", "reused", "regenerated"];
const PIXELS = /^Pixels:\s*(rebuilt|reused|regenerated)\b/im;
const NUMBER = String.raw`(\d+(?:\.\d+)?)`;
const SEP = String.raw`(?:\s*,\s*|\s+)`;
const DRAWN = new RegExp(
`^Drawn:\\s*${NUMBER}${SEP}${NUMBER}${SEP}${NUMBER}${SEP}${NUMBER}(?:.*?${NUMBER}\\s*[–—-]\\s*${NUMBER}\\s*s\\b)?`,
"gim",
);
const COPY = /^Copy:.*["“”]\S/im;
const REGION =
/^Regenerated:\s*(\d+(?:\.\d+)?)(?:\s*,\s*|\s+)(\d+(?:\.\d+)?)(?:\s*,\s*|\s+)(\d+(?:\.\d+)?)(?:\s*,\s*|\s+)(\d+(?:\.\d+)?)/im;
export function parseSegments(markdown: string): Segment[] {
const heads = [...markdown.matchAll(HEADING)];
@@ -25,12 +44,31 @@ export function parseSegments(markdown: string): Segment[] {
const body = markdown.slice(m.index! + m[0].length, heads[i + 1]?.index ?? markdown.length);
const found = PIXELS.exec(body)?.[1]?.toLowerCase();
const pixels = SOURCES.find((source) => source === found) ?? "rebuilt";
const r = REGION.exec(body);
const region: Segment["regenerated"] = r
? [Number(r[1]), Number(r[2]), Number(r[3]), Number(r[4])]
: undefined;
const drawn = [...body.matchAll(DRAWN)].flatMap((d) => {
const box: [number, number, number, number] = [
Number(d[1]),
Number(d[2]),
Number(d[3]),
Number(d[4]),
];
if (box[2] <= 0 || box[3] <= 0) return [];
return [d[5] ? { box, window: [Number(d[5]), Number(d[6])] as [number, number] } : { box }];
});
return {
id: m[1]!,
start: Number(m[2]),
end: Number(m[3]),
title: (m[4] ?? "").trim(),
pixels,
...(region && pixels !== "regenerated" && region[2]! > 0 && region[3]! > 0
? { regenerated: region }
: {}),
...(drawn.length ? { drawn } : {}),
...(COPY.test(body) ? { copy: true as const } : {}),
};
});
}
skills-manifest.json
+3 -3
View File
@@ -18,7 +18,7 @@
"files": 13
},
"media-use": {
"hash": "99605f8eca2490e4",
"hash": "1a7d1b228edc28b9",
"files": 117
},
"remotion-to-hyperframes": {
@@ -26,8 +26,8 @@
"files": 76
},
"reverse-any-video": {
"hash": "299673b85afcbd17",
"files": 6
"hash": "b7a34caab79eadce",
"files": 4
},
"slideshow": {
"hash": "c8a591e2aa7a6d86",
skills/media-use/references/heygen.md
+14 -10
View File
@@ -16,15 +16,19 @@ A refusal names its fix: `api_key_required` needs an API key, `402` or `insuffic
## Making media: `media-use generate`
Make it without quoting a price or asking for a budget; `--plan` only prints what a call would cost, for your own planning; never pass it on. At most two attempts per piece; a `content_policy_violation` gets one reworded try.
Make it without quoting a price or asking for a budget; `--plan` only prints what a call would cost, for your own planning; never pass it on. Make a piece again only for a fault you can name (a cut inside it, text in it, the wrong framing), once, never to chase a motion or look score; a `content_policy_violation` gets one reworded try.
```bash
# a still, edited from a reference picture: a clean plate with the words removed
npx hyperframes media-use generate --type image --reference assets/frame.png --prompt "the same shot with no text or logos" --out assets/plate.png --plan
# a shot of 5–15 s, starting from a first frame; longer shots are shots chained from each one's last frame
npx hyperframes media-use generate --type video --first-frame assets/s03-first.png --prompt "slow push in" --seconds 8 --out assets/s03.mp4
# a person re-shot from a still, saying a 5–15 s stretch of a recording, lips in sync
npx hyperframes media-use generate --type video --reference assets/person.png --audio assets/s02-voice.m4a --prompt "talks to camera, seated" --seconds 8 --aspect 9:16 --out assets/s02-person.mp4
# a person re-shot from a still over one shot of the original (never across a cut), saying that shot's voice with lips
# in sync and moving as the guide (the original shot itself) does; --mask x,y,w,h[,a-b] (repeatable, the guide's
# pixels and seconds) blanks text in the guide, which HeyGen would copy
npx hyperframes media-use generate --type video --reference assets/person.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --mask 110,615,390,165,0.4-2.0 --prompt "talks to camera, seated" --seconds 8 --aspect 9:16 --out assets/s02-person.mp4
# the same for a person who does not speak: no --audio, the guide clip carries the motion
npx hyperframes media-use generate --type video --reference assets/person.png --motion assets/s05-original.mp4 --prompt "holds up a jar, smiles" --seconds 6 --aspect 9:16 --out assets/s05-person.mp4
# a person in a photo speaking a script in a HeyGen voice
npx hyperframes media-use generate --type avatar --photo assets/person.png --script "…" --voice <voice-id> --out assets/person.mp4
# a voice reading text
@@ -35,13 +39,13 @@ npx hyperframes media-use generate --type music --prompt "warm lo-fi, soft keys"
The file lands at `--out` and the media ledger records it as generated, with the route, model, job, prompt and the files it was made from. Pictures and shots carry no text or logos: ask for clean plates and draw words in HyperFrames.
| Type | Model | Price |
| ------ | -------------------------------------------------------- | ----------------------------------------------------------------------------------------------- |
| image | nano-banana-2 (default), or gpt-image-2.5 with `--model` | $0.067 an image; gpt-image-2.5 $0.036–0.053 by aspect, plus $0.0082 a reference after the first |
| video | heygen-video-1, 768p | $0.018 a second |
| avatar | talking photo | $4.83 a minute of speech |
| music | lyria-3-pro | $0.08 a track |
| voice | the voice's engine | charged per character, not priced ahead |
| Type | Model | Price |
| ------ | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| image | nano-banana-2 (default), or gpt-image-2.5 with `--model` | $0.067 an image; gpt-image-2.5 $0.036–0.053 by aspect, plus $0.0082 a reference after the first |
| video | heygen-video-1, 768p | $0.02 a second from a prompt or first frame; $0.05 a second with a reference picture, voice or guide clip, and the guide's seconds count too (1080p costs about twice) |
| avatar | talking photo | $4.83 a minute of speech |
| music | lyria-3-pro | $0.08 a track |
| voice | the voice's engine | charged per character, not priced ahead |
HeyGen charges in units of $1/60, rounded up; `--plan` prints that charged price. Voice ids come from `heygen voice list` or `npx hyperframes media-use resolve --type voice`; a cloned voice from `heygen voice clone create` (`operations.md`).
skills/reverse-any-video/SKILL.md
+18 -50
View File
@@ -9,62 +9,30 @@ metadata:
# Reverse a video
The result is a HyperFrames project whose render matches the original and stays editable: text is text, graphics are elements with named colours, and every media file records where it came from. It is done when `npx hyperframes check` passes and `diff` against the original passes every segment. Build to the `/hyperframes` contract; this skill adds how to read a video you only have as pixels. Reverse videos the person has the right to use.
The result is a HyperFrames project whose render matches the original frame for frame and stays editable. Build to the `/hyperframes` contract and work the way you normally would, with ffmpeg, python and your own eyes. This skill holds only what earlier rebuilds got wrong or spent hours on. Reverse videos the person has the right to use.
## 0. Prepare
Sign in first; start only when `npx hyperframes auth status` shows a sign-in. Open with `references/start.md`. Rebuild the video as it is; remix it only when the person asks.
Sign in first; start only when `npx hyperframes auth status` shows a sign-in. Open with `references/start.md`.
## The original decides the frame
```bash
npx hyperframes init <project>
npx hyperframes measure probe <video> --normalize <project>/.reverse/source.mp4
```
1. `npx hyperframes measure probe <video>`: the root composition is that size exactly, `data-fps` is its rate (NTSC stays a fraction: `30000/1001`) and its duration is the video's frames ÷ fps, not the audio's length. `init`'s default canvas is wrong for most videos.
2. `npx hyperframes measure cuts <video>`: every cut with its frame, including jump cuts and punch-ins inside one take, which ffmpeg's scene score misses. Every shot starts at its listed time. Write times as frame ÷ fps rounded down (frame 83 at 30 fps is `2.766666`): `2.767` lands a frame late.
3. The sound is the original's: copy the video into the project and play it as one `<audio src="original.mp4">` over the whole film (no extracted or re-encoded track); every clip is muted.
4. Read consecutive frames wherever something changes: each cut, entrance, exit and transition, a few frames either side (`npx hyperframes measure frames --range a,b --roi x,y,w,h`). Sheets a second or a quarter apart miss 1–3-frame timing, which way things enter and leave, blurs and dissolves at cuts, and one-frame flashes.
`.reverse/source.mp4` is the evidence: measured and compared, never published. Give the root composition the original's size and render at its frame rate (the probe's `canvas`; NTSC rates stay fractions such as `30000/1001`). Over 60 s, work in sections split at cuts.
## Every layer
## 1. Analyse: measure first, then look
- **A component** (text, captions, stickers, cards, UI and screen recordings, charts, maps) is rebuilt in HTML.
- **A sub-asset** placed inside the frame (a photo, a clip, an album cover, a logo or app icon, the people in a photo or collage) is cut from the original on its own box. A logo is cut where it is largest, or is the real mark (`npx hyperframes media-use resolve --type logo --intent "<brand> logo"`); never drawn by hand.
- **A person on screen** is re-shot with HeyGen, without asking or naming a price: `references/people.md`.
- Never play the original's whole frame.
You see stills, not motion; evenly spaced frames blur cuts and drop flashes. Let measurement decide where to look (`references/watch.md`):
What decides whether drawn things match, in `references/match.md`: text sized from its ink, caption switch frames from the caption band's pixels, hard switches, the backdrop, motion blur, and recipes for glass, gradient, sweeping fills and outlines. Look at every large word through its whole life before you build it.
1. `npx hyperframes measure events .reverse/source.mp4 --map`: cuts, flashes, what appears where, holds and the sound's hits, grouped into windows.
2. `measure frames --every 1` for the overview, `measure frames --events` for each window's consecutive frames, `measure trail` to see a move in one picture.
3. Numbers from `measure box | ink | curve | objects | rhythm | look`; words from `npx hyperframes transcribe`; beats from `npx hyperframes audio`.
## Check once, at the end
Start with `npx hyperframes analyze .reverse/source.mp4 --preset reverse`: HeyGen drafts the same timeline; correct it against the frames.
1. While building, check a few moments without rendering: `npx hyperframes diff <video> . --at t,t,…` (the original beside yours, scored).
2. When every shot has content: `npx hyperframes check`, fix its errors, then `npx hyperframes render` in the foreground with a 10-minute timeout (a longer render: start it detached, `setsid nohup npx hyperframes render -o <out> > render.log 2>&1 &`, and wait on the log in the foreground), then `npx hyperframes diff <video> <render>`. It lists the original's cuts that yours misses and the extra ones, the duration, the sound's offset, stretches where content is missing, and the worst moments with the failing cell beside the original's.
3. Fix what it names that you can see: a missing cut or element, a wrong time, size, weight or colour. Render and diff once more, then stop. Small score differences are not worth another round.
Write `REVERSE.md` (`references/spec.md`): the brief, the recurring objects, then one section per segment with its copy, actions, times and where its pixels come from. Every number comes from a measurement; tag each claim observed, measured or hypothesis.
## 2. Rebuild by layer
| Layer | Signed in (default) | Offline, or asked local only |
| --- | --- | --- |
| Text, shapes, UI, charts, transitions | HTML + GSAP, registry first | — |
| Burned-in captions | a caption track styled like the original, timed from the transcript | — |
| Logos | SVG geometry; cut from the original when photographic | — |
| Photos, footage, AI shots | generated from its frame (`/media-use`) | cut on its own (`measure extract --roi`) |
| A presenter | re-shot with HeyGen from a still and their voice | cut from the original |
| Sound | original voice (`measure extract --out assets/sound.m4a`), generated music | the original's track |
`references/layers.md` covers placement, burned-in text over reused footage, fonts and regeneration. One sub-composition per segment (one scene when a single camera spans several); each string one element bound to a variable; colours as `:root` variables.
## 3. Check, cheapest first
| Step | Command | Cost |
| --- | --- | --- |
| 1 | `npx hyperframes check`, with a `*.motion.json` of the measured onsets | no render |
| 2 | `npx hyperframes diff .reverse/source.mp4 . --at <t,…>` at onsets, mid-moves and both sides of cuts | stills |
| 3 | `npx hyperframes diff .reverse/source.mp4 . --range <a,b>` for the segment you changed | its frames |
| 4 | `npx hyperframes render`, then `diff .reverse/source.mp4 <render> --segments REVERSE.md --sbs` | one render |
Fix what `worst.jpg` shows: missing elements, then position and size, then timing, then colour. Stop when every segment passes, or after two rounds without gain on the worst segment, and write what is left in the report (`references/check.md`).
Segments handed to subagents: wait for every one in the same turn, then assemble `REVERSE.md` and run step 4 yourself. A headless run (`claude -p`) ends with your turn and drops background work; a segment left empty still scores high on a plain background, which `diff` flags as missing content.
## Rules
- What cannot be drawn is generated with HeyGen when signed in, without asking or a price; cut only offline. A clip carrying a chart, title or UI is a copy, not a reverse.
- Build mechanisms, not traced paths: no coordinate lists or per-frame positions.
- Copy every string exactly: case, punctuation, line breaks.
- Keep hard cuts hard, holds held and one-frame flashes one frame long.
- What is on screen at a segment's first frame starts there; it does not animate in.
- Generated pictures carry no text or logos; HyperFrames draws those.
Work alone: one agent holding the whole film beats subagents that each re-read and re-watch it and wait on each other. Write a short `REVERSE.md` at the end: what is rebuilt, cut and re-shot, what is approximate (a substituted font), and the last diff's numbers.
skills/reverse-any-video/references/check.md
-72
View File
@@ -1,72 +0,0 @@
# Checking the rebuild against the original
Check from cheapest to most expensive, and fix only the segment that fails. Each step passes before the next runs; after a fix, start again from the step that failed.
| Step | Command | What it catches |
| ---- | -------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
| 1 | `npx hyperframes check --json` | contract and layout errors, and the measured onsets and order in `*.motion.json`; no render |
| 2 | `npx hyperframes diff .reverse/source.mp4 . --at <t,…>` | stills at chosen moments, captured from the project without rendering |
| 3 | `npx hyperframes diff .reverse/source.mp4 . --range <a,b>` | every frame of one segment, captured the same way |
| 4 | `npx hyperframes render --fps <fps>`, then `npx hyperframes diff .reverse/source.mp4 <render> --segments REVERSE.md --sbs` | every frame, cuts, duration, a verdict per segment, `sbs.mp4` |
## Motion assertions from the measurements
Put what you measured where `check` can hold you to it: one `index.motion.json` beside `index.html`, its selectors reaching into every scene, its times in film seconds.
```json
{
"assertions": [
{ "kind": "appearsBy", "selector": "#headline", "bySec": 0.57 },
{ "kind": "before", "a": "#headline", "b": "#cta" },
{ "kind": "staysInFrame", "selector": ".card" }
]
}
```
`check` samples the timeline about every 0.05 s, so `bySec` is the measured onset plus 0.05. Leave `keepsMoving` out: the original's holds are part of what you copy. A contrast or text-size finding on an element the original shows the same way is the original's look: keep it and mark the element `data-layout-ignore`.
## Moments to compare at
At each event's first frame and the frame before it, mid-move, at the settle, and on both sides of every cut; `measure events --map` lists them. A still pass at a quiet moment says little.
## Reading `diff`
- `ssim.mean` and `ssim.p05`: the whole frame, averaged and at its worst 5%.
- `ssim.cell_p05`: each frame's worst cell of a 4x4 grid. A missing, late or misplaced element drags one cell and barely moves the whole-frame score, so read this first.
- `missing_content`: the share of frames where a cell the original fills is flat in yours, and the longest such stretches (`spans`, seconds). Two near-blank patches score alike, so an empty scene on a plain background passes SSIM and the cell gate; over 2% fails.
- `frames`: the path of a tab-separated table of every compared frame's time, SSIM and worst cell, to find the stretches that fail.
- `worst`: the lowest moments, at least 0.5 s apart, each with the failing cell (`roi`, in source pixels) and `cell.picture`, that cell at full size, original beside yours. `worst.jpg` shows the whole frames with the cell boxed: look at these before changing anything.
- `cuts`: the original's cuts with no cut within a frame in yours (`missing`), and yours with none in the original (`extra`).
- `sound`: how far yours' sound sits from the original's (positive is late).
- `segments`: a verdict per `REVERSE.md` segment. `regenerated` segments are judged by `energy.mean_ratio` (how much it moves, 0.8–1.25) and `look`. On a project folder, `--segments` alone compares the segments' whole span.
Passing, provisional until more films calibrate it: mean ≥ 0.92, p05 ≥ 0.85, worst cell p05 ≥ 0.75 for rebuilt and reused segments; reused pixels should land near 1. Short fades and flights of small elements are where the worst cells end up. A regenerated person also gets `measure sync` (within +45 / −125 ms) and `measure face` on the render, and chained shots `measure continuity`. Every cut within a frame, the same duration, and the sound within a frame. Override with `--min-mean`, `--min-p05` and `--min-cell` only for a reason you write in the report.
## Fixing
Biggest difference first: a missing element, then position and size, then timing, then colour and texture.
| What `diff` shows | Likely cause | Fix |
| ----------------------------------- | ------------------------------------------------ | --------------------------------------------------------------- |
| low everywhere from the first frame | canvas size, aspect or background differ | match the probe's size; check the root's fill |
| fine at first, drifting later | frame rate or durations differ | render at the original's `fps`; times as multiples of one frame |
| one cell low during a move only | ease or timing | `measure curve` on both, compare ease, start and duration |
| one cell low throughout a segment | an element missing, misplaced or mis-sized | `measure box` on both at that time |
| low only in text cells | font, size, weight or tracking | see Fonts in `layers.md` |
| cut missing or one frame off | a clip's `data-start` off a frame boundary | start clips at `k / fps` |
| colour only | palette or grade | `measure look --compare`; colours from `measure box` |
| reused segment not near 1 | wrong window or crop, or a transform on the clip | re-extract with the window and box from the map |
Stop when every segment passes, or when two rounds in a row leave the worst segment no better. Write what is left, and why, in the report.
## Effects a seeked timeline reproduces
- **Directional motion blur** (a whip, words sliding in): CSS `blur()` is symmetric and wrong. An SVG `<filter>` with `<feGaussianBlur stdDeviation="0 0">` on the moving wrapper, its `stdDeviation` tweened (`"24 0"` is a horizontal smear); for another angle, rotate an outer wrapper and counter-rotate the inner one.
- **Ghost trail:** 2–4 clones behind the element, offset along the motion by a growing distance, opacity about 0.5, 0.3, 0.15.
- **Zoom blur:** a few scaled clones (1.02–1.15) with falling opacity around the zoom's origin.
- **Chromatic fringe:** red and cyan clones offset 2–4 px opposite ways, `mix-blend-mode: screen`.
- **Glow:** stacked `text-shadow` or `drop-shadow`, tweening radius and alpha, not a blur of the core.
- **Grain:** over drawn layers, SVG `feTurbulence` with a fixed seed, stepped per frame for moving grain; on footage, `npx hyperframes media-treatment` (`/media-use`).
- **Glass type over footage:** a second copy of the shot (the same `src` and timing) inside the letters: an inline SVG `<clipPath>` holding the `<text>` clips it, and an SVG filter of `feTurbulence` (`baseFrequency` about 0.02, a fixed seed) into `feDisplacementMap` (scale 40–60) bends it. A darker copy of the text multiplied over it (15–35%, tinted toward the original's), a 2–3 px dark stroke, a light top edge (a gradient copy at 30–50% opacity, `mix-blend-mode: screen`) and a soft shadow make it read as glass. The copy sits in the same camera group as the shot, so it moves with it.
- **Masks and wipes:** `clip-path` or `mask-image` with a tweened gradient stop.
- **Flash:** a full-frame layer for exactly the measured frames.
skills/reverse-any-video/references/layers.md
-64
View File
@@ -1,64 +0,0 @@
# Rebuilding each layer
Every video is three kinds of layer: what can be drawn (text, shapes, UI, charts, transitions), pixels that cannot be drawn (photos, footage, AI shots, people), and sound. A motion graphic is nearly all the first; a talking head is a person with graphics over them; an AI film is nearly all the second. Mark each segment's `Pixels:` in `REVERSE.md` as you decide.
## Drawn layers
- Search the registry before hand-building a named effect, transition, caption style, chart or card (`/hyperframes-registry`); an installed item is source you restyle to the measured look.
- One sub-composition per segment, mounted at the segment's start; when one continuous camera carries several segments, they share one scene with a timeline label per segment. Each string is one element with its text bound to a variable; colours are `:root` variables named by role; fonts are variables too.
- Sizes and positions come from `measure box` and `measure objects` in source pixels: the canvas is the source's size, so they go in as measured.
- Times are multiples of one frame (`k / fps`), written rounded down: 67 / 24 is `2.791666`, since `2.791667` starts a frame late. An onset half a frame off lands on the wrong frame.
- Effects that a seeked timeline reproduces are in `check.md`.
### Fonts
Match the family by traits (geometry, contrast, x-height, terminals), then the weight. Prefer a bundled font (`/hyperframes` `references/format.md`), then a Google Fonts family downloaded into `assets/fonts/` with `@font-face`. Size it by measurement: render the candidate at a known size, `measure ink` the same text in the original and in a snapshot of yours, and scale by the ratio of their heights. A substituted font is approximate: say so in the report.
### Burned-in captions
Rebuild them as a caption track (`/hyperframes` `references/captions.md`) with the original's grouping, line breaks, position and style rather than the reference's defaults; take the words from the frames and their times from `npx hyperframes transcribe` (which can mishear names), and check each group's first and last frame against the original's with `measure events --roi` over the caption band.
### Logos
Draw simple marks as SVG from their measured geometry (`measure objects` for rings, grids and rows of parts). A photographic or textured logo is cut from the original as a still.
## Pixels from the original
Footage, photos, AI shots and people are cut from the original unless the person asks for new ones. Cut only what cannot be drawn, each piece on its own and cropped to its box: an album cover, a photo, a video window. A clip never carries drawn things along with it; an animated map, a chart, a title or a UI in the original is rebuilt even where it moves like footage.
```bash
npx hyperframes measure extract .reverse/source.mp4 --from 4.0 --to 9.5 --roi 0,0,1080,1350 --out assets/s03-presenter.mp4
npx hyperframes measure extract .reverse/source.mp4 --at 2.4 --roi 120,340,840,620 --out assets/s02-photo.png
```
Each file is recorded in the project's media ledger as cut from the original, with its window and region, and the output's `place` gives the `data-start`, `data-duration` and box to put it at. Place a clip as a muted `<video class="clip">` inside a wrapper at that box; it starts at the window's start, so it needs no trim.
- **Pictures that overlap** (popups, stacked cards): cut each at a moment it is whole and alone, before anything covers it, and cut a clip from the moment it first shows, or it plays out of step.
- **All footage, nothing drawn:** one clip per shot, cut at the cuts `diff` finds (`measure cuts`), in one scene, plus the sound.
- **Whole-frame footage with graphics over it:** one clip of the full frame only where a camera image fills the frame, the graphics rebuilt on top. A frame that is mostly drawn (a map, a chart, a UI) is rebuilt, and only its photos and video windows are cut, each to its own box.
- **Footage inside a moving frame** (a card that slides, a phone mock-up that zooms): when the content exists at rest, cut it where it is largest and least covered, and move it with your own transforms so the motion stays in your timeline. When it only appears while the frame moves, cut the union of its boxes over the move as one clip, place it unscaled outside your camera group, and clip it to the frame's box as your camera moves that box. Content that plays inside its frame is a clip, not a still.
- **A presenter talking to camera:** re-shot with HeyGen (below), not cut: their frame carries the titles and captions burned over them. When HeyGen is not available, cut them on their own and note the burned-in text.
- **Text burned into the footage:** crop around it when it sits outside the content. When it sits over the content, keep the footage and lay the rebuilt text exactly over the burned-in one: the render matches, but editing that text shows the old one underneath. Note it in the report, and that a clean plate can be regenerated on request.
## Sound
The original's sound is one file, `measure extract .reverse/source.mp4 --out assets/sound.m4a`, placed as one `<audio id="sound">` over the whole film. Reused clips then stay muted, or the voice plays twice. A film with no sound gets no `<audio>`. Graphics that land on hits are checked against `npx hyperframes audio` onsets.
## New pixels
Signed in, every camera or generated pixel is remade with HeyGen by default: presenters re-shot, stills and shots generated, music generated to the original's length and tempo. Cut from the original only offline or when the person asks for local only. Regenerated media is made with `npx hyperframes media-use generate` (`/media-use`). Ground every generation in the original:
- **A still:** `--type image --reference <the original frame>`, asked to keep the composition and leave out what HeyGen should not draw: a clean plate.
- **A shot:** `--type video --first-frame <the shot's first frame>` at the shot's length, 5–15 s; a longer shot is chained shots, each starting from the last frame of the one before (`measure extract --at` cuts both).
- **A person speaking:** `--type video --reference <a still of them> --audio <their voice for that stretch> --prompt "…" --seconds <n> --aspect <the original's>`: the original words and timing, lips in sync, no burned-in text. Cut the voice at pauses into 5–15 s stretches (`transcribe` gives the word times), one shot per stretch at its start, then match each shot to the original:
1. **Ask for the same shot.** The prompt names the shot size (from `measure face` height: close-up, medium, wide), what the person does (sits, walks toward the camera, turns), the camera (`measure rhythm`: still, push-in, pan, handheld), the place and the light.
2. **Frame it.** `measure face` on the original's stretch and on the new shot: scale and move the shot so the face boxes line up. Shot, cutout and the text between them sit in one camera group, so everything moves together.
3. **Move it like the original.** `measure rhythm --start a --end b` names each shot's camera (push in, pull out, pan, handheld) with its zoom and pan; the face heights from step 2 show its curve over time. Tween the camera group by that zoom and pan across the shot, on top of the framing. Handheld is a small seeded wiggle the size of rhythm's path, never per-frame positions.
4. **Put text behind them where the original does.** Where the person covers the original's text, cut them out of the new shot (`npx hyperframes remove-background <shot> -o <cutout>.webm`) and stack shot, text, cutout. This only reads when the framing matches: a person framed smaller never reaches the text.
5. **Check the motion.** `measure flow … --compare <render>` over the stretch: the energy ratio and timing say whether it moves like the original. Generated shots come out steadier than handheld footage, so the camera group carries the difference.
`--type avatar` speaks a new script instead.
- **Music or voice:** `--type music` at the original's length and tempo (`npx hyperframes audio` gives both); `--type voice` for a voice-over in a chosen voice.
Pictures carry no text or logos: ask for clean plates and draw the words in HyperFrames. Mark the segment `Pixels: regenerated`; `diff` then judges how much and when it moves and its look, since its pixels cannot match.
skills/reverse-any-video/references/match.md
+39
View File
@@ -0,0 +1,39 @@
# Matching what is drawn
Where earlier rebuilds lost points on drawn layers, and what fixed it.
## Timing
- Graphics change on exact frames. An element that is fully there on frame 241 is set there, not faded in; tween only what you saw move over several frames. A camera that lands in one frame is a `set`, not a tween.
- Captions: take each group's first and last frame from the caption band's pixels, every frame (`npx hyperframes measure events <video> --roi <band>`, or a frame difference over the band). The words come from `npx hyperframes transcribe`, which can mishear names; the switch times come from the pixels, since captions often lead or trail the voice. Switch hard unless the original fades.
- A blur flash, a dissolve or a wipe at a cut is part of the cut: rebuild it over its measured frames.
## Size, type and position
- Size text from its ink, not by eye: crop the line in a source frame and the same line in a snapshot of yours, compare the ink boxes' height and width, and scale size, then `letter-spacing`, until both agree within a few pixels. A substitute font usually sets wider; compare numerals too. Then compare the ink's darkness for weight, and look for an outline or shadow.
- Match the family by its traits (geometry, contrast, x-height, terminals) and try two or three candidates at the measured size before choosing. Prefer a bundled font, then a Google Fonts family in `assets/fonts/`. Say in the report which fonts are substitutes.
- Positions come from measured boxes in source pixels; the canvas is the source's size, so they go in as measured.
## Look
- Motion graphics: the backdrop covers most of the frame. Sample a grid of the empty backdrop at several moments and match its gradient and brightness over time.
- Fast moves in the original are motion-blurred; sharp tiles at speed score as wrong. Blur along the move while it is fast (below).
- Dots and pixel textures: measure the lattice (pitch, offset, colour) from a frame and render from that grid.
- Flat graphics that move like footage (a map under a camera, a diagram that scrolls): trace each plate into flat-colour shapes and fit the camera move; it matches far better than redrawing from outside data.
## Every large word, through its life
Look at a strip of consecutive frames over its box from the frame it appears to the frame it leaves (`measure frames --range a,b --roi <its box> --max 16`). Read how it arrives (pop, scale, fade, type-on, word by word, wipe), how it changes while on screen (a fill sweeping across, a sheen, a counter, a colour change: direction and duration) and how it leaves. Then zoom in at full size and name what the letters are made of: solid, gradient, glass, outline, glow, knockout.
## Recipes a seeked timeline reproduces
- **Fill that sweeps across the letters:** two copies of the word; the lit one on top, `clip-path: inset(0 100% 0 0)` tweened to `inset(0 0 0 0)` over the measured time and direction.
- **Gradient type:** `background: linear-gradient(<direction>, <both end colours>)` with `background-clip: text` and a transparent fill.
- **Glass type over footage** (the letters show the footage behind them bent, with a dark rim and a light edge): a second copy of the shot inside the letters, clipped by an SVG `<clipPath>` holding the `<text>`, bent by `feTurbulence` (`baseFrequency` about 0.02, fixed seed) into `feDisplacementMap` (scale 40–60); a darker copy of the text multiplied over it at 15–35%, a 2–3 px dark stroke, a light top edge. It sits in the shot's camera group.
- **Outline type:** `-webkit-text-stroke` at the measured width, transparent fill.
- **Directional motion blur:** an SVG `<feGaussianBlur stdDeviation="0 0">` on the moving wrapper, its `stdDeviation` tweened (`"24 0"` smears horizontally); CSS `blur()` is symmetric and wrong.
- **Ghost trail:** 2–4 clones behind the element along the move, opacity about 0.5, 0.3, 0.15.
- **Glow:** stacked `text-shadow` or `drop-shadow`, tweening radius and alpha.
- **Masks and wipes:** `clip-path` or `mask-image` with a tweened stop.
- **Flash:** a full-frame layer for exactly the measured frames.
- **Grain:** SVG `feTurbulence` with a fixed seed, stepped per frame for moving grain. Grain lowers SSIM whatever you do: match its look, not its pixels.
skills/reverse-any-video/references/people.md
+22
View File
@@ -0,0 +1,22 @@
# Re-shooting a person
Every person on screen, speaking or not, is re-shot with HeyGen so the original's face and the text burned over it never ship. What makes a re-shot person match is the original's own clip driving the motion, and one generated shot per original shot. Without a HeyGen key, cut them on their own box instead and say the burned-in text came along.
## The recipe
1. **One shot per original shot.** From `measure cuts`: a generated shot never spans a cut, jump cuts and punch-ins included. One take across a jump cut deletes the cut; a motion guide that crosses a cut makes HeyGen cut somewhere of its own. HeyGen makes 5–15 s: a shorter shot is asked for at 5 s and trimmed; a longer one is split where the original reframes, else chained: the next part's reference is the last frame of the one before.
2. **A plate per shot:** a real frame of that shot at the source's aspect, with every drawn thing and every other face (a photo on the wall, a sticker) painted out, since HeyGen may animate the wrong face (`cv2.inpaint` for small marks, a temporal median where the person moves); it keeps the room sharp and true. Where drawn things are large or cross the person, an image edit cleans better (`npx hyperframes media-use generate --type image --reference <frame>`, asked to remove the text and keep everything else); check it kept the room sharp.
3. **Generate every shot at once,** in the background, and build the graphics while they render (each comes back in about a minute):
```bash
npx hyperframes media-use generate --type video --reference <plate> --motion <the original shot> \
--mask <x,y,w,h,a-b> --audio <the original voice over that shot> --aspect <the source's> --seconds <n> --out <shot>.mp4
```
`--motion` is the original shot itself, cut at its cut times: HeyGen follows its body and camera motion and takes the face from the plate. It also copies text burned into the guide, so give one `--mask` per drawn box in that shot (the clip's pixels and seconds). Mask the text's own box, never a full-width band: a gesture under a mask is lost. A person who does not speak gets no `--audio`.
4. **Check each shot as it arrives:** no cut inside it (generate prints `cuts_inside`, and refuses a guide with a cut), none of the original's text, the face box within 5% of the original's (`npx hyperframes measure face`). Regenerate a failing shot once; then trim it to its good part.
5. **Frame it:** scale and move each shot so its face box sits on the original's at the shot's start and end (`npx hyperframes measure face <shot> --against <video> --from a --to b` prints the `scale`, `x`, `y` to set), tweening between them only where the original's camera moves. The framing jumps at cuts and nowhere else.
6. **Text behind the person:** where the original's text passes behind them, cut them out of the shot (`npx hyperframes remove-background <shot> -o <cutout>.webm`) and stack shot, text, cutout.
Keep the original's voice as the soundtrack and the shots muted. Do not clone voices or make avatars for a re-shoot: the original voice drives the lips. The people inside a photo, a collage or a screenshot are sub-assets, cut with it.
skills/reverse-any-video/references/spec.md
-68
View File
@@ -1,68 +0,0 @@
# REVERSE.md
The rebuild's spec, and at the end its report. Someone who never saw the original should be able to rebuild it from this file alone; you rebuild from it too, so a correction found while building goes back in. `npx hyperframes diff --segments REVERSE.md` reads its segment headings and `Pixels:` lines.
Evidence tags, after the claim they support:
- `[O]` observed in the frames, not measured
- `[M]` measured with a `measure` command (keep the number)
- `[H]` a hypothesis about how it was made: an ease name, a font, a camera pivot
Numbers only where they change what gets built: onsets and cuts in seconds with two decimals (frame-exact times are multiples of 1/fps), sizes and positions as shares of the frame or in source pixels, counts, overshoot or none. Qualities (paper grain, soft glow, slight jitter) are words. No per-frame positions or traced paths: a move is its start, end, duration, ease and overshoot.
## Layout
```text
# <film> — reverse
Source: 1080x1920, 30 fps, 14.2 s, sound: voice and music.
## Brief
Storyline: what the film shows from the first frame to the last, in 3–6 sentences.
Look: visual language, texture, lighting.
Palette: 3–6 colours with hex and role.
Type: typefaces by traits (geometric sans, high-contrast serif), weights, case, tracking; names only as [H].
Motion rules: how things enter, leave and hand off; speed, easing feel, cuts on beat, smooth or stepped.
Footage and images: each photographic or filmed element, one line each.
Never: 3–6 substitutions a rebuild is likely to make that would break the look ("no fades where the
original hard-cuts").
## Objects
One line per element that recurs or must be drawn exactly: name — construction (count, shape, arrangement,
size as a share of the frame, colour, type traits) — where it appears and in what state. Segments use the name.
## Sequence
| id | window (s) | what happens | pixels |
## S01 · 0.00–2.40 s · Title card
Pixels: rebuilt
Start: what is on screen at the segment's first frame and where, in paint order; "already on screen" for
anything carried over.
Copy: every string, quoted exactly (case, punctuation, line breaks), with its element and type.
Actions:
- 0.12–0.48 headline slides up 8% of the frame, power3.out, no overshoot [M]
Relations: what moves together, masks, occlusion, camera against object.
End → next: the last state and how it hands off (hard cut, carried element, match move).
Must-check:
- 3–6 single sentences, each checkable on a frame, naming what a rebuild most likely gets wrong here.
## Unknowns
- What could not be settled and what was assumed instead.
## Report
```
## Segments
Follow the original's own units: split at cuts and layout changes, never inside one continuous move. Usually 2–8 s each; a 60 s film has 6–14. Windows are contiguous and cover 0 to the duration. A long film is sections of at most 60 s, each with its own segments.
`Pixels:` says where the segment's picture comes from, and so how `diff` judges it:
| Pixels | Means | Judged by |
| ------------------- | --------------------------------------------------------------------- | ---------------------------------------- |
| `rebuilt` (default) | drawn in HTML: text, shapes, UI | SSIM, whole frame and worst cell |
| `reused` | cut from the original with `measure extract`, overlays rebuilt on top | SSIM, which should come out near 1 |
| `regenerated` | made with HeyGen: the signed-in default, or what the person asked for | how much and when it moves, and its look |
## Report
Written after the last check: each segment's `diff` verdict and scores; what is approximate (a substituted font, a hand-drawn logo); reused footage with burned-in text under rebuilt text (editing that text shows the old one); what was not rebuilt and why; time and money spent. End with one sentence on what HeyGen would add, only when it would: "S03 and S05 carry burned-in captions; with a HeyGen sign-in they can be regenerated as clean plates."
skills/reverse-any-video/references/start.md
+5 -5
View File
@@ -6,11 +6,11 @@ block they can copy.
## Not signed in yet
Say why HeyGen matters before asking: signing in is what gets the best remake. HeyGen watches the whole video and
drafts its timeline, and can re-shoot the presenter when the video has one. For example:
Say why HeyGen matters before asking: signing in is what gets the best remake. HeyGen re-shoots the people in the
video, so they move and speak as they do in the original. For example:
> To rebuild this video at its best, I'll work with HeyGen: it watches the whole video, drafts the timeline, and can
> re-shoot the presenter. Sign in once by typing this in the prompt:
> To rebuild this video at its best, I'll work with HeyGen: it re-shoots the people in it, so they move and speak just
> like the original. Sign in once by typing this in the prompt:
>
> ```
> ! npx hyperframes auth login
@@ -18,7 +18,7 @@ drafts its timeline, and can re-shoot the presenter when the video has one. For
>
> Your browser opens the HeyGen sign-in. Once you're in, reply "done".
>
> Then it takes about 20–50 minutes: I study the video shot by shot, rebuild every layer as a project you can edit,
> Then it takes about 20–50 minutes: I study the video frame by frame, rebuild every layer as a project you can edit,
> and check it against the original until it matches.
## Signed in
skills/reverse-any-video/references/watch.md
-70
View File
@@ -1,70 +0,0 @@
# Watching a video you only see as stills
A video model watches motion; you look at pictures. Looking at more pictures does not close that gap. What closes it is letting measurement choose which frames to look at, and turning motion into a picture. Evenly spaced frames, one to five a second, made earlier rebuilds see hard cuts as dissolves, miss one-frame flashes and land entrances 2–12 frames off. The rebuilds that matched the original frame for frame had looked at every frame of every window where something changed.
## The loop
1. **Map.** `npx hyperframes measure events .reverse/source.mp4 --map` reads every frame at 160 px. It returns the shots, the long holds, the sound's onsets and beats, and `windows`: stretches where something changes, each with what changes (`cut`, `flash 1f`, `appear r1c2,r1c3`, `change …`) and the sound's onsets inside it. Cells are a 4x4 grid, `r0c0` top left. The full event list is in the JSON file it names.
2. **Overview.** `measure frames .reverse/source.mp4 --every 1` puts one frame a second on a sheet: the structure, the layouts, the look.
3. **Each window.** `measure frames .reverse/source.mp4 --events` gives one strip per window with its consecutive frames, numbered. A long window is thinned, but the frames before, at and after each event are always kept. Read a strip left to right: which frame is the first with the new state, and what happened in between.
4. **A move in one picture.** `measure trail .reverse/source.mp4 --from <a> --to <b> --roi <tight round the element>` lays the window's frames over each other, earliest faintest, with the moving part's centre numbered, plus a heat map of where the picture changed. A slide leaves a row of ghosts, a scale leaves nested outlines, a mask reveal leaves an edge that sweeps while the content stays put.
5. **Zoom.** `measure frames --at <t> --roi <x,y,w,h> --width 0` crops at full size to read text, glyph shapes and edges.
6. **Measure.** The numbers in the spec come from the commands below, not from a picture.
7. **Settle the mechanism**, then write the segment in `REVERSE.md` and close it: later segments do not need its pictures.
Budget about 8 pictures a segment, more where the map shows many windows. Pictures are the most expensive thing in your context; long films go section by section, and when your harness has sub-agents, one section each, with this file and the map.
## Mechanism questions
Settle each one that applies with a picture or a number, not with a first impression:
1. **Scale or reveal?** Does the artwork grow, or does a mask open over artwork that stays put? Compare the content's size at the start and end of the reveal.
2. **Camera or object?** Does the whole picture move (`measure rhythm` names push in, pull out, pan, handheld per shot), or one element over a still background? One moving card is not a camera move. A camera that travels (a dolly or tracking shot) slides near things past far ones at different speeds; a zoom or pan moves everything together. Zooms and pans can be added afterwards on a camera group; travel cannot, so a re-shot person's prompt has to ask for it.
3. **Cut or transition?** Hard cut, dissolve, wipe, a moving occluder, a tile reveal, a match move. The map's `cut` has a `frames` length when it spans several frames (a whip, a short dissolve).
4. **How does it appear?** Pop at full size, scale in with or without overshoot, fade, type-on by character, word by word, slide from an edge. Look at the frames either side of the onset, then `measure box --track` or `measure curve` for the ease.
5. **How does it move?** Direction, travel as a share of the frame, duration, ease family, overshoot or none, holds. Follow the same instance through lead-in, peak and settle; matching end points says nothing about the path.
6. **Smooth or stepped?** `measure rhythm`'s `update_fps` near half the frame rate means animation on twos. A film animated on twos looks wrong rebuilt smooth.
7. **Blur or glow?** Tell a sharp core, a one-sided directional smear, a symmetric glow and focus blur apart; give onset, peak and decay.
8. **What is the type made of?** Zoom in at full size. Letters showing a photo, lines or other text are a knockout: what shows through is its own layer. Letters showing the footage behind them bent, with a dark rim and a light edge, are glass ("Effects" in `check.md`). Name solid, gradient, glass, chrome, blur or glow in the spec; a flat grey stand-in reads as a different film.
9. **Particles and dust:** colour, dot size, density at start and peak, where dense or empty, drift direction and speed.
10. **Cursor:** tip position (`measure track`), arrow, hand and I-beam swaps, press and release, and the interface's response. Arriving is not clicking.
11. **3D or flat?** A narrowing box can be rotation, crop or scale; check its four corners before calling it 3D.
12. **Footage or drawn?** Photographic and filmed content is cut from the original; what matters for the rebuild is its box, its timing and how the frame around it moves.
## What a spec must say, or the rebuild gets it wrong
- **Identity across cuts.** An element that recurs, or changes role (a circle that becomes the O of a word), is named once in Objects and called by that name everywhere.
- **Construction of marks and logos.** Element count, shape, arrangement (`measure objects`: ring, grid, row, stack), sizes relative to each other, colour, the wordmark's weight, case and tracking.
- **Groups.** How many, arranged how, overlapping how, in what order they appear, how often they change, in which direction, and what stays still. "Cards shuffle" cannot be built; "5 cards, each 12% of the frame high, fanned 20° apart, dealt left to right one every 0.20 s, each sliding up 10% of the frame with power2.out" can.
- **Frame rhythm.** Smooth or stepped, once for the film when it holds throughout.
- **What each segment starts with.** Elements already on screen at a segment's first frame are there, not animated in from empty.
- **Exact copy.** Case, punctuation and line breaks as shown.
- **Relations that carry meaning.** Which end of a connector attaches to what, which caption belongs to which face, which mask follows which card.
## Reading the measurements
| Command | Answers | Read |
| ---------------------------- | ------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------- |
| `measure events` | when each region changes | `frame` is exact; `appear`/`disappear` are steps between two steady looks; `change` is other motion; `hold` is stillness of 0.3 s or more |
| `measure cuts` | where shots begin | `frames` > 1 is a multi-frame transition, not a one-frame shot |
| `measure box` | one graphic's box, colours, text lines, cap height; with `--track`, its moves | draw the ROI round the graphic with some ground around it; each move has an ease and an `rmse` |
| `measure curve` | one move's ease, overshoot and timing | over 0.12 `rmse` the ease is unclear: say so rather than name one |
| `measure ink` | where text ink sits, first and last visible | font size: render a candidate at a known size, scan the same text there, size = candidate size × original height ÷ candidate height |
| `measure objects` | how many elements and their arrangement | positions (centres) and sizes are shares of the frame; touching elements count as one |
| `measure rhythm` | moving share, holds, update rate, camera per shot | camera moves are a best whole-frame fit: confirm on frames |
| `measure look` | brightness, contrast, saturation, warmth | compare a rebuild's grade with `--compare` |
| `measure track` | a cursor, hand or product followed frame by frame from its box at one moment | it stops at a cut or a lost match: give a new `--at` and `--roi` where it reappears |
| `measure flow` | how a textured element or the camera moves, from optical flow, as tweens; `--compare` against a render | for footage and textured cards where `box` finds no flat colour |
| `measure face` | how big the face is and where | a regenerated presenter keeps the likeness from 0.25 of the frame's height |
| `measure sync` | whether the mouth and the voice line up | positive ms is a late picture; trust `high` confidence, repeat `medium` on another window |
| `measure continuity` | jumps, flicker and extra shapes in yours that the original lacks | for regenerated and chained shots |
| `npx hyperframes transcribe` | words and their times | captions and anything timed to speech |
| `npx hyperframes audio` | onsets, beats, loudness | graphics that land on hits; the map already lists onsets per window |
## With a HeyGen draft
`npx hyperframes analyze .reverse/source.mp4 --preset reverse` has HeyGen's video model watch the film and writes a draft: segments with where their pixels come from, copy, actions with times, recurring elements, media boxes, risks, and the questions a frame check should settle. It sees about 6 frames a second at low resolution, so its times are within a few frames and fine detail is missing. Treat it as the first pass of steps 1–3: answer its questions with strips and trails, correct its times with the map, and write what you confirmed. Corrections are the most useful thing you produce.
## If you cannot see pictures
Every step here needs an agent that can look at images. Without that, `analyze` (`/media-use`) is the only way to watch; when it is not available either, say plainly that this video cannot be reversed in this environment.