diff --git a/packages/cli/src/commands/analyze.test.ts b/packages/cli/src/commands/analyze.test.ts new file mode 100644 index 0000000000..c250412332 --- /dev/null +++ b/packages/cli/src/commands/analyze.test.ts @@ -0,0 +1,32 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { findFFmpeg } from "../browser/ffmpeg.js"; +import { probeVideo } from "../measure/media.js"; +import { prepare } from "./analyze.js"; + +const ffmpeg = findFFmpeg(); +const dir = mkdtempSync(join(tmpdir(), "hf-analyze-test-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +describe("analyze", () => { + it.skipIf(!ffmpeg)("sends the whole part when it slows it down", async () => { + const src = join(dir, "src.mp4"); + execFileSync(ffmpeg!, [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=d=4:s=320x240:r=30", + src, + ]); + const info = await probeVideo(src); + const out = join(dir, "part.mp4"); + writeFileSync(out, prepare(info, [1, 3], 3, dir)); + expect((await probeVideo(out)).duration).toBeCloseTo(6, 0); + }); +}); diff --git a/packages/cli/src/commands/analyze.ts b/packages/cli/src/commands/analyze.ts index ab3b4feb28..406cf3084f 100644 --- a/packages/cli/src/commands/analyze.ts +++ b/packages/cli/src/commands/analyze.ts @@ -12,10 +12,10 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { basename, dirname, extname, join, resolve } from "node:path"; -import { defineCommand } from "../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "./_examples.js"; import { tryResolveCredential } from "../auth/resolver.js"; -import { idempotencyKey, runJob } from "../heygen/v3.js"; +import { idempotencyKey, runJob, type Job } from "../heygen/v3.js"; import { DESCRIBE_SCHEMA, mergeDrafts, @@ -39,8 +39,8 @@ export const examples: Example[] = [ ]; const ROUTE = "/v3/video-analyses"; -const MODEL = "gemini-4-argon"; -const HEYGEN_FPS = 2; +export const MODEL = "gemini-4-argon"; +export const HEYGEN_FPS = 2; const HEYGEN_MAX_S = 90; const PART_S = 30; const INLINE_BYTES = 16 * 1024 * 1024; @@ -65,25 +65,92 @@ export function partsOf(duration: number, cuts: number[]): [number, number][] { export const slowBy = (fps: number, seconds: number) => Math.max(1, Math.min(fps / HEYGEN_FPS, HEYGEN_MAX_S / seconds)); -/** One part as HeyGen reads it: slowed, 720 px on its long side, silent when slowed, under the inline cap. */ -function prepare(info: VideoInfo, [a, b]: [number, number], slow: number, dir: string): Buffer { - const fit = info.width >= info.height ? "scale=720:-2" : "scale=-2:720"; +/** The first of a few qualities that fits HeyGen's inline cap; `args` are ffmpeg's for that quality and output. */ +export function underInlineCap( + dir: string, + name: string, + args: (crf: number, out: string) => string[], + seconds: number, +): Buffer { for (const crf of [23, 30, 36]) { - const out = join(dir, `part-${a}-${crf}.mp4`); - runFfmpeg([ - ...["-y", "-ss", String(a), "-i", info.path, "-t", String(b - a)], - ...["-vf", `${fit},setsar=1,setpts=${slow}*PTS`, "-r", String(HEYGEN_FPS * 2)], - ...(slow === 1 && info.audio ? ["-c:a", "aac", "-b:a", "96k"] : ["-an"]), - ...["-c:v", "libx264", "-crf", String(crf), "-pix_fmt", "yuv420p", out], - ]); + const out = join(dir, `${name}-${crf}.mp4`); + runFfmpeg(args(crf, out)); if (statSync(out).size <= INLINE_BYTES) return readFileSync(out); } throw new Fix( - `a ${round(b - a, 1)} s part stays over HeyGen's 16 MB inline cap`, + `a ${round(seconds, 1)} s part stays over HeyGen's 16 MB inline cap`, "analyze a shorter window with --start/--end", ); } +/** One part as HeyGen reads it: slowed, 720 px on its long side, silent when slowed, under the inline cap. */ +export function prepare( + info: VideoInfo, + [a, b]: [number, number], + slow: number, + dir: string, +): Buffer { + const fit = info.width >= info.height ? "scale=720:-2" : "scale=-2:720"; + return underInlineCap( + dir, + `part-${a}`, + (crf, out) => [ + // -t bounds the slowed output: the part times the slow-down, or the model sees only the first 1/slow of it. + ...["-y", "-ss", String(a), "-i", info.path, "-t", String((b - a) * slow)], + ...["-vf", `${fit},setsar=1,setpts=${slow}*PTS`, "-r", String(HEYGEN_FPS * 2)], + ...(slow === 1 && info.audio ? ["-c:a", "aac", "-b:a", "96k"] : ["-an"]), + ...["-c:v", "libx264", "-crf", String(crf), "-pix_fmt", "yuv420p", out], + ], + b - a, + ); +} + +export interface Asked { + answer: unknown; + job: string; + usage: unknown; +} + +/** One clip and one question to HeyGen's video model; the same request picks up the same job. */ +export async function askVideo( + video: Buffer, + prompt: string, + schema: unknown, + opts: { + model?: string; + oauth?: boolean; + what: string; + project?: string; + onWait?: (job: Job) => void; + }, +): Promise { + const body = { + model: opts.model ?? MODEL, + video: { type: "base64", media_type: "video/mp4", data: video.toString("base64") }, + prompt, + response_schema: schema, + max_output_tokens: 32768, + }; + const job = await runJob( + ROUTE, + body, + idempotencyKey(ROUTE, [video, prompt, JSON.stringify(schema), body.model]), + (id) => `${ROUTE}/${encodeURIComponent(id)}`, + { log: { projectDir: opts.project, what: opts.what }, onWait: opts.onWait }, + ).catch((err: unknown) => { + if (err instanceof Fix && /refused the credential/.test(err.message) && opts.oauth) + throw new Fix( + "HeyGen's video analysis is not available with this sign-in yet", + "continue without analyze: measure the video and look at its frames", + ); + throw err; + }); + const output = job.output; + const answer = + typeof output === "object" && output !== null && "json" in output ? output.json : output; + return { answer, job: job.id, usage: job.usage ?? null }; +} + function defaultOut(video: string): string { const stem = basename(video, extname(video)); return existsSync(".reverse") @@ -92,7 +159,7 @@ function defaultOut(video: string): string { } /** The project a job log belongs to: the nearest folder up from here with index.html. */ -function projectHere(): string | undefined { +export function projectHere(): string | undefined { for (let dir = resolve("."); ; dir = dirname(dir)) { if (existsSync(join(dir, "index.html"))) return dir; if (dirname(dir) === dir) return undefined; @@ -172,52 +239,23 @@ export default defineCommand({ `Video: ${info.width}x${info.height}.`, ]; const prompt = [question, ...notes].join("\n\n"); - const body = { - model: String(args.model ?? MODEL), - video: { type: "base64", media_type: "video/mp4", data: video.toString("base64") }, - prompt, - response_schema: schema, - max_output_tokens: 32768, - }; console.error( `analyze: part ${i + 1}/${parts.length}, ${round(part[0], 2)}–${round(part[1], 2)} s, slowed ${round(slow, 2)}x`, ); - const job = await runJob( - ROUTE, - body, - idempotencyKey(ROUTE, [video, prompt, JSON.stringify(schema), body.model]), - (id) => `${ROUTE}/${encodeURIComponent(id)}`, - { - log: { - projectDir: project, - what: `analyze ${basename(info.path)} ${round(part[0], 2)}-${round(part[1], 2)}s`, - }, - onWait: (j) => console.error(`analyze: part ${i + 1} ${j.status}…`), - }, - ).catch((err: unknown) => { - if ( - err instanceof Fix && - /refused the credential/.test(err.message) && - credential.type === "oauth" - ) - throw new Fix( - "HeyGen's video analysis is not available with this sign-in yet", - "continue without analyze: measure the video and look at its frames", - ); - throw err; + const asked = await askVideo(video, prompt, schema, { + model: args.model === undefined ? undefined : String(args.model), + oauth: credential.type === "oauth", + project, + what: `analyze ${basename(info.path)} ${round(part[0], 2)}-${round(part[1], 2)}s`, + onWait: (j) => console.error(`analyze: part ${i + 1} ${j.status}…`), }); - const output = job.output; - const answer = - typeof output === "object" && output !== null && "json" in output - ? output.json - : output; - answers.push(toFilmTime(answer, slow, part[0])); + answers.push(toFilmTime(asked.answer, slow, part[0])); done.push({ start: round(part[0], 3), end: round(part[1], 3), slow: round(slow, 2), - job: job.id, - usage: job.usage ?? null, + job: asked.job, + usage: asked.usage, }); } } finally { diff --git a/packages/cli/src/commands/diff.ts b/packages/cli/src/commands/diff.ts index bcc4f7745d..411b680472 100644 --- a/packages/cli/src/commands/diff.ts +++ b/packages/cli/src/commands/diff.ts @@ -1,5 +1,6 @@ +import { statSync } from "node:fs"; import { basename, extname, join, resolve } from "node:path"; -import { defineCommand } from "../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "./_examples.js"; import { c } from "../ui/colors.js"; import { failCommand, setCommandExitCode } from "../utils/commandResult.js"; @@ -10,10 +11,16 @@ import { DEFAULT_MIN_MEAN, DEFAULT_MIN_P05, diffVideos, + findLedger, type DiffResult, } from "../measure/diff.js"; -import { Fix, formatJson, outDir, parseNumbers, parseRange } from "../measure/media.js"; +import { Fix, formatJson, outDir, parseNumbers, parseRange, probeVideo } from "../measure/media.js"; import { readSegments } from "../measure/segments.js"; +import { tryResolveCredential } from "../auth/resolver.js"; +import { attachDiagnoses, explainTargets, targetsOf } from "../heygen/explain.js"; +import { projectHere } from "./analyze.js"; +import { repeated } from "./media-use/generate.js"; +import { parseMask } from "../heygen/generate.js"; export const examples: Example[] = [ [ @@ -28,10 +35,18 @@ export const examples: Example[] = [ "Score chosen moments of a project without rendering it", "hyperframes diff .reverse/source.mp4 . --at 0.5,2.4,3.1", ], + [ + "Score one drawn element (a title's box) by itself at chosen moments of a project", + "hyperframes diff .reverse/source.mp4 . --at 1,1.5,2 --drawn 120,670,485,115", + ], [ "Judge each REVERSE.md segment of the final render and write a side-by-side", "hyperframes diff .reverse/source.mp4 renders/final.mp4 --segments REVERSE.md --sbs", ], + [ + "Also ask HeyGen's video model what differs in each failing drawn element (a lead, not a verdict)", + "hyperframes diff .reverse/source.mp4 renders/final.mp4 --segments REVERSE.md --explain", + ], ]; function number(value: unknown, flag: string): number | undefined { @@ -40,6 +55,31 @@ function number(value: unknown, flag: string): number | undefined { return n; } +/** `x,y,w,h` or `x,y,w,h,a-b` in the original's pixels and seconds. */ +function parseDrawn(text: string): ReturnType { + try { + return parseMask(text); + } catch { + throw new Fix(`--drawn ${text} is not x,y,w,h[,a-b]`, "--drawn 120,670,485,115,0-2.77"); + } +} + +function personLine(p: NonNullable[number]["person"] & object): string { + const off = [ + ...(p.framing ?? []) + .filter((f) => !f.pass) + .map( + (f) => `framing off ${f.window[0]}–${f.window[1]} s (shift ${f.shift}, size ×${f.scale})`, + ), + ...(p.text ?? []).filter((t) => t.carried).map((t) => `${t.clip} carries ${t.drawn}'s text`), + ...(p.sync ?? []) + .filter((x) => x.pass === false) + .map((x) => `lips ${x.lag_ms} ms off at ${x.window[0]} s`), + ]; + const moved = p.energy ? `, motion ×${p.energy.mean_ratio} (reported)` : ""; + return `person ${p.pass === null ? "—" : p.pass ? "✓" : "✗"}${off.length ? ` ${off.join("; ")}` : ""}${moved}`; +} + function printHuman(r: DiffResult) { const mark = r.pass ? c.success("✓ pass") : c.error("✗ fail"); console.log( @@ -49,8 +89,16 @@ function printHuman(r: DiffResult) { console.log( ` missing content in ${Math.round(r.missing_content.share * 100)}% of frames: ${r.missing_content.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}`, ); + if (r.missing_content.texture.share > 0) + console.log( + ` missing texture (reported, not failed) in ${Math.round(r.missing_content.texture.share * 100)}% of frames: ${r.missing_content.texture.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}`, + ); for (const m of r.worst) console.log(` worst ${m.t}s ssim ${m.ssim} cell ${m.cell.roi.join(",")} at ${m.cell.ssim}`); + if (r.whole_frame_reuse?.spans.length) + console.log( + ` whole frame of the original in ${Math.round(r.whole_frame_reuse.share * 100)}% of frames${r.whole_frame_reuse.pass ? "" : `, with its drawn things at ${r.whole_frame_reuse.drawn.map(([a, b]) => `${a}–${b} s`).join(", ")} (a copy)`}`, + ); if (r.cuts) console.log( ` cuts: ${r.cuts.reference.length} in the original${r.cuts.pass ? ", all matched" : `; missing ${r.cuts.missing.join(", ") || "none"}, extra ${r.cuts.extra.join(", ") || "none"}`}`, @@ -63,16 +111,54 @@ function printHuman(r: DiffResult) { console.log( ` sound: ${"offset_ms" in r.sound ? (r.sound.offset_ms === null ? (r.sound.note ?? "not aligned") : `${r.sound.offset_ms} ms off`) : `original ${r.sound.original ? "has" : "has no"} sound, yours ${r.sound.yours ? "has" : "has no"}`}${r.sound.pass ? "" : " (fails)"}`, ); + if (r.components) + console.log( + ` components: ${r.components.passed} of ${r.components.total} pass${r.components.mean === null ? "" : `, mean ${r.components.mean}`}`, + ); + if (r.person) { + const g = r.person.gates; + const mark = (v: boolean | null, failed: string[]) => + v === null ? "unmeasured" : v ? "ok" : `fails in ${failed.join(", ")}`; + console.log( + ` person: framing ${mark(g.framing, r.person.failed.framing)}, text in the clip ${mark(g.text, r.person.failed.text)}, lips ${mark(g.sync, r.person.failed.sync)}; reported: motion ×${r.person.energy ?? "?"}, ${r.person.shots_at_cuts.at_cuts} of ${r.person.shots_at_cuts.total} clips start at a cut${r.person.shots_at_cuts.cuts_without_shot.length ? `, no new clip at ${r.person.shots_at_cuts.cuts_without_shot.join(", ")} s` : ""}`, + ); + } console.log(` every frame: ${r.frames}`); for (const s of r.segments ?? []) { const state = s.pass === null ? c.dim("—") : s.pass ? c.success("✓") : c.error("✗"); - const detail = s.ssim - ? `ssim ${s.ssim.mean} / p05 ${s.ssim.p05} / cell ${s.ssim.cell_p05}` - : s.energy - ? `motion ×${s.energy.mean_ratio}, luma ×${s.look?.luma_ratio}` - : (s.note ?? ""); - console.log(` ${state} ${s.id} ${s.window[0]}–${s.window[1]} s ${s.pixels}: ${detail}`); + const detail = + [ + s.ssim && `ssim ${s.ssim.mean} / p05 ${s.ssim.p05} / cell ${s.ssim.cell_p05}`, + s.energy && `motion ×${s.energy.mean_ratio}, luma ×${s.look?.luma_ratio}`, + s.person && personLine(s.person), + ] + .filter(Boolean) + .join("; ") || + (s.note ?? ""); + const region = s.regenerated ? `; person in ${s.regenerated.roi.join(",")}` : ""; + console.log( + ` ${state} ${s.id} ${s.window[0]}–${s.window[1]} s ${s.pixels}${s.regenerated ? " outside the region" : ""}: ${detail}${region}`, + ); } + for (const d of r.drawn ?? []) { + const state = d.pass === null ? c.dim("—") : d.pass ? c.success("✓") : c.error("✗"); + const detail = + d.pass === null + ? "no ink found in the box (crop it tight around the letters)" + : `score ${d.score} / p05 ${d.p05}, off by ${d.offset!.join(",")} px${d.ink === undefined ? "" : `, ink ×${d.ink}`}`; + console.log( + ` ${state} ${d.id} ${d.box.join(",")} ${d.window[0]}–${d.window[1]} s: ${detail}`, + ); + } + for (const s of r.segments ?? []) + for (const [id, d] of [ + ...(s.drawn ?? []).map((e) => [e.id, e.diagnosis] as const), + [s.id, s.worst_cell?.diagnosis] as const, + ]) + if (d) + console.log( + ` ${c.dim("?")} ${id} ${d.window.map(([a, b]) => `${a}–${b} s`).join(", ")} (unverified): ${d.matches ? "matches" : d.differences.map((x) => `${x.aspect}: original ${x.original}; yours ${x.yours}`).join(" · ") || "no answer"}`, + ); for (const w of r.warnings) console.log(` ${c.warn("⚠")} ${w}`); if (r.sheet) console.log(` sheet ${r.sheet}`); if (r.sbs) console.log(` side by side ${r.sbs}`); @@ -99,6 +185,16 @@ export default defineCommand({ description: "REVERSE.md: a verdict per `## S01 · a–b s` segment (Pixels: regenerated compares motion and look)", }, + ledger: { + type: "string", + description: + "The project's media ledger, to find whole-frame cuts of the original (default: .media/manifest.jsonl beside the project or REVERSE.md)", + }, + project: { + type: "string", + description: + "The project folder, to find the ledger's generated clips and check them for the original's text (default: the folder holding .media)", + }, "min-mean": { type: "string", description: `Mean SSIM to pass (default ${DEFAULT_MIN_MEAN})` }, "min-p05": { type: "string", @@ -122,22 +218,63 @@ export default defineCommand({ type: "string", description: "Render-ready timeout in ms for a project folder (default 5000)", }, + explain: { + type: "boolean", + description: + "Ask HeyGen's video model what differs in each failing drawn element (and failing segment's worst cell); experimental, needs a HeyGen API key; an unverified lead", + default: false, + }, + "explain-only": { + type: "string", + description: + "Ask about these element or segment ids instead, passing or not, e.g. S02.D6,S04", + }, + drawn: { + type: "string", + description: + "A drawn element's box x,y,w,h in the original's pixels, optionally its seconds a-b (repeatable): scored by itself whatever is behind it, with its offset and ink (yours over the original's: under 1 is lighter, thinner or smaller)", + }, json: { type: "boolean", description: "Print one JSON result", default: false }, }, - async run({ args }) { + async run({ args, rawArgs }) { const json = args.json === true; try { if (args.at !== undefined && args.range !== undefined) throw new Fix("--at and --range both given", "pass one of them"); const yours = resolve(String(args.yours)); const stem = basename(yours, extname(yours)) || "project"; + const segmentsFile = args.segments === undefined ? undefined : resolve(String(args.segments)); + const only = + args["explain-only"] === undefined + ? undefined + : String(args["explain-only"]) + .split(",") + .map((id) => id.trim()) + .filter(Boolean); + const explain = args.explain === true || only !== undefined; + const credential = explain ? await tryResolveCredential() : undefined; + if (explain && !credential) + throw new Fix( + "--explain needs a HeyGen sign-in", + "set HEYGEN_API_KEY, or diff without --explain and look at the failing boxes' frames", + ); + if (explain && (segmentsFile === undefined || statSync(yours).isDirectory())) + throw new Fix( + "--explain reads a render's failing elements", + "diff a rendered video with --segments REVERSE.md --explain", + ); const result = await diffVideos({ reference: resolve(String(args.reference)), yours, at: args.at === undefined ? undefined : parseNumbers(args.at, "--at"), range: args.range === undefined ? undefined : parseRange(args.range), - segments: - args.segments === undefined ? undefined : readSegments(resolve(String(args.segments))), + segments: segmentsFile === undefined ? undefined : readSegments(segmentsFile), + ledger: + args.ledger === undefined + ? findLedger(yours, segmentsFile) + : resolve(String(args.ledger)), + project: args.project === undefined ? undefined : resolve(String(args.project)), + drawn: repeated(rawArgs, "--drawn").map(parseDrawn), thresholds: { mean: number(args["min-mean"], "--min-mean"), p05: number(args["min-p05"], "--min-p05"), @@ -149,6 +286,20 @@ export default defineCommand({ timeoutMs: number(args.timeout, "--timeout"), log: (message) => console.error(message), }); + if (explain) { + const targets = targetsOf(result, only); + console.error(`diff: asking HeyGen about ${targets.length} elements`); + attachDiagnoses( + result, + await explainTargets(targets, { + reference: probeVideo(result.reference), + yours: probeVideo(result.yours), + oauth: credential!.type === "oauth", + project: projectHere(), + log: (message) => console.error(message), + }), + ); + } if (json) console.log(formatJson(withMeta({ ok: true, ...result }))); else printHuman(result); if (!result.pass) setCommandExitCode(1); diff --git a/packages/cli/src/commands/measure.ts b/packages/cli/src/commands/measure.ts index b923e09784..7888f8d3d3 100644 --- a/packages/cli/src/commands/measure.ts +++ b/packages/cli/src/commands/measure.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "./_examples.js"; export const examples: Example[] = [ diff --git a/packages/cli/src/commands/measure/box.ts b/packages/cli/src/commands/measure/box.ts index c3116bbd3e..899e6a7fa8 100644 --- a/packages/cli/src/commands/measure/box.ts +++ b/packages/cli/src/commands/measure/box.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { num, diff --git a/packages/cli/src/commands/measure/continuity.ts b/packages/cli/src/commands/measure/continuity.ts index df8e97141f..53d200c75c 100644 --- a/packages/cli/src/commands/measure/continuity.ts +++ b/packages/cli/src/commands/measure/continuity.ts @@ -1,5 +1,5 @@ import { resolve } from "node:path"; -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, num, outArg, runMeasure } from "../../measure/command.js"; import { continuity } from "../../measure/continuity.js"; diff --git a/packages/cli/src/commands/measure/curve.ts b/packages/cli/src/commands/measure/curve.ts index ce8e61c1c7..0afceaa9ac 100644 --- a/packages/cli/src/commands/measure/curve.ts +++ b/packages/cli/src/commands/measure/curve.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, diff --git a/packages/cli/src/commands/measure/cuts.ts b/packages/cli/src/commands/measure/cuts.ts index 53ac6747f8..e148bca0b6 100644 --- a/packages/cli/src/commands/measure/cuts.ts +++ b/packages/cli/src/commands/measure/cuts.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, runMeasure, videoArg, window } from "../../measure/command.js"; import { findCuts, signatures } from "../../measure/cuts.js"; diff --git a/packages/cli/src/commands/measure/energy.ts b/packages/cli/src/commands/measure/energy.ts index 681b1f10a7..73595da846 100644 --- a/packages/cli/src/commands/measure/energy.ts +++ b/packages/cli/src/commands/measure/energy.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, num, runMeasure } from "../../measure/command.js"; import { compareEnergy, energyCurve } from "../../measure/energy.js"; diff --git a/packages/cli/src/commands/measure/events.ts b/packages/cli/src/commands/measure/events.ts index 5067562731..067e30cb5f 100644 --- a/packages/cli/src/commands/measure/events.ts +++ b/packages/cli/src/commands/measure/events.ts @@ -1,6 +1,6 @@ import { writeFileSync } from "node:fs"; import { join } from "node:path"; -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, diff --git a/packages/cli/src/commands/measure/extract.ts b/packages/cli/src/commands/measure/extract.ts index 9822b79606..6f6c847180 100644 --- a/packages/cli/src/commands/measure/extract.ts +++ b/packages/cli/src/commands/measure/extract.ts @@ -4,7 +4,7 @@ import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, num, roiArg, runMeasure, videoAndRoi, videoArg } from "../../measure/command.js"; import { Fix, frameAt, round, runFfmpeg, type Roi, type VideoInfo } from "../../measure/media.js"; diff --git a/packages/cli/src/commands/measure/face.ts b/packages/cli/src/commands/measure/face.ts index 20679162db..d4c7cfcff0 100644 --- a/packages/cli/src/commands/measure/face.ts +++ b/packages/cli/src/commands/measure/face.ts @@ -1,14 +1,24 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, num, runMeasure, videoArg, window } from "../../measure/command.js"; -import { faceBoxes, faceSummary } from "../../measure/face.js"; -import { probeVideo } from "../../measure/media.js"; +import { + FIT_MIN_FACES, + againstStart, + faceBoxes, + faceFit, + faceSummary, +} from "../../measure/face.js"; +import { Fix, probeVideo, round } from "../../measure/media.js"; export const examples: Example[] = [ [ "How big and where the presenter's face is in a shot", "hyperframes measure face ref.mp4 --start 4 --end 9.5", ], + [ + "The scale and x, y that put a generated clip's face on the original's face over 2-6 s", + "hyperframes measure face presenter.mp4 --against ref.mp4 --from 2 --to 6", + ], ]; export default defineCommand({ @@ -19,29 +29,77 @@ export default defineCommand({ }, args: { video: videoArg, - start: { type: "string", description: "First second (default 0)" }, - end: { type: "string", description: "Last second (default: the end)" }, + start: { + type: "string", + description: + "First second (default 0; with --against, 0 for a shot of its own and --from for a render as long as the original)", + }, + end: { + type: "string", + description: + "Last second (default: the end; with --against, --start plus the length of --from/--to)", + }, samples: { type: "string", - description: "Frames sampled, evenly, the last one included (default 12)", + description: "Frames sampled, evenly, the last one included (default 8)", }, cast: { type: "boolean", - description: "The person is one face in a crowd: grade the smallest face", + description: + "The person is one face in a crowd: grade the smallest face (and look for faces down to 20 px)", default: false, }, + against: { + type: "string", + description: + "The original video: print the GSAP scale and x, y (canvas pixels) that put this clip's face on the original's over --from/--to", + }, + from: { type: "string", description: "With --against: the original's first second" }, + to: { type: "string", description: "With --against: the original's last second" }, json: jsonArg, }, async run({ args }) { await runMeasure(async () => { const info = probeVideo(String(args.video)); + const cast = args.cast === true; + const samples = Math.max(1, num(args.samples, "--samples", 8)); + const opts = cast ? { minFace: 0 } : {}; + if (args.against !== undefined) { + const ref = probeVideo(String(args.against)); + const [a, b] = window(ref, args.from, args.to, ["--from", "--to"]); + const s = num(args.start, "--start", againstStart(info.duration, ref.duration, a)); + const e = Math.min(info.duration, s + (b - a)); + if (!(e > s)) + throw new Fix(`--start ${s} is past this clip's end`, `--start 0 (or leave it out)`); + const [fo, fc] = [ + await faceBoxes(ref, a, b, samples, opts), + await faceBoxes(info, s, e, samples, opts), + ]; + const rel = (f: { t: number; boxes: number[][] }[], t0: number) => + f.map((x) => ({ t: x.t - t0, boxes: x.boxes })); + const fit = faceFit(rel(fo, a), rel(fc, s), ref, info, cast); + return { + video: info.path, + against: ref.path, + window: [round(s, 3), round(e, 3)], + against_window: [a, b], + canvas: [ref.width, ref.height], + ...fit, + ...(fit.transform === null + ? { + note: `fewer than ${FIT_MIN_FACES} moments with a face in both: try a wider window or more --samples`, + } + : {}), + reading: + "gsap.set(clip, { scale, x, y }) on a clip element that fills the canvas (object-fit: cover) with the default transform origin (its centre); x, y and the faces' centre and height are in the canvas's (the original's) pixels, clip_face before the transform; residual is the median face-centre miss left after it, in face heights", + }; + } const [a, b] = window(info, args.start, args.end); - const samples = await faceBoxes(info, a, b, Math.max(1, num(args.samples, "--samples", 12))); return { video: info.path, start: a, end: b, - ...faceSummary(samples, args.cast === true), + ...faceSummary(await faceBoxes(info, a, b, samples, opts), cast), reading: "face_height is the face's height over the frame's; boxes are x, y, w, h as shares of the frame", }; diff --git a/packages/cli/src/commands/measure/flow.ts b/packages/cli/src/commands/measure/flow.ts index 930cc0df5f..db98c48436 100644 --- a/packages/cli/src/commands/measure/flow.ts +++ b/packages/cli/src/commands/measure/flow.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, diff --git a/packages/cli/src/commands/measure/frames.ts b/packages/cli/src/commands/measure/frames.ts index 2324b10bc7..2a27ed3842 100644 --- a/packages/cli/src/commands/measure/frames.ts +++ b/packages/cli/src/commands/measure/frames.ts @@ -1,6 +1,6 @@ import { mkdtempSync, readdirSync, renameSync, rmSync } from "node:fs"; import { join } from "node:path"; -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, diff --git a/packages/cli/src/commands/measure/ink.ts b/packages/cli/src/commands/measure/ink.ts index 822015e9e7..67c9b7961c 100644 --- a/packages/cli/src/commands/measure/ink.ts +++ b/packages/cli/src/commands/measure/ink.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, num, runMeasure, videoAndRoi, videoArg, window } from "../../measure/command.js"; import { scan, summary } from "../../measure/ink.js"; diff --git a/packages/cli/src/commands/measure/look.ts b/packages/cli/src/commands/measure/look.ts index 0a52a690d7..599336bbc0 100644 --- a/packages/cli/src/commands/measure/look.ts +++ b/packages/cli/src/commands/measure/look.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, diff --git a/packages/cli/src/commands/measure/objects.ts b/packages/cli/src/commands/measure/objects.ts index 90d94f67a0..7c3d4377d2 100644 --- a/packages/cli/src/commands/measure/objects.ts +++ b/packages/cli/src/commands/measure/objects.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, num, roiArg, runMeasure, videoAndRoi, videoArg } from "../../measure/command.js"; import { objects } from "../../measure/objects.js"; diff --git a/packages/cli/src/commands/measure/probe.ts b/packages/cli/src/commands/measure/probe.ts index 027dd8c45a..850a101857 100644 --- a/packages/cli/src/commands/measure/probe.ts +++ b/packages/cli/src/commands/measure/probe.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, runMeasure, videoArg } from "../../measure/command.js"; import { diff --git a/packages/cli/src/commands/measure/rhythm.ts b/packages/cli/src/commands/measure/rhythm.ts index 4926a32f26..0d057203eb 100644 --- a/packages/cli/src/commands/measure/rhythm.ts +++ b/packages/cli/src/commands/measure/rhythm.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, diff --git a/packages/cli/src/commands/measure/sync.ts b/packages/cli/src/commands/measure/sync.ts index 6bf4e7f14c..e00bbaded5 100644 --- a/packages/cli/src/commands/measure/sync.ts +++ b/packages/cli/src/commands/measure/sync.ts @@ -1,4 +1,4 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, diff --git a/packages/cli/src/commands/measure/track.ts b/packages/cli/src/commands/measure/track.ts index 61896753ef..e31f802993 100644 --- a/packages/cli/src/commands/measure/track.ts +++ b/packages/cli/src/commands/measure/track.ts @@ -1,5 +1,5 @@ import { join } from "node:path"; -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, num, outArg, runMeasure, videoArg } from "../../measure/command.js"; import { Fix, outDir, parseRoi, probeVideo, round, runFfmpeg } from "../../measure/media.js"; diff --git a/packages/cli/src/commands/measure/trail.ts b/packages/cli/src/commands/measure/trail.ts index 893ef83fa5..30ab059a58 100644 --- a/packages/cli/src/commands/measure/trail.ts +++ b/packages/cli/src/commands/measure/trail.ts @@ -2,7 +2,7 @@ // onion skin (earliest faintest), the moving part's path with numbered points, and a heat map of where the picture // changed. One look tells a slide from a scale from a mask reveal; `keyframes --shot` does the same for a project. import { join } from "node:path"; -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, diff --git a/packages/cli/src/commands/media-use.ts b/packages/cli/src/commands/media-use.ts index aec0c1a2bf..f3eb26a050 100644 --- a/packages/cli/src/commands/media-use.ts +++ b/packages/cli/src/commands/media-use.ts @@ -96,7 +96,6 @@ function subcommand(name: MediaUseVerb, invoke: InvokeMediaUse) { export function createMediaUseCommand(invoke: InvokeMediaUse = invokeEngine) { const subCommands = { ...Object.fromEntries(MEDIA_USE_VERBS.map((name) => [name, () => subcommand(name, invoke)])), - // Made on HeyGen through the hyperframes sign-in, not the resolve engine. generate: () => import("./media-use/generate.js").then((m) => m.default), }; return defineCommand({ diff --git a/packages/cli/src/commands/media-use/generate.ts b/packages/cli/src/commands/media-use/generate.ts index 9f38b821b6..66b51622ea 100644 --- a/packages/cli/src/commands/media-use/generate.ts +++ b/packages/cli/src/commands/media-use/generate.ts @@ -1,10 +1,12 @@ -import { defineCommand } from "../../utils/defineCommand.js"; +import { defineCommand } from "citty"; import type { Example } from "../_examples.js"; import { jsonArg, num, runMeasure } from "../../measure/command.js"; import { Fix } from "../../measure/media.js"; import { GENERATE_TYPES, + checkGuide, generate, + parseMask, type GenerateRequest, type GenerateType, } from "../../heygen/generate.js"; @@ -21,7 +23,7 @@ export const examples: Example[] = [ ], [ "The original's presenter re-shot clean, saying an 8 s stretch of the original voice", - 'hyperframes media-use generate --type video --reference assets/presenter-clean.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --prompt "the woman talks to camera, seated, soft daylight" --seconds 8 --aspect 9:16 --out assets/s02-presenter.mp4', + 'hyperframes media-use generate --type video --reference assets/presenter-clean.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --mask 110,615,390,165 --prompt "the woman talks to camera, seated, soft daylight" --seconds 8 --aspect 9:16 --out assets/s02-presenter.mp4', ], [ "The person in a photo saying a script in a chosen voice", @@ -31,6 +33,17 @@ export const examples: Example[] = [ const isType = (v: string): v is GenerateType => GENERATE_TYPES.some((t) => t === v); +/** Every value of a flag given more than once (citty keeps only the last). */ +export function repeated(raw: string[], flag: string): string[] { + return raw.flatMap((a, i) => + a === flag && raw[i + 1] !== undefined + ? [raw[i + 1]!] + : a.startsWith(`${flag}=`) + ? [a.slice(flag.length + 1)] + : [], + ); +} + export default defineCommand({ meta: { name: "generate", @@ -62,6 +75,11 @@ export default defineCommand({ type: "string", description: "video: a clip whose camera and body motion the shot follows (up to 15 s)", }, + mask: { + type: "string", + description: + "video: x,y,w,h[,a-b] of the --motion clip to blank before sending (its drawn text; a-b in the clip's seconds); repeat for more", + }, "first-frame": { type: "string", description: "video: the picture the shot starts from" }, photo: { type: "string", description: "avatar: the person's photo" }, script: { type: "string", description: "avatar, voice: the words spoken" }, @@ -88,7 +106,7 @@ export default defineCommand({ }, json: jsonArg, }, - async run({ args }) { + async run({ args, rawArgs }) { await runMeasure(async () => { const type = String(args.type); if (!isType(type)) @@ -104,6 +122,7 @@ export default defineCommand({ firstFrame: args["first-frame"], audio: args.audio, motion: args.motion, + masks: repeated(rawArgs, "--mask").map(parseMask), photo: args.photo, script: args.script, voice: args.voice, @@ -114,7 +133,8 @@ export default defineCommand({ instrumental: !args.vocals, }; const price = priceOf(request); - if (args.plan) + if (args.plan) { + checkGuide(request); return { plan: true, type, @@ -122,6 +142,7 @@ export default defineCommand({ price, next: "after the person agrees to this price, run the same command without --plan", }; + } const made = await generate(request, (status) => console.error(`generate: ${status}…`)); return { ...made, price }; }); diff --git a/packages/cli/src/commands/snapshot.ts b/packages/cli/src/commands/snapshot.ts index 0d70a74a67..0de51249a7 100644 --- a/packages/cli/src/commands/snapshot.ts +++ b/packages/cli/src/commands/snapshot.ts @@ -706,7 +706,6 @@ export async function captureSnapshots( review?: boolean; /** Progress lines; `--json` sends them to stderr so stdout stays one envelope. */ log?: (message: string) => void; - /** False for working frames another command reads and discards: nothing enters project history. */ record?: boolean; }, ): Promise { diff --git a/packages/cli/src/heygen/explain.test.ts b/packages/cli/src/heygen/explain.test.ts new file mode 100644 index 0000000000..ff17f74393 --- /dev/null +++ b/packages/cli/src/heygen/explain.test.ts @@ -0,0 +1,140 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; +import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; + +vi.mock("../cloud/auth.js", () => ({ + resolveCloudAuthHeaders: async () => ({ "x-api-key": "k" }), + resolveCloudBaseUrl: () => "https://api.test", +})); + +const { attachDiagnoses, explainQuestion, explainTargets, layoutOf, piecesOf, targetsOf } = + await import("./explain.js"); +const { diffVideos } = await import("../measure/diff.js"); +const { parseSegments } = await import("../measure/segments.js"); +const { probeVideo } = await import("../measure/media.js"); +const ffmpeg = findFFmpeg(); +const noTools = !ffmpeg || !findFFprobe(); +const dir = mkdtempSync(join(tmpdir(), "hf-explain-test-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); +afterEach(() => vi.unstubAllGlobals()); + +const reply = (body: unknown) => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); + +describe("explain layout", () => { + it("stacks a wide box, puts a tall one side by side, and keeps the picture near-square and large", () => { + const wide = layoutOf([147, 590, 786, 150], 1080, 1920); + expect(wide.layout).toBe("original above yours"); + expect(wide.crop).toEqual([140, 584, 798, 162]); + expect(Math.max(...wide.size)).toBe(1200); + expect(Math.max(...wide.size) / Math.min(...wide.size)).toBeLessThanOrEqual(1.26); + // The element spans the picture's width. + expect(wide.size[0]).toBe(1200); + const tall = layoutOf([108, 174, 60, 118], 720, 1280); + expect(tall.layout).toBe("original left of yours"); + expect(tall.filter).toContain("hstack"); + // A small box is scaled up, never down to a thumbnail. + expect(Math.min(...tall.size)).toBeGreaterThan(600); + }); + + it("widens a short window a little and asks a long one at its two ends", () => { + expect(piecesOf([2.64, 3.44], 36)).toEqual([[2.39, 3.69]]); + expect(piecesOf([11.14, 19.69], 36)).toEqual([ + [10.89, 12.89], + [17.94, 19.94], + ]); + expect(piecesOf([0, 1], 1)).toEqual([[0, 1]]); + }); +}); + +describe.skipIf(noTools)("diff --explain", () => { + it("asks the fixed question about each failing drawn element at native size and files the answer under it", async () => { + // Three white strokes over a plain blue picture; yours draws them 24 px to the right. + const strokes = (x0: number) => + [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(","); + const make = (name: string, x0: number) => { + const out = join(dir, name); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=0x2050a0:s=320x180:r=30:d=3"], + ...["-vf", strokes(x0), "-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "0", out], + ]); + return out; + }; + const [ref, yours] = [make("ref.mp4", 120), make("off.mp4", 144)]; + const md = + "## S01 · 0–3 s · Label over a presenter\nPixels: regenerated\nRegenerated: 0, 0, 320, 180\nDrawn: 118, 108, 42, 28, 1.0–2.0 s\nDrawn: 10, 10, 40, 20, 0–1 s\n"; + const result = await diffVideos({ + reference: ref, + yours, + segments: parseSegments(md), + outDir: join(dir, "diff"), + }); + const targets = targetsOf(result); + // The empty second box passes, so only the label is asked about. + expect(targets).toEqual([{ id: "S01.D1", box: [118, 108, 42, 28], window: [1, 2] }]); + expect(targetsOf(result, ["S01.D2"]).map((t) => t.id)).toEqual(["S01.D2"]); + + const sent: { url: string; body?: Record }[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + sent.push({ url, ...(init?.body ? { body: JSON.parse(String(init.body)) } : {}) }); + // The clip is slowed 6x, so 4.5 s in it is 0.75 s after the window starts. + return reply({ + data: { + id: "a1", + status: "completed", + usage: { input_tokens: 812, output_tokens: 90 }, + output: { + json: { + matches: false, + differences: [ + { aspect: "position", original: "left", yours: "right", start: 4.5, end: 9 }, + ], + }, + }, + }, + }); + }); + const found = await explainTargets(targets, { + reference: probeVideo(ref), + yours: probeVideo(yours), + }); + expect(sent).toHaveLength(1); + expect(sent[0]!.url).toBe("https://api.test/v3/video-analyses"); + const body = sent[0]!.body!; + const prompt = String(body.prompt); + expect(prompt.startsWith(explainQuestion("original above yours"))).toBe(true); + // Neither the score nor the verdict reaches the model. + expect(prompt).not.toMatch(/score|ssim|fail|pass|0\.\d/i); + // The video is the two crops, upscaled near-square, slowed for 2 fps. + const video = join(dir, "sent.mp4"); + const sentVideo = body.video; + if (typeof sentVideo !== "object" || sentVideo === null || !("data" in sentVideo)) + throw new Error("no video sent"); + writeFileSync(video, Buffer.from(String(sentVideo.data), "base64")); + const info = probeVideo(video); + expect([info.width, info.height]).toEqual(layoutOf([118, 108, 42, 28], 320, 180).size); + expect(info.duration).toBeGreaterThan(8); + expect(info.audio).toBe(false); + + attachDiagnoses(result, found); + const d = result.segments![0]!.drawn![0]!; + expect(d.diagnosis).toMatchObject({ + unverified: true, + window: [[0.75, 2.25]], + matches: false, + differences: [ + { aspect: "position", original: "left", yours: "right", start: 1.5, end: 2.25 }, + ], + layout: "original above yours", + input_tokens: 812, + jobs: ["a1"], + }); + expect(result.segments![0]!.drawn![1]!.diagnosis).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/heygen/explain.ts b/packages/cli/src/heygen/explain.ts new file mode 100644 index 0000000000..e32d3a13ca --- /dev/null +++ b/packages/cli/src/heygen/explain.ts @@ -0,0 +1,270 @@ +// `diff --explain`: pixels find which drawn element fails, HeyGen's video model says what differs in it. Its route +// samples at most 2 fps at low resolution, so each frame is a thumbnail: the element's box is cut from both videos at +// native resolution, the original above yours (left of yours for a tall box), padded to near-square and scaled until +// the element is about 600 px, over a short window slowed so 2 fps sees about 12 fps. The question is fixed and never +// carries the score, the verdict or a guess at the cause, which would prime the answer. An answer is a lead: unverified +// until the frames and the pixels confirm it. +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { askVideo, HEYGEN_FPS, slowBy, underInlineCap } from "../commands/analyze.js"; +import type { DiffResult } from "../measure/diff.js"; +import { round, type Roi, type VideoInfo } from "../measure/media.js"; +import { toFilmTime } from "./reverseDraft.js"; + +// Pixels kept around the box, the grey line between the two sides, the widest picture sent, and its long side. +const MARGIN = 6; +const GAP = 8; +const MAX_ASPECT = 1.25; +const LONG_SIDE = 1200; +// The window is widened this much each side to see the element arrive and leave; longer than MAX_S, its first and +// last PIECE_S seconds are asked apart. Frames a second the model should see. +const LEAD_S = 0.25; +const MAX_S = 4; +const PIECE_S = 2; +const EXPLAIN_FPS = 12; +const PARALLEL = 4; + +/** What a difference is about: how the element looks, or how it moves. */ +export const APPEARANCE = [ + "words", + "font", + "weight", + "size", + "position", + "colour", + "material", + "doubled", +]; +export const MOTION = ["appears", "changes", "leaves", "timing"]; + +const str = { type: "string" }; +const num = { type: "number" }; +export const EXPLAIN_SCHEMA = { + type: "object", + properties: { + matches: { type: "boolean" }, + differences: { + type: "array", + items: { + type: "object", + properties: { + aspect: { type: "string", enum: [...APPEARANCE, ...MOTION, "other"] }, + original: str, + yours: str, + start: num, + end: num, + }, + required: ["aspect", "original", "yours", "start", "end"], + }, + }, + }, + required: ["matches", "differences"], +}; + +export type Layout = "original above yours" | "original left of yours"; + +/** The one question, worded for where each side sits. */ +export function explainQuestion(layout: Layout): string { + const [first, second] = layout === "original above yours" ? ["TOP", "BOTTOM"] : ["LEFT", "RIGHT"]; + return `${first} is the original video, ${second} is a recreation of the same moment. Both are cut to the same box around one drawn element: text, a number, a badge or a card. The person, the scenery and anything behind the element were remade on purpose: ignore them. + +Compare only the drawn element. List each way the ${second} one differs from the ${first} one: the words or letters, font, weight or italic, size, position, colour, material (solid, gradient, glass, chrome, outline, glow, shadow), doubled or ghosted letters, how it appears, how it changes while on screen, how it leaves, and timing. Say what each side shows. If the drawn element matches, set matches to true and list nothing. Report only what you can see in the frames.`; +} + +/** The box with a margin, on even pixels inside the frame. */ +function cropOf([x, y, w, h]: Roi, fw: number, fh: number): Roi { + const even = (v: number) => Math.floor(v / 2) * 2; + const x0 = even(Math.max(0, x - MARGIN)); + const y0 = even(Math.max(0, y - MARGIN)); + return [ + x0, + y0, + Math.max(2, even(Math.min(fw, x + w + MARGIN) - x0)), + Math.max(2, even(Math.min(fh, y + h + MARGIN) - y0)), + ]; +} + +/** How a box is laid out for the model: both sides, padded to near-square, scaled to LONG_SIDE. */ +export function layoutOf( + box: Roi, + fw: number, + fh: number, +): { crop: Roi; layout: Layout; size: [number, number]; filter: string } { + const crop = cropOf(box, fw, fh); + const [, , w, h] = crop; + const above = w >= h; + const [sw, sh] = above ? [w, 2 * h + GAP] : [2 * w + GAP, h]; + const even = (v: number) => Math.ceil(v / 2) * 2; + const [pw, ph] = [even(Math.max(sw, sh / MAX_ASPECT)), even(Math.max(sh, sw / MAX_ASPECT))]; + const k = LONG_SIDE / Math.max(pw, ph); + const size: [number, number] = [even(pw * k), even(ph * k)]; + const cut = `crop=${w}:${h}:${crop[0]}:${crop[1]},setsar=1`; + const filter = [ + `[0:v]${cut},pad=${above ? `${w}:${h + GAP}` : `${w + GAP}:${h}`}:0:0:color=gray[a]`, + `[1:v]scale=${fw}:${fh},${cut}[b]`, + `[a][b]${above ? "vstack" : "hstack"},pad=${pw}:${ph}:(ow-iw)/2:(oh-ih)/2:color=black,scale=${size[0]}:${size[1]}:flags=lanczos`, + ].join(";"); + return { crop, layout: above ? "original above yours" : "original left of yours", size, filter }; +} + +/** The film seconds asked about: the window widened a little, or its two ends when it is long. */ +export function piecesOf([a, b]: [number, number], duration: number): [number, number][] { + const [s, e] = [Math.max(0, a - LEAD_S), Math.min(duration, b + LEAD_S)]; + if (e - s <= MAX_S) return [[round(s, 3), round(e, 3)]]; + return [ + [round(s, 3), round(s + PIECE_S, 3)], + [round(e - PIECE_S, 3), round(e, 3)], + ]; +} + +export interface Difference { + aspect: string; + original: string; + yours: string; + start?: number; + end?: number; +} + +export interface Diagnosis { + /** A lead from a video model, never a verdict: look at the box's frames and confirm with pixels before fixing. */ + unverified: true; + /** The film seconds the model watched (one or two pieces). */ + window: [number, number][]; + matches: boolean | null; + differences: Difference[]; + layout: Layout; + /** The picture sent, in pixels, and the input tokens the route counted (low ones mean it saw thumbnails). */ + size: [number, number]; + input_tokens: number | null; + jobs: string[]; +} + +export interface Target { + id: string; + box: Roi; + window: [number, number]; +} + +/** What `--explain` asks about: every failing drawn element and failing SSIM segment's worst cell, or those named. */ +export function targetsOf(result: DiffResult, only?: string[]): Target[] { + const wanted = (id: string, pass: boolean | null) => (only ? only.includes(id) : pass === false); + return (result.segments ?? []).flatMap((s) => [ + ...(s.drawn ?? []) + .filter((d) => wanted(d.id, d.pass)) + .map((d) => ({ id: d.id, box: d.box, window: d.window })), + ...(s.worst_cell && wanted(s.id, s.pass) + ? [ + { + id: s.id, + box: s.worst_cell.roi, + window: [ + Math.max(s.window[0], s.worst_cell.t - PIECE_S / 2), + Math.min(s.window[1], s.worst_cell.t + PIECE_S / 2), + ] as [number, number], + }, + ] + : []), + ]); +} + +const isRecord = (v: unknown): v is Record => + typeof v === "object" && v !== null && !Array.isArray(v); + +export interface ExplainOptions { + reference: VideoInfo; + yours: VideoInfo; + model?: string; + oauth?: boolean; + project?: string; + log?: (message: string) => void; +} + +/** One target asked piece by piece. */ +async function explainOne(t: Target, o: ExplainOptions, dir: string): Promise { + const { layout, size, filter } = layoutOf(t.box, o.reference.width, o.reference.height); + const pieces = piecesOf(t.window, Math.min(o.reference.duration, o.yours.duration)); + const differences: Difference[] = []; + const matches: boolean[] = []; + const jobs: string[] = []; + let tokens: number | null = null; + for (const [n, [a, b]] of pieces.entries()) { + const slow = slowBy(EXPLAIN_FPS, b - a); + const video = underInlineCap( + dir, + `${t.id}-${n}`, + (crf, out) => [ + ...["-y", "-ss", String(a), "-t", String(b - a), "-i", o.reference.path], + ...["-ss", String(a), "-t", String(b - a), "-i", o.yours.path], + ...["-filter_complex", `${filter},setpts=${slow}*PTS`, "-r", String(HEYGEN_FPS * 2)], + ...["-an", "-c:v", "libx264", "-crf", String(crf), "-pix_fmt", "yuv420p", out], + ], + b - a, + ); + const prompt = [ + explainQuestion(layout), + `This clip is ${round((b - a) * slow, 2)} s long${slow > 1 ? `: it plays at 1/${round(slow, 2)} speed and has no sound; judge motion as if at normal speed` : ""}. Give times in seconds of this clip.`, + ].join("\n\n"); + o.log?.(`explain: ${t.id} ${a}–${b} s`); + const asked = await askVideo(video, prompt, EXPLAIN_SCHEMA, { + model: o.model, + oauth: o.oauth, + project: o.project, + what: `diff --explain ${basename(o.yours.path)} ${t.id} ${a}-${b}s`, + }); + jobs.push(asked.job); + const used = isRecord(asked.usage) ? asked.usage.input_tokens : undefined; + if (typeof used === "number") tokens = (tokens ?? 0) + used; + const answer = toFilmTime(asked.answer, slow, a); + if (!isRecord(answer)) continue; + if (typeof answer.matches === "boolean") matches.push(answer.matches); + for (const d of Array.isArray(answer.differences) ? answer.differences : []) + if (isRecord(d)) + differences.push({ + aspect: String(d.aspect ?? "other"), + original: String(d.original ?? ""), + yours: String(d.yours ?? ""), + ...(typeof d.start === "number" ? { start: d.start } : {}), + ...(typeof d.end === "number" ? { end: d.end } : {}), + }); + } + return { + unverified: true, + window: pieces, + matches: matches.length ? matches.every(Boolean) && !differences.length : null, + differences, + layout, + size, + input_tokens: tokens, + jobs, + }; +} + +/** Each target's diagnosis, a few at a time, by id. */ +export async function explainTargets( + targets: Target[], + o: ExplainOptions, +): Promise> { + const out = new Map(); + const dir = mkdtempSync(join(tmpdir(), "hf-explain-")); + try { + const queue = [...targets]; + await Promise.all( + Array.from({ length: Math.min(PARALLEL, queue.length) }, async () => { + for (let t = queue.shift(); t; t = queue.shift()) + out.set(t.id, await explainOne(t, o, dir)); + }), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + return out; +} + +/** Each diagnosis written under its box in the result. */ +export function attachDiagnoses(result: DiffResult, found: Map): void { + for (const s of result.segments ?? []) { + for (const d of s.drawn ?? []) d.diagnosis = found.get(d.id) ?? d.diagnosis; + if (s.worst_cell) s.worst_cell.diagnosis = found.get(s.id) ?? s.worst_cell.diagnosis; + } +} diff --git a/packages/cli/src/heygen/generate.test.ts b/packages/cli/src/heygen/generate.test.ts index 9b649e2243..d9fcc1a132 100644 --- a/packages/cli/src/heygen/generate.test.ts +++ b/packages/cli/src/heygen/generate.test.ts @@ -10,7 +10,7 @@ vi.mock("../cloud/auth.js", () => ({ resolveCloudBaseUrl: () => "https://api.test", })); -const { generate, urlOf } = await import("./generate.js"); +const { generate, parseMask, urlOf } = await import("./generate.js"); const { priceOf } = await import("./pricing.js"); const ffmpeg = findFFmpeg(); const project = mkdtempSync(join(tmpdir(), "hf-generate-test-")); @@ -20,6 +20,17 @@ afterEach(() => vi.unstubAllGlobals()); const json = (body: unknown) => new Response(JSON.stringify(body), { status: 200 }); +/** A 30 fps clip of these lavfi sources one after another: each new source is a cut. */ +function shots(name: string, sources: string[], size = "180x320"): string { + const path = join(project, name); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y"], + ...sources.flatMap((src) => ["-f", "lavfi", "-i", `${src}:s=${size}:r=30`]), + ...["-filter_complex", `concat=n=${sources.length}:v=1`, "-pix_fmt", "yuv420p", path], + ]); + return path; +} + describe("generate", () => { it("finds the file URL wherever the route keeps it", () => { expect( @@ -135,6 +146,146 @@ describe("generate", () => { ).rejects.toThrow(/--first-frame cannot be used/); }); + it.skipIf(!ffmpeg)("blanks the motion clip's masked regions before sending it", async () => { + // A guide with white letters at 40,100 over a moving picture; the mask covers them for its first 0.5 s. + const guide = join(project, "titled.mp4"); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=d=1:s=180x320", "-vf"], + "drawbox=x=40:y=100:w=100:h=10:color=white:t=fill,drawbox=x=40:y=120:w=100:h=10:color=white:t=fill", + guide, + ]); + const face = join(project, "face2.png"); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=gray:s=36x64", "-frames:v", "1"], + face, + ]); + const sent: Record[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + if (init?.body) sent.push(JSON.parse(String(init.body))); + if (url === "https://files.test/m.mp4") return new Response(new Uint8Array([7])); + return json({ + data: { id: "v3", status: "completed", video_url: "https://files.test/m.mp4" }, + }); + }); + await generate({ + type: "video", + out: join(project, "assets/masked.mp4"), + prompt: "walks", + references: [face], + motion: guide, + masks: [parseMask("30,90,120,50,0-0.5")], + }); + const [video] = sent[0]!.reference_videos as { data: string }[]; + const sentClip = join(project, "sent.mp4"); + writeFileSync(sentClip, Buffer.from(video!.data, "base64")); + // The letters are two white rows of bars: the largest step between neighbouring rows' brightness is their edge. + // Filled from around, no such step is left; after 0.5 s they are back. + const edge = (t: number) => { + const px = execFileSync(ffmpeg!, [ + ...["-v", "error", "-ss", String(t), "-i", sentClip, "-frames:v", "1"], + ...["-vf", "crop=100:40:40:95,format=gray", "-f", "rawvideo", "-"], + ]); + const rows = Array.from({ length: 40 }, (_, y) => px.subarray(y * 100, y * 100 + 100)); + const mean = rows.map((r) => r.reduce((a, v) => a + v, 0) / r.length); + return Math.max(...mean.slice(1).map((m, y) => Math.abs(m - mean[y]!))); + }; + expect(edge(0.2)).toBeLessThan(30); + expect(edge(0.8)).toBeGreaterThan(60); + expect(readFileSync(join(project, ".media/manifest.jsonl"), "utf8")).toContain( + '"motion_masks":[[30,90,120,50,0,0.5]]', + ); + expect(() => parseMask("30,90,120")).toThrow(/x,y,w,h/); + const { repeated } = await import("../commands/media-use/generate.js"); + expect( + repeated(["--mask", "1,1,2,2", "--out", "a.mp4", "--mask=3,3,4,4,0-1"], "--mask"), + ).toEqual(["1,1,2,2", "3,3,4,4,0-1"]); + await expect( + generate({ type: "video", out: "x.mp4", prompt: "p", masks: [parseMask("1,1,2,2")] }), + ).rejects.toThrow(/--mask blanks regions of the --motion clip/); + }); + + it.skipIf(!ffmpeg)("refuses a guide with a cut inside before sending anything", async () => { + const guide = shots("two-shots.mp4", ["color=c=red:d=1", "testsrc2=d=1"]); + const face = join(project, "face3.png"); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=gray:s=36x64", "-frames:v", "1"], + face, + ]); + const fetched = vi.fn(); + vi.stubGlobal("fetch", fetched); + const request = { + type: "video" as const, + out: join(project, "assets/refused.mp4"), + prompt: "talks", + references: [face], + motion: guide, + }; + await expect(generate(request)).rejects.toMatchObject({ + message: `--motion ${guide} has a cut at 1.00 s`, + fix: "guide has a cut at 1.00 s: generate 0–1.00 s and 1.00–2.00 s as two shots, each with its own --motion clip", + }); + expect(fetched).not.toHaveBeenCalled(); + expect(existsSync(request.out)).toBe(false); + }); + + it.skipIf(!ffmpeg)("reports where HeyGen cut inside the returned shot", async () => { + // The guide's last frame is another picture: a cut within a frame of the end is no second shot, so it is sent. + const guide = shots("stray-frame.mp4", ["testsrc2=d=1", "color=c=blue:d=0.02"]); + const made = shots("heygen-cut.mp4", ["testsrc2=d=1", "color=c=red:d=1"]); + const clean = shots("heygen-clean.mp4", ["testsrc2=d=2"]); + const face = join(project, "face4.png"); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=gray:s=36x64", "-frames:v", "1"], + face, + ]); + let file = made; + const sent: Record[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + if (init?.body) sent.push(JSON.parse(String(init.body))); + if (url === "https://files.test/c.mp4") return new Response(readFileSync(file)); + return json({ + data: { id: `c${sent.length}`, status: "completed", video_url: "https://files.test/c.mp4" }, + }); + }); + const request = { type: "video" as const, prompt: "walks", references: [face], motion: guide }; + const cut = await generate({ ...request, out: join(project, "assets/cut.mp4") }); + expect(sent).toHaveLength(1); + expect(cut.cuts_inside).toEqual([1]); + expect(readFileSync(join(project, ".media/manifest.jsonl"), "utf8")).toContain( + '"cuts_inside":[1]', + ); + file = clean; + const one = await generate({ ...request, out: join(project, "assets/one.mp4"), seconds: 6 }); + expect(one.cuts_inside).toEqual([]); + }); + + it.skipIf(!ffmpeg)( + "asks for 768p whatever the source's size, as 1080p bills about twice", + async () => { + const guide = shots("tall.mp4", ["testsrc2=d=1"], "1080x1920"); + const sent: Record[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + if (init?.body) sent.push(JSON.parse(String(init.body))); + if (url === "https://files.test/t.mp4") return new Response(new Uint8Array([7])); + return json({ + data: { id: "t1", status: "completed", video_url: "https://files.test/t.mp4" }, + }); + }); + const r = await generate({ + type: "video", + out: join(project, "assets/tall.mp4"), + prompt: "walks", + motion: guide, + aspect: "9:16", + }); + expect(sent[0]).toMatchObject({ resolution: "768p", aspect_ratio: "9:16" }); + // An unreadable download is kept and its cuts reported as unknown. + expect(r.cuts_inside).toBeNull(); + // A reference shot bills its seconds and the guide's at $0.05: 5 s and the 1 s guide. + expect(priceOf({ type: "video", out: "a.mp4", motion: guide }).usd).toBe(0.3); + }, + ); + it("waits for a queued shot and downloads it", async () => { const answers = [ json({ data: { video_id: "v1", status: "pending" } }), diff --git a/packages/cli/src/heygen/generate.ts b/packages/cli/src/heygen/generate.ts index 36db27bbc2..ecad29fc7a 100644 --- a/packages/cli/src/heygen/generate.ts +++ b/packages/cli/src/heygen/generate.ts @@ -4,7 +4,8 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, extname, join, relative, resolve } from "node:path"; -import { Fix, runFfmpeg } from "../measure/media.js"; +import { findCuts, signatures } from "../measure/cuts.js"; +import { Fix, probeVideo, round, runFfmpeg, videoClip } from "../measure/media.js"; import { idempotencyKey, runJob, type Job } from "./v3.js"; export type GenerateType = "image" | "video" | "avatar" | "voice" | "music"; @@ -26,6 +27,8 @@ export interface GenerateRequest { audio?: string; /** video: a clip whose camera and body motion the shot follows (the original's same stretch, up to 15 s). */ motion?: string; + /** video: regions of the motion clip blanked before it is sent (its drawn text), so the shot does not copy them. */ + masks?: Mask[]; /** video: the shot starts from this picture. */ firstFrame?: string; /** avatar: the photo of the person who speaks. */ @@ -40,6 +43,38 @@ export interface GenerateRequest { bpm?: number; } +/** A region of the motion clip in its pixels, and optionally the seconds of the clip it covers. */ +export interface Mask { + box: [number, number, number, number]; + window?: [number, number]; +} + +/** `x,y,w,h` or `x,y,w,h,a-b` (seconds of the motion clip). */ +export function parseMask(text: string): Mask { + const m = + /^\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:,\s*(\d+(?:\.\d+)?)\s*-\s*(\d+(?:\.\d+)?)\s*)?$/.exec( + text, + ); + if (!m || Number(m[3]) <= 0 || Number(m[4]) <= 0) + throw new Fix(`--mask ${text} is not x,y,w,h[,a-b]`, "--mask 190,405,700,175,1.0-2.7"); + const box: Mask["box"] = [Number(m[1]), Number(m[2]), Number(m[3]), Number(m[4])]; + return m[5] ? { box, window: [Number(m[5]), Number(m[6])] } : { box }; +} + +/** + * Filters that fill each mask's region from the pixels around it (ffmpeg delogo), so no letter survives and the + * motion around it stays. delogo needs a pixel of picture on every side, so boxes are kept 1 px inside the frame. + */ +export function maskFilters(masks: Mask[], width: number, height: number): string[] { + return masks.flatMap(({ box: [x, y, w, h], window }) => { + const [x0, y0] = [Math.max(1, x), Math.max(1, y)]; + const [x1, y1] = [Math.min(width - 2, x + w), Math.min(height - 2, y + h)]; + if (x1 - x0 < 1 || y1 - y0 < 1) return []; + const when = window ? `:enable='between(t,${window[0]},${window[1]})'` : ""; + return [`delogo=x=${x0}:y=${y0}:w=${x1 - x0}:h=${y1 - y0}${when}`]; + }); +} + interface Route { path: string; poll: (id: string) => string; @@ -56,6 +91,9 @@ type Media = ( ) => { type: "base64"; media_type: string; data: string }; const DEFAULTS = { image: "nano-banana-2", video: "heygen-video-1", music: "lyria-3-pro" } as const; +export const VIDEO_RESOLUTION = "768p"; +/** HeyGen follows at most this many seconds of a guide clip; the rest is not sent. */ +export const GUIDE_MAX_S = 15; const MUSIC_LENGTHS = [30, 60, 120, 180]; const need = (value: T | undefined, flag: string, example: string): T => { @@ -97,7 +135,9 @@ export const ROUTES: Record = { mode: spoken ? "reference_to_video" : r.firstFrame ? "image_to_video" : "text_to_video", prompt: need(r.prompt, "--prompt", '--prompt "slow push in on the desk"'), duration: Math.min(15, Math.max(5, Math.round(r.seconds ?? 5))), - resolution: "768p", + // 1080p bills about twice 768p (2026-10-23, 5 s 9:16 jobs: text $0.10 vs $0.20, a reference picture $0.25 + // vs $0.45, a 1 s guide clip $0.30 vs $0.55), so every shot stays at 768p whatever the source's size. + resolution: VIDEO_RESOLUTION, ...(r.firstFrame ? { image: media(r.firstFrame, "image") } : {}), ...(r.firstFrame ? {} : { aspect_ratio: r.aspect ?? "16:9" }), ...(r.references?.length @@ -172,8 +212,11 @@ export function urlOf(job: Job, paths: string[]): string | null { return null; } -/** A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small. */ -function inline(work: string): Media { +/** + * A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion + * clip with its masks blanked. + */ +function inline(work: string, masks: Mask[] = []): Media { return (path, kind) => { if (!existsSync(path)) throw new Fix(`no file at ${path}`, "pass a picture or sound from the project"); @@ -197,12 +240,17 @@ function inline(work: string): Media { else if (kind === "audio") runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]); // A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s. - else - runFfmpeg([ - ...["-y", "-i", path, "-t", "15", "-an", "-vf"], + else { + const { width, height } = masks.length ? probeVideo(path) : { width: 0, height: 0 }; + const filters = [ + ...maskFilters(masks, width, height), "scale='if(gt(iw,ih),min(720,iw),-2)':'if(gt(iw,ih),-2,min(720,ih))'", + ]; + runFfmpeg([ + ...["-y", "-i", path, "-t", "15", "-an", "-vf", filters.join(",")], ...["-c:v", "libx264", "-crf", "28", "-preset", "veryfast", "-pix_fmt", "yuv420p", out], ]); + } return { type: "base64", media_type: { image: "image/jpeg", audio: "audio/mpeg", video: "video/mp4" }[kind], @@ -211,6 +259,42 @@ function inline(work: string): Media { }; } +/** + * The seconds where a new shot starts in a video's first `seconds`, and the length read. A cut within a frame of + * either end leaves a stray frame, not a second shot, and does not count. + */ +export function cutsInside(path: string, seconds?: number): { cuts: number[]; length: number } { + const info = probeVideo(path); + const clip = videoClip(info.path, 0, seconds, info); + const { frames } = signatures(clip.input, clip.filters); + const cuts = findCuts(frames) + .filter((c) => c.frame > 1 && c.frame < frames.length - 1) + .map((c) => round(c.frame / info.fps, 3)); + return { cuts, length: round(frames.length / info.fps, 3) }; +} + +const COUNT = ["", "one", "two", "three", "four", "five", "six"]; + +/** + * Refuses a --motion clip holding more than one shot before anything is sent: HeyGen follows the guide's cut into + * a cut of its own, so each shot is its own generation. + */ +export function checkGuide(r: Pick): void { + if (!r.motion) return; + const { cuts, length } = cutsInside(r.motion, GUIDE_MAX_S); + if (!cuts.length) return; + const s = (t: number) => t.toFixed(2); + const and = (items: string[]) => `${items.slice(0, -1).join(", ")} and ${items.at(-1)}`; + const at = + cuts.length > 1 ? `cuts at ${and(cuts.map((t) => `${s(t)} s`))}` : `a cut at ${s(cuts[0]!)} s`; + const edges = [0, ...cuts, length]; + const shots = edges.slice(1).map((e, i) => `${i ? s(edges[i]!) : "0"}–${s(e)} s`); + throw new Fix( + `--motion ${r.motion} has ${at}`, + `guide has ${at}: generate ${and(shots)} as ${COUNT[shots.length] ?? shots.length} shots, each with its own --motion clip`, + ); +} + /** The nearest folder at or above `from` holding index.html. */ function projectOf(from: string): string | null { for (let dir = resolve(from); ; dir = dirname(dir)) { @@ -225,6 +309,8 @@ export interface Generated { job: string; route: string; ledger?: string; + /** video: seconds where HeyGen cut inside the returned shot; null when the file could not be read. */ + cuts_inside?: number[] | null; } export async function generate( @@ -239,7 +325,10 @@ export async function generate( const project = projectOf(dirname(out)); const work = mkdtempSync(join(tmpdir(), "hf-generate-")); try { - const body = route.body(r, inline(work)); + if (r.masks?.length && !r.motion) + throw new Fix("--mask blanks regions of the --motion clip", "pass --motion with --mask"); + checkGuide(r); + const body = route.body(r, inline(work, r.masks)); const key = idempotencyKey(route.path, [JSON.stringify(body)]); const job = await runJob(route.path, body, key, (id) => route.poll(encodeURIComponent(id)), { log: { @@ -262,6 +351,14 @@ export async function generate( "run the same command again; it re-reads the job", ); writeFileSync(out, Buffer.from(await res.arrayBuffer())); + // HeyGen sometimes cuts inside a shot on its own; the shot is paid for and kept, the cuts are reported. + let cuts: number[] | null | undefined; + if (r.type === "video") + try { + cuts = cutsInside(out).cuts; + } catch { + cuts = null; + } let ledger: string | undefined; if (project) { const { recordInPlace } = await import("../media-use/lib/manifest.mjs"); @@ -284,12 +381,23 @@ export async function generate( ...(r.photo ? { photo: describe(r.photo) } : {}), ...(r.audio ? { audio: describe(r.audio) } : {}), ...(r.motion ? { motion: describe(r.motion) } : {}), + ...(r.masks?.length + ? { motion_masks: r.masks.map((m) => [...m.box, ...(m.window ?? [])]) } + : {}), ...(r.voice ? { voice: r.voice } : {}), + ...(cuts !== undefined ? { cuts_inside: cuts } : {}), }, }).id; regenerateIndex(project); } - return { out, type: r.type, job: job.id, route: route.path, ...(ledger ? { ledger } : {}) }; + return { + out, + type: r.type, + job: job.id, + route: route.path, + ...(ledger ? { ledger } : {}), + ...(cuts !== undefined ? { cuts_inside: cuts } : {}), + }; } finally { rmSync(work, { recursive: true, force: true }); } diff --git a/packages/cli/src/heygen/pricing.ts b/packages/cli/src/heygen/pricing.ts index 5105bf4299..27faeddc10 100644 --- a/packages/cli/src/heygen/pricing.ts +++ b/packages/cli/src/heygen/pricing.ts @@ -1,7 +1,8 @@ // Prices before a paid call, from the HyperFrames app's creation pricing (hyperframes-internal // packages/desktop/shared/creationPricing.mjs, 2026-10-22). HeyGen charges in units of $1/60, rounded up; the label // rounds up to the cent. Routes HeyGen does not price ahead say so instead of guessing. -import type { GenerateRequest } from "./generate.js"; +import { probeVideo, round } from "../measure/media.js"; +import { GUIDE_MAX_S, type GenerateRequest } from "./generate.js"; export interface Price { usd: number | null; @@ -21,6 +22,9 @@ const GPT_IMAGE: Record = { }; const NANO_IMAGE = 0.067; const VIDEO_768P_PER_S = 0.018; +// A reference shot (pictures, voice or guide clip) bills more, and its guide clip's seconds bill too: measured +// 2026-10-23 from the wallet, 5 s with a picture $0.25, 10 s $0.50, 5 s with a 1 s guide $0.30, with a 3 s guide $0.40. +const VIDEO_REF_768P_PER_S = 0.05; const MUSIC_PER_TRACK = 0.08; const AVATAR_IV_PER_MIN = 4.83; // An avatar's length is estimated from its script, as the app does. @@ -40,6 +44,13 @@ export function priceOf(r: GenerateRequest): Price { } case "video": { const seconds = Math.min(15, Math.max(5, Math.round(r.seconds ?? 5))); + if (r.audio || r.references?.length || r.motion) { + const guide = r.motion ? Math.min(GUIDE_MAX_S, probeVideo(r.motion).pictureDuration) : 0; + return { + usd: cents((seconds + guide) * VIDEO_REF_768P_PER_S), + basis: `heygen-video-1 reference shot at 768p, ${seconds} s${guide ? ` and the ${round(guide, 2)} s guide` : ""} at $0.05 a second`, + }; + } return { usd: cents(seconds * VIDEO_768P_PER_S), basis: `heygen-video-1 at 768p, ${seconds} s at $0.018 a second`, diff --git a/packages/cli/src/measure/cuts.test.ts b/packages/cli/src/measure/cuts.test.ts new file mode 100644 index 0000000000..87102d0185 --- /dev/null +++ b/packages/cli/src/measure/cuts.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { confirmsCut, cutsIn, findCuts, SIG_H, SIG_W } from "./cuts.js"; + +/** Frame changes: `base` everywhere, with the given changes at their frames. */ +const changes = (n: number, base: number, at: Record) => + Array.from({ length: n }, (_, i) => (i === 0 ? 0 : (at[i] ?? base))); + +describe("cutsIn", () => { + it("finds a hard cut and a two-frame whip as one cut", () => { + expect(cutsIn(changes(60, 1, { 20: 40, 40: 30, 41: 25 }))).toEqual([ + { frame: 20, frames: 1, change: 40 }, + { frame: 40, frames: 2, change: 30 }, + ]); + }); + + it("finds a soft same-angle punch-in that nothing near it rivals", () => { + // dan 9.743 s: 11.3 grey levels against neighbours of about 1. + expect(cutsIn(changes(60, 1, { 30: 11.3 }))).toEqual([ + { frame: 30, frames: 1, change: 11.3, soft: true }, + ]); + }); + + it("leaves a smaller change among moving frames or a stepped animation alone", () => { + // Hands moving: neighbours up to 5, so 11 is not alone. + expect(cutsIn(changes(60, 1, { 25: 5, 30: 11 }))).toEqual([]); + // A 15 fps animation in a 30 fps file changes every other frame by about as much. + const stepped = Array.from({ length: 60 }, (_, i) => (i % 2 ? 0 : 10.5)); + expect(cutsIn(stepped)).toEqual([]); + expect(cutsIn(changes(60, 1, { 30: 9 }))).toEqual([]); + }); +}); + +describe("confirmsCut", () => { + it("matches a soft cut with a clear change within a frame on the other side", () => { + const cut = { frame: 30, frames: 1, change: 11.3, soft: true as const }; + expect(confirmsCut(changes(60, 1, { 31: 7 }), cut)).toBe(true); + expect(confirmsCut(changes(60, 1, { 32: 7 }), cut)).toBe(false); + expect(confirmsCut(changes(60, 1, { 30: 4 }), cut)).toBe(false); + expect(confirmsCut(changes(60, 2, { 30: 7 }), cut)).toBe(false); + }); +}); + +describe("findCuts", () => { + it("reads changes from grey frames", () => { + const frame = (v: number) => new Uint8Array(SIG_W * SIG_H).fill(v); + const frames = [ + ...Array.from({ length: 20 }, () => frame(100)), + ...Array.from({ length: 20 }, () => frame(140)), + ]; + expect(findCuts(frames)).toEqual([{ frame: 20, frames: 1, change: 40 }]); + }); +}); diff --git a/packages/cli/src/measure/cuts.ts b/packages/cli/src/measure/cuts.ts index 9b7a615e9b..8481701cbb 100644 --- a/packages/cli/src/measure/cuts.ts +++ b/packages/cli/src/measure/cuts.ts @@ -1,6 +1,7 @@ // Cuts from the frame-to-frame difference itself, ported from the Studio's videoShots.mjs: ffmpeg's scene score // missed two of the James Blake source's cuts at 0.25, so a cut is a frame far unlike the one before and several -// times unlike its neighbours' changes. +// times unlike its neighbours' changes. A punch-in at the same angle changes less (dan's 9.743 s: 11 grey levels), +// so a smaller change also counts when nothing else near it changes anywhere near as much: a soft cut. import { decode, meanAbsDiff, type Frames } from "./media.js"; // Small enough to read a minute quickly, large enough to tell a cut from motion. @@ -8,6 +9,25 @@ export const SIG_W = 64; export const SIG_H = 36; const CUT_LEVEL = 12; const CUT_RATIO = 4; +// A soft cut: at least SOFT_LEVEL, SOFT_RATIO times its neighbours' median and SOFT_ALONE times the largest of them. +const SOFT_LEVEL = 10; +const SOFT_RATIO = 8; +const SOFT_ALONE = 4; +// The other side confirms a soft cut with a change of CONFIRM_LEVEL, CUT_RATIO times its neighbours' median and +// CONFIRM_SHARE of the cut's own change, within a frame of it. +const CONFIRM_LEVEL = 6; +const CONFIRM_SHARE = 0.5; +const NEIGHBOURS = 12; + +export interface Cut { + frame: number; + /** A whip or a two-frame dissolve changes on consecutive frames: one cut this many frames long. */ + frames: number; + /** Its largest change, in mean grey levels. */ + change: number; + /** Found only by the soft rule. */ + soft?: true; +} /** Every frame of ffmpeg's `input` (after `filters`) as SIG_W x SIG_H grey levels. */ export function signatures(input: string[], filters: string[] = []): Frames { @@ -19,26 +39,54 @@ const median = (values: number[]) => { return sorted[Math.floor(sorted.length / 2)] ?? 0; }; -/** - * The frame indexes where a new shot starts, each with its change (mean grey levels) and length: a whip or a - * two-frame dissolve changes on consecutive frames, which is one cut `frames` long, not a shot of one frame. - */ -export function findCuts( - frames: Uint8Array[], -): { frame: number; frames: number; change: number }[] { - const diffs = frames.map((f, i) => (i ? meanAbsDiff(frames[i - 1]!, f) : 0)); - const hits = diffs.flatMap((d, i) => { - if (i === 0 || d < CUT_LEVEL) return []; - const around = diffs.slice(Math.max(1, i - 12), i).concat(diffs.slice(i + 1, i + 13)); - return d >= CUT_RATIO * Math.max(1, median(around)) ? [{ frame: i, change: d }] : []; +/** Each frame's mean grey difference from the one before (0 for the first). */ +export function frameChanges(frames: Uint8Array[]): number[] { + return frames.map((f, i) => (i ? meanAbsDiff(frames[i - 1]!, f) : 0)); +} + +const neighbours = (diffs: number[], i: number) => + diffs.slice(Math.max(1, i - NEIGHBOURS), i).concat(diffs.slice(i + 1, i + NEIGHBOURS + 1)); + +/** The frame indexes where a new shot starts, from each frame's change (`frameChanges`). */ +export function cutsIn(diffs: number[]): Cut[] { + const hits = diffs.flatMap((d, i): { frame: number; change: number; soft?: true }[] => { + if (i === 0) return []; + const around = neighbours(diffs, i); + const typical = Math.max(1, median(around)); + if (d >= CUT_LEVEL && d >= CUT_RATIO * typical) return [{ frame: i, change: d }]; + const alone = d >= SOFT_ALONE * Math.max(0, ...around); + return d >= SOFT_LEVEL && d >= SOFT_RATIO * typical && alone + ? [{ frame: i, change: d, soft: true }] + : []; }); - const out: { frame: number; frames: number; change: number }[] = []; - for (const h of hits) { + const out: Cut[] = []; + for (const { soft, ...h } of hits) { const last = out.at(-1); if (last && h.frame === last.frame + last.frames) { last.frames++; last.change = Math.max(last.change, h.change); - } else out.push({ ...h, frames: 1 }); + if (!soft) delete last.soft; + } else out.push({ ...h, frames: 1, ...(soft ? { soft } : {}) }); } return out; } + +/** The frame indexes where a new shot starts, each with its change and length. */ +export function findCuts(frames: Uint8Array[]): Cut[] { + return cutsIn(frameChanges(frames)); +} + +/** + * Whether these changes show a clear change within a frame of `frame`, though maybe under a cut's thresholds: a + * soft cut found on the other side counts as matched by it, as a soft cut is a change near the line. + */ +export function confirmsCut(diffs: number[], cut: Cut): boolean { + return [cut.frame - 1, cut.frame, cut.frame + 1].some((i) => { + const d = diffs[i]; + if (i < 1 || d === undefined) return false; + return ( + d >= Math.max(CONFIRM_LEVEL, CONFIRM_SHARE * cut.change) && + d >= CUT_RATIO * Math.max(1, median(neighbours(diffs, i))) + ); + }); +} diff --git a/packages/cli/src/measure/diff.test.ts b/packages/cli/src/measure/diff.test.ts index 56070fa1a2..f25a7536c5 100644 --- a/packages/cli/src/measure/diff.test.ts +++ b/packages/cli/src/measure/diff.test.ts @@ -1,10 +1,21 @@ import { execFileSync } from "node:child_process"; -import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, describe, expect, it } from "vitest"; import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; -import { diffVideos, pickedFrames, selectFilter } from "./diff.js"; +import { + cellsOutside, + diffVideos, + longRuns, + pickedFrames, + regionRoi, + selectFilter, + soundOffset, + wholeFrameCuts, +} from "./diff.js"; +import { probeVideo } from "./media.js"; +import { carriedFrame, shotFraming, syncPass } from "./person.js"; import { parseSegments } from "./segments.js"; const ffmpeg = findFFmpeg(); @@ -42,6 +53,145 @@ describe("parseSegments", () => { { id: "S04", start: 6.5, end: 8, title: "", pixels: "rebuilt" }, ]); }); + + it("reads a regenerated region inside a rebuilt segment, in pixels or shares", () => { + const md = [ + "## S01 · 0–2 s · Board under the presenter", + "Pixels: rebuilt", + "Regenerated: 0, 95, 720, 600 (the re-shot presenter)", + "## S02 · 2–4 s", + "Regenerated: 0 0.5 1 0.5", + "## S03 · 4–6 s", + "Pixels: regenerated", + "Regenerated: 0, 0, 10, 10", + ].join("\n"); + const [a, b, c] = parseSegments(md); + expect(a!.regenerated).toEqual([0, 95, 720, 600]); + expect(b!.regenerated).toEqual([0, 0.5, 1, 0.5]); + expect(c!.regenerated).toBeUndefined(); + }); + + it("reads drawn elements over a regenerated person, and whether a segment quotes copy", () => { + const md = [ + "## S01 · 0–2 s · Title over the presenter", + "Pixels: rebuilt", + "Regenerated: 0, 0, 608, 1080", + 'Copy: "IS NO" / "LONGER"', + "Drawn: 110, 615, 390, 165 (the title)", + "Drawn: 40 40 120 120, logo 0.5–1.5 s", + "## S02 · 2–4 s", + "Copy: none, footage only", + ].join("\n"); + const [a, b] = parseSegments(md); + expect(a!.drawn).toEqual([ + { box: [110, 615, 390, 165] }, + { box: [40, 40, 120, 120], window: [0.5, 1.5] }, + ]); + expect(a!.copy).toBe(true); + expect(b!.drawn).toBeUndefined(); + expect(b!.copy).toBeUndefined(); + }); +}); + +describe("whole-frame cuts", () => { + it("keeps the ledger's cuts and stills from the original that cover the whole frame", () => { + const ledger = [ + { + type: "video", + provenance: { provider: "original", window: [0, 12.267], roi: [0, 0, 720, 1280] }, + }, + { type: "video", provenance: { provider: "original", window: [1, 2] } }, + { + type: "video", + provenance: { provider: "original", window: [3, 4], roi: [0, 0, 720, 704] }, + }, + { type: "image", provenance: { provider: "original", at: 3, roi: [0, 0, 720, 1280] } }, + { type: "image", provenance: { provider: "original", at: 5, roi: [60, 260, 480, 560] } }, + { type: "video", provenance: { provider: "heygen", window: [0, 5] } }, + ] + .map((e) => JSON.stringify(e)) + .join("\n"); + expect(wholeFrameCuts(`${ledger}\nnot json`, 720, 1280)).toEqual({ + windows: [ + [0, 12.267], + [1, 2], + ], + stills: [3], + }); + }); + + it("drops runs of frames shorter than the minimum", () => { + const on = [true, true, false, true, true, true, true, true]; + expect(longRuns(on, [0, 1, 2, 3, 4, 5, 9, 10], 3)).toEqual([ + false, + false, + false, + true, + true, + true, + false, + false, + ]); + }); +}); + +describe("regenerated regions", () => { + it("scales shares to pixels, clips to the frame and leaves the covered cells to the region", () => { + expect(regionRoi([0, 0.5, 1, 0.5], 320, 180)).toEqual([0, 90, 320, 90]); + expect(regionRoi([600, 1200, 400, 200], 720, 1280)).toEqual([600, 1200, 120, 80]); + // 80x44 cells: the bottom half covers rows 2 and 3 (row 2 from y 88, all but 2 px of it). + expect(cellsOutside([0, 90, 320, 90], 320, 180)).toEqual([0, 1, 2, 3, 4, 5, 6, 7]); + }); +}); + +describe("person gates", () => { + const at = (t: number, ...boxes: number[][]) => ({ t, boxes }); + const face = [0.4, 0.3, 0.2, 0.12]; + const original = [0, 0.4, 0.8, 1.2].map((t) => at(t, face)); + + it("fails a shot's framing only when the face sits clearly off the original's", () => { + const moved = (dy: number, grow = 1) => + [0, 0.4, 0.8, 1.2].map((t) => at(t, [0.4, 0.3 + dy, 0.2 * grow, 0.12 * grow])); + expect(shotFraming([0, 1.5], original, moved(0.01))).toMatchObject({ pass: true }); + // Half a face height lower, or 40% bigger. + expect(shotFraming([0, 1.5], original, moved(0.06))).toMatchObject({ pass: false }); + expect(shotFraming([0, 1.5], original, moved(0, 1.4))).toMatchObject({ pass: false }); + // One stray moment off (a hand over the face, a photo) is not a framing. + const stray = original.map((m, i) => (i === 1 ? at(m.t, [0.4, 0.6, 0.2, 0.12]) : m)); + expect(shotFraming([0, 1.5], original, stray)).toMatchObject({ pass: true }); + // Another face in yours (a picture on the wall) does not stand in for the person. + const crowd = [0, 0.4, 0.8, 1.2].map((t) => at(t, [0.05, 0.05, 0.3, 0.2], face)); + expect(shotFraming([0, 1.5], original, crowd)).toMatchObject({ + shift: 0, + pass: true, + }); + // Too few faces to judge. + expect(shotFraming([0, 0.5], original, original)).toBeUndefined(); + }); + + it("judges lips only on a confident reading, inside +45 / −125 ms", () => { + expect(syncPass({ best_lag_ms: 30, confidence: "high" })).toBe(true); + expect(syncPass({ best_lag_ms: -120, confidence: "high" })).toBe(true); + expect(syncPass({ best_lag_ms: 80, confidence: "high" })).toBe(false); + expect(syncPass({ best_lag_ms: -200, confidence: "high" })).toBe(false); + expect(syncPass({ best_lag_ms: -300, confidence: "medium" })).toBeNull(); + }); + + it("finds the original's letters in a generated clip, not the place they share", () => { + // A textured place (the same in the original, the clean still and the clip) and white letters over it. + const [w, h] = [60, 30]; + const place = new Uint8Array(w * h * 3).map((_, i) => 60 + ((i * 37) % 50)); + const lettered = (px: Uint8Array) => { + const out = new Uint8Array(px); + for (let y = 8; y < 22; y++) + for (const x0 of [10, 25, 40]) + for (let x = x0; x < x0 + 4; x++) out.set([255, 255, 255], (y * w + x) * 3); + return out; + }; + const original = lettered(place); + expect(carriedFrame(original, lettered(place), place, w, h, undefined)).toBeGreaterThan(0.9); + expect(carriedFrame(original, place, place, w, h, undefined)).toBeLessThan(0.1); + }); }); describe("frame picks", () => { @@ -119,6 +269,32 @@ describe.skipIf(noTools)("diffVideos", () => { expect(r.pass).toBe(false); const same = await diffVideos({ reference: page, yours: page, outDir: join(dir, "page") }); expect(same.missing_content).toMatchObject({ share: 0, pass: true }); + // A regenerated segment's picture is its own: what it leaves out is not missing. + const regen = await diffVideos({ + reference: page, + yours, + segments: parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n"), + outDir: join(dir, "emptied-regen"), + }); + expect(regen.missing_content).toMatchObject({ share: 0, pass: true }); + }); + + it("reports missing texture apart from missing shapes, without failing it", async () => { + // A fine 4 px checker in the bottom-left cell: flat at an eighth of the width, so texture, not structure. + const checker = + "if(lt(X\\,80)*gte(Y\\,135)\\,if(mod(floor(X/4)+floor(Y/4)\\,2)\\,60\\,200)\\,235)"; + const grain = clip("grain.mp4", [ + ...["-f", "lavfi", "-i", "color=c=white:s=320x180:r=30:d=2", "-vf"], + `format=gray,geq=lum='${checker}',format=yuv420p`, + ]); + const plain = clip("plain.mp4", [ + ...["-f", "lavfi", "-i", "color=c=white:s=320x180:r=30:d=2", "-vf"], + "format=gray,geq=lum=235,format=yuv420p", + ]); + const r = await diffVideos({ reference: grain, yours: plain, outDir: join(dir, "grain") }); + expect(r.missing_content).toMatchObject({ share: 0, pass: true }); + expect(r.missing_content.texture.share).toBeGreaterThan(0.9); + expect(r.warnings.some((w) => w.includes("reported, not failed"))).toBe(true); }); it("scores only a range, on the original's frame numbers", async () => { @@ -155,6 +331,374 @@ describe.skipIf(noTools)("diffVideos", () => { expect(r.pass).toBe(false); }); + it("matches a soft cut in the original with a hard one in yours", async () => { + // Grey 0x80 to 0x8b changes the signature by 10 levels: a soft cut. Yours jumps further on the same frame. + const step = (name: string, to: string, at: number) => + clip(name, [ + ...["-f", "lavfi", "-i", `color=c=0x808080:s=320x180:r=30:d=${at}`], + ...["-f", "lavfi", "-i", `color=c=${to}:s=320x180:r=30:d=${3 - at}`], + ...["-filter_complex", "[0][1]concat=n=2:v=1[v]", "-map", "[v]"], + ]); + const soft = step("soft-ref.mp4", "0x8b8b8b", 1); + const r = await diffVideos({ + reference: soft, + yours: step("soft-hard.mp4", "0xa0a0a0", 1), + outDir: join(dir, "soft"), + }); + expect(r.cuts).toMatchObject({ reference: [1], missing: [], extra: [], pass: true }); + const late = await diffVideos({ + reference: soft, + yours: step("soft-late.mp4", "0xa0a0a0", 1.2), + outDir: join(dir, "soft-late"), + }); + expect(late.cuts).toMatchObject({ missing: [1], extra: [1.2], pass: false }); + }); + + it("judges a rebuilt segment's regenerated region by motion and look, the rest by SSIM", async () => { + // The bottom half mirrored, as a re-shot presenter differs in pixels but moves the same. + const mixed = clip("mixed.mp4", [ + ...["-i", ref, "-filter_complex"], + "[0]split[a][b];[b]crop=320:90:0:90,hflip[f];[a][f]overlay=0:90", + ]); + const md = "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\n"; + const plain = await diffVideos({ + reference: ref, + yours: mixed, + segments: parseSegments(md), + outDir: join(dir, "mixed-plain"), + }); + expect(plain.segments?.[0]?.pass).toBe(false); + const r = await diffVideos({ + reference: ref, + yours: mixed, + segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`), + outDir: join(dir, "mixed"), + }); + const [s] = r.segments!; + expect(s!.ssim!.mean).toBeGreaterThan(0.99); + expect(s!.regenerated?.roi).toEqual([0, 90, 320, 90]); + expect(s!.person?.energy?.mean_ratio).toBeCloseTo(1, 1); + expect(s!.pass).toBe(true); + // A wrong box outside the region still fails the segment. + const boxed = clip("mixed-box.mp4", [ + ...["-i", mixed, "-vf"], + "drawbox=x=20:y=10:w=60:h=40:color=red:t=fill", + ]); + const bad = await diffVideos({ + reference: ref, + yours: boxed, + segments: parseSegments(`${md}Regenerated: 0, 0.5, 1, 0.5\n`), + outDir: join(dir, "mixed-box"), + }); + expect(bad.segments?.[0]?.pass).toBe(false); + }); + + it("judges each drawn element inside a regenerated region by itself, not by what is behind it", async () => { + // A label (three white strokes) over a soft bottom half, a regenerated person (here 5 frames late); yours + // draws it 24 px off. + const strokes = (x0: number) => + [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(","); + const label = strokes(120); + const soft = (delay: number) => + `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`; + const base = clip("label-ref.mp4", [...["-i", ref, "-filter_complex"], `${soft(0)},${label}`]); + const right = clip("label-right.mp4", [ + ...["-i", ref, "-filter_complex"], + `${soft(5)},${label}`, + ]); + const off = clip("label-off.mp4", [ + ...["-i", ref, "-filter_complex"], + `${soft(5)},${strokes(144)}`, + ]); + const md = + "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\n"; + const judge = (yours: string, text: string, name: string) => + diffVideos({ + reference: base, + yours, + segments: parseSegments(text), + outDir: join(dir, name), + }); + // Without a Drawn line the label hides in the person's box. + expect((await judge(off, md, "drawn-hidden")).segments?.[0]?.pass).toBe(true); + const drawn = `${md}Drawn: 118, 108, 42, 28\n`; + const good = (await judge(right, drawn, "drawn-right")).segments![0]!; + expect(good.pass).toBe(true); + expect(good.drawn?.[0]).toMatchObject({ id: "S01.D1", box: [118, 108, 42, 28], pass: true }); + expect(good.scored).toContain("1 drawn elements, each scored by itself"); + const bad = await judge(off, drawn, "drawn-off"); + expect(bad.segments?.[0]?.pass).toBe(false); + expect(bad.segments?.[0]?.drawn?.[0]?.pass).toBe(false); + // A box over the whole frame is flagged. + const whole = await judge(right, md.replace("0, 90, 320, 90", "0, 0, 320, 180"), "drawn-whole"); + expect(whole.warnings.some((w) => w.includes("covers the whole frame"))).toBe(true); + }); + + it("scores a drawn element by itself without segments, and its ink says when yours is thinner", async () => { + // The same three-stroke label over two different flat backdrops; yours draws it at half the stroke width. + const strokes = (w: number) => + [0, 14, 28].map((x) => `drawbox=x=${120 + x}:y=110:w=${w}:h=24:color=white:t=fill`).join(","); + const backdrop = (colour: string) => [ + "-f", + "lavfi", + "-i", + `color=c=${colour}:s=320x180:d=3:r=30`, + ]; + const base = clip("solo-ref.mp4", [...backdrop("0x305070"), "-vf", strokes(6)]); + const right = clip("solo-right.mp4", [...backdrop("0x604030"), "-vf", strokes(6)]); + const thin = clip("solo-thin.mp4", [...backdrop("0x604030"), "-vf", strokes(3)]); + const box: [number, number, number, number] = [116, 106, 44, 32]; + const good = await diffVideos({ + reference: base, + yours: right, + at: [0.5, 1, 1.5, 2], + drawn: [{ box }, { box, window: [1, 3] }], + outDir: join(dir, "solo-right"), + }); + expect(good.segments).toBeUndefined(); + expect(good.drawn).toHaveLength(2); + expect(good.drawn?.[0]).toMatchObject({ id: "D1", box, window: [0, 3], frames: 4, pass: true }); + expect(good.drawn?.[1]).toMatchObject({ id: "D2", window: [1, 3], frames: 3, pass: true }); + const bad = await diffVideos({ + reference: base, + yours: thin, + at: [0.5, 1, 1.5, 2], + drawn: [{ box }], + outDir: join(dir, "solo-thin"), + }); + expect(good.drawn?.[0]).toMatchObject({ score: 1, offset: [0, 0], ink: 1 }); + // Half-weight letters in the right place: the score barely drops, the ink says it. + expect(bad.drawn?.[0]?.score).toBeLessThan(0.9); + expect(bad.drawn?.[0]?.ink).toBe(0.5); + }); + + it("fails the original's whole frame played where copy is meant to be rebuilt", async () => { + const ledger = join(dir, "manifest.jsonl"); + writeFileSync( + ledger, + JSON.stringify({ + type: "video", + path: "assets/all.mp4", + provenance: { provider: "original", window: [0, 3], roi: [0, 0, 320, 180] }, + }) + "\n", + ); + const titled = '## S01 · 0–3 s · Title\nPixels: rebuilt\nCopy: "TITLE"\n'; + const copy = await diffVideos({ + reference: ref, + yours: ref, + segments: parseSegments(titled), + ledger, + outDir: join(dir, "copy"), + }); + expect(copy.whole_frame_reuse).toMatchObject({ spans: [[0, 3]], drawn: [[0, 3]], pass: false }); + expect(copy.segments?.[0]?.pass).toBe(false); + expect(copy.pass).toBe(false); + // Footage reused with its burned-in title is a copy too (Miao, 2026-10-23: no whole-frame cut with drawn things). + const burned = await diffVideos({ + reference: ref, + yours: ref, + segments: parseSegments( + '## S01 · 0–3 s · Title over footage\nPixels: reused\nCopy: "TITLE"\n', + ), + ledger, + outDir: join(dir, "burned"), + }); + expect(burned.whole_frame_reuse).toMatchObject({ share: 1, drawn: [[0, 3]], pass: false }); + // Footage with nothing drawn on it may be cut whole. + const footage = await diffVideos({ + reference: ref, + yours: ref, + segments: parseSegments("## S01 · 0–3 s · Footage\nPixels: reused\n"), + ledger, + outDir: join(dir, "footage"), + }); + expect(footage.whole_frame_reuse).toMatchObject({ share: 1, drawn: [], pass: true }); + expect(footage.pass).toBe(true); + // A cut that only fed a generation (a motion guide) is not played: yours differs from it. + const flipped = clip("guided.mp4", [...["-i", ref, "-vf"], "hflip"]); + const guided = await diffVideos({ + reference: ref, + yours: flipped, + segments: parseSegments(titled), + ledger, + outDir: join(dir, "guided"), + }); + expect(guided.whole_frame_reuse).toMatchObject({ spans: [], pass: true }); + // A generated shot that starts from the original's first frame matches it for a few frames only: not reuse. + const started = clip("started.mp4", [ + ...["-i", ref, "-filter_complex"], + "[0]split[a][b];[b]hflip[f];[a][f]overlay=enable='gte(t\\,0.2)'", + ]); + const start = await diffVideos({ + reference: ref, + yours: started, + segments: parseSegments(titled), + ledger, + outDir: join(dir, "started"), + }); + expect(start.whole_frame_reuse).toMatchObject({ spans: [], pass: true }); + }); + + it("finds a whole-frame still of the original shown in yours", async () => { + const ledger = join(dir, "stills.jsonl"); + writeFileSync( + ledger, + [ + { type: "image", provenance: { provider: "original", at: 1, roi: [0, 0, 320, 180] } }, + { type: "image", provenance: { provider: "original", at: 2.5 } }, + ] + .map((e) => JSON.stringify(e)) + .join("\n"), + ); + // Yours holds the original's frame at 1 s for the whole clip. + const held = clip("held.mp4", [ + ...["-i", ref, "-vf"], + "select=eq(n\\,30),loop=loop=89:size=1,setpts=N/30/TB", + ]); + const r = await diffVideos({ + reference: ref, + yours: held, + segments: parseSegments('## S01 · 0–3 s · Collage\nPixels: rebuilt\nCopy: "TITLE"\n'), + ledger, + outDir: join(dir, "held"), + }); + expect(r.whole_frame_reuse).toMatchObject({ stills: [1], spans: [[0, 3]], pass: false }); + }); + + it("reads sound that starts late as late, and the picture's length as the duration", async () => { + // Clicks at uneven times, with the sound running 0.4 s past the picture. + const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3.4"; + const voiced = (name: string, delay: number) => + clip(name, [ + ...["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"], + ...["-itsoffset", String(delay), "-f", "lavfi", "-i", clicks], + ...["-map", "0:v", "-map", "1:a", "-c:a", "aac"], + ]); + const on = voiced("voiced.mp4", 0); + const same = await diffVideos({ reference: on, yours: on, outDir: join(dir, "voiced") }); + expect(same.duration).toMatchObject({ reference: 3, pass: true }); + expect(same.sound).toMatchObject({ offset_ms: 0, pass: true }); + const late = await diffVideos({ + reference: on, + yours: voiced("late.mp4", 0.2), + outDir: join(dir, "late"), + }); + expect(late.sound).toMatchObject({ offset_ms: 200, pass: false }); + }); + + it("places the sound by its decoded timestamps, not the start time the probe reports", () => { + // ffmpeg 5.1 drops an HE-AAC edit list's skip twice: the first sample it keeps is timestamped 0.1 s in while + // the probe still says the track starts at 0. Here a sound timestamped 0.25 s in, probed as starting at 0, + // against the same sound with 0.25 s of silence baked into its samples: in sync. + const clicks = "aevalsrc=sin(2*PI*660*t)*lt(mod(t*t*3\\,1)\\,0.08):s=48000:d=3"; + const picture = ["-f", "lavfi", "-i", "testsrc2=d=3:r=30:s=320x180"]; + const stamped = clip("stamped.mp4", [ + ...picture, + ...["-itsoffset", "0.25", "-f", "lavfi", "-i", clicks], + ...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"], + ]); + const baked = clip("baked.mp4", [ + ...picture, + ...["-f", "lavfi", "-i", clicks, "-af", "adelay=250:all=1"], + ...["-map", "0:v", "-map", "1:a", "-c:a", "pcm_s16le", "-f", "mov"], + ]); + const probedAtZero = { ...probeVideo(stamped), audioStart: 0 }; + expect(soundOffset(stamped, baked, probedAtZero, probeVideo(baked))).toMatchObject({ + offset_ms: 0, + pass: true, + }); + }); + + it("passes a re-shot person whose motion is off when the components are right", async () => { + // The bottom half (the person) frozen on its first frame: no motion at all. Pixels above match. + const frozen = clip("frozen.mp4", [ + ...["-i", ref, "-filter_complex"], + "[0]split[a][b];[b]crop=320:90:0:90,trim=end_frame=1,tpad=stop=89:stop_mode=clone[f];[a][f]overlay=0:90", + ]); + const r = await diffVideos({ + reference: ref, + yours: frozen, + segments: parseSegments( + "## S01 · 0–3 s · Board over a presenter\nPixels: rebuilt\nRegenerated: 0, 0.5, 1, 0.5\n", + ), + outDir: join(dir, "frozen"), + }); + const [s] = r.segments!; + expect(s!.person?.energy?.mean_ratio).toBeLessThan(0.5); + expect(s!.person?.gates).toEqual({ framing: null, text: null, sync: null }); + expect(s!.pass).toBe(true); + expect(r.components).toMatchObject({ passed: 1, total: 1 }); + expect(r.person?.energy).toBeLessThan(0.5); + expect(r.pass).toBe(true); + }); + + it("fails a re-shot person whose generated clip carries the original's text", async () => { + // The original: a white label over a soft lower half (the person). Yours draws the label over a re-shot half. + const strokes = (x0: number) => + [0, 14, 28].map((x) => `drawbox=x=${x0 + x}:y=110:w=6:h=24:color=white:t=fill`).join(","); + const label = strokes(120); + const soft = (delay: number) => + `[0]split[a][b];[b]crop=320:90:0:90,gblur=sigma=10,hue=s=0.4,tpad=start=${delay}:start_mode=clone[f];[a][f]overlay=0:90:shortest=1`; + const base = clip("carried-ref.mp4", [ + ...["-i", ref, "-filter_complex"], + `${soft(0)},${label}`, + ]); + const project = join(dir, "carried"); + mkdirSync(join(project, "assets"), { recursive: true }); + // The generated clips: one made from a guide with the label on it, one from a masked guide. Yours draws the + // label over each; over the first, 8 px off, the letters show doubled. + clip("carried/assets/dirty.mp4", [...["-i", ref, "-filter_complex"], `${soft(5)},${label}`]); + clip("carried/assets/clean.mp4", [...["-i", ref, "-filter_complex"], soft(5)]); + execFileSync(ffmpeg!, [ + ...["-hide_banner", "-loglevel", "error", "-y", "-i", ref, "-filter_complex"], + `${soft(0)},trim=end_frame=1`, + ...["-frames:v", "1", join(project, "assets", "still.png")], + ]); + const ledger = (path: string) => { + const file = join(project, `${path}.jsonl`); + writeFileSync( + file, + JSON.stringify({ + type: "video", + path: `assets/${path}.mp4`, + provenance: { provider: "heygen", references: ["assets/still.png"], window: [0, 3] }, + }) + "\n", + ); + return file; + }; + const md = + "## S01 · 0–3 s · Label over a presenter\nPixels: rebuilt\nRegenerated: 0, 90, 320, 90\nDrawn: 118, 108, 42, 28\n"; + const judge = (path: string, x0: number) => + diffVideos({ + reference: base, + yours: clip(`carried-yours-${path}.mp4`, [ + ...["-i", join(project, "assets", `${path}.mp4`), "-vf"], + strokes(x0), + ]), + segments: parseSegments(md), + ledger: ledger(path), + project, + outDir: join(dir, `carried-${path}`), + }); + const dirty = await judge("dirty", 128); + expect(dirty.segments?.[0]?.person?.text?.[0]).toMatchObject({ + drawn: "S01.D1", + carried: true, + }); + expect(dirty.segments?.[0]?.pass).toBe(false); + expect(dirty.person).toMatchObject({ gates: { text: false }, failed: { text: ["S01"] } }); + expect(dirty.pass).toBe(false); + const clean = await judge("clean", 120); + expect(clean.segments?.[0]?.drawn?.[0]?.pass).toBe(true); + expect(clean.segments?.[0]?.person?.gates.text).toBe(true); + expect(clean.segments?.[0]?.pass).toBe(true); + expect(clean.person?.shots_at_cuts).toMatchObject({ + at_cuts: 1, + total: 1, + cuts_without_shot: [], + }); + }); + it("judges a regenerated segment by motion and look, not pixels", async () => { const shifted = clip("shifted.mp4", [...["-i", ref, "-vf"], "hflip"]); const segments = parseSegments("## S01 · 0–3 s · AI shot\nPixels: regenerated\n"); diff --git a/packages/cli/src/measure/diff.ts b/packages/cli/src/measure/diff.ts index ed94f40391..66fb340a6a 100644 --- a/packages/cli/src/measure/diff.ts +++ b/packages/cli/src/measure/diff.ts @@ -14,8 +14,15 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; -import { findCuts, signatures } from "./cuts.js"; +import { basename, dirname, join, resolve } from "node:path"; +import { confirmsCut, cutsIn, frameChanges, signatures } from "./cuts.js"; +import { + drawnFrames, + drawnVerdict, + readDrawn, + scoreDrawnFrame, + type DrawnVerdict, +} from "./drawn.js"; import { compareEnergy, energyCurve, type EnergyComparison } from "./energy.js"; import { compareLook, look, sameLook, type LookComparison } from "./look.js"; import { @@ -28,9 +35,20 @@ import { videoClip, writeImage, type Clip, + type Roi, type VideoInfo, } from "./media.js"; +import { + AT_CUT_S, + clipUses, + generatedClips, + personVerdict, + placeClips, + type GeneratedClip, + type PersonVerdict, +} from "./person.js"; import type { Segment } from "./segments.js"; +import type { Diagnosis } from "../heygen/explain.js"; // To be calibrated on the reverse bench; light-reverse's rebuilt moments scored 0.885–0.958 mean SSIM. export const DEFAULT_MIN_MEAN = 0.92; @@ -43,19 +61,37 @@ export const DEFAULT_MIN_CELL = 0.75; // A regenerated shot moves about as much as the original's. const ENERGY_RATIO: [number, number] = [0.8, 1.25]; const GRID = 4; +const ALL_CELLS = Array.from({ length: GRID * GRID }, (_, c) => c); const WORST_GAP_S = 0.5; const SHEET_W = 960; const CUT_SLACK = 1; +// A segment's cell is left to its regenerated region when the region covers more than this share of it. +const REGION_CELL_SHARE = 0.1; // Missing content: cells are read at this width, from at most this many frames. A cell whose luma spread in the // original reaches MISSING_DETAIL while yours is flat (under MISSING_FLAT and MISSING_KEEP of the original's) is // empty; over MISSING_MAX_SHARE of frames with an empty cell fails. A weaker blur or a missing small label is not -// empty: SSIM and its cell gate grade those (provisional, calibrated on the reverse bench). +// empty: SSIM and its cell gate grade those (provisional, calibrated on the reverse bench). An empty cell is missing +// structure when the original's cell still spreads MISSING_DETAIL at an eighth of that width (shapes, cards, a +// map's land), and missing texture otherwise (a credit line, sea-floor grain): texture is reported, not failed. const MISSING_W = 192; const MISSING_SAMPLES = 1800; const MISSING_DETAIL = 8; const MISSING_KEEP = 0.35; const MISSING_FLAT = 4; const MISSING_MAX_SHARE = 0.02; +const STRUCTURE_SCALE = 8; +// Whole-frame reuse: a cut from the original covering this share of its frame plays the whole frame, and a frame +// whose median cell scores REUSE_SSIM shows it. Drawn things come along with it (a copy) for longer than +// REUSE_MAX_FRAMES fails. A Regenerated box over this share of the frame judges nothing by pixels. A cut matched for +// less than REUSE_MIN_S is not reuse: a generated shot starts from a clean still of the original's first frame. A whole- +// frame still of the original shows where yours, read in grey at STILL_W wide, is within STILL_DIFF of it in most +// of the 4x4 cells. +const WHOLE_FRAME_SHARE = 0.9; +const REUSE_SSIM = 0.9; +const REUSE_MAX_FRAMES = 2; +const REUSE_MIN_S = 0.5; +const STILL_W = 96; +const STILL_DIFF = 4; /** Which of the original's frames are compared. */ export type Pick = @@ -82,6 +118,12 @@ export interface DiffOptions { outDir: string; /** Render-ready timeout for a project folder's capture. */ timeoutMs?: number; + /** The project's media ledger (`.media/manifest.jsonl`): its cuts from the original show whole-frame reuse. */ + ledger?: string; + /** The project folder, to find the ledger's generated clips (default: yours, or the folder holding `.media`). */ + project?: string; + /** Drawn elements to score by themselves, segments or not: boxes in the original's pixels, optional windows. */ + drawn?: { box: [number, number, number, number]; window?: [number, number] }[]; log?: (message: string) => void; } @@ -108,8 +150,22 @@ export interface SegmentVerdict { pixels: Segment["pixels"]; pass: boolean | null; ssim?: Scores; + /** A regenerated segment with no person in it (a product shot): judged by motion and look. */ energy?: EnergyComparison; look?: LookComparison; + /** What `pass` covers, in words: which pixels SSIM scored, and what was judged otherwise. */ + scored?: string; + /** A rebuilt or reused segment's regenerated region; `ssim` then scores the cells outside it. */ + regenerated?: { roi: [number, number, number, number] }; + /** + * The re-shot person: three gates decide (framing, text carried in the generated clip, lip sync); motion energy + * and look are reported, not judged. + */ + person?: PersonVerdict & { energy?: EnergyComparison; look?: LookComparison }; + /** Each `Drawn:` element over the regenerated person, scored by itself (drawn.ts), in the order written. */ + drawn?: (DrawnVerdict & { id: string; diagnosis?: Diagnosis })[]; + /** A segment scored by SSIM: the 4x4 cell with the lowest mean over it, and the moment it is lowest. */ + worst_cell?: { roi: Roi; ssim: number; t: number; diagnosis?: Diagnosis }; note?: string; } @@ -123,11 +179,24 @@ export interface CutReport { } export interface MissingContent { - /** Share of compared frames with a cell the original fills and yours leaves flat. */ + /** Share of compared frames with a cell whose shapes the original has and yours leaves flat. */ share: number; /** The longest such stretches, in seconds. */ spans: [number, number][]; pass: boolean; + /** Frames where yours leaves flat only the original's fine texture in a cell: reported, not failed. */ + texture: { share: number; spans: [number, number][] }; +} + +export interface WholeFrameReuse { + /** Stretches where yours plays a whole-frame cut of the original, in seconds. */ + spans: [number, number][]; + /** The times (in the original) of the ledger's whole-frame stills that show in yours. */ + stills: number[]; + share: number; + /** Those inside segments that quote copy: the original's drawn things played along, a copy. */ + drawn: [number, number][]; + pass: boolean; } export interface DiffResult { @@ -139,11 +208,18 @@ export interface DiffResult { thresholds: Thresholds; ssim: Scores; missing_content: MissingContent; + whole_frame_reuse?: WholeFrameReuse; worst: Moment[]; cuts?: CutReport; duration?: { reference: number; yours: number; pass: boolean }; sound?: ReturnType; segments?: SegmentVerdict[]; + /** What decides the verdict: every drawn element and each rebuilt segment's area outside the person. */ + components?: { passed: number; total: number; mean: number | null }; + /** The re-shot person over the film: the gates that decide, and what is only reported. */ + person?: PersonSummary; + /** Each `drawn` option's element (D1, D2, …), scored by itself: reported, not judged. */ + drawn?: (DrawnVerdict & { id: string })[]; sheet?: string; sbs?: string; log: string; @@ -152,6 +228,26 @@ export interface DiffResult { warnings: string[]; } +export interface PersonSummary { + /** Each gate over every segment: false when one fails, null when none could be measured. */ + gates: { framing: boolean | null; text: boolean | null; sync: boolean | null }; + /** Segments failing each gate. */ + failed: { framing: string[]; text: string[]; sync: string[] }; + /** Median of the segments' motion energy ratios (yours over the original's): reported, not judged. */ + energy: number | null; + /** + * Generated clips that start at one of the original's cuts, of those placed on the film, and the original's cuts + * inside the person's segments where no clip starts (a jump cut played as one take): reported. + */ + shots_at_cuts: { + at_cuts: number; + total: number; + off: { clip: string; start: number }[]; + cuts_without_shot: number[]; + }; + clips: GeneratedClip[]; +} + interface Series { /** Whole-frame SSIM per compared frame. */ frame: number[]; @@ -247,42 +343,67 @@ function ssimSeries(ref: Clip, yours: Clip, logPath: string): Series { /** * Stretches where a cell the original fills is flat in yours. SSIM scores two near-blank patches alike, so a scene * left empty on a plain background passes SSIM and its cell gate; this compares each cell's luma spread instead. + * `judged(k)` lists the cells of the original's frame k this applies to: a regenerated person's gesture is theirs. */ -function missingContent(ref: Clip, yours: Clip, ks: number[], fps: number): MissingContent { +function missingContent( + ref: Clip, + yours: Clip, + ks: number[], + fps: number, + judged: (k: number) => number[] = () => ALL_CELLS, +): MissingContent { const stride = Math.max(1, Math.ceil(ks.length / MISSING_SAMPLES)); const w = MISSING_W; const h = Math.max(GRID * 2, Math.round((w * ref.height) / ref.width / GRID) * GRID); + const [cw, ch] = [ + w / STRUCTURE_SCALE, + Math.max(GRID, Math.round(h / STRUCTURE_SCALE / GRID) * GRID), + ]; const thin = stride > 1 ? [`select=not(mod(n\\,${stride}))`, `setpts=N/(${fps / stride})/TB`] : []; - const spreads = (clip: Clip) => - decode(clip.input, [...clip.filters, ...thin, `scale=${w}:${h}`], w, h, true).frames.map((px) => - cellSpreads(px, w, h), + const spreads = (clip: Clip, sw: number, sh: number) => + decode(clip.input, [...clip.filters, ...thin, `scale=${sw}:${sh}`], sw, sh, true).frames.map( + (px) => cellSpreads(px, sw, sh), ); - const [a, b] = [spreads(ref), spreads(yours)]; + const [a, b, coarse] = [spreads(ref, w, h), spreads(yours, w, h), spreads(ref, cw, ch)]; // The thinning select can pass one frame past the pick; times come from ks, so stop with it. - const n = Math.min(a.length, b.length, Math.ceil(ks.length / stride)); - const empty = Array.from({ length: n }, (_, j) => - a[j]!.some( - (s, c) => s >= MISSING_DETAIL && b[j]![c]! < Math.min(MISSING_FLAT, MISSING_KEEP * s), - ), - ); - const spans: [number, number][] = []; - empty.forEach((e, j) => { - if (!e) return; - const t = round(ks[j * stride]! / fps, 3); - const last = spans.at(-1); - if (last && empty[j - 1]) - last[1] = round((ks[Math.min(ks.length - 1, (j + 1) * stride - 1)]! + 1) / fps, 3); - else spans.push([t, round((ks[Math.min(ks.length - 1, (j + 1) * stride - 1)]! + 1) / fps, 3)]); + const n = Math.min(a.length, b.length, coarse.length, Math.ceil(ks.length / stride)); + const kinds = Array.from({ length: n }, (_, j) => { + const empty = judged(ks[j * stride]!).filter((c) => { + const s = a[j]![c]!; + return s >= MISSING_DETAIL && b[j]![c]! < Math.min(MISSING_FLAT, MISSING_KEEP * s); + }); + return { + structure: empty.some((c) => coarse[j]![c]! >= MISSING_DETAIL), + texture: empty.length > 0, + }; }); - const share = n ? empty.filter(Boolean).length / n : 0; - return { - share: round(share, 4), - spans: spans + const spansOf = (on: boolean[]) => { + const spans: [number, number][] = []; + on.forEach((e, j) => { + if (!e) return; + const t = round(ks[j * stride]! / fps, 3); + const end = round((ks[Math.min(ks.length - 1, (j + 1) * stride - 1)]! + 1) / fps, 3); + const last = spans.at(-1); + if (last && on[j - 1]) last[1] = end; + else spans.push([t, end]); + }); + return spans .sort((x, y) => y[1] - y[0] - (x[1] - x[0])) .slice(0, 8) - .sort((x, y) => x[0] - y[0]), + .sort((x, y) => x[0] - y[0]); + }; + const structure = kinds.map((k) => k.structure); + const texture = kinds.map((k) => k.texture && !k.structure); + const share = n ? structure.filter(Boolean).length / n : 0; + return { + share: round(share, 4), + spans: spansOf(structure), pass: share <= MISSING_MAX_SHARE, + texture: { + share: round(n ? texture.filter(Boolean).length / n : 0, 4), + spans: spansOf(texture), + }, }; } @@ -342,22 +463,24 @@ function outline(px: Uint8Array, w: number, [x, y, bw, bh]: number[], t = 3) { } function cutReport(ref: Clip, yours: Clip, k0: number, fps: number): CutReport { - const a = findCuts(signatures(ref.input, ref.filters).frames).map((c) => c.frame); - const b = findCuts(signatures(yours.input, yours.filters).frames).map((c) => c.frame); + const [da, db] = [ref, yours].map((c) => frameChanges(signatures(c.input, c.filters).frames)); + const [a, b] = [cutsIn(da!), cutsIn(db!)]; const used = new Set(); const missing: number[] = []; - for (const f of a) { - const hit = b.find((g) => !used.has(g) && Math.abs(g - f) <= CUT_SLACK); - if (hit === undefined) missing.push(f); - else used.add(hit); + for (const cut of a) { + const hit = b.find((g) => !used.has(g.frame) && Math.abs(g.frame - cut.frame) <= CUT_SLACK); + // A soft cut is near the line: a clear change on the other side matches it. + if (hit) used.add(hit.frame); + else if (!(cut.soft && confirmsCut(db!, cut))) missing.push(cut.frame); } + const extra = b.filter((g) => !used.has(g.frame) && !(g.soft && confirmsCut(da!, g))); const t = (f: number) => round((k0 + f) / fps, 3); return { - reference: a.map(t), - yours: b.map(t), + reference: a.map((c) => t(c.frame)), + yours: b.map((c) => t(c.frame)), missing: missing.map(t), - extra: b.filter((g) => !used.has(g)).map(t), - pass: missing.length === 0 && b.length === used.size, + extra: extra.map((c) => t(c.frame)), + pass: missing.length === 0 && extra.length === 0, }; } @@ -406,47 +529,469 @@ async function captureProject( }; } -function regeneratedVerdict(s: Segment, reference: string, yours: string, ref: VideoInfo) { +function regeneratedVerdict( + s: Segment, + reference: string, + yours: string, + ref: VideoInfo, + roi?: Roi, +) { const seconds = s.end - s.start; const [a, b] = [videoClip(reference, s.start, seconds, ref), videoClip(yours, s.start, seconds)]; - const energy = compareEnergy(energyCurve(a), energyCurve(b)); - const looks = compareLook(look(a), look(b)); + // Yours may be another size: the region is the same share of its frame. + const mine = roi && scaleRoi(roi, b.width / ref.width, b.height / ref.height); + const energy = compareEnergy(energyCurve(a, roi), energyCurve(b, mine)); + const looks = compareLook(look(a, 2, roi), look(b, 2, mine)); const moves = energy.mean_ratio >= ENERGY_RATIO[0] && energy.mean_ratio <= ENERGY_RATIO[1]; return { pass: moves && sameLook(looks), energy, look: looks }; } +const scaleRoi = ([x, y, w, h]: Roi, sx: number, sy: number): Roi => { + const [x0, y0] = [Math.round((x * sx) / 2) * 2, Math.round((y * sy) / 2) * 2]; + return [ + x0, + y0, + Math.max(2, Math.round((w * sx) / 2) * 2), + Math.max(2, Math.round((h * sy) / 2) * 2), + ]; +}; + +/** A segment's regenerated region in the original's pixels, clipped to the frame; shares when every value is ≤ 1. */ +export function regionRoi( + box: [number, number, number, number], + width: number, + height: number, +): Roi { + const [sx, sy] = box.every((v) => v <= 1) ? [width, height] : [1, 1]; + const roi = scaleRoi(box, sx, sy); + const [x, y] = [Math.min(roi[0], width - 2), Math.min(roi[1], height - 2)]; + return [x, y, Math.min(roi[2], width - x), Math.min(roi[3], height - y)]; +} + +/** The grid cells (row by row) a region covers no more than REGION_CELL_SHARE of. */ +export function cellsOutside(roi: Roi, width: number, height: number): number[] { + const [cw, ch] = cellSize(width, height); + return ALL_CELLS.filter((c) => { + const [x, y] = [(c % GRID) * cw, Math.floor(c / GRID) * ch]; + const ox = Math.max(0, Math.min(x + cw, roi[0] + roi[2]) - Math.max(x, roi[0])); + const oy = Math.max(0, Math.min(y + ch, roi[1] + roi[3]) - Math.max(y, roi[1])); + return ox * oy <= REGION_CELL_SHARE * cw * ch; + }); +} + +interface VerdictContext { + reference: string; + yours: string; + ref: VideoInfo; + mode: DiffResult["mode"]; + t: Thresholds; + /** A segment's drawn elements, scored by themselves. */ + drawn: Map; + /** A segment's re-shot person, by the three gates. */ + person: Map; +} + function segmentVerdicts( segments: Segment[], series: Series, worstCell: number[], ks: number[], - ctx: { - reference: string; - yours: string; - ref: VideoInfo; - mode: DiffResult["mode"]; - t: Thresholds; - }, + ctx: VerdictContext, ): SegmentVerdict[] { return segments.map((s) => { const head = { id: s.id, window: [s.start, s.end] as [number, number], pixels: s.pixels }; - if (s.pixels === "regenerated") - return ctx.mode === "video" - ? { ...head, ...regeneratedVerdict(s, ctx.reference, ctx.yours, ctx.ref) } - : { - ...head, - pass: null, - note: "regenerated pixels are judged by motion and look: diff a render", - }; + if (s.pixels === "regenerated" && !s.drawn) { + if (ctx.mode !== "video") + return { + ...head, + pass: null, + note: "regenerated pixels are judged on a render: diff a render", + }; + const moved = regeneratedVerdict(s, ctx.reference, ctx.yours, ctx.ref); + const person = ctx.person.get(s); + // No face to judge (a product shot): motion and look decide. + if (!person || person.pass === null) return { ...head, ...moved }; + return { + ...head, + pass: person.pass, + scored: "the person: framing, text carried in the generated clip, lip sync", + person: { ...person, energy: moved.energy, look: moved.look }, + }; + } const [a, b] = [frameAt(s.start, ctx.ref.fps), frameAt(s.end, ctx.ref.fps)]; const mine = ks.flatMap((k, i) => (k >= a && k < b && i < series.frame.length ? [i] : [])); if (!mine.length) return { ...head, pass: null, note: "no compared frame falls in this segment" }; + if (s.regenerated || s.pixels === "regenerated") + return mixedVerdict(s, head, mine, series, ctx); const st = scores( mine.map((i) => series.frame[i]!), mine.map((i) => worstCell[i]!), ); - return { ...head, pass: passes(st, ctx.t), ssim: st }; + return { + ...head, + pass: passes(st, ctx.t), + ssim: st, + worst_cell: lowestCell(mine, series, ks, ctx.ref), + }; + }); +} + +/** Of these frames, the cell with the lowest mean SSIM, and the time it is lowest. */ +export function lowestCell( + mine: number[], + series: Series, + ks: number[], + ref: { width: number; height: number; fps: number }, +): { roi: Roi; ssim: number; t: number } { + const means = series.cells[mine[0]!]!.map( + (_, c) => mine.reduce((sum, i) => sum + series.cells[i]![c]!, 0) / mine.length, + ); + const c = means.indexOf(Math.min(...means)); + const at = mine.reduce((lo, i) => (series.cells[i]![c]! < series.cells[lo]![c]! ? i : lo)); + const [cw, ch] = cellSize(ref.width, ref.height); + return { + roi: [(c % GRID) * cw, Math.floor(c / GRID) * ch, cw, ch], + ssim: round(means[c]!, 4), + t: round(ks[at]! / ref.fps, 3), + }; +} + +/** + * A segment with a regenerated person (its box, or the whole frame): the person by motion and look, the cells + * outside the box by SSIM, and each drawn element over the person by itself. + */ +function mixedVerdict( + s: Segment, + head: { id: string; window: [number, number]; pixels: Segment["pixels"] }, + mine: number[], + series: Series, + ctx: VerdictContext, +): SegmentVerdict { + const whole = s.pixels === "regenerated"; + const roi: Roi = whole + ? [0, 0, ctx.ref.width, ctx.ref.height] + : regionRoi(s.regenerated!, ctx.ref.width, ctx.ref.height); + const rest = whole ? [] : cellsOutside(roi, ctx.ref.width, ctx.ref.height); + const moved = + ctx.mode === "video" + ? regeneratedVerdict(s, ctx.reference, ctx.yours, ctx.ref, whole ? undefined : roi) + : undefined; + const gates = ctx.person.get(s); + const person = + gates || moved + ? { + ...(gates ?? { gates: { framing: null, text: null, sync: null }, pass: null }), + energy: moved?.energy, + look: moved?.look, + } + : undefined; + const st = rest.length + ? scores( + mine.map((i) => rest.reduce((sum, c) => sum + series.cells[i]![c]!, 0) / rest.length), + mine.map((i) => Math.min(...rest.map((c) => series.cells[i]![c]!))), + ) + : undefined; + const drawn = ctx.drawn.get(s) ?? []; + const verdicts = [ + st ? passes(st, ctx.t) : null, + person?.pass ?? null, + ...drawn.map((d) => d.pass), + ]; + const pass = verdicts.includes(false) ? false : verdicts.every((p) => p === null) ? null : true; + const scored = [ + st + ? `SSIM: the ${rest.length} cells outside the regenerated box` + : drawn.length + ? "drawn elements only (no cell is outside the regenerated box)" + : "no pixels (the regenerated box covers every cell)", + "the person: framing, text carried in the generated clip, lip sync (motion and look reported)", + ...(drawn.length ? [`${drawn.length} drawn elements, each scored by itself`] : []), + ].join("; "); + const note = + ctx.mode === "project" + ? { note: `the regenerated ${whole ? "picture" : "region"} is judged on a render` } + : {}; + const parts = { + pass, + scored, + ...(st ? { ssim: st } : {}), + ...(drawn.length ? { drawn } : {}), + ...(person ? { person } : {}), + }; + return { ...head, ...parts, ...(whole ? {} : { regenerated: { roi } }), ...note }; +} + +/** + * The drawn elements over a regenerated person, each scored by itself on its window inside its segment (drawn.ts), + * from one decode of each side. + */ +function scoreDrawn( + segments: Segment[], + refClip: Clip, + yours: Clip, + ks: number[], + ref: VideoInfo, + available: number, +): Map { + const jobs = segments.flatMap((s) => drawnOf(s, ref).map((d) => ({ s, ...d }))); + const verdicts = scoreElements(jobs, refClip, yours, ks, ref, available); + const out = new Map(); + jobs.forEach((j, n) => out.set(j.s, [...(out.get(j.s) ?? []), verdicts[n]!])); + return out; +} + +/** Drawn elements scored by themselves on their windows, in order, from one decode of each side. */ +function scoreElements( + elements: { id: string; box: Roi; window: [number, number] }[], + refClip: Clip, + yours: Clip, + ks: number[], + ref: VideoInfo, + available: number, +): (DrawnVerdict & { id: string })[] { + const jobs = elements.map((d) => ({ + ...d, + frames: drawnFrames(ks, ref.fps, ...d.window, available), + })); + const live = jobs.filter((j) => j.frames.length); + const [a, b] = [refClip, yours].map((clip) => readDrawn(clip, live, ref.width, ref.height)); + return jobs.map((j) => { + const n = live.indexOf(j); + const pairs = + n < 0 ? [] : a![n]!.map((fa, i) => [fa, b![n]![i]] as const).filter(([, fb]) => fb); + const frames = pairs.map(([fa, fb]) => scoreDrawnFrame(fa, fb!, j.box[2], j.box[3])); + return { id: j.id, ...drawnVerdict(j.box, j.window, frames) }; + }); +} + +/** A segment's drawn elements over a regenerated person: id, box in the original's pixels, window. */ +function drawnOf(s: Segment, ref: VideoInfo): { id: string; box: Roi; window: [number, number] }[] { + if (!(s.regenerated || s.pixels === "regenerated")) return []; + return (s.drawn ?? []).map((d, n) => ({ + id: `${s.id}.D${n + 1}`, + box: regionRoi(d.box, ref.width, ref.height), + window: [Math.max(s.start, d.window?.[0] ?? s.start), Math.min(s.end, d.window?.[1] ?? s.end)], + })); +} + +/** The film's headline: the components that decide it, and the person's gates and what is reported. */ +export function headline( + segments: SegmentVerdict[], + clips: GeneratedClip[], + t: Thresholds, + cuts: number[] = [], +): { components: NonNullable; person: PersonSummary } { + const parts = segments.flatMap((v) => [ + ...(v.ssim ? [{ pass: passes(v.ssim, t), score: v.ssim.mean }] : []), + ...(v.drawn ?? []).flatMap((d) => + d.pass === null ? [] : [{ pass: d.pass, score: d.score ?? 0 }], + ), + ]); + const people = segments.flatMap((v) => + v.person ? [{ id: v.id, window: v.window, ...v.person }] : [], + ); + const gate = (key: "framing" | "text" | "sync") => { + const g = people.map((p) => p.gates[key]); + return g.includes(false) ? false : g.some((x) => x === true) ? true : null; + }; + const ratios = people.flatMap((p) => (p.energy ? [p.energy.mean_ratio] : [])); + const placed = clips.filter((c) => c.at_cut !== undefined); + return { + components: { + passed: parts.filter((p) => p.pass).length, + total: parts.length, + mean: parts.length + ? round(parts.reduce((sum, p) => sum + p.score, 0) / parts.length, 4) + : null, + }, + person: { + gates: { framing: gate("framing"), text: gate("text"), sync: gate("sync") }, + failed: { + framing: people.filter((p) => p.gates.framing === false).map((p) => p.id), + text: people.filter((p) => p.gates.text === false).map((p) => p.id), + sync: people.filter((p) => p.gates.sync === false).map((p) => p.id), + }, + energy: ratios.length + ? round([...ratios].sort((a, b) => a - b)[ratios.length >> 1]!, 3) + : null, + shots_at_cuts: { + at_cuts: placed.filter((c) => c.at_cut).length, + total: placed.length, + off: placed.flatMap((c) => (c.at_cut ? [] : [{ clip: c.path, start: c.offset! }])), + cuts_without_shot: placed.length + ? cuts.filter( + (k) => + people.some((p) => k > p.window[0] && k < p.window[1]) && + !placed.some((c) => Math.abs(c.offset! - k) <= AT_CUT_S), + ) + : [], + }, + clips, + }, + }; +} + +/** A media ledger's cuts from the original that show its whole frame: video windows and the times of stills. */ +export function wholeFrameCuts( + ledger: string, + width: number, + height: number, +): { windows: [number, number][]; stills: number[] } { + const isRecord = (v: unknown): v is Record => + typeof v === "object" && v !== null; + const numbers = (v: unknown) => + Array.isArray(v) ? v.filter((x): x is number => typeof x === "number") : []; + type Cut = { window: [number, number] } | { at: number }; + const cuts = ledger.split("\n").flatMap((line): Cut[] => { + let entry: unknown; + try { + entry = JSON.parse(line); + } catch { + return []; + } + if (!isRecord(entry) || !isRecord(entry.provenance) || entry.provenance.provider !== "original") + return []; + const p = entry.provenance; + const [window, roi] = [numbers(p.window), numbers(p.roi)]; + // A cut with no region is the whole frame. + const area = roi.length === 4 ? roi[2]! * roi[3]! : width * height; + if (area < WHOLE_FRAME_SHARE * width * height) return []; + if (entry.type === "video" && window.length === 2) + return [{ window: [window[0]!, window[1]!] }]; + return entry.type === "image" && typeof p.at === "number" ? [{ at: p.at }] : []; + }); + return { + windows: cuts.flatMap((c) => ("window" in c ? [c.window] : [])), + stills: [...new Set(cuts.flatMap((c) => ("at" in c ? [c.at] : [])))].sort((a, b) => a - b), + }; +} + +/** For each still (a time in the original), which compared frames of yours show it. */ +function stillsShown(ref: Clip, yours: Clip, stills: number[], fps: number): boolean[][] { + if (!stills.length) return []; + const h = Math.max(GRID, Math.round((STILL_W * ref.height) / ref.width / GRID) * GRID); + const read = (clip: Clip, extra: string[] = [], limit?: number) => + decode( + clip.input, + [...clip.filters, ...extra, `scale=${STILL_W}:${h}`], + STILL_W, + h, + true, + limit, + ).frames; + const mine = read(yours); + return stills.map((at) => { + const still = read(ref, [`select=eq(n\\,${frameAt(at, fps)})`], 1)[0]; + if (!still) return mine.map(() => false); + return mine.map((px) => { + const cells = cellDiffs(px, still, STILL_W, h).sort((a, b) => a - b); + return pct(cells, 0.5) <= STILL_DIFF; + }); + }); +} + +/** The mean absolute difference of two grey pictures in each 4x4 cell. */ +function cellDiffs(a: Uint8Array, b: Uint8Array, w: number, h: number): number[] { + const [cw, ch] = [Math.floor(w / GRID), Math.floor(h / GRID)]; + return ALL_CELLS.map((c) => { + const [x0, y0] = [(c % GRID) * cw, Math.floor(c / GRID) * ch]; + let sum = 0; + for (let y = y0; y < y0 + ch; y++) + for (let x = x0; x < x0 + cw; x++) sum += Math.abs(a[y * w + x]! - b[y * w + x]!); + return sum / (cw * ch); + }); +} + +/** The ledger beside a project or its REVERSE.md (the grade keeps a copy as media-manifest.jsonl). */ +export function findLedger(yours: string, segmentsFile?: string): string | undefined { + const dirs = [ + ...(existsSync(yours) && statSync(yours).isDirectory() ? [yours] : []), + ...(segmentsFile ? [dirname(segmentsFile)] : []), + ]; + return dirs + .flatMap((d) => [join(d, ".media", "manifest.jsonl"), join(d, "media-manifest.jsonl")]) + .find((f) => existsSync(f)); +} + +/** Runs of compared frames that are on, as [start, end) seconds. */ +function frameSpans(on: boolean[], ks: number[], fps: number): [number, number][] { + const spans: [number, number][] = []; + on.forEach((v, i) => { + if (!v) return; + const last = spans.at(-1); + if (last && on[i - 1] && ks[i] === ks[i - 1]! + 1) last[1] = round((ks[i]! + 1) / fps, 3); + else spans.push([round(ks[i]! / fps, 3), round((ks[i]! + 1) / fps, 3)]); + }); + return spans; +} + +/** + * Where yours plays one of the original's whole-frame cuts (inside its window, most cells near the original's own + * pixels, for REUSE_MIN_S or longer) or shows one of its whole-frame stills, and where that frame's segment draws + * copy: the original's drawn things came along with the footage. + */ +function wholeFrameReuse( + cuts: { windows: [number, number][]; stills: number[] }, + stills: boolean[][], + series: Series, + ks: number[], + fps: number, + segments?: Segment[], +): WholeFrameReuse { + const inside = (k: number, a: number, b: number) => k >= frameAt(a, fps) && k < frameAt(b, fps); + const played = series.frame.map( + (_, i) => + cuts.windows.some(([a, b]) => inside(ks[i]!, a, b)) && + pct( + [...series.cells[i]!].sort((x, y) => x - y), + 0.5, + ) >= REUSE_SSIM, + ); + const long = longRuns(played, ks, Math.ceil(REUSE_MIN_S * fps)); + const shown = long.map((v, i) => v || stills.some((on) => on[i])); + const drawn = shown.map( + (v, i) => v && !!segments?.some((s) => s.copy && inside(ks[i]!, s.start, s.end)), + ); + return { + spans: frameSpans(shown, ks, fps), + stills: cuts.stills.filter((_, j) => stills[j]?.some(Boolean)), + share: round(shown.filter(Boolean).length / Math.max(1, shown.length), 4), + drawn: frameSpans(drawn, ks, fps), + pass: drawn.filter(Boolean).length <= REUSE_MAX_FRAMES, + }; +} + +/** `on` with its runs of consecutive frames shorter than `min` turned off. */ +export function longRuns(on: boolean[], ks: number[], min: number): boolean[] { + const out = [...on]; + for (let i = 0; i < on.length; ) { + if (!on[i]) { + i++; + continue; + } + let j = i + 1; + while (j < on.length && on[j] && ks[j] === ks[j - 1]! + 1) j++; + if (j - i < min) out.fill(false, i, j); + i = j; + } + return out; +} + +/** Regenerated boxes that hide what should be judged by pixels. */ +function regionWarnings(segments: Segment[], ref: VideoInfo): string[] { + return segments.flatMap((s) => { + const box = s.regenerated && regionRoi(s.regenerated, ref.width, ref.height); + if (s.drawn || !(box || s.pixels === "regenerated")) return []; + if (box && box[2] * box[3] >= WHOLE_FRAME_SHARE * ref.width * ref.height) + return [ + `${s.id}: its Regenerated box covers the whole frame, so nothing in it is judged by pixels: box the person, and give each drawn element over them a Drawn: x, y, w, h line`, + ]; + return s.copy + ? [ + `${s.id} draws copy over a regenerated person: a drawn element inside the person's box is judged by pixels only with its own Drawn: x, y, w, h line`, + ] + : []; }); } @@ -517,22 +1062,11 @@ async function worstMoments( /** Loudness every 10 ms of the first 20 s, as a rising-edge curve: where sounds start. */ function onsetCurve(path: string): number[] { - // Padded from time 0, so a track that starts late reads late. + // On the sound's own timestamps (first_pts=0 pads up to its first decoded sample), not the probe's start_time: + // ffmpeg 5.1 drops an HE-AAC edit list's skip twice, keeps the rest timestamped right, and still probes 0. const raw = runFfmpeg([ - "-i", - path, - "-vn", - "-af", - "aresample=async=1:first_pts=0", - "-t", - "20", - "-ac", - "1", - "-ar", - "8000", - "-f", - "f32le", - "-", + ...["-i", path, "-vn", "-af", "aresample=async=1:first_pts=0"], + ...["-t", "20", "-ac", "1", "-ar", "8000", "-f", "f32le", "-"], ]); const samples = new Float32Array(raw.buffer, raw.byteOffset, Math.floor(raw.byteLength / 4)); const rms: number[] = []; @@ -545,7 +1079,7 @@ function onsetCurve(path: string): number[] { } /** How far yours' sound sits from the original's (positive: yours is late), from where sounds start. */ -function soundOffset(reference: string, yours: string, ref: VideoInfo, mine: VideoInfo) { +export function soundOffset(reference: string, yours: string, ref: VideoInfo, mine: VideoInfo) { if (!ref.audio || !mine.audio) return { original: ref.audio, yours: mine.audio, pass: ref.audio === mine.audio }; const [a, b] = [onsetCurve(reference), onsetCurve(yours)]; @@ -690,11 +1224,23 @@ export async function diffVideos(options: DiffOptions): Promise { cell: opts.thresholds?.cell ?? DEFAULT_MIN_CELL, }; const overall = scores(series.frame, worstCell); - const missing = missingContent(refClip, yours, ks, ref.fps); + const missing = missingContent(refClip, yours, ks, ref.fps, (k) => { + const s = opts.segments?.find( + (seg) => k >= frameAt(seg.start, ref.fps) && k < frameAt(seg.end, ref.fps), + ); + if (s?.pixels === "regenerated") return []; + return s?.regenerated + ? cellsOutside(regionRoi(s.regenerated, ref.width, ref.height), ref.width, ref.height) + : ALL_CELLS; + }); if (!missing.pass) warnings.push( `${Math.round(missing.share * 100)}% of frames leave empty what the original fills (${missing.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}): build what is missing there`, ); + if (missing.texture.share > MISSING_MAX_SHARE) + warnings.push( + `${Math.round(missing.texture.share * 100)}% of frames lack fine texture or small detail the original has (${missing.texture.spans.map(([a, b]) => `${a}–${b} s`).join(", ")}): reported, not failed`, + ); const picks = worstIndexes(worstCell, ks, ref.fps, opts.worst ?? 4); const { worst, sheet } = await worstMoments(picks, { refClip, @@ -708,11 +1254,44 @@ export async function diffVideos(options: DiffOptions): Promise { const duration = yoursInfo && pick.kind === "all" ? { - reference: round(ref.duration, 3), - yours: round(yoursInfo.duration, 3), - pass: Math.abs(ref.duration - yoursInfo.duration) <= 1.5 / ref.fps, + reference: round(ref.pictureDuration, 3), + yours: round(yoursInfo.pictureDuration, 3), + pass: Math.abs(ref.pictureDuration - yoursInfo.pictureDuration) <= 1.5 / ref.fps, } : undefined; + const ledgerText = + opts.ledger && existsSync(opts.ledger) ? readFileSync(opts.ledger, "utf8") : undefined; + const people = new Map(); + let clips: ReturnType = []; + if (opts.segments && yoursInfo) { + const project = + opts.project ?? + (opts.ledger && basename(dirname(opts.ledger)) === ".media" + ? dirname(dirname(opts.ledger)) + : undefined); + const refCuts = cuts?.reference ?? []; + const uses = project + ? await clipUses(project).catch(() => new Map()) + : new Map(); + clips = ledgerText ? placeClips(generatedClips(ledgerText), project, ref, refCuts, uses) : []; + if (clips.length && clips.every((c) => !c.info)) + warnings.push( + "the ledger's generated clips are not found, so text carried in them is not checked: pass --project ", + ); + for (const s of opts.segments) + if (s.regenerated || s.pixels === "regenerated") + people.set( + s, + await personVerdict({ + ref, + yours: yoursInfo, + window: [s.start, s.end], + cuts: refCuts, + clips, + drawn: drawnOf(s, ref), + }), + ); + } const segments = opts.segments ? segmentVerdicts(opts.segments, series, worstCell, ks, { reference: opts.reference, @@ -720,8 +1299,51 @@ export async function diffVideos(options: DiffOptions): Promise { ref, mode, t, + drawn: scoreDrawn(opts.segments, refClip, yours, ks, ref, series.frame.length), + person: people, }) : undefined; + if (opts.segments) warnings.push(...regionWarnings(opts.segments, ref)); + const elements = opts.drawn?.length + ? scoreElements( + opts.drawn.map((d, n) => ({ + id: `D${n + 1}`, + box: regionRoi(d.box, ref.width, ref.height), + window: d.window ?? [0, round(ref.pictureDuration, 3)], + })), + refClip, + yours, + ks, + ref, + series.frame.length, + ) + : undefined; + const ledger = ledgerText ? wholeFrameCuts(ledgerText, ref.width, ref.height) : undefined; + const reuse = + ledger && (ledger.windows.length || ledger.stills.length) + ? wholeFrameReuse( + ledger, + stillsShown({ ...refClip, filters: [] }, yours, ledger.stills, ref.fps), + series, + ks, + ref.fps, + opts.segments, + ) + : undefined; + if (reuse && !reuse.pass) { + const seconds = reuse.drawn.reduce((sum, [a, b]) => sum + b - a, 0); + warnings.push( + `${round(seconds, 2)} s play the original's whole frame with its drawn things in it (${reuse.drawn.map(([a, b]) => `${a}–${b} s`).join(", ")}): a copy, not a reverse. Cut only the camera image, each piece on its own box, and rebuild what is drawn`, + ); + for (const v of segments ?? []) + if (reuse.drawn.some(([a, b]) => a < v.window[1] && b > v.window[0])) { + v.pass = false; + v.note = "plays the original's whole frame with its drawn things: a copy"; + } + } else if (reuse?.spans.length && !opts.segments) + warnings.push( + `${Math.round(reuse.share * 100)}% of frames play the original's whole frame: pass --segments REVERSE.md to tell footage from copied drawings`, + ); const sound = yoursInfo && pick.kind === "all" ? soundOffset(opts.reference, opts.yours, ref, yoursInfo) @@ -732,12 +1354,21 @@ export async function diffVideos(options: DiffOptions): Promise { : undefined; const judged = segments ? segments.every((s) => s.pass !== false) : passes(overall, t); + const top = segments + ? headline( + segments, + clips.map(({ info: _info, file: _file, still: _still, ...c }) => c), + t, + cuts?.reference, + ) + : undefined; const complete = series.frame.length === ks.length; return { pass: judged && complete && missing.pass && + (reuse?.pass ?? true) && (cuts?.pass ?? true) && (duration?.pass ?? true) && (sound?.pass ?? true), @@ -748,11 +1379,14 @@ export async function diffVideos(options: DiffOptions): Promise { thresholds: t, ssim: overall, missing_content: missing, + ...(reuse ? { whole_frame_reuse: reuse } : {}), worst, ...(cuts ? { cuts } : {}), ...(duration ? { duration } : {}), ...(sound ? { sound } : {}), + ...(top ? top : {}), ...(segments ? { segments } : {}), + ...(elements ? { drawn: elements } : {}), ...(sheet ? { sheet } : {}), ...(sbs ? { sbs } : {}), log: logPath, diff --git a/packages/cli/src/measure/drawn.test.ts b/packages/cli/src/measure/drawn.test.ts new file mode 100644 index 0000000000..cb8c6ceb37 --- /dev/null +++ b/packages/cli/src/measure/drawn.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { DRAWN_MIN_MEAN, drawnVerdict, scoreDrawnFrame } from "./drawn.js"; + +const [W, H] = [160, 48]; + +/** A crop: a background, then bars standing in for letters (x, width) in one colour. */ +function crop( + background: (x: number, y: number) => [number, number, number], + bars: [number, number][], + ink: [number, number, number] = [255, 255, 255], + dy = 0, +): Uint8Array { + const px = new Uint8Array(W * H * 3); + for (let y = 0; y < H; y++) + for (let x = 0; x < W; x++) { + const on = y >= 10 + dy && y < 38 + dy && bars.some(([a, w]) => x >= a && x < a + w); + px.set(on ? ink : background(x, y), (y * W + x) * 3); + } + return px; +} + +// Two different backgrounds behind the same letters: a warm room and a cool, striped one. +const room = (x: number, y: number): [number, number, number] => [90 + (x % 7), 60 + (y % 5), 40]; +const street = (x: number, _y: number): [number, number, number] => [ + 30, + 70 + ((x >> 3) % 2) * 25, + 110, +]; +const word: [number, number][] = [10, 30, 50, 70, 90, 110, 130].map((x) => [x, 8]); + +describe("scoreDrawnFrame", () => { + it("scores the same letters over a different background as a match", () => { + const f = scoreDrawnFrame(crop(room, word), crop(street, word), W, H); + expect(f.score).toBeGreaterThan(DRAWN_MIN_MEAN); + expect(f.offset).toEqual([0, 0]); + }); + + it("forgives a pixel or two of offset and says how far", () => { + const f = scoreDrawnFrame(crop(room, word), crop(street, word, undefined, 2), W, H); + expect(f.score).toBeGreaterThan(DRAWN_MIN_MEAN); + expect(f.offset).toEqual([0, 2]); + }); + + it("fails lighter weight, wider tracking, a doubled letter and another fill", () => { + const light = word.map(([x]): [number, number] => [x, 4]); + const wide = word.map(([x, w], i): [number, number] => [x + i * 3, w]); + const doubled = [...word, ...word.map(([x, w]): [number, number] => [x + 12, w])]; + for (const yours of [ + crop(street, light), + crop(street, wide), + crop(street, doubled), + crop(street, word, [250, 210, 0]), + ]) + expect(scoreDrawnFrame(crop(room, word), yours, W, H).score).toBeLessThan(DRAWN_MIN_MEAN); + }); + + it("scores a missing element as nothing matched, and nothing on both sides as no frame", () => { + expect(scoreDrawnFrame(crop(room, word), crop(street, []), W, H).score).toBeLessThan(0.5); + const flat = () => [128, 128, 128] as [number, number, number]; + expect(scoreDrawnFrame(crop(flat, []), crop(flat, []), W, H).score).toBeNull(); + }); +}); + +describe("drawnVerdict", () => { + it("passes on the mean and the 5th percentile of the frames, and reports ink weight", () => { + const good = scoreDrawnFrame(crop(room, word), crop(street, word), W, H); + const gone = scoreDrawnFrame(crop(room, word), crop(street, []), W, H); + const v = drawnVerdict([0, 0, W, H], [0, 1], Array(10).fill(good)); + expect(v).toMatchObject({ pass: true, frames: 10, ink: 1 }); + const bad = drawnVerdict([0, 0, W, H], [0, 1], [...Array(9).fill(good), gone]); + expect(bad.pass).toBe(false); + expect(drawnVerdict([0, 0, W, H], [0, 1], [])).toMatchObject({ pass: null, frames: 0 }); + }); +}); diff --git a/packages/cli/src/measure/drawn.ts b/packages/cli/src/measure/drawn.ts new file mode 100644 index 0000000000..7ea920d85a --- /dev/null +++ b/packages/cli/src/measure/drawn.ts @@ -0,0 +1,231 @@ +// A drawn element (text, a badge, a card) over a regenerated person, scored by itself: what is behind it differs on +// purpose, so SSIM over its box measures the background. Each frame finds the element's ink on both sides (pixels +// far from the median colour of their neighbourhood, so a letter stands out from any background behind it), aligns +// the two within a few pixels, and asks how much of either side's ink the other matches in colour. Wrong weight, +// tracking, slant, size, fill, a doubled letter or a late onset all leave ink unmatched; a different background +// leaves none. Calibrated on the reverse bench (10-23): correct titles and captions 0.82–0.94, wrong ones 0.40–0.66. +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { round, runFfmpeg, type Clip, type Roi } from "./media.js"; + +export const DRAWN_MIN_MEAN = 0.8; +export const DRAWN_MIN_P05 = 0.65; +// A pixel is ink when its colour is this far (RGB distance) from its neighbourhood's median. +const INK_CONTRAST = 60; +// Two pixels match within this RGB distance, one pixel apart at most, after aligning the sides this many pixels. +const MATCH_COLOUR = 45; +const ALIGN = 3; +// Fewer ink pixels than this on both sides: nothing drawn in that frame on either side. +const MIN_INK = 30; +// Elements are read at most this often. +const MAX_FPS = 10; + +export interface DrawnFrame { + /** Share of both sides' ink the other side matches (F1 of matched ink), or null when neither side has ink. */ + score: number | null; + /** Where yours sits against the original, x and y in pixels (within ±3; further lowers the score). */ + offset: [number, number]; + inkOriginal: number; + inkYours: number; +} + +export interface DrawnVerdict { + box: Roi; + window: [number, number]; + frames: number; + pass: boolean | null; + /** Mean and 5th percentile of the frames' scores. */ + score?: number; + p05?: number; + /** Median of the frames' offsets. */ + offset?: [number, number]; + /** Yours' ink over the original's, median of frames where both have ink: under 1 is lighter or smaller. */ + ink?: number; +} + +/** Each pixel's colour distance from the median colour of the blocks around it. */ +export function contrast(px: Uint8Array, w: number, h: number): Float32Array { + const bs = Math.max(4, Math.floor(Math.max(8, Math.floor(h / 2)) / 2)); + const [gw, gh] = [Math.ceil(w / bs), Math.ceil(h / bs)]; + const med = new Float32Array(gw * gh * 3); + const hist = new Uint32Array(256); + for (let by = 0; by < gh; by++) + for (let bx = 0; bx < gw; bx++) { + const [x0, x1] = [Math.max(0, (bx - 1) * bs), Math.min(w, (bx + 2) * bs)]; + const [y0, y1] = [Math.max(0, (by - 1) * bs), Math.min(h, (by + 2) * bs)]; + const half = ((x1 - x0) * (y1 - y0)) / 2; + for (let c = 0; c < 3; c++) { + hist.fill(0); + for (let y = y0; y < y1; y++) + for (let x = x0; x < x1; x++) hist[px[(y * w + x) * 3 + c]!]!++; + let [v, seen] = [0, 0]; + while (v < 255 && seen + hist[v]! < half) seen += hist[v++]!; + med[(by * gw + bx) * 3 + c] = v; + } + } + const out = new Float32Array(w * h); + for (let y = 0; y < h; y++) + for (let x = 0; x < w; x++) { + const [o, m] = [(y * w + x) * 3, (Math.floor(y / bs) * gw + Math.floor(x / bs)) * 3]; + out[y * w + x] = Math.hypot( + px[o]! - med[m]!, + px[o + 1]! - med[m + 1]!, + px[o + 2]! - med[m + 2]!, + ); + } + return out; +} + +const inkOf = (px: Uint8Array, w: number, h: number) => { + const c = contrast(px, w, h); + return Uint8Array.from(c, (v) => (v > INK_CONTRAST ? 1 : 0)); +}; + +/** One frame of an element: the original's and yours' RGB crops of the same box. */ +export function scoreDrawnFrame(a: Uint8Array, b: Uint8Array, w: number, h: number): DrawnFrame { + const [ia, ib] = [inkOf(a, w, h), inkOf(b, w, h)]; + const [na, nb] = [ia.reduce((s, v) => s + v, 0), ib.reduce((s, v) => s + v, 0)]; + if (na + nb < MIN_INK) return { score: null, offset: [0, 0], inkOriginal: na, inkYours: nb }; + // The shift of yours whose ink overlaps the original's most. + let best = { iou: -1, dx: 0, dy: 0 }; + for (let dy = -ALIGN; dy <= ALIGN; dy++) + for (let dx = -ALIGN; dx <= ALIGN; dx++) { + let [both, either] = [0, 0]; + for (let y = 0; y < h; y++) { + const sy = y - dy; + for (let x = 0; x < w; x++) { + const sx = x - dx; + const vb = sy >= 0 && sy < h && sx >= 0 && sx < w ? ib[sy * w + sx]! : 0; + const va = ia[y * w + x]!; + both += va & vb; + either += va | vb; + } + } + const iou = either ? both / either : 0; + if (iou > best.iou) best = { iou, dx, dy }; + } + // Ink on either side counts as matched when the other side has a pixel of its colour within one pixel. + const at = (x: number, y: number) => + (Math.min(h - 1, Math.max(0, y)) * w + Math.min(w - 1, Math.max(0, x))) * 3; + const near = (p: Uint8Array, o: number, q: Uint8Array, x: number, y: number) => { + for (let ry = -1; ry <= 1; ry++) + for (let rx = -1; rx <= 1; rx++) { + const k = at(x + rx, y + ry); + if (Math.hypot(p[o]! - q[k]!, p[o + 1]! - q[k + 1]!, p[o + 2]! - q[k + 2]!) < MATCH_COLOUR) + return true; + } + return false; + }; + let [hitA, hitB, inkA, inkB] = [0, 0, 0, 0]; + for (let y = 0; y < h; y++) + for (let x = 0; x < w; x++) { + const i = y * w + x; + const [sx, sy] = [x - best.dx, y - best.dy]; + const inB = sy >= 0 && sy < h && sx >= 0 && sx < w && ib[sy * w + sx] === 1; + if (ia[i]) { + inkA++; + if (near(a, i * 3, b, sx, sy)) hitA++; + } + if (inB) { + inkB++; + if (near(b, at(sx, sy), a, x, y)) hitB++; + } + } + const [recall, precision] = [inkA ? hitA / inkA : 0, inkB ? hitB / inkB : 0]; + return { + score: recall + precision ? (2 * recall * precision) / (recall + precision) : 0, + offset: [0 - best.dx, 0 - best.dy], + inkOriginal: na, + inkYours: nb, + }; +} + +const median = (v: number[]) => [...v].sort((x, y) => x - y)[v.length >> 1] ?? 0; + +/** The verdict of an element from its frames. */ +export function drawnVerdict( + box: Roi, + window: [number, number], + frames: DrawnFrame[], +): DrawnVerdict { + const head = { box, window, frames: frames.length }; + const scored = frames.flatMap((f) => (f.score === null ? [] : [f])); + if (!scored.length) return { ...head, pass: null }; + const sorted = scored.map((f) => f.score!).sort((x, y) => x - y); + const mean = sorted.reduce((s, v) => s + v, 0) / sorted.length; + const p05 = sorted[Math.min(sorted.length - 1, Math.floor(0.05 * sorted.length))]!; + const both = scored.filter((f) => f.inkOriginal >= MIN_INK && f.inkYours >= MIN_INK); + return { + ...head, + pass: mean >= DRAWN_MIN_MEAN && p05 >= DRAWN_MIN_P05, + score: round(mean, 3), + p05: round(p05, 3), + offset: [median(scored.map((f) => f.offset[0])), median(scored.map((f) => f.offset[1]))], + ...(both.length ? { ink: round(median(both.map((f) => f.inkYours / f.inkOriginal)), 2) } : {}), + }; +} + +export interface DrawnJob { + box: Roi; + /** Indexes into the compared frames (the clips' frames after their own filters), in order. */ + frames: number[]; +} + +/** + * Each job's crops from a clip scaled to `width`x`height` (the original's size), one decode for all of them: + * `frames[j][n]` is job j's n-th frame, RGB at the box's size. + */ +export function readDrawn( + clip: Clip, + jobs: DrawnJob[], + width: number, + height: number, +): Uint8Array[][] { + if (!jobs.length) return []; + const work = mkdtempSync(join(tmpdir(), "hf-diff-drawn-")); + try { + const tags = jobs.map((_, j) => `[d${j}]`).join(""); + const graph = [ + `[0:v]${[...clip.filters, "setsar=1", `scale=${width}:${height}:flags=bicubic`].join(",")},split=${jobs.length}${tags}`, + ...jobs.map(({ box: [x, y, w, h], frames }, j) => { + const pick = frames.map((k) => `eq(n\\,${k})`).join("+"); + return `[d${j}]select=${pick},crop=${w}:${h}:${x}:${y},format=rgb24[e${j}]`; + }), + ].join(";"); + const outs = jobs.flatMap((_, j) => [ + ...["-map", `[e${j}]`, "-fps_mode", "passthrough", "-f", "rawvideo", `e${j}.rgb`], + ]); + runFfmpeg([...clip.input, "-an", "-filter_complex", graph, ...outs], undefined, work); + return jobs.map(({ box: [, , w, h] }, j) => { + const raw = readFileSync(join(work, `e${j}.rgb`)); + const size = w * h * 3; + const frames: Uint8Array[] = []; + for (let i = 0; i + size <= raw.length; i += size) frames.push(raw.subarray(i, i + size)); + return frames; + }); + } finally { + rmSync(work, { recursive: true, force: true }); + } +} + +/** The compared frames (indexes into `ks`) inside [a, b) seconds, thinned to MAX_FPS. */ +export function drawnFrames( + ks: number[], + fps: number, + a: number, + b: number, + available: number, +): number[] { + const stride = Math.max(1, Math.round(fps / MAX_FPS)); + const inside = ks.flatMap((k, i) => + i < available && k / fps >= a - 1e-6 && k / fps < b - 1e-6 ? [i] : [], + ); + // Every stride-th frame of a run of consecutive frames; moments already that far apart (--at) all count. + let last = -Infinity; + return inside.filter((i) => { + if (ks[i]! - last < stride) return false; + last = ks[i]!; + return true; + }); +} diff --git a/packages/cli/src/measure/face.test.ts b/packages/cli/src/measure/face.test.ts new file mode 100644 index 0000000000..d6bdbe0c89 --- /dev/null +++ b/packages/cli/src/measure/face.test.ts @@ -0,0 +1,149 @@ +import { execFileSync } from "node:child_process"; +import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir, userInfo } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; +import { againstStart, faceBoxes, faceFit, minWindow, readSamples, sampleFrames } from "./face.js"; +import { probeVideo, runFfmpeg } from "./media.js"; +import { OPENCV_DIR } from "./opencv.js"; + +// As in opencv.test.ts: borrow this machine's verified opencv.js cache, skip without one. +const cached = join(userInfo().homedir, ".cache", "hyperframes", "opencv", "4.10.0"); +if (existsSync(join(cached, "opencv.js")) && !existsSync(join(OPENCV_DIR, "opencv.js"))) + cpSync(cached, OPENCV_DIR, { recursive: true }); +const media = !!findFFmpeg() && !!findFFprobe(); +const opencv = media && existsSync(join(OPENCV_DIR, "opencv.js")); +const dir = mkdtempSync(join(tmpdir(), "hf-face-test-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +describe("sampleFrames", () => { + it("spreads the samples evenly, the last frame included, each once", () => { + expect(sampleFrames(30, 0, 1, 4)).toEqual([0, 10, 19, 29]); + expect(sampleFrames(30, 2, 2.1, 12)).toEqual([60, 61, 62]); + expect(sampleFrames(24, 1, 1.01, 8)).toEqual([24]); + }); +}); + +describe("minWindow", () => { + it("is a step of the cascades' 20 px x 1.1 ladder, no larger than the share", () => { + expect(minWindow(1280, 0)).toBe(20); + expect(minWindow(400, 0.04)).toBe(20); + // 4% of 1280 is 51.2 px: 20 x 1.1^9 = 47.2 is the last step under it. + expect(minWindow(1280, 0.04)).toBe(47); + expect(minWindow(1080, 0.04)).toBe(43); + }); +}); + +describe("againstStart", () => { + it("reads a generated shot from its own start and a full render on film time", () => { + // A 5.2 s HeyGen shot for the original's 9.5-12.27 s: its face is at 0-2.77 s of the clip, not 9.5. + expect(againstStart(5.17, 12.27, 9.5)).toBe(0); + // A shot longer than the window but shorter than the film is still its own clip (it used to be read from --from). + expect(againstStart(7.29, 12.27, 2.77)).toBe(0); + expect(againstStart(12.27, 12.27, 9.5)).toBe(9.5); + expect(againstStart(12.3, 12.27, 9.5)).toBe(9.5); + }); +}); + +describe("faceFit", () => { + const at = (t: number, ...boxes: number[][]) => ({ t, boxes }); + const canvas = { width: 720, height: 1280 }; + + it("gives the scale about the centre and the x, y that land the clip's face on the original's", () => { + // The clip's face, 0.1 high at (0.5, 0.4); the original's twice the size, 10 px right and 20 px up of where + // scaling about the centre puts it: centre y 640 + 2 (512 - 640) - 20 = 364. + const clip = [0, 0.5, 1, 1.5].map((t) => at(t, [0.45, 0.35, 0.1, 0.1])); + const [w, h] = [0.2, 0.2]; + const [cx, cy] = [(360 + 10) / 720, 364 / 1280]; + const original = [0, 0.5, 1, 1.5].map((t) => + at(t, [cx - w / 2, cy - h / 2, w, h], [0.05, 0.05, 0.03, 0.03]), + ); + const fit = faceFit(original, clip, canvas, canvas); + expect(fit.moments).toBe(4); + expect(fit.transform).toEqual({ scale: 2, x: 10, y: -20 }); + expect(fit.residual).toBe(0); + }); + + it("fits a clip of another size through object-fit: cover, and shrugs off a stray moment", () => { + // 768x1344 under a 720x1280 canvas: cover scales it by 1280/1344 and crops its sides; its face then sits + // where the original's is, at the same height, so nothing moves. + const s0 = 1280 / 1344; + const ox = (720 - 768 * s0) / 2; + const [fx, fy, fh] = [360, 500, 200]; + const clipBox = [ + (fx - ox - fh / 2) / (768 * s0), + (fy - fh / 2) / 1280, + fh / (768 * s0), + fh / 1280, + ]; + const origBox = [(fx - fh / 2) / 720, (fy - fh / 2) / 1280, fh / 720, fh / 1280]; + const clip = [0, 0.25, 0.5, 0.75, 1].map((t) => at(t, clipBox)); + const original = [0, 0.25, 0.5, 0.75, 1].map((t, i) => + at(t + 0.02, i === 2 ? [0.1, 0.1, 0.4, 0.4] : origBox), + ); + const fit = faceFit(original, clip, canvas, { width: 768, height: 1344 }); + expect(fit.transform).toEqual({ scale: 1, x: 0, y: 0 }); + }); + + it("gives no transform without faces on both sides at enough moments", () => { + const fit = faceFit( + [at(0, [0.4, 0.3, 0.2, 0.1]), at(1)], + [at(0, [0.4, 0.3, 0.2, 0.1]), at(1, [0.4, 0.3, 0.2, 0.1])], + canvas, + canvas, + ); + expect(fit).toEqual({ moments: 1, transform: null }); + // Moments farther apart than a tenth of a second do not pair. + expect( + faceFit( + [at(0, [0.4, 0.3, 0.2, 0.1]), at(1, [0.4, 0.3, 0.2, 0.1])], + [at(0.5, [0.4, 0.3, 0.2, 0.1]), at(1.5, [0.4, 0.3, 0.2, 0.1])], + canvas, + canvas, + ).transform, + ).toBeNull(); + }); +}); + +describe.skipIf(!media)("readSamples", () => { + // Every frame different, keyframes 2.5 s apart and B-frames: what a seek has to get right. + const video = join(dir, "frames.mp4"); + if (media) + execFileSync(findFFmpeg()!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=160x96:r=30:d=8"], + ...["-c:v", "libx264", "-g", "75", "-bf", "2", "-pix_fmt", "yuv420p", video], + ]); + + it("seeks to the same frames that decoding from the start picks", () => { + const info = probeVideo(video); + // Nearby samples (one pass) and samples seconds apart (a seek each), off and on keyframes. + const ks = [1, 2, 40, 74, 75, 76, 140, 141, 210, 239]; + const got = readSamples(info, ks, 80, 48); + const raw = runFfmpeg([ + ...["-i", video, "-an", "-vf"], + `select=${ks.map((k) => `eq(n\\,${k})`).join("+")},scale=80:48,format=gray`, + ...["-fps_mode", "passthrough", "-f", "rawvideo", "-"], + ]); + expect(got).toHaveLength(ks.length); + got.forEach((f, i) => + expect(Buffer.compare(f, raw.subarray(i * 80 * 48, (i + 1) * 80 * 48))).toBe(0), + ); + }); +}); + +describe.skipIf(!opencv)("faceBoxes", () => { + const video = join(dir, "blank.mp4"); + if (opencv) + execFileSync(findFFmpeg()!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=0x406080:s=320x568:r=30:d=4"], + ...["-c:v", "libx264", "-pix_fmt", "yuv420p", video], + ]); + + it("labels every sampled frame and finds no face in a blank one", async () => { + const r = await faceBoxes(probeVideo(video), 1, 3, 5); + expect(r.map((s) => s.frame)).toEqual([30, 45, 60, 74, 89]); + expect(r.map((s) => s.t)).toEqual([1, 1.5, 2, 2.467, 2.967]); + expect(r.every((s) => s.boxes.length === 0)).toBe(true); + }); +}); diff --git a/packages/cli/src/measure/face.ts b/packages/cli/src/measure/face.ts index 5a1cc735db..5580ecdc62 100644 --- a/packages/cli/src/measure/face.ts +++ b/packages/cli/src/measure/face.ts @@ -1,13 +1,24 @@ // How big the face is and where, ported from hf-media's face_size: OpenCV's frontal and profile Haar cascades on // evenly spaced frames, graded by face height over frame height. A regenerated presenter keeps the likeness from // FACE_HOLDS up; below FACE_FLOOR a video model cannot carry one at all (the harness's face-size eval, 2026-10-08). -import { frameAt, readClip, round, videoClip, type VideoInfo } from "./media.js"; +import { frameAt, round, runFfmpeg, type VideoInfo } from "./media.js"; import { greyMat, loadOpenCv } from "./opencv.js"; export const FACE_HOLDS = 0.25; export const FACE_SLIDES = 0.15; export const FACE_FLOOR = 0.06; +/** + * The smallest face looked for by default, as a share of the frame's height: well under FACE_FLOOR (a smaller face + * fails either way), and it skips the cascades' smallest, costliest windows. Those start at 20 px and grow by 10% a + * step, so a minimum on one of the steps finds the faces the full search finds from that size up, with the same + * boxes; scaling the frame down instead moves every box by up to a step. + */ +export const FACE_MIN = 0.04; const WORK_W = 960; +const STEP = 1.1; +const WINDOW_PX = 20; +// Samples closer than this share one decode from the first of them; farther ones get a seek of their own. +const RUN_S = 2; type Verdict = "pass" | "warn" | "fail" | "unmeasured"; const ORDER: Verdict[] = ["fail", "unmeasured", "warn", "pass"]; @@ -18,24 +29,87 @@ export function faceVerdict(height: number | null, cast = false): Verdict { return height < (cast ? FACE_SLIDES : FACE_FLOOR) ? "fail" : "warn"; } -/** Each sampled frame's faces as [x, y, w, h] shares of the frame. */ -export async function faceBoxes(info: VideoInfo, start: number, end: number, samples: number) { - const cv = await loadOpenCv((m) => console.error(m)); - const [k0, k1] = [ - frameAt(start, info.fps), - Math.max(frameAt(start, info.fps), Math.ceil(end * info.fps - 1e-6) - 1), - ]; +/** `samples` frame indices spread evenly over [start, end), the last frame included. */ +export function sampleFrames(fps: number, start: number, end: number, samples: number): number[] { + const k0 = frameAt(start, fps); + const k1 = Math.max(k0, Math.ceil(end * fps - 1e-6) - 1); const n = Math.min(samples, k1 - k0 + 1); - const ks = [ + return [ ...new Set( Array.from({ length: n }, (_, i) => k0 + Math.round((i * (k1 - k0)) / Math.max(1, n - 1))), ), ]; - const clip = { - ...videoClip(info.path, 0, undefined, info), - filters: [`select=${ks.map((k) => `eq(n\\,${k})`).join("+")}`, `setpts=N/(${info.fpsText})/TB`], - }; - const { frames, w, h } = readClip(clip, { width: Math.min(WORK_W, info.width), gray: true }); +} + +/** The largest window on the cascades' 20 px x 1.1^k ladder that is no more than `share` of a frame `h` high. */ +export function minWindow(h: number, share: number): number { + const steps = Math.floor(Math.log(Math.max(1, (share * h) / WINDOW_PX)) / Math.log(STEP) + 1e-9); + return Math.round(WINDOW_PX * STEP ** steps); +} + +/** + * Frames `ks` (indices from the first frame) as grey bytes at `w` x `h`, which must keep the video's aspect. Each run + * of nearby frames is one ffmpeg pass that seeks to its first frame and decodes only to its last; a variable-rate + * video, whose indices a seek cannot find, is read from its start. + */ +export function readSamples(info: VideoInfo, ks: number[], w: number, h: number): Uint8Array[] { + const runs: number[][] = []; + for (const k of ks) { + const run = runs.at(-1); + if (run && (info.vfr || (k - run.at(-1)!) / info.fps <= RUN_S)) run.push(k); + else runs.push([k]); + } + const size = w * h; + return runs.flatMap((run) => { + // Half a frame early: an input seek drops the frames before it, so the first one left is frame run[0]. + const k0 = info.vfr ? 0 : run[0]!; + const seek = k0 > 0 ? ["-ss", ((k0 - 0.5) / info.fps).toFixed(6)] : []; + const filters = [ + `select=${run.map((k) => `eq(n\\,${k - k0})`).join("+")}`, + ...(w !== info.width || h !== info.height ? [`scale=${w}:${h}`] : []), + "format=gray", + ]; + const raw = runFfmpeg([ + ...seek, + "-i", + info.path, + "-an", + "-vf", + filters.join(","), + "-fps_mode", + "passthrough", + "-frames:v", + String(run.length), + "-f", + "rawvideo", + "-", + ]); + const frames: Uint8Array[] = []; + for (let i = 0; i + size <= raw.length; i += size) frames.push(raw.subarray(i, i + size)); + return frames; + }); +} + +/** + * Each sampled frame's faces as [x, y, w, h] shares of the frame. `minFace` is the smallest face looked for as a + * share of the frame's height (default FACE_MIN; 0 looks down to the cascades' 20 px, for crowds). + */ +export async function faceBoxes( + info: VideoInfo, + start: number, + end: number, + samples: number, + opts: { minFace?: number } = {}, +) { + const cv = await loadOpenCv((m) => console.error(m)); + const ks = sampleFrames(info.fps, start, end, samples); + const w = Math.min(WORK_W, info.width); + const h = + w === info.width + ? info.height + : Math.max(2, Math.round((info.height * w) / info.width / 2) * 2); + const frames = readSamples(info, ks, w, h); + const min = minWindow(h, opts.minFace ?? FACE_MIN); const cascades = ["haarcascade_frontalface_alt2.xml", "haarcascade_profileface.xml"].map( (name) => { const c = new cv.CascadeClassifier(); @@ -50,7 +124,7 @@ export async function faceBoxes(info: VideoInfo, start: number, end: number, sam try { for (const c of cascades) { const found = new cv.RectVector(); - c.detectMultiScale(gray, found, 1.1, 5, 0, new cv.Size(20, 20)); + c.detectMultiScale(gray, found, STEP, 5, 0, new cv.Size(min, min)); for (let j = 0; j < found.size(); j++) { const r = found.get(j); boxes.push([r.x / w, r.y / h, r.width / w, r.height / h]); @@ -67,16 +141,20 @@ export async function faceBoxes(info: VideoInfo, start: number, end: number, sam } } +/** The subject's face among a frame's: the largest, or the smallest when the person is cast in a crowd. */ +export function subjectFace(boxes: number[][], cast = false): number[] | null { + return boxes.reduce( + (a, b) => (!a || (cast ? b[3]! < a[3]! : b[3]! > a[3]!) ? b : a), + null, + ); +} + /** The subject's face (largest, or the smallest when the person is cast in a crowd), its median and last height. */ export function faceSummary( samples: { frame: number; t: number; boxes: number[][] }[], cast = false, ) { - const pick = (boxes: number[][]) => - boxes.reduce( - (a, b) => (!a || (cast ? b[3]! < a[3]! : b[3]! > a[3]!) ? b : a), - null, - ); + const pick = (boxes: number[][]) => subjectFace(boxes, cast); const seen = samples.flatMap((s) => { const b = pick(s.boxes); return b ? [{ t: s.t, box: b.map((v) => round(v, 4)) }] : []; @@ -101,3 +179,79 @@ export function faceSummary( : {}), }; } + +const median = (v: number[]) => [...v].sort((x, y) => x - y)[v.length >> 1]!; + +/** Fewest moments with a face on both sides that a fit is given from. */ +export const FIT_MIN_FACES = 2; +const PAIR_S = 0.1; + +/** + * Where --against starts reading the clip: a render as long as the original runs on film time, so at --from; a + * shot of its own (a generated clip) starts at its own 0. + */ +export function againstStart(clipDuration: number, originalDuration: number, from: number): number { + return Math.abs(clipDuration - originalDuration) <= 0.5 ? from : 0; +} + +/** + * The GSAP transform that puts a clip's face on the original's: `scale` about the clip element's centre, then `x`, `y` + * in the original's (the canvas's) pixels, for a clip element that fills the canvas with object-fit: cover. Moments + * pair by time (`t` from each side's window start, within PAIR_S); each side's subject face is taken per moment and + * their centres and heights medianed, so one stray detection does not move the fit. `residual` is the median distance + * left between the faces after it, in the original's face heights. + */ +export function faceFit( + original: { t: number; boxes: number[][] }[], + clip: { t: number; boxes: number[][] }[], + canvas: { width: number; height: number }, + clipSize: { width: number; height: number }, + cast = false, +) { + const pairs = original.flatMap((o) => { + const near = clip.reduce<(typeof clip)[number] | null>( + (p, q) => (!p || Math.abs(q.t - o.t) < Math.abs(p.t - o.t) ? q : p), + null, + ); + if (!near || Math.abs(near.t - o.t) > PAIR_S) return []; + const [a, b] = [subjectFace(o.boxes, cast), subjectFace(near.boxes, cast)]; + return a && b ? [{ a, b }] : []; + }); + if (pairs.length < FIT_MIN_FACES) return { moments: pairs.length, transform: null }; + const { width: W, height: H } = canvas; + // object-fit: cover, centred: the clip's pixel (u, v) lands on the canvas at (ox + u s0, oy + v s0). + const s0 = Math.max(W / clipSize.width, H / clipSize.height); + const [cw, ch] = [clipSize.width * s0, clipSize.height * s0]; + const [ox, oy] = [(W - cw) / 2, (H - ch) / 2]; + const face = (b: number[], w: number, h: number, x0: number, y0: number) => ({ + x: x0 + (b[0]! + b[2]! / 2) * w, + y: y0 + (b[1]! + b[3]! / 2) * h, + h: b[3]! * h, + }); + const os = pairs.map((p) => face(p.a, W, H, 0, 0)); + const cs = pairs.map((p) => face(p.b, cw, ch, ox, oy)); + const mid = (v: { x: number; y: number; h: number }[]) => ({ + x: median(v.map((f) => f.x)), + y: median(v.map((f) => f.y)), + h: median(v.map((f) => f.h)), + }); + const [o, c] = [mid(os), mid(cs)]; + const scale = o.h / c.h; + // About the element's centre (GSAP's default transform origin): p -> centre + scale (p - centre) + (x, y). + const x = o.x - (W / 2 + scale * (c.x - W / 2)); + const y = o.y - (H / 2 + scale * (c.y - H / 2)); + const residual = median( + pairs.map((_, i) => { + const [p, q] = [os[i]!, cs[i]!]; + const [mx, my] = [W / 2 + scale * (q.x - W / 2) + x, H / 2 + scale * (q.y - H / 2) + y]; + return Math.hypot(mx - p.x, my - p.y) / p.h; + }), + ); + return { + moments: pairs.length, + transform: { scale: round(scale, 3), x: round(x, 1), y: round(y, 1) }, + original_face: { centre: [round(o.x, 1), round(o.y, 1)], height: round(o.h, 1) }, + clip_face: { centre: [round(c.x, 1), round(c.y, 1)], height: round(c.h, 1) }, + residual: round(residual, 3), + }; +} diff --git a/packages/cli/src/measure/media.ts b/packages/cli/src/measure/media.ts index e96b04a005..39ec256a62 100644 --- a/packages/cli/src/measure/media.ts +++ b/packages/cli/src/measure/media.ts @@ -48,8 +48,12 @@ export interface VideoInfo { /** The frame rate as ffmpeg writes it, e.g. 30000/1001. */ fpsText: string; duration: number; + /** The picture's length: the container's can run on with a longer sound track. */ + pictureDuration: number; frames: number; audio: boolean; + /** Seconds the sound starts after the picture (0 when it starts with it or there is none). */ + audioStart: number; /** Frames over seconds, which a variable-rate video's nominal rate does not tell. */ avgFps: number; /** The container's frame rate and the stream's average differ: frames are not evenly spaced. */ @@ -67,6 +71,7 @@ interface ProbeStream { avg_frame_rate?: string; nb_frames?: string; duration?: string; + start_time?: string; color_transfer?: string; side_data_list?: { rotation?: number }[]; tags?: { rotate?: string }; @@ -108,6 +113,8 @@ export function probeVideo(path: string): VideoInfo { const avg = rate(video.avg_frame_rate); const duration = Number(json.format?.duration ?? video.duration ?? 0); const counted = Number(video.nb_frames ?? 0); + const sound = streams.find((s) => s.codec_type === "audio"); + const frames = counted > 0 ? counted : Math.round(duration * fps); return { path, width: (turned ? video.height : video.width) ?? 0, @@ -115,8 +122,10 @@ export function probeVideo(path: string): VideoInfo { fps, fpsText: fpsText.endsWith("/1") ? fpsText.slice(0, -2) : fpsText, duration, - frames: counted > 0 ? counted : Math.round(duration * fps), - audio: streams.some((s) => s.codec_type === "audio"), + pictureDuration: Number(video.duration ?? 0) || frames / (avg || fps), + frames, + audio: Boolean(sound), + audioStart: Math.max(0, Number(sound?.start_time ?? 0) - Number(video.start_time ?? 0) || 0), avgFps: avg || fps, vfr: avg > 0 && Math.abs(avg - fps) / fps > 0.01, hdr: /smpte2084|arib-std-b67/.test(video.color_transfer ?? ""), diff --git a/packages/cli/src/measure/person.ts b/packages/cli/src/measure/person.ts new file mode 100644 index 0000000000..62d9e634b5 --- /dev/null +++ b/packages/cli/src/measure/person.ts @@ -0,0 +1,576 @@ +// A re-shot person never matches the original by pixels, so pixels do not judge them. Three gates do: the framing +// is clearly wrong (face centre or height off the original's, per shot), the generated clip carries the original's +// words (its motion guide's text came along), or the lips are out of sync. The rest (motion energy, look, whether +// each generated clip starts at one of the original's cuts) is reported, not judged. +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { contrast } from "./drawn.js"; +import { faceBoxes } from "./face.js"; +import { probeVideo, round, runFfmpeg, type Roi, type VideoInfo } from "./media.js"; +import { avSync, type SyncResult } from "./sync.js"; + +/** + * A shot's framing is clearly wrong when, over the shot, the face centre sits more than FRAMING_SHIFT face heights + * from the original's or its height is outside FRAMING_SCALE of it (medians). Calibrated on the reverse bench (10-23, + * faces at 4 a second, shots of a second or more): goat round 3's 8.0–10.0 s shot is 0.23 off and ×0.82, dan round 2 + * at most 0.16 and ×0.93–1.10, cindy round 3 0.02; dan round 3 misses its 35 s close-up at ×0.77 (and is 0.21–0.23 + * off at 3.6 s and 30 s, which face boxes cannot tell from goat). Shorter shots have too few faces to judge. + */ +export const FRAMING_SHIFT = 0.3; +export const FRAMING_SCALE: [number, number] = [0.8, 1.25]; +const FRAMING_MIN_S = 1; +const FRAMING_MIN_FACES = 3; +const FACE_RATE = 4; +/** Picture against sound (positive: the picture is late), the broadcast tolerance; judged on high confidence only. */ +export const SYNC_MS: [number, number] = [-125, 45]; +const SYNC_MIN_S = 2; +const SYNC_MIN_SPEECH = 0.3; +/** + * A generated clip carries a drawn element's text when it shows the original's letters inside the element's box: + * of the original's sharp ink (contrast over TEXT_CONTRAST) where it differs from the clean still the clip was made + * from and where yours draws over the clip (each by DRAWN_DIFF: the letters, not the place), the share the clip has + * within two pixels as sharp ink of the same colour (TEXT_COLOUR). The clip shares the person and the place with the + * original, so only carried letters match there. Calibrated on ig03 round 2 (10-23): t12's "Transparency", "a deep + * understanding", "of this market" 0.30, 0.45, 0.56; every clean box 0.24 or less. + */ +export const CARRIED_TEXT = 0.35; +const TEXT_CONTRAST = 140; +const DRAWN_DIFF = 60; +const TEXT_COLOUR = 15; +const TEXT_MIN_INK = 50; +const CARRIED_SAMPLES = 8; +/** A generated clip found by its sound needs this correlation with the original's; it starts at a cut within this. */ +const LOCATE_MIN = 0.6; +export const AT_CUT_S = 0.1; +const ENVELOPE_RATE = 100; + +export interface GeneratedClip { + /** As the ledger writes it, relative to the project. */ + path: string; + /** The still it was made from (a clean frame of the original), relative to the project. */ + reference?: string; + /** Film time of the clip's first frame, and how well its sound matched the original's there. */ + offset?: number; + match?: number; + window?: [number, number]; + /** Where the film shows it: film window, the clip's time at its start, and its playback rate. */ + uses?: ClipUse[]; + /** Whether it starts within AT_CUT_S of the film's start or one of the original's cuts. */ + at_cut?: boolean; + note?: string; +} + +export interface ClipUse { + window: [number, number]; + from: number; + rate: number; +} + +export interface ShotFraming { + window: [number, number]; + /** Medians: face centre distance in the original's face heights, and yours' face height over the original's. */ + shift: number; + scale: number; + pass: boolean; +} + +export interface CarriedText { + drawn: string; + clip: string; + window: [number, number]; + /** How well the generated clip's ink inside the box matches the original's text there. */ + score: number; + carried: boolean; +} + +export interface PersonVerdict { + /** Each gate: false when it fails, null when it could not be measured. */ + gates: { framing: boolean | null; text: boolean | null; sync: boolean | null }; + pass: boolean | null; + framing?: ShotFraming[]; + text?: CarriedText[]; + sync?: { + window: [number, number]; + lag_ms: number | null; + original_lag_ms: number | null; + confidence: string; + pass: boolean | null; + }[]; +} + +const isRecord = (v: unknown): v is Record => typeof v === "object" && v !== null; + +/** The ledger's generated videos (HeyGen), with the film window it records when it does. */ +export function generatedClips(ledger: string): GeneratedClip[] { + return ledger.split("\n").flatMap((line) => { + let entry: unknown; + try { + entry = JSON.parse(line); + } catch { + return []; + } + if (!isRecord(entry) || entry.type !== "video" || typeof entry.path !== "string") return []; + const p = entry.provenance; + if (!isRecord(p) || p.provider !== "heygen") return []; + const w = Array.isArray(p.window) ? p.window.filter((v) => typeof v === "number") : []; + const refs = Array.isArray(p.references) + ? p.references.filter((v) => typeof v === "string") + : []; + return [ + { + path: entry.path, + ...(refs[0] ? { reference: refs[0] } : {}), + ...(w.length === 2 ? { offset: w[0]!, window: [w[0]!, w[1]!] as [number, number] } : {}), + }, + ]; + }); +} + +/** Loudness every 10 ms of a file's sound. */ +function envelope(path: string): number[] { + const raw = runFfmpeg([...["-i", path, "-vn", "-ac", "1", "-ar", "8000", "-f", "f32le", "-"]]); + const s = new Float32Array( + raw.buffer.slice(raw.byteOffset, raw.byteOffset + (raw.byteLength & ~3)), + ); + const step = 8000 / ENVELOPE_RATE; + const out: number[] = []; + for (let i = 0; i + step <= s.length; i += step) { + let e = 0; + for (let j = i; j < i + step; j++) e += s[j]! ** 2; + out.push(Math.sqrt(e / step)); + } + return out; +} + +/** + * Where `b` sits inside `a` (index of a where b[0] lands) and the correlation there, over lags that keep at least + * nine tenths of b inside a. + */ +export function bestLag(a: number[], b: number[]): { lag: number; r: number } { + let best = { lag: 0, r: -2 }; + const keep = Math.ceil(0.9 * b.length); + for (let lag = -(b.length - keep); lag <= a.length - keep; lag++) { + const [i0, i1] = [Math.max(0, -lag), Math.min(b.length, a.length - lag)]; + const n = i1 - i0; + if (n < Math.max(50, keep)) continue; + let [ma, mb] = [0, 0]; + for (let i = i0; i < i1; i++) { + ma += a[i + lag]!; + mb += b[i]!; + } + [ma, mb] = [ma / n, mb / n]; + let [sab, sa, sb] = [0, 0, 0]; + for (let i = i0; i < i1; i++) { + const [x, y] = [a[i + lag]! - ma, b[i]! - mb]; + sab += x * y; + sa += x * x; + sb += y * y; + } + const r = sa && sb ? sab / Math.sqrt(sa * sb) : -2; + if (r > best.r) best = { lag, r }; + } + return best; +} + +/** + * Each generated clip placed on the film. Its own start (where HeyGen's shot begins) is the ledger's window when it + * records one, else where the clip's sound (the original's voice it was made from) lines up with the original's. + * Where the film shows it comes from the project's timeline (`uses`, by the clip's path), else its whole window. + * Files resolve against the project. + */ +export function placeClips( + clips: GeneratedClip[], + project: string | undefined, + ref: VideoInfo, + cuts: number[], + shown: Map = new Map(), +): (GeneratedClip & { file?: string; info?: VideoInfo; still?: string })[] { + let whole: number[] | undefined; + return clips.map((c) => { + const file = project ? join(project, c.path) : undefined; + let info: VideoInfo | undefined; + try { + info = file ? probeVideo(file) : undefined; + } catch { + info = undefined; + } + if (!info || !file) return { ...c, note: "the clip's file is not in the project" }; + const uses = shown.get(normalize(c.path)); + let placed: GeneratedClip = c; + if (c.offset === undefined && info.audio && ref.audio) { + whole ??= envelope(ref.path); + const { lag, r } = bestLag(whole, envelope(file)); + const offset = round(lag / ENVELOPE_RATE + ref.audioStart - info.audioStart, 3); + placed = + r >= LOCATE_MIN + ? { ...c, offset, match: round(r, 2) } + : { ...c, match: round(r, 2), note: "its sound does not match the original's" }; + } + if (placed.offset === undefined && uses?.length) + placed = { ...placed, offset: round(uses[0]!.window[0] - uses[0]!.from / uses[0]!.rate, 3) }; + if (placed.offset === undefined) + return { + ...placed, + file, + info, + note: placed.note ?? "no window in the ledger, no sound and no place in the project", + }; + const offset = placed.offset; + const end = Math.min(ref.pictureDuration, offset + info.pictureDuration); + return { + ...placed, + window: placed.window ?? [round(Math.max(0, offset), 3), round(end, 3)], + uses: uses ?? [ + { + window: [round(Math.max(0, offset), 3), round(end, 3)], + from: Math.max(0, -offset), + rate: 1, + }, + ], + at_cut: [0, ...cuts].some((t) => Math.abs(t - offset) <= AT_CUT_S), + file, + info, + ...(c.reference && existsSync(join(project!, c.reference)) + ? { still: join(project!, c.reference) } + : {}), + }; + }); +} + +/** Where a project's timeline shows each video file, by project-relative path. */ +export async function clipUses(project: string): Promise> { + const out = new Map(); + const index = join(project, "index.html"); + if (!existsSync(index)) return out; + const [{ ensureDOMParser }, { describeProject }] = await Promise.all([ + import("../utils/dom.js"), + import("../timeline/describeProject.js"), + ]); + ensureDOMParser(); + const timeline = await describeProject(index); + for (const row of timeline.tracks.find((t) => t.kind === "video")?.rows ?? []) { + if (!row.src || row.absEnd <= row.absStart) continue; + const key = normalize(row.src); + const use = { + window: [round(row.absStart, 3), round(row.absEnd, 3)] as [number, number], + from: row.mediaStart ?? 0, + rate: row.playbackRate ?? 1, + }; + out.set(key, [...(out.get(key) ?? []), use]); + } + return out; +} + +/** A project-relative path written one way. */ +export const normalize = (p: string) => p.replace(/^\.\//, "").replace(/\\/g, "/"); + +/** RGB crops of one box at these times: the picture scaled to cover `w`x`h` (the original's size) first. */ +function boxAt( + info: VideoInfo, + times: number[], + box: Roi, + w: number, + h: number, +): (Uint8Array | undefined)[] { + const ks = times.map((t) => Math.max(0, Math.min(info.frames - 1, Math.round(t * info.fps)))); + const unique = [...new Set(ks)].sort((x, y) => x - y); + const work = mkdtempSync(join(tmpdir(), "hf-person-")); + try { + const filters = [ + `select=${unique.map((k) => `eq(n\\,${k})`).join("+")}`, + `scale=${w}:${h}:force_original_aspect_ratio=increase:flags=bicubic`, + `crop=${w}:${h}`, + `crop=${box[2]}:${box[3]}:${box[0]}:${box[1]}`, + "format=rgb24", + ].join(","); + runFfmpeg( + [ + ...["-i", info.path, "-an", "-vf", filters, "-fps_mode", "passthrough"], + ...["-f", "rawvideo", "out.rgb"], + ], + undefined, + work, + ); + const raw = readFileSync(join(work, "out.rgb")); + const size = box[2] * box[3] * 3; + return ks.map((k) => { + const i = unique.indexOf(k) * size; + return i + size <= raw.length ? raw.subarray(i, i + size) : undefined; + }); + } finally { + rmSync(work, { recursive: true, force: true }); + } +} + +/** Within `r` pixels of (x, y), some pixel passes `test`. */ +function nearby( + w: number, + h: number, + x: number, + y: number, + r: number, + test: (k: number) => boolean, +) { + for (let dy = -r; dy <= r; dy++) + for (let dx = -r; dx <= r; dx++) { + const [xx, yy] = [Math.min(w - 1, Math.max(0, x + dx)), Math.min(h - 1, Math.max(0, y + dy))]; + if (test(yy * w + xx)) return true; + } + return false; +} + +const dist = (p: Uint8Array, i: number, q: Uint8Array, k: number) => + Math.hypot(p[i * 3]! - q[k * 3]!, p[i * 3 + 1]! - q[k * 3 + 1]!, p[i * 3 + 2]! - q[k * 3 + 2]!); + +/** One frame: the share of the original's letters (apart from the still and from what yours draws) the clip shows. */ +export function carriedFrame( + original: Uint8Array, + clip: Uint8Array, + still: Uint8Array | undefined, + w: number, + h: number, + yours: Uint8Array | undefined, +): number | null { + const [sharp, inked] = [contrast(original, w, h), contrast(clip, w, h)]; + let [n, hit] = [0, 0]; + for (let y = 0; y < h; y++) + for (let x = 0; x < w; x++) { + const i = y * w + x; + if (sharp[i]! <= TEXT_CONTRAST) continue; + if (still && !nearby(w, h, x, y, 2, (k) => dist(original, k, still, k) > DRAWN_DIFF)) + continue; + if (yours && !nearby(w, h, x, y, 2, (k) => dist(yours, k, clip, k) > DRAWN_DIFF)) continue; + n++; + const same = (k: number) => + inked[k]! > TEXT_CONTRAST && dist(original, i, clip, k) < TEXT_COLOUR; + if (nearby(w, h, x, y, 2, same)) hit++; + } + return n >= TEXT_MIN_INK ? hit / n : null; +} + +/** One box of a still picture, scaled to cover `w`x`h` (the original's size) first. */ +function stillBox(path: string, box: Roi, w: number, h: number): Uint8Array | undefined { + try { + const raw = runFfmpeg([ + ...["-i", path, "-frames:v", "1", "-vf"], + `scale=${w}:${h}:force_original_aspect_ratio=increase:flags=bicubic,crop=${w}:${h},crop=${box[2]}:${box[3]}:${box[0]}:${box[1]},format=rgb24`, + ...["-f", "rawvideo", "-"], + ]); + return raw.length >= box[2] * box[3] * 3 ? new Uint8Array(raw) : undefined; + } catch { + return undefined; + } +} + +/** + * Whether a generated clip carries the original's text inside a drawn element's box over its window: the clip + * alone, against the original there. A clean clip shows the person and the place behind the letters; a clip made + * from a motion guide with words on it shows those words. + */ +export function carriedText( + ref: VideoInfo, + yours: VideoInfo, + clip: { path: string; info: VideoInfo; uses: ClipUse[]; still?: string }, + drawn: { id: string; box: Roi; window: [number, number] }, +): CarriedText | undefined { + // The film times both show, each with the clip's own time there. + const spans = clip.uses.flatMap((u) => { + const [a, b] = [Math.max(drawn.window[0], u.window[0]), Math.min(drawn.window[1], u.window[1])]; + return b - a >= 1 / ref.fps ? [{ a, b, u }] : []; + }); + const total = spans.reduce((s, x) => s + x.b - x.a, 0); + if (!total) return undefined; + // Evenly spread samples over the shown spans. + const times = Array.from({ length: CARRIED_SAMPLES }, (_, i) => { + let at = ((i + 0.5) * total) / CARRIED_SAMPLES; + for (const x of spans) { + if (at <= x.b - x.a) + return { t: x.a + at, c: x.u.from + (at + x.a - x.u.window[0]) * x.u.rate }; + at -= x.b - x.a; + } + const x = spans.at(-1)!; + return { t: x.b, c: x.u.from + (x.b - x.u.window[0]) * x.u.rate }; + }); + const orig = boxAt( + ref, + times.map((x) => x.t), + drawn.box, + ref.width, + ref.height, + ); + const generated = boxAt( + clip.info, + times.map((x) => x.c), + drawn.box, + ref.width, + ref.height, + ); + const mine = boxAt( + yours, + times.map((x) => x.t), + drawn.box, + ref.width, + ref.height, + ); + const still = clip.still ? stillBox(clip.still, drawn.box, ref.width, ref.height) : undefined; + const scores = orig.flatMap((o, i) => { + const g = generated[i]; + const s = o && g ? carriedFrame(o, g, still, drawn.box[2], drawn.box[3], mine[i]) : null; + return s === null ? [] : [s]; + }); + if (!scores.length) return undefined; + const score = scores.reduce((s, v) => s + v, 0) / scores.length; + return { + drawn: drawn.id, + clip: clip.path, + window: [round(spans[0]!.a, 3), round(spans.at(-1)!.b, 3)], + score: round(score, 3), + carried: score >= CARRIED_TEXT, + }; +} + +const median = (v: number[]) => [...v].sort((x, y) => x - y)[v.length >> 1]!; + +const centre = (b: number[]) => [b[0]! + b[2]! / 2, b[1]! + b[3]! / 2] as const; + +/** The original's shots inside a window: split at its cuts. */ +export function shotsIn(window: [number, number], cuts: number[]): [number, number][] { + const edges = [window[0], ...cuts.filter((t) => t > window[0] && t < window[1]), window[1]]; + return edges.slice(1).map((e, i) => [edges[i]!, e]); +} + +/** + * One shot's framing from both sides' faces, sampled at the same moments (boxes as shares of a frame `aspect` wide + * per unit high): per moment, the original's subject (its largest face) against yours' face nearest it; the shot's + * medians decide, so one stray detection (a photo, a hand) does not. + */ +export function shotFraming( + window: [number, number], + original: { t: number; boxes: number[][] }[], + yours: { t: number; boxes: number[][] }[], + aspect = 1, +): ShotFraming | undefined { + const inside = (s: { t: number }) => s.t >= window[0] && s.t < window[1]; + // Across, a share of the frame is `aspect` times longer than down. + const apart = (b: number[], x: number, y: number) => + Math.hypot((centre(b)[0] - x) * aspect, centre(b)[1] - y); + const [a, b] = [original.filter(inside), yours.filter(inside)]; + const moments = a.flatMap((o, i) => { + const y = b[i]; + if (!o.boxes.length || !y?.boxes.length) return []; + const subject = o.boxes.reduce((p, q) => (q[3]! > p[3]! ? q : p)); + const [ox, oy] = centre(subject); + const near = y.boxes.reduce((p, q) => (apart(q, ox, oy) < apart(p, ox, oy) ? q : p)); + return [{ shift: apart(near, ox, oy) / subject[3]!, scale: near[3]! / subject[3]! }]; + }); + if (moments.length < FRAMING_MIN_FACES) return undefined; + const shift = round(median(moments.map((m) => m.shift)), 3); + const scale = round(median(moments.map((m) => m.scale)), 3); + return { + window: [round(window[0], 3), round(window[1], 3)], + shift, + scale, + pass: shift <= FRAMING_SHIFT && scale >= FRAMING_SCALE[0] && scale <= FRAMING_SCALE[1], + }; +} + +/** Lips in sync: the lag inside SYNC_MS, judged only when the reading is confident (it is a motion proxy). */ +export function syncPass(r: Pick): boolean | null { + if (r.confidence !== "high" || r.best_lag_ms === null) return null; + return r.best_lag_ms >= SYNC_MS[0] && r.best_lag_ms <= SYNC_MS[1]; +} + +/** A segment's person: framing per shot, carried text per drawn element, lip sync per long speaking shot. */ +export async function personVerdict(opts: { + ref: VideoInfo; + yours: VideoInfo; + window: [number, number]; + cuts: number[]; + clips: (GeneratedClip & { info?: VideoInfo; still?: string })[]; + drawn: { id: string; box: Roi; window: [number, number] }[]; +}): Promise { + const { ref, yours, window } = opts; + const shots = shotsIn(window, opts.cuts); + const n = Math.max(2, Math.round((window[1] - window[0]) * FACE_RATE)); + // A face inside a drawn element (a photo on a card) is not the person. + const drawnFace = (t: number, b: number[]) => + opts.drawn.some(({ box: [x, y, w, h], window: [a, z] }) => { + const [cx, cy] = [(b[0]! + b[2]! / 2) * ref.width, (b[1]! + b[3]! / 2) * ref.height]; + return t >= a && t < z && cx >= x && cx < x + w && cy >= y && cy < y + h; + }); + const people = (samples: Awaited>) => + samples.map((f) => ({ ...f, boxes: f.boxes.filter((b) => !drawnFace(f.t, b)) })); + const [fo, fy] = [ + people(await faceBoxes(ref, window[0], window[1], n)), + people(await faceBoxes(yours, window[0], window[1], n)), + ]; + const framing = shots.flatMap((s) => + s[1] - s[0] >= FRAMING_MIN_S + ? [shotFraming(s, fo, fy, ref.width / ref.height)].filter((f) => f !== undefined) + : [], + ); + const text = opts.drawn.flatMap((d) => + opts.clips.flatMap((c) => + c.info && c.uses + ? [ + carriedText( + ref, + yours, + { path: c.path, info: c.info, uses: c.uses, still: c.still }, + d, + ), + ].filter((t) => t !== undefined) + : [], + ), + ); + const sync = shots.flatMap((s) => { + if (s[1] - s[0] < SYNC_MIN_S || !yours.audio) return []; + const mouth = ( + info: VideoInfo, + samples: { t: number; boxes: number[][] }[], + ): Roi | undefined => { + const boxes = samples + .filter((f) => f.t >= s[0] && f.t < s[1] && f.boxes.length) + .map((f) => f.boxes.reduce((x, y) => (y[3]! > x[3]! ? y : x))); + if (boxes.length < FRAMING_MIN_FACES) return undefined; + const [x, y, w, h] = [0, 1, 2, 3].map((i) => median(boxes.map((b) => b[i]!))); + const roi: Roi = [ + Math.round(((x! + 0.2 * w!) * info.width) / 2) * 2, + Math.round(((y! + 0.62 * h!) * info.height) / 2) * 2, + Math.max(8, Math.round((0.6 * w! * info.width) / 2) * 2), + Math.max(8, Math.round((0.38 * h! * info.height) / 2) * 2), + ]; + return roi[0] + roi[2] <= info.width && roi[1] + roi[3] <= info.height ? roi : undefined; + }; + const [mo, my] = [mouth(ref, fo), mouth(yours, fy)]; + if (!my) return []; + const mine = avSync(yours, my, s[0], s[1]); + if (mine.speech_active_fraction < SYNC_MIN_SPEECH) return []; + const theirs = mo && ref.audio ? avSync(ref, mo, s[0], s[1]) : undefined; + return [ + { + window: [round(s[0], 3), round(s[1], 3)] as [number, number], + lag_ms: mine.best_lag_ms, + original_lag_ms: theirs?.best_lag_ms ?? null, + confidence: mine.confidence, + pass: syncPass(mine), + }, + ]; + }); + const gate = (v: (boolean | null)[]) => + v.includes(false) ? false : v.some((p) => p === true) ? true : null; + const gates = { + framing: gate(framing.map((f) => f.pass)), + text: gate(text.map((t) => !t.carried)), + sync: gate(sync.map((s) => s.pass)), + }; + const all = Object.values(gates); + return { + gates, + pass: all.includes(false) ? false : all.some((g) => g === true) ? true : null, + ...(framing.length ? { framing } : {}), + ...(text.length ? { text } : {}), + ...(sync.length ? { sync } : {}), + }; +} diff --git a/packages/cli/src/measure/segments.ts b/packages/cli/src/measure/segments.ts index 2bfc91cf00..cc981a9eb2 100644 --- a/packages/cli/src/measure/segments.ts +++ b/packages/cli/src/measure/segments.ts @@ -1,5 +1,9 @@ // The segments of a REVERSE.md: `## S01 · 0.00–2.40 s · title` headings, each with an optional -// `Pixels: rebuilt | reused | regenerated` line saying where that window's picture comes from. +// `Pixels: rebuilt | reused | regenerated` line saying where that window's picture comes from, and an optional +// `Regenerated: x, y, w, h` line for a rebuilt or reused segment with a regenerated region (a re-shot presenter +// under drawn graphics): the region's box in the source's pixels, or as shares of the frame when all are ≤ 1. Each +// `Drawn: x, y, w, h` line is a drawn element over a regenerated person, still judged by pixels, with an optional +// `a–b s` when it shows for part of the segment. A `Copy:` line that quotes a string says the segment draws text. import { existsSync, readFileSync } from "node:fs"; import { Fix } from "./media.js"; @@ -12,12 +16,27 @@ export interface Segment { end: number; title: string; pixels: PixelSource; + /** A regenerated region inside a rebuilt or reused segment: [x, y, w, h], pixels or shares of the frame. */ + regenerated?: [number, number, number, number]; + /** Drawn elements over a regenerated person: their boxes, judged by pixels, and when they show. */ + drawn?: { box: [number, number, number, number]; window?: [number, number] }[]; + /** The segment quotes copy: it has drawn text. */ + copy?: true; } const HEADING = /^##\s+(S\d+)\s*[·|-]\s*(\d+(?:\.\d+)?)\s*[–—-]\s*(\d+(?:\.\d+)?)\s*s\b\s*(?:[·|-]\s*(.*))?$/gm; const SOURCES: readonly PixelSource[] = ["rebuilt", "reused", "regenerated"]; const PIXELS = /^Pixels:\s*(rebuilt|reused|regenerated)\b/im; +const NUMBER = String.raw`(\d+(?:\.\d+)?)`; +const SEP = String.raw`(?:\s*,\s*|\s+)`; +const DRAWN = new RegExp( + `^Drawn:\\s*${NUMBER}${SEP}${NUMBER}${SEP}${NUMBER}${SEP}${NUMBER}(?:.*?${NUMBER}\\s*[–—-]\\s*${NUMBER}\\s*s\\b)?`, + "gim", +); +const COPY = /^Copy:.*["“”]\S/im; +const REGION = + /^Regenerated:\s*(\d+(?:\.\d+)?)(?:\s*,\s*|\s+)(\d+(?:\.\d+)?)(?:\s*,\s*|\s+)(\d+(?:\.\d+)?)(?:\s*,\s*|\s+)(\d+(?:\.\d+)?)/im; export function parseSegments(markdown: string): Segment[] { const heads = [...markdown.matchAll(HEADING)]; @@ -25,12 +44,31 @@ export function parseSegments(markdown: string): Segment[] { const body = markdown.slice(m.index! + m[0].length, heads[i + 1]?.index ?? markdown.length); const found = PIXELS.exec(body)?.[1]?.toLowerCase(); const pixels = SOURCES.find((source) => source === found) ?? "rebuilt"; + const r = REGION.exec(body); + const region: Segment["regenerated"] = r + ? [Number(r[1]), Number(r[2]), Number(r[3]), Number(r[4])] + : undefined; + const drawn = [...body.matchAll(DRAWN)].flatMap((d) => { + const box: [number, number, number, number] = [ + Number(d[1]), + Number(d[2]), + Number(d[3]), + Number(d[4]), + ]; + if (box[2] <= 0 || box[3] <= 0) return []; + return [d[5] ? { box, window: [Number(d[5]), Number(d[6])] as [number, number] } : { box }]; + }); return { id: m[1]!, start: Number(m[2]), end: Number(m[3]), title: (m[4] ?? "").trim(), pixels, + ...(region && pixels !== "regenerated" && region[2]! > 0 && region[3]! > 0 + ? { regenerated: region } + : {}), + ...(drawn.length ? { drawn } : {}), + ...(COPY.test(body) ? { copy: true as const } : {}), }; }); } diff --git a/skills-manifest.json b/skills-manifest.json index 71ceb047f9..a507520983 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,7 +18,7 @@ "files": 13 }, "media-use": { - "hash": "99605f8eca2490e4", + "hash": "1a7d1b228edc28b9", "files": 117 }, "remotion-to-hyperframes": { @@ -26,8 +26,8 @@ "files": 76 }, "reverse-any-video": { - "hash": "299673b85afcbd17", - "files": 6 + "hash": "b7a34caab79eadce", + "files": 4 }, "slideshow": { "hash": "c8a591e2aa7a6d86", diff --git a/skills/media-use/references/heygen.md b/skills/media-use/references/heygen.md index 62cdce9c14..654c8988a4 100644 --- a/skills/media-use/references/heygen.md +++ b/skills/media-use/references/heygen.md @@ -16,15 +16,19 @@ A refusal names its fix: `api_key_required` needs an API key, `402` or `insuffic ## Making media: `media-use generate` -Make it without quoting a price or asking for a budget; `--plan` only prints what a call would cost, for your own planning; never pass it on. At most two attempts per piece; a `content_policy_violation` gets one reworded try. +Make it without quoting a price or asking for a budget; `--plan` only prints what a call would cost, for your own planning; never pass it on. Make a piece again only for a fault you can name (a cut inside it, text in it, the wrong framing), once, never to chase a motion or look score; a `content_policy_violation` gets one reworded try. ```bash # a still, edited from a reference picture: a clean plate with the words removed npx hyperframes media-use generate --type image --reference assets/frame.png --prompt "the same shot with no text or logos" --out assets/plate.png --plan # a shot of 5–15 s, starting from a first frame; longer shots are shots chained from each one's last frame npx hyperframes media-use generate --type video --first-frame assets/s03-first.png --prompt "slow push in" --seconds 8 --out assets/s03.mp4 -# a person re-shot from a still, saying a 5–15 s stretch of a recording, lips in sync -npx hyperframes media-use generate --type video --reference assets/person.png --audio assets/s02-voice.m4a --prompt "talks to camera, seated" --seconds 8 --aspect 9:16 --out assets/s02-person.mp4 +# a person re-shot from a still over one shot of the original (never across a cut), saying that shot's voice with lips +# in sync and moving as the guide (the original shot itself) does; --mask x,y,w,h[,a-b] (repeatable, the guide's +# pixels and seconds) blanks text in the guide, which HeyGen would copy +npx hyperframes media-use generate --type video --reference assets/person.png --audio assets/s02-voice.m4a --motion assets/s02-original.mp4 --mask 110,615,390,165,0.4-2.0 --prompt "talks to camera, seated" --seconds 8 --aspect 9:16 --out assets/s02-person.mp4 +# the same for a person who does not speak: no --audio, the guide clip carries the motion +npx hyperframes media-use generate --type video --reference assets/person.png --motion assets/s05-original.mp4 --prompt "holds up a jar, smiles" --seconds 6 --aspect 9:16 --out assets/s05-person.mp4 # a person in a photo speaking a script in a HeyGen voice npx hyperframes media-use generate --type avatar --photo assets/person.png --script "…" --voice --out assets/person.mp4 # a voice reading text @@ -35,13 +39,13 @@ npx hyperframes media-use generate --type music --prompt "warm lo-fi, soft keys" The file lands at `--out` and the media ledger records it as generated, with the route, model, job, prompt and the files it was made from. Pictures and shots carry no text or logos: ask for clean plates and draw words in HyperFrames. -| Type | Model | Price | -| ------ | -------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| image | nano-banana-2 (default), or gpt-image-2.5 with `--model` | $0.067 an image; gpt-image-2.5 $0.036–0.053 by aspect, plus $0.0082 a reference after the first | -| video | heygen-video-1, 768p | $0.018 a second | -| avatar | talking photo | $4.83 a minute of speech | -| music | lyria-3-pro | $0.08 a track | -| voice | the voice's engine | charged per character, not priced ahead | +| Type | Model | Price | +| ------ | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| image | nano-banana-2 (default), or gpt-image-2.5 with `--model` | $0.067 an image; gpt-image-2.5 $0.036–0.053 by aspect, plus $0.0082 a reference after the first | +| video | heygen-video-1, 768p | $0.02 a second from a prompt or first frame; $0.05 a second with a reference picture, voice or guide clip, and the guide's seconds count too (1080p costs about twice) | +| avatar | talking photo | $4.83 a minute of speech | +| music | lyria-3-pro | $0.08 a track | +| voice | the voice's engine | charged per character, not priced ahead | HeyGen charges in units of $1/60, rounded up; `--plan` prints that charged price. Voice ids come from `heygen voice list` or `npx hyperframes media-use resolve --type voice`; a cloned voice from `heygen voice clone create` (`operations.md`). diff --git a/skills/reverse-any-video/SKILL.md b/skills/reverse-any-video/SKILL.md index a833ea3300..893f5efb89 100644 --- a/skills/reverse-any-video/SKILL.md +++ b/skills/reverse-any-video/SKILL.md @@ -9,62 +9,30 @@ metadata: # Reverse a video -The result is a HyperFrames project whose render matches the original and stays editable: text is text, graphics are elements with named colours, and every media file records where it came from. It is done when `npx hyperframes check` passes and `diff` against the original passes every segment. Build to the `/hyperframes` contract; this skill adds how to read a video you only have as pixels. Reverse videos the person has the right to use. +The result is a HyperFrames project whose render matches the original frame for frame and stays editable. Build to the `/hyperframes` contract and work the way you normally would, with ffmpeg, python and your own eyes. This skill holds only what earlier rebuilds got wrong or spent hours on. Reverse videos the person has the right to use. -## 0. Prepare +Sign in first; start only when `npx hyperframes auth status` shows a sign-in. Open with `references/start.md`. Rebuild the video as it is; remix it only when the person asks. -Sign in first; start only when `npx hyperframes auth status` shows a sign-in. Open with `references/start.md`. +## The original decides the frame -```bash -npx hyperframes init -npx hyperframes measure probe