diff --git a/packages/cli/src/heygen/generate.test.ts b/packages/cli/src/heygen/generate.test.ts index 413fa1b18b..b7735f020e 100644 --- a/packages/cli/src/heygen/generate.test.ts +++ b/packages/cli/src/heygen/generate.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { findFFmpeg } from "../browser/ffmpeg.js"; +import { probeVideo } from "../measure/media.js"; vi.mock("../cloud/auth.js", () => ({ resolveCloudAuthHeaders: async () => ({ "x-api-key": "k" }), @@ -13,7 +14,7 @@ vi.mock("../cloud/auth.js", () => ({ vi.stubEnv("HEYGEN_API_KEY", "k"); -const { checkRequest, generate, parseMask, urlOf } = await import("./generate.js"); +const { checkRequest, generate, parseMask, structureRatio, urlOf } = await import("./generate.js"); const { checkLimit, creditsOf, priceOf } = await import("./pricing.js"); const ffmpeg = findFFmpeg(); const project = mkdtempSync(join(tmpdir(), "hf-generate-test-")); @@ -149,6 +150,115 @@ describe("generate", () => { ).rejects.toThrow(/--first-frame cannot be used/); }); + /** A static sine texture (smooth fills cannot copy it), with a white bar sliding through x 30..150 when `bar`. */ + function textured(name: string, size: string, bar: boolean): string { + const path = join(project, name); + const slide = bar ? ",drawbox=x='30+t*40':y=100:w=20:h=30:color=white:t=fill" : ""; + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", `color=c=gray:s=${size}:r=30:d=3`], + ...[ + "-vf", + `geq=lum='128+100*sin(X/3)*sin(Y/3)':cb=128:cr=128${slide}`, + "-pix_fmt", + "yuv420p", + path, + ], + ]); + return path; + } + + const answers = (file: string) => { + const sent: Record[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + if (init?.body) sent.push(JSON.parse(String(init.body))); + if (url === "https://files.test/r.mp4") return new Response(readFileSync(file)); + return json({ + data: { id: "r1", status: "completed", video_url: "https://files.test/r.mp4" }, + }); + }); + return sent; + }; + + it.skipIf(!ffmpeg)( + "fills a masked box with the picture behind moving text, not a smear", + async () => { + const guide = textured("slide.mp4", "180x320", true); + const sent = answers(guide); + await generate({ + type: "video", + out: join(project, "assets/slide-out.mp4"), + prompt: "p", + motion: guide, + masks: [parseMask("30,90,120,50")], + }); + const [video] = sent[0]!.reference_videos as { data: string }[]; + const sentClip = join(project, "slide-sent.mp4"); + writeFileSync(sentClip, Buffer.from(video!.data, "base64")); + const box = (f: string) => + execFileSync(ffmpeg!, [ + ...["-v", "error", "-ss", "1.5", "-i", f, "-frames:v", "1"], + ...["-vf", "crop=120:50:30:90,format=gray", "-f", "rawvideo", "-"], + ]); + const [got, clean] = [box(sentClip), box(textured("slide-clean.mp4", "180x320", false))]; + const diff = got.reduce((a, v, i) => a + Math.abs(v - clean[i]!), 0) / got.length; + expect(diff).toBeLessThan(12); + }, + ); + + it("reads a box with more edges than its surroundings as structure", () => { + const flat = new Uint8Array(100 * 100).fill(100); + expect(structureRatio(flat, 100, [40, 40, 20, 20])).toEqual({ inside: 0, ring: 0 }); + for (let y = 40; y < 60; y++) + for (let x = 40; x < 60; x++) flat[y * 100 + x] = (x + y) % 2 ? 255 : 0; + const { inside, ring } = structureRatio(flat, 100, [40, 40, 20, 20]); + expect(inside).toBeGreaterThan(100); + expect(inside).toBeGreaterThan(10 * ring); + }); + + it.skipIf(!ffmpeg)( + "says so when the result still shows structure inside a masked box", + async () => { + const guide = textured("flatguide.mp4", "180x320", false); + const busy = join(project, "busy.mp4"); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=gray:s=180x320:r=30:d=1"], + ...[ + "-vf", + "geq=lum='if(between(X,30,150)*between(Y,90,140),128+120*sin(X*2)*sin(Y*2),128)':cb=128:cr=128", + ], + ...["-pix_fmt", "yuv420p", busy], + ]); + answers(busy); + const flatMask = parseMask("30,90,120,50"); + const r = await generate({ + type: "video", + out: join(project, "assets/busy-out.mp4"), + prompt: "p", + motion: guide, + masks: [flatMask], + aspect: "9:16", + }); + expect(r.notes?.join(" ")).toContain("mask 30,90,120,50 still shows structure"); + }, + ); + + it.skipIf(!ffmpeg)( + "crops a 1344x768 answer to the requested 16:9, never stretches it", + async () => { + answers(textured("wide.mp4", "1344x768", false)); + const r = await generate({ + type: "video", + out: join(project, "assets/wide-out.mp4"), + prompt: "p", + }); + const { width, height } = probeVideo(r.out); + expect([width, height]).toEqual([1344, 756]); + expect(r.notes?.[0]).toBe( + "HeyGen returned 1344x768 for 16:9; cropped to 1344x756 (centre), not stretched", + ); + }, + ); + it.skipIf(!ffmpeg)("blanks the motion clip's masked regions before sending it", async () => { // A guide with white letters at 40,100 over a moving picture; the mask covers them for its first 0.5 s. const guide = join(project, "titled.mp4"); diff --git a/packages/cli/src/heygen/generate.ts b/packages/cli/src/heygen/generate.ts index a2ccce360d..e957982702 100644 --- a/packages/cli/src/heygen/generate.ts +++ b/packages/cli/src/heygen/generate.ts @@ -1,7 +1,15 @@ // New media on HeyGen for `media-use generate`: a still (optionally edited from a reference), a shot (optionally from // a first frame), a person speaking from one photo, a voice reading text, or music. Every call is priced first with // --plan, keyed for idempotency, downloaded into the project and recorded in the media ledger as generated. -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { dirname, extname, join, relative, resolve } from "node:path"; import { findCuts, signatures } from "../measure/cuts.js"; @@ -62,17 +70,51 @@ export function parseMask(text: string): Mask { } /** - * Filters that fill each mask's region from the pixels around it (ffmpeg delogo), so no letter survives and the - * motion around it stays. delogo needs a pixel of picture on every side, so boxes are kept 1 px inside the frame. + * Fills a box from the pixels around it (ffmpeg delogo): no letter survives, but the box is smeared. Delogo needs a + * pixel of picture on every side, so boxes are kept 1 px inside the frame. */ -function maskFilters(masks: Mask[], width: number, height: number): string[] { - return masks.flatMap(({ box: [x, y, w, h], window }) => { - const [x0, y0] = [Math.max(1, x), Math.max(1, y)]; - const [x1, y1] = [Math.min(width - 2, x + w), Math.min(height - 2, y + h)]; - if (x1 - x0 < 1 || y1 - y0 < 1) return []; +function delogoFilters(masks: Mask[], width: number, height: number): string { + return masks + .flatMap(({ box: [x, y, w, h], window }) => { + const [x0, y0] = [Math.max(1, x), Math.max(1, y)]; + const [x1, y1] = [Math.min(width - 2, x + w), Math.min(height - 2, y + h)]; + if (x1 - x0 < 1 || y1 - y0 < 1) return []; + const when = window ? `:enable='between(t,${window[0]},${window[1]})'` : ""; + return [`delogo=x=${x0}:y=${y0}:w=${x1 - x0}:h=${y1 - y0}${when}`]; + }) + .join(","); +} + +/** A box's temporal median over about a second either side: moving text becomes the picture behind it. */ +function maskFilters(masks: Mask[], fps: number): string { + const radius = Math.min(127, Math.max(2, Math.round(fps))); + const steps = masks.map(({ box: [x, y, w, h], window }, i) => { const when = window ? `:enable='between(t,${window[0]},${window[1]})'` : ""; - return [`delogo=x=${x0}:y=${y0}:w=${x1 - x0}:h=${y1 - y0}${when}`]; + const [w2, h2] = [w - (w % 2), h - (h % 2)]; + return `[v${i}]split[a${i}][b${i}];[b${i}]crop=${w2}:${h2}:${x}:${y},tmedian=radius=${radius}[m${i}];[a${i}][m${i}]overlay=${x}:${y}${when}[v${i + 1}]`; }); + return `[0:v]null[v0];${steps.join(";")}`; +} + +/** Mean absolute pixel step inside a box and in a ring around it: the box holds more edges when inside > ring. */ +export function structureRatio( + gray: Uint8Array, + width: number, + box: [number, number, number, number], +): { inside: number; ring: number } { + const height = gray.length / width; + const [x0, y0, x1, y1] = [box[0], box[1], box[0] + box[2], box[1] + box[3]]; + const pad = Math.max(box[2], box[3]) >> 1; + const sum = [0, 0]; + const n = [0, 0]; + for (let y = Math.max(0, y0 - pad); y < Math.min(height - 1, y1 + pad); y++) + for (let x = Math.max(0, x0 - pad); x < Math.min(width - 1, x1 + pad); x++) { + const k = x >= x0 && x < x1 && y >= y0 && y < y1 ? 0 : 1; + const i = y * width + x; + sum[k]! += Math.abs(gray[i]! - gray[i + 1]!) + Math.abs(gray[i]! - gray[i + width]!); + n[k]!++; + } + return { inside: sum[0]! / (n[0] || 1), ring: sum[1]! / (n[1] || 1) }; } interface Route { @@ -220,7 +262,7 @@ function needFile(path: string): void { /** * A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion - * clip with its masks blanked. + * clip with its masks filled. */ function inline(work: string, masks: Mask[] = []): Media { return (path, kind) => { @@ -246,13 +288,10 @@ function inline(work: string, masks: Mask[] = []): Media { runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]); // A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s. else { - const { width, height } = masks.length ? probeVideo(path) : { width: 0, height: 0 }; - const filters = [ - ...maskFilters(masks, width, height), - "scale='if(gt(iw,ih),min(720,iw),-2)':'if(gt(iw,ih),-2,min(720,ih))'", - ]; + const guide = masks.length ? fillMasks(path, masks, work) : path; + const scale = "scale='if(gt(iw,ih),min(720,iw),-2)':'if(gt(iw,ih),-2,min(720,ih))'"; runFfmpeg([ - ...["-y", "-i", path, "-t", "15", "-an", "-vf", filters.join(",")], + ...["-y", "-i", guide, "-t", "15", "-an", "-vf", scale], ...["-c:v", "libx264", "-crf", "28", "-preset", "veryfast", "-pix_fmt", "yuv420p", out], ]); } @@ -278,6 +317,80 @@ function cutsInside(path: string, seconds?: number): { cuts: number[]; length: n return { cuts, length: round(frames.length / info.fps, 3) }; } +/** The masks whose box, in `clip`'s frames, holds more edges than the picture around it at some sampled time. */ +function structured(clip: string, guide: { width: number; height: number }, masks: Mask[]) { + const { width: w, height: h } = probeVideo(clip); + const gray = runFfmpeg(["-i", clip, "-vf", "fps=2", "-pix_fmt", "gray", "-f", "rawvideo", "-"]); + // The box as a fraction of the guide's frame, in the clip's pixels (its size and aspect may differ). + const [kx, ky] = [w / guide.width, h / guide.height]; + return masks.flatMap((mask) => { + const [bx, by, bw, bh] = mask.box; + const box = [bx * kx, by * ky, bw * kx, bh * ky].map(Math.round) as Mask["box"]; + let worst = 0; + for (let i = 0; (i + 1) * w * h <= gray.length; i++) { + if (mask.window && (i / 2 < mask.window[0] || i / 2 > mask.window[1])) continue; + const { inside, ring } = structureRatio(gray.subarray(i * w * h, (i + 1) * w * h), w, box); + if (inside > 6 && inside > 1.5 * ring) worst = Math.max(worst, inside / (ring || 1)); + } + return worst ? [{ mask, worst }] : []; + }); +} + +/** A copy of the guide with each mask's box filled: the temporal median, and delogo's smear where text stays put. */ +function fillMasks(path: string, masks: Mask[], work: string): string { + const info = probeVideo(path); + const encode = (from: string, name: string, filter: string[]) => { + const out = join(work, name); + runFfmpeg([ + ...["-y", "-i", from, "-an"], + ...filter, + ...["-crf", "14", "-pix_fmt", "yuv420p", out], + ]); + return out; + }; + const median = encode(path, "median.mp4", [ + ...["-t", "15", "-filter_complex", maskFilters(masks, info.fps)], + ...["-map", `[v${masks.length}]`], + ]); + const left = structured(median, info, masks).map((s) => s.mask); + if (!left.length) return median; + return encode(median, "delogo.mp4", ["-vf", delogoFilters(left, info.width, info.height)]); +} + +/** + * HeyGen answers a 16:9 request at 1344x768 (1.75:1). The result is cropped, centred, to the requested ratio + * (1344x756) rather than stretched; returns a note on what was done, or nothing when it already fits. + */ +export function fitAspect(out: string, aspect: string): string | undefined { + const [aw, ah] = aspect.split(":").map(Number); + if (!aw || !ah) return; + const { width, height } = probeVideo(out); + const want = aw / ah; + if (Math.abs(width / height - want) / want < 0.002) return; + const even = (n: number) => n - (n % 2); + const [w, h] = + width / height > want ? [even(height * want), height] : [width, even(width / want)]; + const fitted = `${out}.fit${extname(out)}`; + runFfmpeg([ + ...[ + "-y", + "-i", + out, + "-vf", + `crop=${w}:${h}`, + "-c:v", + "libx264", + "-crf", + "12", + "-preset", + "slow", + ], + ...["-pix_fmt", "yuv420p", "-c:a", "copy", fitted], + ]); + renameSync(fitted, out); + return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`; +} + const COUNT = ["", "one", "two", "three", "four", "five", "six"]; /** @@ -331,6 +444,8 @@ export interface Generated { ledger?: string; /** video: seconds where HeyGen cut inside the returned shot; null when the file could not be read. */ cuts_inside?: number[] | null; + /** What was changed or looks wrong in the result: a crop to the requested aspect, text that may have come back. */ + notes?: string[]; } // fallow-ignore-next-line complexity @@ -378,6 +493,25 @@ export async function generate( } catch { cuts = null; } + const notes: string[] = []; + if (r.type === "video" && (r.aspect || !r.firstFrame)) + try { + const note = fitAspect(out, r.aspect ?? "16:9"); + if (note) notes.push(note); + } catch (e) { + notes.push( + `kept HeyGen's size, could not crop to ${r.aspect ?? "16:9"}: ${e instanceof Error ? e.message : e}`, + ); + } + if (r.masks?.length && r.motion) + try { + for (const { mask, worst } of structured(out, probeVideo(r.motion), r.masks)) + notes.push( + `mask ${mask.box.join(",")} still shows structure in the result (${worst.toFixed(1)}x its surroundings): the text may have come back, regenerate without the guide`, + ); + } catch { + notes.push("could not check the masked boxes of the result"); + } let ledger: string | undefined; if (project) { const { recordInPlace } = await import("../media-use/lib/manifest.mjs"); @@ -416,6 +550,7 @@ export async function generate( route: route.path, ...(ledger ? { ledger } : {}), ...(cuts !== undefined ? { cuts_inside: cuts } : {}), + ...(notes.length ? { notes } : {}), }; } finally { rmSync(work, { recursive: true, force: true });