ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-11-04 13:07:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cli): fill --mask boxes from the picture behind and crop generated clips to the requested aspect (#6892)

Browse Source 
* fix(cli): fill --mask boxes from the picture behind and crop generated clips to the requested aspect

* fix(cli): shorten the mask fill comment
This commit is contained in:
Miguel Ángel
2026-10-31 19:48:28 +00:00
committed by GitHub
parent 2239771efd
commit 919e138fc2
2 changed files with 262 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli/src/heygen/generate.test.ts
+111 -1
View File
@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
import { findFFmpeg } from "../browser/ffmpeg.js";
import { probeVideo } from "../measure/media.js";
vi.mock("../cloud/auth.js", () => ({
resolveCloudAuthHeaders: async () => ({ "x-api-key": "k" }),
@@ -13,7 +14,7 @@ vi.mock("../cloud/auth.js", () => ({
vi.stubEnv("HEYGEN_API_KEY", "k");
const { checkRequest, generate, parseMask, urlOf } = await import("./generate.js");
const { checkRequest, generate, parseMask, structureRatio, urlOf } = await import("./generate.js");
const { checkLimit, creditsOf, priceOf } = await import("./pricing.js");
const ffmpeg = findFFmpeg();
const project = mkdtempSync(join(tmpdir(), "hf-generate-test-"));
@@ -149,6 +150,115 @@ describe("generate", () => {
).rejects.toThrow(/--first-frame cannot be used/);
});
/** A static sine texture (smooth fills cannot copy it), with a white bar sliding through x 30..150 when `bar`. */
function textured(name: string, size: string, bar: boolean): string {
const path = join(project, name);
const slide = bar ? ",drawbox=x='30+t*40':y=100:w=20:h=30:color=white:t=fill" : "";
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", `color=c=gray:s=${size}:r=30:d=3`],
...[
"-vf",
`geq=lum='128+100*sin(X/3)*sin(Y/3)':cb=128:cr=128${slide}`,
"-pix_fmt",
"yuv420p",
path,
],
]);
return path;
}
const answers = (file: string) => {
const sent: Record<string, unknown>[] = [];
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
if (init?.body) sent.push(JSON.parse(String(init.body)));
if (url === "https://files.test/r.mp4") return new Response(readFileSync(file));
return json({
data: { id: "r1", status: "completed", video_url: "https://files.test/r.mp4" },
});
});
return sent;
};
it.skipIf(!ffmpeg)(
"fills a masked box with the picture behind moving text, not a smear",
async () => {
const guide = textured("slide.mp4", "180x320", true);
const sent = answers(guide);
await generate({
type: "video",
out: join(project, "assets/slide-out.mp4"),
prompt: "p",
motion: guide,
masks: [parseMask("30,90,120,50")],
});
const [video] = sent[0]!.reference_videos as { data: string }[];
const sentClip = join(project, "slide-sent.mp4");
writeFileSync(sentClip, Buffer.from(video!.data, "base64"));
const box = (f: string) =>
execFileSync(ffmpeg!, [
...["-v", "error", "-ss", "1.5", "-i", f, "-frames:v", "1"],
...["-vf", "crop=120:50:30:90,format=gray", "-f", "rawvideo", "-"],
]);
const [got, clean] = [box(sentClip), box(textured("slide-clean.mp4", "180x320", false))];
const diff = got.reduce((a, v, i) => a + Math.abs(v - clean[i]!), 0) / got.length;
expect(diff).toBeLessThan(12);
},
);
it("reads a box with more edges than its surroundings as structure", () => {
const flat = new Uint8Array(100 * 100).fill(100);
expect(structureRatio(flat, 100, [40, 40, 20, 20])).toEqual({ inside: 0, ring: 0 });
for (let y = 40; y < 60; y++)
for (let x = 40; x < 60; x++) flat[y * 100 + x] = (x + y) % 2 ? 255 : 0;
const { inside, ring } = structureRatio(flat, 100, [40, 40, 20, 20]);
expect(inside).toBeGreaterThan(100);
expect(inside).toBeGreaterThan(10 * ring);
});
it.skipIf(!ffmpeg)(
"says so when the result still shows structure inside a masked box",
async () => {
const guide = textured("flatguide.mp4", "180x320", false);
const busy = join(project, "busy.mp4");
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=gray:s=180x320:r=30:d=1"],
...[
"-vf",
"geq=lum='if(between(X,30,150)*between(Y,90,140),128+120*sin(X*2)*sin(Y*2),128)':cb=128:cr=128",
],
...["-pix_fmt", "yuv420p", busy],
]);
answers(busy);
const flatMask = parseMask("30,90,120,50");
const r = await generate({
type: "video",
out: join(project, "assets/busy-out.mp4"),
prompt: "p",
motion: guide,
masks: [flatMask],
aspect: "9:16",
});
expect(r.notes?.join(" ")).toContain("mask 30,90,120,50 still shows structure");
},
);
it.skipIf(!ffmpeg)(
"crops a 1344x768 answer to the requested 16:9, never stretches it",
async () => {
answers(textured("wide.mp4", "1344x768", false));
const r = await generate({
type: "video",
out: join(project, "assets/wide-out.mp4"),
prompt: "p",
});
const { width, height } = probeVideo(r.out);
expect([width, height]).toEqual([1344, 756]);
expect(r.notes?.[0]).toBe(
"HeyGen returned 1344x768 for 16:9; cropped to 1344x756 (centre), not stretched",
);
},
);
it.skipIf(!ffmpeg)("blanks the motion clip's masked regions before sending it", async () => {
// A guide with white letters at 40,100 over a moving picture; the mask covers them for its first 0.5 s.
const guide = join(project, "titled.mp4");
packages/cli/src/heygen/generate.ts
+151 -16
View File
@@ -1,7 +1,15 @@
// New media on HeyGen for `media-use generate`: a still (optionally edited from a reference), a shot (optionally from
// a first frame), a person speaking from one photo, a voice reading text, or music. Every call is priced first with
// --plan, keyed for idempotency, downloaded into the project and recorded in the media ledger as generated.
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
renameSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, extname, join, relative, resolve } from "node:path";
import { findCuts, signatures } from "../measure/cuts.js";
@@ -62,17 +70,51 @@ export function parseMask(text: string): Mask {
}
/**
* Filters that fill each mask's region from the pixels around it (ffmpeg delogo), so no letter survives and the
* motion around it stays. delogo needs a pixel of picture on every side, so boxes are kept 1 px inside the frame.
* Fills a box from the pixels around it (ffmpeg delogo): no letter survives, but the box is smeared. Delogo needs a
* pixel of picture on every side, so boxes are kept 1 px inside the frame.
*/
function maskFilters(masks: Mask[], width: number, height: number): string[] {
return masks.flatMap(({ box: [x, y, w, h], window }) => {
const [x0, y0] = [Math.max(1, x), Math.max(1, y)];
const [x1, y1] = [Math.min(width - 2, x + w), Math.min(height - 2, y + h)];
if (x1 - x0 < 1 || y1 - y0 < 1) return [];
function delogoFilters(masks: Mask[], width: number, height: number): string {
return masks
.flatMap(({ box: [x, y, w, h], window }) => {
const [x0, y0] = [Math.max(1, x), Math.max(1, y)];
const [x1, y1] = [Math.min(width - 2, x + w), Math.min(height - 2, y + h)];
if (x1 - x0 < 1 || y1 - y0 < 1) return [];
const when = window ? `:enable='between(t,${window[0]},${window[1]})'` : "";
return [`delogo=x=${x0}:y=${y0}:w=${x1 - x0}:h=${y1 - y0}${when}`];
})
.join(",");
}
/** A box's temporal median over about a second either side: moving text becomes the picture behind it. */
function maskFilters(masks: Mask[], fps: number): string {
const radius = Math.min(127, Math.max(2, Math.round(fps)));
const steps = masks.map(({ box: [x, y, w, h], window }, i) => {
const when = window ? `:enable='between(t,${window[0]},${window[1]})'` : "";
return [`delogo=x=${x0}:y=${y0}:w=${x1 - x0}:h=${y1 - y0}${when}`];
const [w2, h2] = [w - (w % 2), h - (h % 2)];
return `[v${i}]split[a${i}][b${i}];[b${i}]crop=${w2}:${h2}:${x}:${y},tmedian=radius=${radius}[m${i}];[a${i}][m${i}]overlay=${x}:${y}${when}[v${i + 1}]`;
});
return `[0:v]null[v0];${steps.join(";")}`;
}
/** Mean absolute pixel step inside a box and in a ring around it: the box holds more edges when inside > ring. */
export function structureRatio(
gray: Uint8Array,
width: number,
box: [number, number, number, number],
): { inside: number; ring: number } {
const height = gray.length / width;
const [x0, y0, x1, y1] = [box[0], box[1], box[0] + box[2], box[1] + box[3]];
const pad = Math.max(box[2], box[3]) >> 1;
const sum = [0, 0];
const n = [0, 0];
for (let y = Math.max(0, y0 - pad); y < Math.min(height - 1, y1 + pad); y++)
for (let x = Math.max(0, x0 - pad); x < Math.min(width - 1, x1 + pad); x++) {
const k = x >= x0 && x < x1 && y >= y0 && y < y1 ? 0 : 1;
const i = y * width + x;
sum[k]! += Math.abs(gray[i]! - gray[i + 1]!) + Math.abs(gray[i]! - gray[i + width]!);
n[k]!++;
}
return { inside: sum[0]! / (n[0] || 1), ring: sum[1]! / (n[1] || 1) };
}
interface Route {
@@ -220,7 +262,7 @@ function needFile(path: string): void {
/**
* A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion
* clip with its masks blanked.
* clip with its masks filled.
*/
function inline(work: string, masks: Mask[] = []): Media {
return (path, kind) => {
@@ -246,13 +288,10 @@ function inline(work: string, masks: Mask[] = []): Media {
runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]);
// A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s.
else {
const { width, height } = masks.length ? probeVideo(path) : { width: 0, height: 0 };
const filters = [
...maskFilters(masks, width, height),
"scale='if(gt(iw,ih),min(720,iw),-2)':'if(gt(iw,ih),-2,min(720,ih))'",
];
const guide = masks.length ? fillMasks(path, masks, work) : path;
const scale = "scale='if(gt(iw,ih),min(720,iw),-2)':'if(gt(iw,ih),-2,min(720,ih))'";
runFfmpeg([
...["-y", "-i", path, "-t", "15", "-an", "-vf", filters.join(",")],
...["-y", "-i", guide, "-t", "15", "-an", "-vf", scale],
...["-c:v", "libx264", "-crf", "28", "-preset", "veryfast", "-pix_fmt", "yuv420p", out],
]);
}
@@ -278,6 +317,80 @@ function cutsInside(path: string, seconds?: number): { cuts: number[]; length: n
return { cuts, length: round(frames.length / info.fps, 3) };
}
/** The masks whose box, in `clip`'s frames, holds more edges than the picture around it at some sampled time. */
function structured(clip: string, guide: { width: number; height: number }, masks: Mask[]) {
const { width: w, height: h } = probeVideo(clip);
const gray = runFfmpeg(["-i", clip, "-vf", "fps=2", "-pix_fmt", "gray", "-f", "rawvideo", "-"]);
// The box as a fraction of the guide's frame, in the clip's pixels (its size and aspect may differ).
const [kx, ky] = [w / guide.width, h / guide.height];
return masks.flatMap((mask) => {
const [bx, by, bw, bh] = mask.box;
const box = [bx * kx, by * ky, bw * kx, bh * ky].map(Math.round) as Mask["box"];
let worst = 0;
for (let i = 0; (i + 1) * w * h <= gray.length; i++) {
if (mask.window && (i / 2 < mask.window[0] || i / 2 > mask.window[1])) continue;
const { inside, ring } = structureRatio(gray.subarray(i * w * h, (i + 1) * w * h), w, box);
if (inside > 6 && inside > 1.5 * ring) worst = Math.max(worst, inside / (ring || 1));
}
return worst ? [{ mask, worst }] : [];
});
}
/** A copy of the guide with each mask's box filled: the temporal median, and delogo's smear where text stays put. */
function fillMasks(path: string, masks: Mask[], work: string): string {
const info = probeVideo(path);
const encode = (from: string, name: string, filter: string[]) => {
const out = join(work, name);
runFfmpeg([
...["-y", "-i", from, "-an"],
...filter,
...["-crf", "14", "-pix_fmt", "yuv420p", out],
]);
return out;
};
const median = encode(path, "median.mp4", [
...["-t", "15", "-filter_complex", maskFilters(masks, info.fps)],
...["-map", `[v${masks.length}]`],
]);
const left = structured(median, info, masks).map((s) => s.mask);
if (!left.length) return median;
return encode(median, "delogo.mp4", ["-vf", delogoFilters(left, info.width, info.height)]);
}
/**
* HeyGen answers a 16:9 request at 1344x768 (1.75:1). The result is cropped, centred, to the requested ratio
* (1344x756) rather than stretched; returns a note on what was done, or nothing when it already fits.
*/
export function fitAspect(out: string, aspect: string): string | undefined {
const [aw, ah] = aspect.split(":").map(Number);
if (!aw || !ah) return;
const { width, height } = probeVideo(out);
const want = aw / ah;
if (Math.abs(width / height - want) / want < 0.002) return;
const even = (n: number) => n - (n % 2);
const [w, h] =
width / height > want ? [even(height * want), height] : [width, even(width / want)];
const fitted = `${out}.fit${extname(out)}`;
runFfmpeg([
...[
"-y",
"-i",
out,
"-vf",
`crop=${w}:${h}`,
"-c:v",
"libx264",
"-crf",
"12",
"-preset",
"slow",
],
...["-pix_fmt", "yuv420p", "-c:a", "copy", fitted],
]);
renameSync(fitted, out);
return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`;
}
const COUNT = ["", "one", "two", "three", "four", "five", "six"];
/**
@@ -331,6 +444,8 @@ export interface Generated {
ledger?: string;
/** video: seconds where HeyGen cut inside the returned shot; null when the file could not be read. */
cuts_inside?: number[] | null;
/** What was changed or looks wrong in the result: a crop to the requested aspect, text that may have come back. */
notes?: string[];
}
// fallow-ignore-next-line complexity
@@ -378,6 +493,25 @@ export async function generate(
} catch {
cuts = null;
}
const notes: string[] = [];
if (r.type === "video" && (r.aspect || !r.firstFrame))
try {
const note = fitAspect(out, r.aspect ?? "16:9");
if (note) notes.push(note);
} catch (e) {
notes.push(
`kept HeyGen's size, could not crop to ${r.aspect ?? "16:9"}: ${e instanceof Error ? e.message : e}`,
);
}
if (r.masks?.length && r.motion)
try {
for (const { mask, worst } of structured(out, probeVideo(r.motion), r.masks))
notes.push(
`mask ${mask.box.join(",")} still shows structure in the result (${worst.toFixed(1)}x its surroundings): the text may have come back, regenerate without the guide`,
);
} catch {
notes.push("could not check the masked boxes of the result");
}
let ledger: string | undefined;
if (project) {
const { recordInPlace } = await import("../media-use/lib/manifest.mjs");
@@ -416,6 +550,7 @@ export async function generate(
route: route.path,
...(ledger ? { ledger } : {}),
...(cuts !== undefined ? { cuts_inside: cuts } : {}),
...(notes.length ? { notes } : {}),
};
} finally {
rmSync(work, { recursive: true, force: true });