fix(cli): media-use sends a picture on a machine with no ffmpeg (#7112)

* fix(cli): media-use sends a picture on a machine with no ffmpeg

* fix(cli): pictures fall back to ffmpeg when sharp cannot load

* fix(cli): drop two comments the names already carry
This commit is contained in:
Miguel Ángel
2026-11-03 23:51:38 +00:00
committed by GitHub
parent 482bcd73ed
commit 3adc9f130d
2 changed files with 177 additions and 36 deletions
+123 -5
View File
@@ -3,7 +3,8 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "no
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { findFFmpeg } from "../browser/ffmpeg.js";
import { FFMPEG_PATH_ENV, findFFmpeg } from "../browser/ffmpeg.js";
import { loadSharp } from "../utils/optionalPackages.js";
import { probeVideo } from "../measure/media.js";
vi.mock("../cloud/auth.js", () => ({
@@ -21,6 +22,16 @@ vi.mock("../cloud/upload.js", () => ({
},
}));
const sharpGone = vi.hoisted(() => ({ on: false }));
vi.mock("../utils/optionalPackages.js", async (load) => {
const real = await load<typeof import("../utils/optionalPackages.js")>();
return {
...real,
loadSharp: (feature: string) =>
sharpGone.on ? Promise.reject(new Error("no sharp here")) : real.loadSharp(feature),
};
});
vi.stubEnv("HEYGEN_API_KEY", "k");
vi.stubEnv("HEYGEN_API_URL", "https://api.test");
@@ -132,6 +143,103 @@ describe("generate", () => {
expect(ledger).toContain('"references":["ref.png"]');
});
it("sends a picture with no ffmpeg on the machine, shrunk to 2048 px", async () => {
const big = join(project, "big.png");
const sharp = await loadSharp("test");
await sharp({ create: { width: 3000, height: 1000, channels: 3, background: "#c33" } })
.png()
.toFile(big);
const before = process.env[FFMPEG_PATH_ENV];
process.env[FFMPEG_PATH_ENV] = join(project, "no-ffmpeg");
const sent: Record<string, unknown>[] = [];
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
if (init?.body) sent.push(JSON.parse(String(init.body)));
if (url === "https://files.test/big.png") return new Response(new Uint8Array([1]));
return json({
data: { id: "img2", status: "completed", images: [{ url: "https://files.test/big.png" }] },
});
});
try {
await generate({
type: "image",
out: join(project, "assets/big-out.png"),
prompt: "p",
references: [big],
});
} finally {
if (before === undefined) delete process.env[FFMPEG_PATH_ENV];
else process.env[FFMPEG_PATH_ENV] = before;
}
const [picture] = (sent[0]!.reference_images as { media_type: string; data: string }[])!;
expect(picture!.media_type).toBe("image/jpeg");
const { width, height, format } = await sharp(Buffer.from(picture!.data, "base64")).metadata();
expect([format, width, height]).toEqual(["jpeg", 2048, 683]);
});
it.skipIf(!ffmpeg)("sends a picture sharp cannot read through ffmpeg", async () => {
const bmp = join(project, "still.bmp");
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=blue:s=40x30", "-frames:v", "1", bmp],
]);
const sent: Record<string, unknown>[] = [];
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
if (init?.body) sent.push(JSON.parse(String(init.body)));
if (url === "https://files.test/bmp.png") return new Response(new Uint8Array([1]));
return json({
data: { id: "img3", status: "completed", images: [{ url: "https://files.test/bmp.png" }] },
});
});
await generate({
type: "image",
out: join(project, "assets/bmp-out.png"),
prompt: "p",
references: [bmp],
});
const [picture] = sent[0]!.reference_images as { data: string }[];
const sharp = await loadSharp("test");
const { width, height, format } = await sharp(Buffer.from(picture!.data, "base64")).metadata();
expect([format, width, height]).toEqual(["jpeg", 40, 30]);
});
it.skipIf(!ffmpeg)("sends pictures through ffmpeg when sharp cannot be loaded", async () => {
answers(textured("no-sharp.mp4", "1344x768", false));
sharpGone.on = true;
try {
const r = await generate({
type: "video",
out: join(project, "assets/no-sharp-out.mp4"),
prompt: "p",
firstFrame: still("no-sharp.png", "120x90"),
});
const { width, height } = probeVideo(r.out);
expect([width, height]).toEqual([1024, 768]);
} finally {
sharpGone.on = false;
}
});
it("prepares only the pictures the route sends: --avatar leaves a stale --photo alone", async () => {
const sent: Record<string, unknown>[] = [];
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
if (init?.body) sent.push(JSON.parse(String(init.body)));
if (url === "https://files.test/av.mp4") return new Response(new Uint8Array([1]));
return json({
data: { id: "av1", status: "completed", video_url: "https://files.test/av.mp4" },
});
});
await settled(
generate({
type: "avatar",
out: join(project, "assets/av-out.mp4"),
avatar: "look-1",
photo: join(project, "gone.png"),
script: "hi",
voice: "v1",
}),
);
expect(sent[0]).toMatchObject({ type: "avatar", avatar_id: "look-1" });
});
it.skipIf(!ffmpeg)("re-shoots a person speaking a stretch of the original voice", async () => {
const [face, voice] = [join(project, "face.png"), join(project, "voice.m4a")];
execFileSync(ffmpeg!, [
@@ -980,10 +1088,20 @@ describe("pickVoice", () => {
if (picked instanceof Error) return picked;
request.voice = (picked as { voice_id: string }).voice_id;
vi.useFakeTimers({ toFake: ["Date", "setTimeout"] });
const run = generate(request).catch((err: unknown) => err);
await vi.advanceTimersByTimeAsync(31 * 60_000);
vi.useRealTimers();
return run;
let done = false;
const run = generate(request)
.catch((err: unknown) => err)
.finally(() => (done = true));
// A photo is read before the first poll, so the clock moves until the run ends, not in one jump.
try {
while (!done) {
await new Promise((r) => setImmediate(r));
await vi.advanceTimersByTimeAsync(60_000);
}
return await run;
} finally {
vi.useRealTimers();
}
};
for (const [type, extra] of [
+54 -31
View File
@@ -24,6 +24,7 @@ import type {
} from "../cloud/_gen/types.js";
import { findCuts, signatures } from "../measure/cuts.js";
import { Fix, probeVideo, projectOf, round, runFfmpeg, videoClip } from "../measure/media.js";
import { loadSharp } from "../utils/optionalPackages.js";
import {
idempotencyKey,
keepVoice,
@@ -344,36 +345,57 @@ function needFile(path: string): void {
throw new Fix(`no file at ${path}`, "pass a picture or sound from the project");
}
const shrink = (path: string, out: string) =>
runFfmpeg([
"-y",
"-i",
path,
"-vf",
"scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease",
"-q:v",
"3",
"-frames:v",
"1",
out,
]);
const FIT_2048 = "scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease";
/**
* A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion
* clip with its masks filled.
*/
function inline(work: string, masks: Mask[] = []): Media {
function picturesOf(route: Route, r: GenerateRequest): string[] {
const paths: string[] = [];
route.body(r, (path, kind) => {
if (kind === "image") paths.push(path);
return {};
});
return paths;
}
const sharpOrNone = () => loadSharp("media-use pictures").catch(() => null);
/** Each picture upright and at most 2048 px, as a JPEG in `work`. sharp, not ffmpeg: a picture alone never needs it. */
async function shrunk(paths: string[], work: string): Promise<Map<string, string>> {
const out = new Map<string, string>();
if (!paths.length) return out;
const sharp = await sharpOrNone();
for (const path of new Set(paths)) {
needFile(path);
const to = join(work, `picture-${out.size}.jpg`);
const viaFfmpeg = () =>
runFfmpeg(["-y", "-i", path, "-vf", FIT_2048, "-q:v", "3", "-frames:v", "1", to]);
if (!sharp) viaFfmpeg();
else
await sharp(path)
.rotate()
.resize(2048, 2048, { fit: "inside", withoutEnlargement: true })
.jpeg({ quality: 90 })
.toFile(to)
.catch(viaFfmpeg);
out.set(path, to);
}
return out;
}
/** A picture, sound or motion clip as the routes take it inline: pictures from `shrunk`, clips with masks filled. */
function inline(work: string, pictures: Map<string, string>, masks: Mask[] = []): Media {
return (path, kind) => {
needFile(path);
const out = join(
work,
`in-${Math.random().toString(36).slice(2)}.${{ image: "jpg", audio: "mp3", video: "mp4" }[kind]}`,
);
if (kind === "image") shrink(path, out);
else if (kind === "audio")
const out =
kind === "image"
? pictures.get(path)!
: join(
work,
`in-${Math.random().toString(36).slice(2)}.${{ audio: "mp3", video: "mp4" }[kind]}`,
);
if (kind === "audio")
runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]);
// A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s.
else {
else if (kind === "video") {
const guide = masks.length ? fillMasks(path, masks, work) : path;
const scale = "scale='if(gt(iw,ih),min(720,iw),-2)':'if(gt(iw,ih),-2,min(720,ih))'";
runFfmpeg([
@@ -525,11 +547,11 @@ function fitAspect(out: string, aspect: string): string | undefined {
return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`;
}
function aspectOf(r: GenerateRequest, work: string): string {
async function aspectOf(r: GenerateRequest, pictures: Map<string, string>): Promise<string> {
if (r.aspect || !r.firstFrame) return r.aspect ?? "16:9";
const sent = join(work, "first-frame.jpg");
shrink(r.firstFrame, sent);
const { width, height } = probeVideo(sent);
const sent = pictures.get(r.firstFrame)!;
const sharp = await sharpOrNone();
const { width, height } = sharp ? await sharp(sent).metadata() : probeVideo(sent);
return `${width}:${height}`;
}
@@ -647,7 +669,8 @@ export async function generate(
try {
const sources = assetsOf(r);
const ids = await uploaded(sources);
const local = inline(work, r.masks);
const pictures = await shrunk(picturesOf(route, r), work);
const local = inline(work, pictures, r.masks);
const body = route.body(r, (path, kind) =>
ids.has(path) ? { type: "asset_id", asset_id: ids.get(path) } : local(path, kind),
);
@@ -687,7 +710,7 @@ export async function generate(
const notes: string[] = [];
if (r.type === "video")
try {
const note = fitAspect(out, aspectOf(r, work));
const note = fitAspect(out, await aspectOf(r, pictures));
if (note) notes.push(note);
} catch (e) {
notes.push(`kept HeyGen's size, could not crop: ${e instanceof Error ? e.message : e}`);