From 3adc9f130d907f37c16fdefcc30b27abc142287d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 3 Nov 2026 23:51:38 +0000 Subject: [PATCH] fix(cli): media-use sends a picture on a machine with no ffmpeg (#7112) * fix(cli): media-use sends a picture on a machine with no ffmpeg * fix(cli): pictures fall back to ffmpeg when sharp cannot load * fix(cli): drop two comments the names already carry --- packages/cli/src/heygen/generate.test.ts | 128 ++++++++++++++++++++++- packages/cli/src/heygen/generate.ts | 85 +++++++++------ 2 files changed, 177 insertions(+), 36 deletions(-) diff --git a/packages/cli/src/heygen/generate.test.ts b/packages/cli/src/heygen/generate.test.ts index 6c0b3014f9..8c27d07749 100644 --- a/packages/cli/src/heygen/generate.test.ts +++ b/packages/cli/src/heygen/generate.test.ts @@ -3,7 +3,8 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "no import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { findFFmpeg } from "../browser/ffmpeg.js"; +import { FFMPEG_PATH_ENV, findFFmpeg } from "../browser/ffmpeg.js"; +import { loadSharp } from "../utils/optionalPackages.js"; import { probeVideo } from "../measure/media.js"; vi.mock("../cloud/auth.js", () => ({ @@ -21,6 +22,16 @@ vi.mock("../cloud/upload.js", () => ({ }, })); +const sharpGone = vi.hoisted(() => ({ on: false })); +vi.mock("../utils/optionalPackages.js", async (load) => { + const real = await load(); + return { + ...real, + loadSharp: (feature: string) => + sharpGone.on ? Promise.reject(new Error("no sharp here")) : real.loadSharp(feature), + }; +}); + vi.stubEnv("HEYGEN_API_KEY", "k"); vi.stubEnv("HEYGEN_API_URL", "https://api.test"); @@ -132,6 +143,103 @@ describe("generate", () => { expect(ledger).toContain('"references":["ref.png"]'); }); + it("sends a picture with no ffmpeg on the machine, shrunk to 2048 px", async () => { + const big = join(project, "big.png"); + const sharp = await loadSharp("test"); + await sharp({ create: { width: 3000, height: 1000, channels: 3, background: "#c33" } }) + .png() + .toFile(big); + const before = process.env[FFMPEG_PATH_ENV]; + process.env[FFMPEG_PATH_ENV] = join(project, "no-ffmpeg"); + const sent: Record[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + if (init?.body) sent.push(JSON.parse(String(init.body))); + if (url === "https://files.test/big.png") return new Response(new Uint8Array([1])); + return json({ + data: { id: "img2", status: "completed", images: [{ url: "https://files.test/big.png" }] }, + }); + }); + try { + await generate({ + type: "image", + out: join(project, "assets/big-out.png"), + prompt: "p", + references: [big], + }); + } finally { + if (before === undefined) delete process.env[FFMPEG_PATH_ENV]; + else process.env[FFMPEG_PATH_ENV] = before; + } + const [picture] = (sent[0]!.reference_images as { media_type: string; data: string }[])!; + expect(picture!.media_type).toBe("image/jpeg"); + const { width, height, format } = await sharp(Buffer.from(picture!.data, "base64")).metadata(); + expect([format, width, height]).toEqual(["jpeg", 2048, 683]); + }); + + it.skipIf(!ffmpeg)("sends a picture sharp cannot read through ffmpeg", async () => { + const bmp = join(project, "still.bmp"); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=blue:s=40x30", "-frames:v", "1", bmp], + ]); + const sent: Record[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + if (init?.body) sent.push(JSON.parse(String(init.body))); + if (url === "https://files.test/bmp.png") return new Response(new Uint8Array([1])); + return json({ + data: { id: "img3", status: "completed", images: [{ url: "https://files.test/bmp.png" }] }, + }); + }); + await generate({ + type: "image", + out: join(project, "assets/bmp-out.png"), + prompt: "p", + references: [bmp], + }); + const [picture] = sent[0]!.reference_images as { data: string }[]; + const sharp = await loadSharp("test"); + const { width, height, format } = await sharp(Buffer.from(picture!.data, "base64")).metadata(); + expect([format, width, height]).toEqual(["jpeg", 40, 30]); + }); + + it.skipIf(!ffmpeg)("sends pictures through ffmpeg when sharp cannot be loaded", async () => { + answers(textured("no-sharp.mp4", "1344x768", false)); + sharpGone.on = true; + try { + const r = await generate({ + type: "video", + out: join(project, "assets/no-sharp-out.mp4"), + prompt: "p", + firstFrame: still("no-sharp.png", "120x90"), + }); + const { width, height } = probeVideo(r.out); + expect([width, height]).toEqual([1024, 768]); + } finally { + sharpGone.on = false; + } + }); + + it("prepares only the pictures the route sends: --avatar leaves a stale --photo alone", async () => { + const sent: Record[] = []; + vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { + if (init?.body) sent.push(JSON.parse(String(init.body))); + if (url === "https://files.test/av.mp4") return new Response(new Uint8Array([1])); + return json({ + data: { id: "av1", status: "completed", video_url: "https://files.test/av.mp4" }, + }); + }); + await settled( + generate({ + type: "avatar", + out: join(project, "assets/av-out.mp4"), + avatar: "look-1", + photo: join(project, "gone.png"), + script: "hi", + voice: "v1", + }), + ); + expect(sent[0]).toMatchObject({ type: "avatar", avatar_id: "look-1" }); + }); + it.skipIf(!ffmpeg)("re-shoots a person speaking a stretch of the original voice", async () => { const [face, voice] = [join(project, "face.png"), join(project, "voice.m4a")]; execFileSync(ffmpeg!, [ @@ -980,10 +1088,20 @@ describe("pickVoice", () => { if (picked instanceof Error) return picked; request.voice = (picked as { voice_id: string }).voice_id; vi.useFakeTimers({ toFake: ["Date", "setTimeout"] }); - const run = generate(request).catch((err: unknown) => err); - await vi.advanceTimersByTimeAsync(31 * 60_000); - vi.useRealTimers(); - return run; + let done = false; + const run = generate(request) + .catch((err: unknown) => err) + .finally(() => (done = true)); + // A photo is read before the first poll, so the clock moves until the run ends, not in one jump. + try { + while (!done) { + await new Promise((r) => setImmediate(r)); + await vi.advanceTimersByTimeAsync(60_000); + } + return await run; + } finally { + vi.useRealTimers(); + } }; for (const [type, extra] of [ diff --git a/packages/cli/src/heygen/generate.ts b/packages/cli/src/heygen/generate.ts index 5099c7faa1..80dde8d230 100644 --- a/packages/cli/src/heygen/generate.ts +++ b/packages/cli/src/heygen/generate.ts @@ -24,6 +24,7 @@ import type { } from "../cloud/_gen/types.js"; import { findCuts, signatures } from "../measure/cuts.js"; import { Fix, probeVideo, projectOf, round, runFfmpeg, videoClip } from "../measure/media.js"; +import { loadSharp } from "../utils/optionalPackages.js"; import { idempotencyKey, keepVoice, @@ -344,36 +345,57 @@ function needFile(path: string): void { throw new Fix(`no file at ${path}`, "pass a picture or sound from the project"); } -const shrink = (path: string, out: string) => - runFfmpeg([ - "-y", - "-i", - path, - "-vf", - "scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease", - "-q:v", - "3", - "-frames:v", - "1", - out, - ]); +const FIT_2048 = "scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease"; -/** - * A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion - * clip with its masks filled. - */ -function inline(work: string, masks: Mask[] = []): Media { +function picturesOf(route: Route, r: GenerateRequest): string[] { + const paths: string[] = []; + route.body(r, (path, kind) => { + if (kind === "image") paths.push(path); + return {}; + }); + return paths; +} + +const sharpOrNone = () => loadSharp("media-use pictures").catch(() => null); + +/** Each picture upright and at most 2048 px, as a JPEG in `work`. sharp, not ffmpeg: a picture alone never needs it. */ +async function shrunk(paths: string[], work: string): Promise> { + const out = new Map(); + if (!paths.length) return out; + const sharp = await sharpOrNone(); + for (const path of new Set(paths)) { + needFile(path); + const to = join(work, `picture-${out.size}.jpg`); + const viaFfmpeg = () => + runFfmpeg(["-y", "-i", path, "-vf", FIT_2048, "-q:v", "3", "-frames:v", "1", to]); + if (!sharp) viaFfmpeg(); + else + await sharp(path) + .rotate() + .resize(2048, 2048, { fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: 90 }) + .toFile(to) + .catch(viaFfmpeg); + out.set(path, to); + } + return out; +} + +/** A picture, sound or motion clip as the routes take it inline: pictures from `shrunk`, clips with masks filled. */ +function inline(work: string, pictures: Map, masks: Mask[] = []): Media { return (path, kind) => { needFile(path); - const out = join( - work, - `in-${Math.random().toString(36).slice(2)}.${{ image: "jpg", audio: "mp3", video: "mp4" }[kind]}`, - ); - if (kind === "image") shrink(path, out); - else if (kind === "audio") + const out = + kind === "image" + ? pictures.get(path)! + : join( + work, + `in-${Math.random().toString(36).slice(2)}.${{ audio: "mp3", video: "mp4" }[kind]}`, + ); + if (kind === "audio") runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]); // A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s. - else { + else if (kind === "video") { const guide = masks.length ? fillMasks(path, masks, work) : path; const scale = "scale='if(gt(iw,ih),min(720,iw),-2)':'if(gt(iw,ih),-2,min(720,ih))'"; runFfmpeg([ @@ -525,11 +547,11 @@ function fitAspect(out: string, aspect: string): string | undefined { return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`; } -function aspectOf(r: GenerateRequest, work: string): string { +async function aspectOf(r: GenerateRequest, pictures: Map): Promise { if (r.aspect || !r.firstFrame) return r.aspect ?? "16:9"; - const sent = join(work, "first-frame.jpg"); - shrink(r.firstFrame, sent); - const { width, height } = probeVideo(sent); + const sent = pictures.get(r.firstFrame)!; + const sharp = await sharpOrNone(); + const { width, height } = sharp ? await sharp(sent).metadata() : probeVideo(sent); return `${width}:${height}`; } @@ -647,7 +669,8 @@ export async function generate( try { const sources = assetsOf(r); const ids = await uploaded(sources); - const local = inline(work, r.masks); + const pictures = await shrunk(picturesOf(route, r), work); + const local = inline(work, pictures, r.masks); const body = route.body(r, (path, kind) => ids.has(path) ? { type: "asset_id", asset_id: ids.get(path) } : local(path, kind), ); @@ -687,7 +710,7 @@ export async function generate( const notes: string[] = []; if (r.type === "video") try { - const note = fitAspect(out, aspectOf(r, work)); + const note = fitAspect(out, await aspectOf(r, pictures)); if (note) notes.push(note); } catch (e) { notes.push(`kept HeyGen's size, could not crop: ${e instanceof Error ? e.message : e}`);