mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): media-use sends a picture on a machine with no ffmpeg (#7112)
* fix(cli): media-use sends a picture on a machine with no ffmpeg * fix(cli): pictures fall back to ffmpeg when sharp cannot load * fix(cli): drop two comments the names already carry
This commit is contained in:
@@ -3,7 +3,8 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "no
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { findFFmpeg } from "../browser/ffmpeg.js";
|
||||
import { FFMPEG_PATH_ENV, findFFmpeg } from "../browser/ffmpeg.js";
|
||||
import { loadSharp } from "../utils/optionalPackages.js";
|
||||
import { probeVideo } from "../measure/media.js";
|
||||
|
||||
vi.mock("../cloud/auth.js", () => ({
|
||||
@@ -21,6 +22,16 @@ vi.mock("../cloud/upload.js", () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
const sharpGone = vi.hoisted(() => ({ on: false }));
|
||||
vi.mock("../utils/optionalPackages.js", async (load) => {
|
||||
const real = await load<typeof import("../utils/optionalPackages.js")>();
|
||||
return {
|
||||
...real,
|
||||
loadSharp: (feature: string) =>
|
||||
sharpGone.on ? Promise.reject(new Error("no sharp here")) : real.loadSharp(feature),
|
||||
};
|
||||
});
|
||||
|
||||
vi.stubEnv("HEYGEN_API_KEY", "k");
|
||||
vi.stubEnv("HEYGEN_API_URL", "https://api.test");
|
||||
|
||||
@@ -132,6 +143,103 @@ describe("generate", () => {
|
||||
expect(ledger).toContain('"references":["ref.png"]');
|
||||
});
|
||||
|
||||
it("sends a picture with no ffmpeg on the machine, shrunk to 2048 px", async () => {
|
||||
const big = join(project, "big.png");
|
||||
const sharp = await loadSharp("test");
|
||||
await sharp({ create: { width: 3000, height: 1000, channels: 3, background: "#c33" } })
|
||||
.png()
|
||||
.toFile(big);
|
||||
const before = process.env[FFMPEG_PATH_ENV];
|
||||
process.env[FFMPEG_PATH_ENV] = join(project, "no-ffmpeg");
|
||||
const sent: Record<string, unknown>[] = [];
|
||||
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
|
||||
if (init?.body) sent.push(JSON.parse(String(init.body)));
|
||||
if (url === "https://files.test/big.png") return new Response(new Uint8Array([1]));
|
||||
return json({
|
||||
data: { id: "img2", status: "completed", images: [{ url: "https://files.test/big.png" }] },
|
||||
});
|
||||
});
|
||||
try {
|
||||
await generate({
|
||||
type: "image",
|
||||
out: join(project, "assets/big-out.png"),
|
||||
prompt: "p",
|
||||
references: [big],
|
||||
});
|
||||
} finally {
|
||||
if (before === undefined) delete process.env[FFMPEG_PATH_ENV];
|
||||
else process.env[FFMPEG_PATH_ENV] = before;
|
||||
}
|
||||
const [picture] = (sent[0]!.reference_images as { media_type: string; data: string }[])!;
|
||||
expect(picture!.media_type).toBe("image/jpeg");
|
||||
const { width, height, format } = await sharp(Buffer.from(picture!.data, "base64")).metadata();
|
||||
expect([format, width, height]).toEqual(["jpeg", 2048, 683]);
|
||||
});
|
||||
|
||||
it.skipIf(!ffmpeg)("sends a picture sharp cannot read through ffmpeg", async () => {
|
||||
const bmp = join(project, "still.bmp");
|
||||
execFileSync(ffmpeg!, [
|
||||
...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=blue:s=40x30", "-frames:v", "1", bmp],
|
||||
]);
|
||||
const sent: Record<string, unknown>[] = [];
|
||||
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
|
||||
if (init?.body) sent.push(JSON.parse(String(init.body)));
|
||||
if (url === "https://files.test/bmp.png") return new Response(new Uint8Array([1]));
|
||||
return json({
|
||||
data: { id: "img3", status: "completed", images: [{ url: "https://files.test/bmp.png" }] },
|
||||
});
|
||||
});
|
||||
await generate({
|
||||
type: "image",
|
||||
out: join(project, "assets/bmp-out.png"),
|
||||
prompt: "p",
|
||||
references: [bmp],
|
||||
});
|
||||
const [picture] = sent[0]!.reference_images as { data: string }[];
|
||||
const sharp = await loadSharp("test");
|
||||
const { width, height, format } = await sharp(Buffer.from(picture!.data, "base64")).metadata();
|
||||
expect([format, width, height]).toEqual(["jpeg", 40, 30]);
|
||||
});
|
||||
|
||||
it.skipIf(!ffmpeg)("sends pictures through ffmpeg when sharp cannot be loaded", async () => {
|
||||
answers(textured("no-sharp.mp4", "1344x768", false));
|
||||
sharpGone.on = true;
|
||||
try {
|
||||
const r = await generate({
|
||||
type: "video",
|
||||
out: join(project, "assets/no-sharp-out.mp4"),
|
||||
prompt: "p",
|
||||
firstFrame: still("no-sharp.png", "120x90"),
|
||||
});
|
||||
const { width, height } = probeVideo(r.out);
|
||||
expect([width, height]).toEqual([1024, 768]);
|
||||
} finally {
|
||||
sharpGone.on = false;
|
||||
}
|
||||
});
|
||||
|
||||
it("prepares only the pictures the route sends: --avatar leaves a stale --photo alone", async () => {
|
||||
const sent: Record<string, unknown>[] = [];
|
||||
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
|
||||
if (init?.body) sent.push(JSON.parse(String(init.body)));
|
||||
if (url === "https://files.test/av.mp4") return new Response(new Uint8Array([1]));
|
||||
return json({
|
||||
data: { id: "av1", status: "completed", video_url: "https://files.test/av.mp4" },
|
||||
});
|
||||
});
|
||||
await settled(
|
||||
generate({
|
||||
type: "avatar",
|
||||
out: join(project, "assets/av-out.mp4"),
|
||||
avatar: "look-1",
|
||||
photo: join(project, "gone.png"),
|
||||
script: "hi",
|
||||
voice: "v1",
|
||||
}),
|
||||
);
|
||||
expect(sent[0]).toMatchObject({ type: "avatar", avatar_id: "look-1" });
|
||||
});
|
||||
|
||||
it.skipIf(!ffmpeg)("re-shoots a person speaking a stretch of the original voice", async () => {
|
||||
const [face, voice] = [join(project, "face.png"), join(project, "voice.m4a")];
|
||||
execFileSync(ffmpeg!, [
|
||||
@@ -980,10 +1088,20 @@ describe("pickVoice", () => {
|
||||
if (picked instanceof Error) return picked;
|
||||
request.voice = (picked as { voice_id: string }).voice_id;
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout"] });
|
||||
const run = generate(request).catch((err: unknown) => err);
|
||||
await vi.advanceTimersByTimeAsync(31 * 60_000);
|
||||
vi.useRealTimers();
|
||||
return run;
|
||||
let done = false;
|
||||
const run = generate(request)
|
||||
.catch((err: unknown) => err)
|
||||
.finally(() => (done = true));
|
||||
// A photo is read before the first poll, so the clock moves until the run ends, not in one jump.
|
||||
try {
|
||||
while (!done) {
|
||||
await new Promise((r) => setImmediate(r));
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
}
|
||||
return await run;
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
};
|
||||
|
||||
for (const [type, extra] of [
|
||||
|
||||
@@ -24,6 +24,7 @@ import type {
|
||||
} from "../cloud/_gen/types.js";
|
||||
import { findCuts, signatures } from "../measure/cuts.js";
|
||||
import { Fix, probeVideo, projectOf, round, runFfmpeg, videoClip } from "../measure/media.js";
|
||||
import { loadSharp } from "../utils/optionalPackages.js";
|
||||
import {
|
||||
idempotencyKey,
|
||||
keepVoice,
|
||||
@@ -344,36 +345,57 @@ function needFile(path: string): void {
|
||||
throw new Fix(`no file at ${path}`, "pass a picture or sound from the project");
|
||||
}
|
||||
|
||||
const shrink = (path: string, out: string) =>
|
||||
runFfmpeg([
|
||||
"-y",
|
||||
"-i",
|
||||
path,
|
||||
"-vf",
|
||||
"scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease",
|
||||
"-q:v",
|
||||
"3",
|
||||
"-frames:v",
|
||||
"1",
|
||||
out,
|
||||
]);
|
||||
const FIT_2048 = "scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease";
|
||||
|
||||
/**
|
||||
* A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion
|
||||
* clip with its masks filled.
|
||||
*/
|
||||
function inline(work: string, masks: Mask[] = []): Media {
|
||||
function picturesOf(route: Route, r: GenerateRequest): string[] {
|
||||
const paths: string[] = [];
|
||||
route.body(r, (path, kind) => {
|
||||
if (kind === "image") paths.push(path);
|
||||
return {};
|
||||
});
|
||||
return paths;
|
||||
}
|
||||
|
||||
const sharpOrNone = () => loadSharp("media-use pictures").catch(() => null);
|
||||
|
||||
/** Each picture upright and at most 2048 px, as a JPEG in `work`. sharp, not ffmpeg: a picture alone never needs it. */
|
||||
async function shrunk(paths: string[], work: string): Promise<Map<string, string>> {
|
||||
const out = new Map<string, string>();
|
||||
if (!paths.length) return out;
|
||||
const sharp = await sharpOrNone();
|
||||
for (const path of new Set(paths)) {
|
||||
needFile(path);
|
||||
const to = join(work, `picture-${out.size}.jpg`);
|
||||
const viaFfmpeg = () =>
|
||||
runFfmpeg(["-y", "-i", path, "-vf", FIT_2048, "-q:v", "3", "-frames:v", "1", to]);
|
||||
if (!sharp) viaFfmpeg();
|
||||
else
|
||||
await sharp(path)
|
||||
.rotate()
|
||||
.resize(2048, 2048, { fit: "inside", withoutEnlargement: true })
|
||||
.jpeg({ quality: 90 })
|
||||
.toFile(to)
|
||||
.catch(viaFfmpeg);
|
||||
out.set(path, to);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** A picture, sound or motion clip as the routes take it inline: pictures from `shrunk`, clips with masks filled. */
|
||||
function inline(work: string, pictures: Map<string, string>, masks: Mask[] = []): Media {
|
||||
return (path, kind) => {
|
||||
needFile(path);
|
||||
const out = join(
|
||||
work,
|
||||
`in-${Math.random().toString(36).slice(2)}.${{ image: "jpg", audio: "mp3", video: "mp4" }[kind]}`,
|
||||
);
|
||||
if (kind === "image") shrink(path, out);
|
||||
else if (kind === "audio")
|
||||
const out =
|
||||
kind === "image"
|
||||
? pictures.get(path)!
|
||||
: join(
|
||||
work,
|
||||
`in-${Math.random().toString(36).slice(2)}.${{ audio: "mp3", video: "mp4" }[kind]}`,
|
||||
);
|
||||
if (kind === "audio")
|
||||
runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]);
|
||||
// A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s.
|
||||
else {
|
||||
else if (kind === "video") {
|
||||
const guide = masks.length ? fillMasks(path, masks, work) : path;
|
||||
const scale = "scale='if(gt(iw,ih),min(720,iw),-2)':'if(gt(iw,ih),-2,min(720,ih))'";
|
||||
runFfmpeg([
|
||||
@@ -525,11 +547,11 @@ function fitAspect(out: string, aspect: string): string | undefined {
|
||||
return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`;
|
||||
}
|
||||
|
||||
function aspectOf(r: GenerateRequest, work: string): string {
|
||||
async function aspectOf(r: GenerateRequest, pictures: Map<string, string>): Promise<string> {
|
||||
if (r.aspect || !r.firstFrame) return r.aspect ?? "16:9";
|
||||
const sent = join(work, "first-frame.jpg");
|
||||
shrink(r.firstFrame, sent);
|
||||
const { width, height } = probeVideo(sent);
|
||||
const sent = pictures.get(r.firstFrame)!;
|
||||
const sharp = await sharpOrNone();
|
||||
const { width, height } = sharp ? await sharp(sent).metadata() : probeVideo(sent);
|
||||
return `${width}:${height}`;
|
||||
}
|
||||
|
||||
@@ -647,7 +669,8 @@ export async function generate(
|
||||
try {
|
||||
const sources = assetsOf(r);
|
||||
const ids = await uploaded(sources);
|
||||
const local = inline(work, r.masks);
|
||||
const pictures = await shrunk(picturesOf(route, r), work);
|
||||
const local = inline(work, pictures, r.masks);
|
||||
const body = route.body(r, (path, kind) =>
|
||||
ids.has(path) ? { type: "asset_id", asset_id: ids.get(path) } : local(path, kind),
|
||||
);
|
||||
@@ -687,7 +710,7 @@ export async function generate(
|
||||
const notes: string[] = [];
|
||||
if (r.type === "video")
|
||||
try {
|
||||
const note = fitAspect(out, aspectOf(r, work));
|
||||
const note = fitAspect(out, await aspectOf(r, pictures));
|
||||
if (note) notes.push(note);
|
||||
} catch (e) {
|
||||
notes.push(`kept HeyGen's size, could not crop: ${e instanceof Error ? e.message : e}`);
|
||||
|
||||
Reference in New Issue
Block a user