diff --git a/packages/cli/src/heygen/generate.test.ts b/packages/cli/src/heygen/generate.test.ts index 034edb65ab..c8d5d1fbe9 100644 --- a/packages/cli/src/heygen/generate.test.ts +++ b/packages/cli/src/heygen/generate.test.ts @@ -260,6 +260,47 @@ describe("generate", () => { }, ); + const still = (name: string, size: string) => { + const path = join(project, name); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", `color=c=red:s=${size}`], + ...["-frames:v", "1", path], + ]); + return path; + }; + + it.skipIf(!ffmpeg)("crops to the first frame's shape when no --aspect is passed", async () => { + answers(textured("first-wide.mp4", "1344x768", false)); + const r = await generate({ + type: "video", + out: join(project, "assets/first-out.mp4"), + prompt: "p", + firstFrame: still("first.png", "120x90"), + }); + const { width, height } = probeVideo(r.out); + expect([width, height]).toEqual([1024, 768]); + }); + + it.skipIf(!ffmpeg)("keeps a portrait shot from a phone photo stored sideways", async () => { + const first = still("phone.jpg", "160x90"); + const jpeg = readFileSync(first); + // An Exif segment with Orientation=6: the file holds landscape pixels, the picture is portrait. + const exif = "ffe10022457869660000" + "4d4d002a00000008000101120003000000010006000000000000"; + writeFileSync( + first, + Buffer.concat([jpeg.subarray(0, 2), Buffer.from(exif, "hex"), jpeg.subarray(2)]), + ); + answers(textured("first-tall.mp4", "90x160", false)); + const r = await generate({ + type: "video", + out: join(project, "assets/phone-out.mp4"), + prompt: "p", + firstFrame: first, + }); + const { width, height } = probeVideo(r.out); + expect([width, height]).toEqual([90, 160]); + }); + it.skipIf(!ffmpeg)("blanks the motion clip's masked regions before sending it", async () => { // A guide with white letters at 40,100 over a moving picture; the mask covers them for its first 0.5 s. const guide = join(project, "titled.mp4"); diff --git a/packages/cli/src/heygen/generate.ts b/packages/cli/src/heygen/generate.ts index dae88496a3..37c407adfe 100644 --- a/packages/cli/src/heygen/generate.ts +++ b/packages/cli/src/heygen/generate.ts @@ -262,6 +262,20 @@ function needFile(path: string): void { throw new Fix(`no file at ${path}`, "pass a picture or sound from the project"); } +const shrink = (path: string, out: string) => + runFfmpeg([ + "-y", + "-i", + path, + "-vf", + "scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease", + "-q:v", + "3", + "-frames:v", + "1", + out, + ]); + /** * A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion * clip with its masks filled. @@ -273,19 +287,7 @@ function inline(work: string, masks: Mask[] = []): Media { work, `in-${Math.random().toString(36).slice(2)}.${{ image: "jpg", audio: "mp3", video: "mp4" }[kind]}`, ); - if (kind === "image") - runFfmpeg([ - "-y", - "-i", - path, - "-vf", - "scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease", - "-q:v", - "3", - "-frames:v", - "1", - out, - ]); + if (kind === "image") shrink(path, out); else if (kind === "audio") runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]); // A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s. @@ -393,6 +395,14 @@ export function fitAspect(out: string, aspect: string): string | undefined { return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`; } +function aspectOf(r: GenerateRequest, work: string): string { + if (r.aspect || !r.firstFrame) return r.aspect ?? "16:9"; + const sent = join(work, "first-frame.jpg"); + shrink(r.firstFrame, sent); + const { width, height } = probeVideo(sent); + return `${width}:${height}`; +} + const MIN_SHOT_S = 0.5; export function foldShort(cuts: number[], length: number): number[] { @@ -502,14 +512,12 @@ export async function generate( cuts = null; } const notes: string[] = []; - if (r.type === "video" && (r.aspect || !r.firstFrame)) + if (r.type === "video") try { - const note = fitAspect(out, r.aspect ?? "16:9"); + const note = fitAspect(out, aspectOf(r, work)); if (note) notes.push(note); } catch (e) { - notes.push( - `kept HeyGen's size, could not crop to ${r.aspect ?? "16:9"}: ${e instanceof Error ? e.message : e}`, - ); + notes.push(`kept HeyGen's size, could not crop: ${e instanceof Error ? e.message : e}`); } if (r.masks?.length && r.motion) try {