fix(cli): a shot from --first-frame comes back in that frame's shape (#6924)

* fix(cli): a shot from --first-frame comes back in that frame's shape

HeyGen can answer 1344x768 for a 16:9 first frame. The crop only ran with --aspect or
with no first frame, so these shots kept its size. Without --aspect the first frame now
sets the shape.

* refactor(cli): drop a comment the function name already says

* fix(cli): the first frame's shape is the one sent, not the one stored

A phone JPEG with Orientation=6 uploads as portrait but stores landscape pixels, so a correct
portrait answer was cropped to 90x50. The shape now comes from the same conversion the upload uses.

fails without the fix: keeps a portrait shot from a phone photo stored sideways
This commit is contained in:
WaterrrForever
2026-11-01 14:44:27 +00:00
committed by GitHub
parent af0d68d612
commit 439d01784c
2 changed files with 67 additions and 18 deletions
+41
View File
@@ -260,6 +260,47 @@ describe("generate", () => {
},
);
const still = (name: string, size: string) => {
const path = join(project, name);
execFileSync(ffmpeg!, [
...["-v", "error", "-y", "-f", "lavfi", "-i", `color=c=red:s=${size}`],
...["-frames:v", "1", path],
]);
return path;
};
it.skipIf(!ffmpeg)("crops to the first frame's shape when no --aspect is passed", async () => {
answers(textured("first-wide.mp4", "1344x768", false));
const r = await generate({
type: "video",
out: join(project, "assets/first-out.mp4"),
prompt: "p",
firstFrame: still("first.png", "120x90"),
});
const { width, height } = probeVideo(r.out);
expect([width, height]).toEqual([1024, 768]);
});
it.skipIf(!ffmpeg)("keeps a portrait shot from a phone photo stored sideways", async () => {
const first = still("phone.jpg", "160x90");
const jpeg = readFileSync(first);
// An Exif segment with Orientation=6: the file holds landscape pixels, the picture is portrait.
const exif = "ffe10022457869660000" + "4d4d002a00000008000101120003000000010006000000000000";
writeFileSync(
first,
Buffer.concat([jpeg.subarray(0, 2), Buffer.from(exif, "hex"), jpeg.subarray(2)]),
);
answers(textured("first-tall.mp4", "90x160", false));
const r = await generate({
type: "video",
out: join(project, "assets/phone-out.mp4"),
prompt: "p",
firstFrame: first,
});
const { width, height } = probeVideo(r.out);
expect([width, height]).toEqual([90, 160]);
});
it.skipIf(!ffmpeg)("blanks the motion clip's masked regions before sending it", async () => {
// A guide with white letters at 40,100 over a moving picture; the mask covers them for its first 0.5 s.
const guide = join(project, "titled.mp4");
+26 -18
View File
@@ -262,6 +262,20 @@ function needFile(path: string): void {
throw new Fix(`no file at ${path}`, "pass a picture or sound from the project");
}
const shrink = (path: string, out: string) =>
runFfmpeg([
"-y",
"-i",
path,
"-vf",
"scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease",
"-q:v",
"3",
"-frames:v",
"1",
out,
]);
/**
* A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion
* clip with its masks filled.
@@ -273,19 +287,7 @@ function inline(work: string, masks: Mask[] = []): Media {
work,
`in-${Math.random().toString(36).slice(2)}.${{ image: "jpg", audio: "mp3", video: "mp4" }[kind]}`,
);
if (kind === "image")
runFfmpeg([
"-y",
"-i",
path,
"-vf",
"scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease",
"-q:v",
"3",
"-frames:v",
"1",
out,
]);
if (kind === "image") shrink(path, out);
else if (kind === "audio")
runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]);
// A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s.
@@ -393,6 +395,14 @@ export function fitAspect(out: string, aspect: string): string | undefined {
return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`;
}
function aspectOf(r: GenerateRequest, work: string): string {
if (r.aspect || !r.firstFrame) return r.aspect ?? "16:9";
const sent = join(work, "first-frame.jpg");
shrink(r.firstFrame, sent);
const { width, height } = probeVideo(sent);
return `${width}:${height}`;
}
const MIN_SHOT_S = 0.5;
export function foldShort(cuts: number[], length: number): number[] {
@@ -502,14 +512,12 @@ export async function generate(
cuts = null;
}
const notes: string[] = [];
if (r.type === "video" && (r.aspect || !r.firstFrame))
if (r.type === "video")
try {
const note = fitAspect(out, r.aspect ?? "16:9");
const note = fitAspect(out, aspectOf(r, work));
if (note) notes.push(note);
} catch (e) {
notes.push(
`kept HeyGen's size, could not crop to ${r.aspect ?? "16:9"}: ${e instanceof Error ? e.message : e}`,
);
notes.push(`kept HeyGen's size, could not crop: ${e instanceof Error ? e.message : e}`);
}
if (r.masks?.length && r.motion)
try {