mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 11:26:02 +00:00
fix(cli): a shot from --first-frame comes back in that frame's shape (#6924)
* fix(cli): a shot from --first-frame comes back in that frame's shape HeyGen can answer 1344x768 for a 16:9 first frame. The crop only ran with --aspect or with no first frame, so these shots kept its size. Without --aspect the first frame now sets the shape. * refactor(cli): drop a comment the function name already says * fix(cli): the first frame's shape is the one sent, not the one stored A phone JPEG with Orientation=6 uploads as portrait but stores landscape pixels, so a correct portrait answer was cropped to 90x50. The shape now comes from the same conversion the upload uses. fails without the fix: keeps a portrait shot from a phone photo stored sideways
This commit is contained in:
@@ -260,6 +260,47 @@ describe("generate", () => {
|
||||
},
|
||||
);
|
||||
|
||||
const still = (name: string, size: string) => {
|
||||
const path = join(project, name);
|
||||
execFileSync(ffmpeg!, [
|
||||
...["-v", "error", "-y", "-f", "lavfi", "-i", `color=c=red:s=${size}`],
|
||||
...["-frames:v", "1", path],
|
||||
]);
|
||||
return path;
|
||||
};
|
||||
|
||||
it.skipIf(!ffmpeg)("crops to the first frame's shape when no --aspect is passed", async () => {
|
||||
answers(textured("first-wide.mp4", "1344x768", false));
|
||||
const r = await generate({
|
||||
type: "video",
|
||||
out: join(project, "assets/first-out.mp4"),
|
||||
prompt: "p",
|
||||
firstFrame: still("first.png", "120x90"),
|
||||
});
|
||||
const { width, height } = probeVideo(r.out);
|
||||
expect([width, height]).toEqual([1024, 768]);
|
||||
});
|
||||
|
||||
it.skipIf(!ffmpeg)("keeps a portrait shot from a phone photo stored sideways", async () => {
|
||||
const first = still("phone.jpg", "160x90");
|
||||
const jpeg = readFileSync(first);
|
||||
// An Exif segment with Orientation=6: the file holds landscape pixels, the picture is portrait.
|
||||
const exif = "ffe10022457869660000" + "4d4d002a00000008000101120003000000010006000000000000";
|
||||
writeFileSync(
|
||||
first,
|
||||
Buffer.concat([jpeg.subarray(0, 2), Buffer.from(exif, "hex"), jpeg.subarray(2)]),
|
||||
);
|
||||
answers(textured("first-tall.mp4", "90x160", false));
|
||||
const r = await generate({
|
||||
type: "video",
|
||||
out: join(project, "assets/phone-out.mp4"),
|
||||
prompt: "p",
|
||||
firstFrame: first,
|
||||
});
|
||||
const { width, height } = probeVideo(r.out);
|
||||
expect([width, height]).toEqual([90, 160]);
|
||||
});
|
||||
|
||||
it.skipIf(!ffmpeg)("blanks the motion clip's masked regions before sending it", async () => {
|
||||
// A guide with white letters at 40,100 over a moving picture; the mask covers them for its first 0.5 s.
|
||||
const guide = join(project, "titled.mp4");
|
||||
|
||||
@@ -262,6 +262,20 @@ function needFile(path: string): void {
|
||||
throw new Fix(`no file at ${path}`, "pass a picture or sound from the project");
|
||||
}
|
||||
|
||||
const shrink = (path: string, out: string) =>
|
||||
runFfmpeg([
|
||||
"-y",
|
||||
"-i",
|
||||
path,
|
||||
"-vf",
|
||||
"scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease",
|
||||
"-q:v",
|
||||
"3",
|
||||
"-frames:v",
|
||||
"1",
|
||||
out,
|
||||
]);
|
||||
|
||||
/**
|
||||
* A picture or a sound as the routes take it inline, pictures shrunk to 2048 px so the body stays small, a motion
|
||||
* clip with its masks filled.
|
||||
@@ -273,19 +287,7 @@ function inline(work: string, masks: Mask[] = []): Media {
|
||||
work,
|
||||
`in-${Math.random().toString(36).slice(2)}.${{ image: "jpg", audio: "mp3", video: "mp4" }[kind]}`,
|
||||
);
|
||||
if (kind === "image")
|
||||
runFfmpeg([
|
||||
"-y",
|
||||
"-i",
|
||||
path,
|
||||
"-vf",
|
||||
"scale='min(2048,iw)':'min(2048,ih)':force_original_aspect_ratio=decrease",
|
||||
"-q:v",
|
||||
"3",
|
||||
"-frames:v",
|
||||
"1",
|
||||
out,
|
||||
]);
|
||||
if (kind === "image") shrink(path, out);
|
||||
else if (kind === "audio")
|
||||
runFfmpeg(["-y", "-i", path, "-vn", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", out]);
|
||||
// A guide clip only needs its motion: 720 px on the long side, no sound, at most 15 s.
|
||||
@@ -393,6 +395,14 @@ export function fitAspect(out: string, aspect: string): string | undefined {
|
||||
return `HeyGen returned ${width}x${height} for ${aspect}; cropped to ${w}x${h} (centre), not stretched`;
|
||||
}
|
||||
|
||||
function aspectOf(r: GenerateRequest, work: string): string {
|
||||
if (r.aspect || !r.firstFrame) return r.aspect ?? "16:9";
|
||||
const sent = join(work, "first-frame.jpg");
|
||||
shrink(r.firstFrame, sent);
|
||||
const { width, height } = probeVideo(sent);
|
||||
return `${width}:${height}`;
|
||||
}
|
||||
|
||||
const MIN_SHOT_S = 0.5;
|
||||
|
||||
export function foldShort(cuts: number[], length: number): number[] {
|
||||
@@ -502,14 +512,12 @@ export async function generate(
|
||||
cuts = null;
|
||||
}
|
||||
const notes: string[] = [];
|
||||
if (r.type === "video" && (r.aspect || !r.firstFrame))
|
||||
if (r.type === "video")
|
||||
try {
|
||||
const note = fitAspect(out, r.aspect ?? "16:9");
|
||||
const note = fitAspect(out, aspectOf(r, work));
|
||||
if (note) notes.push(note);
|
||||
} catch (e) {
|
||||
notes.push(
|
||||
`kept HeyGen's size, could not crop to ${r.aspect ?? "16:9"}: ${e instanceof Error ? e.message : e}`,
|
||||
);
|
||||
notes.push(`kept HeyGen's size, could not crop: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
if (r.masks?.length && r.motion)
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user