mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(cli): measure reads an odd-sized box exactly (#7090)
A crop on yuv420p rounds an odd size or offset down to even. readClip split the 32x32 output as 33x33 bytes, so a 10-frame clip came back as 9.4 frames and every frame after the first was misaligned; an odd offset also shifted the chroma. Raw reads now convert to rgb24 (or gray) before the crop, which is exact at any size and place. extract and diff stay on yuv and add exact=1. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fa1942d5ac
commit
823cd126ee
@@ -62,7 +62,7 @@ async function record(
|
||||
return entry.id;
|
||||
}
|
||||
|
||||
const crop = (roi: Roi) => `crop=${roi[2]}:${roi[3]}:${roi[0]}:${roi[1]}`;
|
||||
const crop = (roi: Roi) => `crop=${roi[2]}:${roi[3]}:${roi[0]}:${roi[1]}:exact=1`;
|
||||
|
||||
/** The frame shown at `at` seconds, cropped, as a picture. */
|
||||
export async function cutStill(info: VideoInfo, at: number, roi: Roi, out: string) {
|
||||
|
||||
@@ -368,7 +368,7 @@ function ssimSeries(ref: Clip, yours: Clip, logPath: string): Series {
|
||||
`[1:v]${side([...yours.filters, `scale=${ref.width}:${ref.height}:flags=bicubic`])},split=${n}${tags("o")}`,
|
||||
"[o0][r0]ssim=stats_file=-:shortest=1[s0]",
|
||||
...Array.from({ length: GRID * GRID }, (_, i) => {
|
||||
const crop = `crop=${cw}:${ch}:${(i % GRID) * cw}:${Math.floor(i / GRID) * ch}`;
|
||||
const crop = `crop=${cw}:${ch}:${(i % GRID) * cw}:${Math.floor(i / GRID) * ch}:exact=1`;
|
||||
const k = i + 1;
|
||||
return `[r${k}]${crop}[rc${k}];[o${k}]${crop}[oc${k}];[oc${k}][rc${k}]ssim=stats_file=cell${i}.log:shortest=1[s${k}]`;
|
||||
}),
|
||||
|
||||
@@ -186,7 +186,7 @@ export function readDrawn(
|
||||
`[0:v]${[...clip.filters, "setsar=1", `scale=${width}:${height}:flags=bicubic`].join(",")},split=${jobs.length}${tags}`,
|
||||
...jobs.map(({ box: [x, y, w, h], frames }, j) => {
|
||||
const pick = sumOf(frames.map((k) => `eq(n\\,${k})`));
|
||||
return `[d${j}]select=${pick},crop=${w}:${h}:${x}:${y},format=rgb24[e${j}]`;
|
||||
return `[d${j}]select=${pick},format=rgb24,crop=${w}:${h}:${x}:${y}[e${j}]`;
|
||||
}),
|
||||
].join(";");
|
||||
const outs = jobs.flatMap((_, j) => [
|
||||
|
||||
@@ -4,7 +4,7 @@ import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { selectFirstFrameAfterEach } from "../commands/snapshot.js";
|
||||
import { probeVideo, readClip, sumOf, videoClip } from "./media.js";
|
||||
import { meanAbsDiff, probeVideo, readClip, readFrames, sumOf, videoClip } from "./media.js";
|
||||
|
||||
const terms = (n: number) => Array.from({ length: n }, (_, i) => `eq(n\\,${i * 2})`);
|
||||
|
||||
@@ -52,3 +52,35 @@ describe.skipIf(!ffmpeg)("a select of many frames", () => {
|
||||
expect(decoded(selectFirstFrameAfterEach(times))).toBe(n);
|
||||
});
|
||||
});
|
||||
|
||||
// A crop on yuv420p rounds an odd size or place down to even: a 33x33 box came back 32x32, so its bytes split into
|
||||
// 9.4 frames, and an odd place shifted the colour.
|
||||
describe.skipIf(!ffmpeg)("an odd box", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-crop-"));
|
||||
const path = join(dir, "clip.mp4");
|
||||
beforeAll(() => {
|
||||
execFileSync("ffmpeg", [
|
||||
...["-v", "error", "-f", "lavfi", "-i", "testsrc=d=1:r=10:s=64x64"],
|
||||
...["-pix_fmt", "yuv420p", "-y", path],
|
||||
]);
|
||||
});
|
||||
afterAll(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
it.each([
|
||||
[1, 1, 33, 33],
|
||||
[3, 5, 31, 17],
|
||||
])("reads the box %i,%i,%i,%i as the pixels the whole frame has there", (x, y, w, h) => {
|
||||
for (const gray of [false, true]) {
|
||||
const n = gray ? 1 : 3;
|
||||
const whole = readFrames(path, { gray }).frames[9]!;
|
||||
const there = Uint8Array.from(
|
||||
Array.from({ length: h }, (_, r) => [
|
||||
...whole.subarray(((y + r) * 64 + x) * n, ((y + r) * 64 + x + w) * n),
|
||||
]).flat(),
|
||||
);
|
||||
const box = readFrames(path, { roi: [x, y, w, h], gray }).frames;
|
||||
expect(box).toHaveLength(10);
|
||||
expect(meanAbsDiff(box[9]!, there)).toBe(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -225,7 +225,10 @@ export function readClip(clip: Clip, opts: Omit<ReadOptions, "start" | "seconds"
|
||||
if (fps) filters.push(`fps=${fps}`);
|
||||
let [w, h] = [clip.width, clip.height];
|
||||
if (roi) {
|
||||
filters.push(`crop=${roi[2]}:${roi[3]}:${roi[0]}:${roi[1]}`);
|
||||
filters.push(
|
||||
`format=${gray ? "gray" : "rgb24"}`,
|
||||
`crop=${roi[2]}:${roi[3]}:${roi[0]}:${roi[1]}`,
|
||||
);
|
||||
[w, h] = [roi[2], roi[3]];
|
||||
}
|
||||
if (width && width !== w) {
|
||||
|
||||
@@ -256,9 +256,9 @@ function boxAt(
|
||||
const filters = [
|
||||
`select=${sumOf(unique.map((k) => `eq(n\\,${k})`))}`,
|
||||
`scale=${w}:${h}:force_original_aspect_ratio=increase:flags=bicubic`,
|
||||
"format=rgb24",
|
||||
`crop=${w}:${h}`,
|
||||
`crop=${box[2]}:${box[3]}:${box[0]}:${box[1]}`,
|
||||
"format=rgb24",
|
||||
].join(",");
|
||||
runFfmpeg(
|
||||
[
|
||||
@@ -331,7 +331,7 @@ function stillBox(path: string, box: Roi, w: number, h: number): Uint8Array | un
|
||||
try {
|
||||
const raw = runFfmpeg([
|
||||
...["-i", path, "-frames:v", "1", "-vf"],
|
||||
`scale=${w}:${h}:force_original_aspect_ratio=increase:flags=bicubic,crop=${w}:${h},crop=${box[2]}:${box[3]}:${box[0]}:${box[1]},format=rgb24`,
|
||||
`scale=${w}:${h}:force_original_aspect_ratio=increase:flags=bicubic,format=rgb24,crop=${w}:${h},crop=${box[2]}:${box[3]}:${box[0]}:${box[1]}`,
|
||||
...["-f", "rawvideo", "-"],
|
||||
]);
|
||||
return raw.length >= box[2] * box[3] * 3 ? new Uint8Array(raw) : undefined;
|
||||
|
||||
Reference in New Issue
Block a user