mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(engine): a video clip shows a new source frame on every frame it is on screen (#7471)
* fix(engine): a clip shows a new source frame on every output frame it covers * refactor(engine): shorten the frame count doc to two lines * style(engine): collapse the single-pass ffmpeg call the formatter wants on one line
This commit is contained in:
@@ -67,8 +67,8 @@ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: s
|
||||
}
|
||||
|
||||
describe("extractionCache constants", () => {
|
||||
it("exposes the v9 schema prefix", () => {
|
||||
expect(SCHEMA_PREFIX).toBe("hfcache-v9-");
|
||||
it("exposes the v10 schema prefix", () => {
|
||||
expect(SCHEMA_PREFIX).toBe("hfcache-v10-");
|
||||
});
|
||||
|
||||
it("exposes the frame filename prefix shared with the extractor", () => {
|
||||
|
||||
@@ -62,7 +62,7 @@ export const GC_MARKER = ".hf-last-gc";
|
||||
* Current schema version. Bump it whenever extraction writes different frames for the same key,
|
||||
* or warm entries keep serving the old frames across a deploy. Each bump's commit says why.
|
||||
*/
|
||||
export const SCHEMA_PREFIX = "hfcache-v9-";
|
||||
export const SCHEMA_PREFIX = "hfcache-v10-";
|
||||
|
||||
/** Truncated hex chars of SHA-256 used for the entry directory name. */
|
||||
const KEY_HEX_CHARS = 16;
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { runFfmpeg } from "../utils/runFfmpeg.js";
|
||||
import { extractAllVideoFrames, type ExtractedFrames } from "./videoFrameExtractor.js";
|
||||
|
||||
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
|
||||
const WIDTH = 32;
|
||||
const HEIGHT = 16;
|
||||
|
||||
describe.skipIf(!HAS_FFMPEG)("a clip's frames match the output frames it is on screen", () => {
|
||||
const fixtureDir = mkdtempSync(join(tmpdir(), "hf-clip-slots-"));
|
||||
const source = join(fixtureDir, "index-30fps.mp4");
|
||||
|
||||
beforeAll(async () => {
|
||||
// Frame k carries luma 16 + 2k, so each extracted frame names its source index.
|
||||
const synth = await runFfmpeg([
|
||||
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i"],
|
||||
`nullsrc=s=${WIDTH}x${HEIGHT}:r=30:d=1.5,geq=lum='16+2*N':cb=128:cr=128`,
|
||||
...["-c:v", "libx264", "-qp", "0", "-pix_fmt", "yuv420p", source],
|
||||
]);
|
||||
if (!synth.success) throw new Error(`index fixture synthesis failed: ${synth.stderr}`);
|
||||
}, 30_000);
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(fixtureDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function sourceIndexes(extracted: ExtractedFrames): number[] {
|
||||
const decoded = spawnSync("ffmpeg", [
|
||||
...["-v", "error", "-i", join(extracted.outputDir, extracted.framePattern)],
|
||||
...["-f", "rawvideo", "-pix_fmt", "gray", "pipe:1"],
|
||||
]);
|
||||
if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
|
||||
const indexes: number[] = [];
|
||||
for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) {
|
||||
indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2));
|
||||
}
|
||||
return indexes;
|
||||
}
|
||||
|
||||
it.each([
|
||||
// On screen at output frames 1..8: 0.01 <= f/30 < 0.28.
|
||||
{ label: "0.01-0.28 s", start: 0.01, end: 0.28, onScreen: [0, 1, 2, 3, 4, 5, 6, 7] },
|
||||
// On screen at output frames 0..8: f/30 < 0.27.
|
||||
{ label: "0-0.27 s", start: 0, end: 0.27, onScreen: [0, 1, 2, 3, 4, 5, 6, 7, 8] },
|
||||
])(
|
||||
"gives a $label clip a new source frame on every output frame it is on screen",
|
||||
async (c) => {
|
||||
const id = `slots-${c.start}-${c.end}`;
|
||||
const result = await extractAllVideoFrames(
|
||||
[
|
||||
{
|
||||
id,
|
||||
src: source,
|
||||
start: c.start,
|
||||
end: c.end,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
},
|
||||
],
|
||||
fixtureDir,
|
||||
{ fps: 30, outputDir: join(fixtureDir, id), format: "png" },
|
||||
);
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(sourceIndexes(result.extracted[0]!)).toEqual(c.onScreen);
|
||||
},
|
||||
30_000,
|
||||
);
|
||||
});
|
||||
@@ -473,11 +473,13 @@ describe("resolveVideoExtractionDuration", () => {
|
||||
});
|
||||
|
||||
describe("extractionFrameCountForDuration", () => {
|
||||
it("uses the same VFR ceil and CFR nearest-boundary rules as FFmpeg", () => {
|
||||
it("ceils VFR windows and counts every output slot a CFR window covers", () => {
|
||||
expect(extractionFrameCountForDuration(0.466666, 30, true)).toBe(14);
|
||||
expect(extractionFrameCountForDuration(0.466666, 30, false)).toBe(14);
|
||||
expect(extractionFrameCountForDuration(0.616666, 30, true)).toBe(19);
|
||||
expect(extractionFrameCountForDuration(0.616666, 30, false)).toBe(18);
|
||||
expect(extractionFrameCountForDuration(0.616666, 30, false)).toBe(19);
|
||||
expect(extractionFrameCountForDuration(0.2667, 30, false)).toBe(9);
|
||||
expect(extractionFrameCountForDuration(0.266667, 30, false)).toBe(8);
|
||||
});
|
||||
|
||||
it.each([
|
||||
@@ -3561,7 +3563,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
|
||||
label: "non-half-integer boundary",
|
||||
duration: 0.616666,
|
||||
offset: 0.1,
|
||||
expectedFrames: 18,
|
||||
expectedFrames: 19,
|
||||
},
|
||||
])(
|
||||
"keeps direct and superset CFR extraction equal at a $label",
|
||||
|
||||
@@ -147,14 +147,10 @@ export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat {
|
||||
return typeof value === "string" && (VIDEO_FRAME_FORMATS as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
const SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES = 1e-4;
|
||||
/**
|
||||
* Resolve the frame count produced for a requested extraction duration.
|
||||
*
|
||||
* Both paths sample with FFmpeg's fps filter. A CFR window ends at the nearest
|
||||
* frame boundary; a VFR window rounds up, keeping the slot a trailing still
|
||||
* covers. Keep this calculation shared by superset slicing and
|
||||
* producer coverage accounting so a complete VFR extraction cannot be
|
||||
* rejected because the two paths disagree by one frame.
|
||||
* Frames a window gets: one per output slot it covers (CFR), or rounded up (VFR).
|
||||
* Superset slicing and the coverage gate share it, so they never disagree by a frame.
|
||||
*/
|
||||
export function extractionFrameCountForDuration(
|
||||
durationSeconds: number,
|
||||
@@ -214,9 +210,11 @@ export function extractionFrameCountForDuration(
|
||||
|
||||
const frameNumerator = microseconds * fpsNumerator;
|
||||
const frameDenominator = 1_000_000n * fpsDenominator;
|
||||
const slotsPerTolerance = BigInt(Math.round(1 / SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES));
|
||||
const frameCount = isVFR
|
||||
? (frameNumerator + frameDenominator - 1n) / frameDenominator
|
||||
: (2n * frameNumerator + frameDenominator) / (2n * frameDenominator);
|
||||
: (slotsPerTolerance * (frameNumerator + frameDenominator) - frameDenominator - 1n) /
|
||||
(slotsPerTolerance * frameDenominator);
|
||||
const frames = Number(frameCount);
|
||||
return Math.max(1, Number.isSafeInteger(frames) ? frames : Number.MAX_SAFE_INTEGER);
|
||||
}
|
||||
@@ -1441,13 +1439,10 @@ async function extractVideoFramesRangeUnredacted(
|
||||
false,
|
||||
);
|
||||
const totalFrames = metadata.isVFR ? windowFrames : Math.min(windowFrames, framesBeforeSourceEnd);
|
||||
// A VFR window rounds its frame count up, and only the segments' `-frames:v` bound
|
||||
// emits that last slot: the fps filter ends a `-t` window at the nearest slot. A
|
||||
// decimal rate counts zero frames here and keeps the `-t` window.
|
||||
const boundVfrFrameCount = metadata.isVFR && totalFrames > 0;
|
||||
const frameCap = sampleAtOutputFps && totalFrames > 0 ? ["-frames:v", String(totalFrames)] : [];
|
||||
// A `-t` window ends at the nearest slot; segments read a frame past and keep exactly their own.
|
||||
// A decimal rate counts zero frames here and keeps the `-t` window.
|
||||
const extract = (hwaccel: string[]) =>
|
||||
sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)
|
||||
sampleAtOutputFps && (options.frameRanges || totalFrames > 0)
|
||||
? runSegmentedExtraction({
|
||||
decodeArgs: [...hwaccel, ...decodeArgs],
|
||||
filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],
|
||||
@@ -1467,20 +1462,17 @@ async function extractVideoFramesRangeUnredacted(
|
||||
})
|
||||
: inExtractionSlot(
|
||||
async () =>
|
||||
runFfmpeg(
|
||||
[...hwaccel, ...args, ...frameCap, ...filterArgsFor(startTime), ...encodeArgs],
|
||||
{
|
||||
...runOptions,
|
||||
onStderr: framesWrittenReader(options.onProgress),
|
||||
timeout: stallTimeoutMs(
|
||||
ffmpegProcessTimeout,
|
||||
options.finalFrameOnly
|
||||
? startTime
|
||||
: await leadInSecondsUnlessAborted(videoPath, startTime, signal),
|
||||
metadata.fps,
|
||||
),
|
||||
},
|
||||
),
|
||||
runFfmpeg([...hwaccel, ...args, ...filterArgsFor(startTime), ...encodeArgs], {
|
||||
...runOptions,
|
||||
onStderr: framesWrittenReader(options.onProgress),
|
||||
timeout: stallTimeoutMs(
|
||||
ffmpegProcessTimeout,
|
||||
options.finalFrameOnly
|
||||
? startTime
|
||||
: await leadInSecondsUnlessAborted(videoPath, startTime, signal),
|
||||
metadata.fps,
|
||||
),
|
||||
}),
|
||||
signal,
|
||||
);
|
||||
options.onProgress?.({
|
||||
|
||||
@@ -92,21 +92,21 @@ describe("expectedFramesForClip", () => {
|
||||
expect(expectedFramesForClip(0, 0.316666, 30)).toBe(10);
|
||||
});
|
||||
|
||||
it("matches the CFR fps filter's nearest-boundary rounding when requested", () => {
|
||||
expect(expectedFramesForClip(0, 0.616666, 30, "nearest")).toBe(18);
|
||||
expect(expectedFramesForClip(0, 0.316666, 30, "nearest")).toBe(9);
|
||||
expect(expectedFramesForClip(0, 0.633333, 30, "nearest")).toBe(19);
|
||||
it("counts every output slot a CFR window covers when requested", () => {
|
||||
expect(expectedFramesForClip(0, 0.616666, 30, "slots")).toBe(19);
|
||||
expect(expectedFramesForClip(0, 0.316666, 30, "slots")).toBe(10);
|
||||
expect(expectedFramesForClip(0, 0.633333, 30, "slots")).toBe(19);
|
||||
});
|
||||
|
||||
it("uses exact NTSC rationals for short CFR and VFR boundaries", () => {
|
||||
expect(expectedFramesForClip(0, 0.25025, { num: 30000, den: 1001 }, "nearest")).toBe(8);
|
||||
expect(expectedFramesForClip(0, 0.25025, { num: 30000, den: 1001 }, "slots")).toBe(8);
|
||||
expect(expectedFramesForClip(0, 0.125125, { num: 24000, den: 1001 })).toBe(3);
|
||||
expect(expectedFramesForClip(0, 0.5005, { num: 24000, den: 1001 })).toBe(12);
|
||||
});
|
||||
|
||||
it("requires one frame for every positive sub-frame clip", () => {
|
||||
expect(expectedFramesForClip(0, 0.001, 30)).toBe(1);
|
||||
expect(expectedFramesForClip(0, 0.001, 30, "nearest")).toBe(1);
|
||||
expect(expectedFramesForClip(0, 0.001, 30, "slots")).toBe(1);
|
||||
expect(expectedFramesForClip(1, 1, 30)).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -166,17 +166,24 @@ describe("computeVideoFrameCoverage", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reports full coverage for a short CFR clip matching FFmpeg boundary rounding", () => {
|
||||
it("expects a short CFR clip to fill every output slot it covers", () => {
|
||||
const videos = [makeVideo({ id: "short", start: 0, end: 0.616666 })];
|
||||
const reports = computeVideoFrameCoverage(videos, [makeExtracted("short", 18)], 30);
|
||||
const reports = computeVideoFrameCoverage(videos, [makeExtracted("short", 19)], 30);
|
||||
expect(reports[0]).toMatchObject({
|
||||
expectedFrames: 18,
|
||||
capturedFrames: 18,
|
||||
expectedFrames: 19,
|
||||
capturedFrames: 19,
|
||||
ratio: 1,
|
||||
});
|
||||
expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects a short CFR clip missing the last output slot it covers", () => {
|
||||
const videos = [makeVideo({ id: "aligned", start: 1 / 30, end: 0.28 })];
|
||||
const reports = computeVideoFrameCoverage(videos, [makeExtracted("aligned", 7)], 30);
|
||||
expect(reports[0]).toMatchObject({ expectedFrames: 8, capturedFrames: 7 });
|
||||
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(/captured 7 of expected 8/);
|
||||
});
|
||||
|
||||
it("tolerates a single FFmpeg boundary frame on a short 18/19 VFR extraction", () => {
|
||||
const videos = [makeVideo({ id: "short-vfr", start: 0, end: 0.616666 })];
|
||||
const reports = computeVideoFrameCoverage(
|
||||
|
||||
@@ -135,7 +135,7 @@ export function resolveVideoCoverageThreshold(
|
||||
|
||||
/**
|
||||
* Count frames in a clip's authored `[start,end)` window. CFR extraction
|
||||
* rounds to the nearest output-frame boundary and VFR extraction rounds up,
|
||||
* fills every output slot the window covers and VFR extraction rounds up,
|
||||
* as `extractionFrameCountForDuration` does. Missing entries
|
||||
* retain the fail-closed `ceil` default. Positive sub-frame clips emit one.
|
||||
*/
|
||||
@@ -143,7 +143,7 @@ export function expectedFramesForClip(
|
||||
start: number,
|
||||
end: number,
|
||||
fps: FpsInput,
|
||||
rounding: "ceil" | "nearest" = "ceil",
|
||||
rounding: "ceil" | "slots" = "ceil",
|
||||
): number {
|
||||
const fpsValue = fpsToNumber(toFps(fps));
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end) || !Number.isFinite(fpsValue)) return 0;
|
||||
@@ -171,7 +171,7 @@ function expectedFramesForVideo(
|
||||
entry: ExtractedFrames | undefined,
|
||||
fps: FpsInput,
|
||||
): number {
|
||||
const rounding = entry && !entry.metadata.isVFR ? "nearest" : "ceil";
|
||||
const rounding = entry && !entry.metadata.isVFR ? "slots" : "ceil";
|
||||
const slotSourceDuration = sourceTimeAt(
|
||||
normalizeRateSpec(video.playbackRate),
|
||||
Math.max(0, video.end - video.start),
|
||||
|
||||
Reference in New Issue
Block a user