fix(engine): a video clip shows a new source frame on every frame it is on screen (#7471)

* fix(engine): a clip shows a new source frame on every output frame it covers

* refactor(engine): shorten the frame count doc to two lines

* style(engine): collapse the single-pass ffmpeg call the formatter wants on one line
This commit is contained in:
Miguel Ángel
2026-11-10 02:16:59 +00:00
committed by GitHub
parent 58686a74c6
commit 235c80bae0
7 changed files with 121 additions and 47 deletions
@@ -67,8 +67,8 @@ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: s
}
describe("extractionCache constants", () => {
it("exposes the v9 schema prefix", () => {
expect(SCHEMA_PREFIX).toBe("hfcache-v9-");
it("exposes the v10 schema prefix", () => {
expect(SCHEMA_PREFIX).toBe("hfcache-v10-");
});
it("exposes the frame filename prefix shared with the extractor", () => {
@@ -62,7 +62,7 @@ export const GC_MARKER = ".hf-last-gc";
* Current schema version. Bump it whenever extraction writes different frames for the same key,
* or warm entries keep serving the old frames across a deploy. Each bump's commit says why.
*/
export const SCHEMA_PREFIX = "hfcache-v9-";
export const SCHEMA_PREFIX = "hfcache-v10-";
/** Truncated hex chars of SHA-256 used for the entry directory name. */
const KEY_HEX_CHARS = 16;
@@ -0,0 +1,73 @@
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { mkdtempSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { spawnSync } from "node:child_process";
import { runFfmpeg } from "../utils/runFfmpeg.js";
import { extractAllVideoFrames, type ExtractedFrames } from "./videoFrameExtractor.js";
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
const WIDTH = 32;
const HEIGHT = 16;
describe.skipIf(!HAS_FFMPEG)("a clip's frames match the output frames it is on screen", () => {
const fixtureDir = mkdtempSync(join(tmpdir(), "hf-clip-slots-"));
const source = join(fixtureDir, "index-30fps.mp4");
beforeAll(async () => {
// Frame k carries luma 16 + 2k, so each extracted frame names its source index.
const synth = await runFfmpeg([
...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i"],
`nullsrc=s=${WIDTH}x${HEIGHT}:r=30:d=1.5,geq=lum='16+2*N':cb=128:cr=128`,
...["-c:v", "libx264", "-qp", "0", "-pix_fmt", "yuv420p", source],
]);
if (!synth.success) throw new Error(`index fixture synthesis failed: ${synth.stderr}`);
}, 30_000);
afterAll(() => {
rmSync(fixtureDir, { recursive: true, force: true });
});
function sourceIndexes(extracted: ExtractedFrames): number[] {
const decoded = spawnSync("ffmpeg", [
...["-v", "error", "-i", join(extracted.outputDir, extracted.framePattern)],
...["-f", "rawvideo", "-pix_fmt", "gray", "pipe:1"],
]);
if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
const indexes: number[] = [];
for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) {
indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2));
}
return indexes;
}
it.each([
// On screen at output frames 1..8: 0.01 <= f/30 < 0.28.
{ label: "0.01-0.28 s", start: 0.01, end: 0.28, onScreen: [0, 1, 2, 3, 4, 5, 6, 7] },
// On screen at output frames 0..8: f/30 < 0.27.
{ label: "0-0.27 s", start: 0, end: 0.27, onScreen: [0, 1, 2, 3, 4, 5, 6, 7, 8] },
])(
"gives a $label clip a new source frame on every output frame it is on screen",
async (c) => {
const id = `slots-${c.start}-${c.end}`;
const result = await extractAllVideoFrames(
[
{
id,
src: source,
start: c.start,
end: c.end,
mediaStart: 0,
loop: false,
hasAudio: false,
},
],
fixtureDir,
{ fps: 30, outputDir: join(fixtureDir, id), format: "png" },
);
expect(result.errors).toEqual([]);
expect(sourceIndexes(result.extracted[0]!)).toEqual(c.onScreen);
},
30_000,
);
});
@@ -473,11 +473,13 @@ describe("resolveVideoExtractionDuration", () => {
});
describe("extractionFrameCountForDuration", () => {
it("uses the same VFR ceil and CFR nearest-boundary rules as FFmpeg", () => {
it("ceils VFR windows and counts every output slot a CFR window covers", () => {
expect(extractionFrameCountForDuration(0.466666, 30, true)).toBe(14);
expect(extractionFrameCountForDuration(0.466666, 30, false)).toBe(14);
expect(extractionFrameCountForDuration(0.616666, 30, true)).toBe(19);
expect(extractionFrameCountForDuration(0.616666, 30, false)).toBe(18);
expect(extractionFrameCountForDuration(0.616666, 30, false)).toBe(19);
expect(extractionFrameCountForDuration(0.2667, 30, false)).toBe(9);
expect(extractionFrameCountForDuration(0.266667, 30, false)).toBe(8);
});
it.each([
@@ -3561,7 +3563,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => {
label: "non-half-integer boundary",
duration: 0.616666,
offset: 0.1,
expectedFrames: 18,
expectedFrames: 19,
},
])(
"keeps direct and superset CFR extraction equal at a $label",
@@ -147,14 +147,10 @@ export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat {
return typeof value === "string" && (VIDEO_FRAME_FORMATS as readonly string[]).includes(value);
}
const SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES = 1e-4;
/**
* Resolve the frame count produced for a requested extraction duration.
*
* Both paths sample with FFmpeg's fps filter. A CFR window ends at the nearest
* frame boundary; a VFR window rounds up, keeping the slot a trailing still
* covers. Keep this calculation shared by superset slicing and
* producer coverage accounting so a complete VFR extraction cannot be
* rejected because the two paths disagree by one frame.
* Frames a window gets: one per output slot it covers (CFR), or rounded up (VFR).
* Superset slicing and the coverage gate share it, so they never disagree by a frame.
*/
export function extractionFrameCountForDuration(
durationSeconds: number,
@@ -214,9 +210,11 @@ export function extractionFrameCountForDuration(
const frameNumerator = microseconds * fpsNumerator;
const frameDenominator = 1_000_000n * fpsDenominator;
const slotsPerTolerance = BigInt(Math.round(1 / SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES));
const frameCount = isVFR
? (frameNumerator + frameDenominator - 1n) / frameDenominator
: (2n * frameNumerator + frameDenominator) / (2n * frameDenominator);
: (slotsPerTolerance * (frameNumerator + frameDenominator) - frameDenominator - 1n) /
(slotsPerTolerance * frameDenominator);
const frames = Number(frameCount);
return Math.max(1, Number.isSafeInteger(frames) ? frames : Number.MAX_SAFE_INTEGER);
}
@@ -1441,13 +1439,10 @@ async function extractVideoFramesRangeUnredacted(
false,
);
const totalFrames = metadata.isVFR ? windowFrames : Math.min(windowFrames, framesBeforeSourceEnd);
// A VFR window rounds its frame count up, and only the segments' `-frames:v` bound
// emits that last slot: the fps filter ends a `-t` window at the nearest slot. A
// decimal rate counts zero frames here and keeps the `-t` window.
const boundVfrFrameCount = metadata.isVFR && totalFrames > 0;
const frameCap = sampleAtOutputFps && totalFrames > 0 ? ["-frames:v", String(totalFrames)] : [];
// A `-t` window ends at the nearest slot; segments read a frame past and keep exactly their own.
// A decimal rate counts zero frames here and keeps the `-t` window.
const extract = (hwaccel: string[]) =>
sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)
sampleAtOutputFps && (options.frameRanges || totalFrames > 0)
? runSegmentedExtraction({
decodeArgs: [...hwaccel, ...decodeArgs],
filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],
@@ -1467,20 +1462,17 @@ async function extractVideoFramesRangeUnredacted(
})
: inExtractionSlot(
async () =>
runFfmpeg(
[...hwaccel, ...args, ...frameCap, ...filterArgsFor(startTime), ...encodeArgs],
{
...runOptions,
onStderr: framesWrittenReader(options.onProgress),
timeout: stallTimeoutMs(
ffmpegProcessTimeout,
options.finalFrameOnly
? startTime
: await leadInSecondsUnlessAborted(videoPath, startTime, signal),
metadata.fps,
),
},
),
runFfmpeg([...hwaccel, ...args, ...filterArgsFor(startTime), ...encodeArgs], {
...runOptions,
onStderr: framesWrittenReader(options.onProgress),
timeout: stallTimeoutMs(
ffmpegProcessTimeout,
options.finalFrameOnly
? startTime
: await leadInSecondsUnlessAborted(videoPath, startTime, signal),
metadata.fps,
),
}),
signal,
);
options.onProgress?.({
@@ -92,21 +92,21 @@ describe("expectedFramesForClip", () => {
expect(expectedFramesForClip(0, 0.316666, 30)).toBe(10);
});
it("matches the CFR fps filter's nearest-boundary rounding when requested", () => {
expect(expectedFramesForClip(0, 0.616666, 30, "nearest")).toBe(18);
expect(expectedFramesForClip(0, 0.316666, 30, "nearest")).toBe(9);
expect(expectedFramesForClip(0, 0.633333, 30, "nearest")).toBe(19);
it("counts every output slot a CFR window covers when requested", () => {
expect(expectedFramesForClip(0, 0.616666, 30, "slots")).toBe(19);
expect(expectedFramesForClip(0, 0.316666, 30, "slots")).toBe(10);
expect(expectedFramesForClip(0, 0.633333, 30, "slots")).toBe(19);
});
it("uses exact NTSC rationals for short CFR and VFR boundaries", () => {
expect(expectedFramesForClip(0, 0.25025, { num: 30000, den: 1001 }, "nearest")).toBe(8);
expect(expectedFramesForClip(0, 0.25025, { num: 30000, den: 1001 }, "slots")).toBe(8);
expect(expectedFramesForClip(0, 0.125125, { num: 24000, den: 1001 })).toBe(3);
expect(expectedFramesForClip(0, 0.5005, { num: 24000, den: 1001 })).toBe(12);
});
it("requires one frame for every positive sub-frame clip", () => {
expect(expectedFramesForClip(0, 0.001, 30)).toBe(1);
expect(expectedFramesForClip(0, 0.001, 30, "nearest")).toBe(1);
expect(expectedFramesForClip(0, 0.001, 30, "slots")).toBe(1);
expect(expectedFramesForClip(1, 1, 30)).toBe(0);
});
});
@@ -166,17 +166,24 @@ describe("computeVideoFrameCoverage", () => {
});
});
it("reports full coverage for a short CFR clip matching FFmpeg boundary rounding", () => {
it("expects a short CFR clip to fill every output slot it covers", () => {
const videos = [makeVideo({ id: "short", start: 0, end: 0.616666 })];
const reports = computeVideoFrameCoverage(videos, [makeExtracted("short", 18)], 30);
const reports = computeVideoFrameCoverage(videos, [makeExtracted("short", 19)], 30);
expect(reports[0]).toMatchObject({
expectedFrames: 18,
capturedFrames: 18,
expectedFrames: 19,
capturedFrames: 19,
ratio: 1,
});
expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow();
});
it("rejects a short CFR clip missing the last output slot it covers", () => {
const videos = [makeVideo({ id: "aligned", start: 1 / 30, end: 0.28 })];
const reports = computeVideoFrameCoverage(videos, [makeExtracted("aligned", 7)], 30);
expect(reports[0]).toMatchObject({ expectedFrames: 8, capturedFrames: 7 });
expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(/captured 7 of expected 8/);
});
it("tolerates a single FFmpeg boundary frame on a short 18/19 VFR extraction", () => {
const videos = [makeVideo({ id: "short-vfr", start: 0, end: 0.616666 })];
const reports = computeVideoFrameCoverage(
@@ -135,7 +135,7 @@ export function resolveVideoCoverageThreshold(
/**
* Count frames in a clip's authored `[start,end)` window. CFR extraction
* rounds to the nearest output-frame boundary and VFR extraction rounds up,
* fills every output slot the window covers and VFR extraction rounds up,
* as `extractionFrameCountForDuration` does. Missing entries
* retain the fail-closed `ceil` default. Positive sub-frame clips emit one.
*/
@@ -143,7 +143,7 @@ export function expectedFramesForClip(
start: number,
end: number,
fps: FpsInput,
rounding: "ceil" | "nearest" = "ceil",
rounding: "ceil" | "slots" = "ceil",
): number {
const fpsValue = fpsToNumber(toFps(fps));
if (!Number.isFinite(start) || !Number.isFinite(end) || !Number.isFinite(fpsValue)) return 0;
@@ -171,7 +171,7 @@ function expectedFramesForVideo(
entry: ExtractedFrames | undefined,
fps: FpsInput,
): number {
const rounding = entry && !entry.metadata.isVFR ? "nearest" : "ceil";
const rounding = entry && !entry.metadata.isVFR ? "slots" : "ceil";
const slotSourceDuration = sourceTimeAt(
normalizeRateSpec(video.playbackRate),
Math.max(0, video.end - video.start),