From 235c80bae0cde99754cab78609cb5d7d210b650e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 10 Nov 2026 02:16:59 +0000 Subject: [PATCH] fix(engine): a video clip shows a new source frame on every frame it is on screen (#7471) * fix(engine): a clip shows a new source frame on every output frame it covers * refactor(engine): shorten the frame count doc to two lines * style(engine): collapse the single-pass ffmpeg call the formatter wants on one line --- .../src/services/extractionCache.test.ts | 4 +- .../engine/src/services/extractionCache.ts | 2 +- .../videoFrameExtractor.slots.test.ts | 73 +++++++++++++++++++ .../src/services/videoFrameExtractor.test.ts | 8 +- .../src/services/videoFrameExtractor.ts | 48 +++++------- .../render/videoFrameCoverage.test.ts | 27 ++++--- .../src/services/render/videoFrameCoverage.ts | 6 +- 7 files changed, 121 insertions(+), 47 deletions(-) create mode 100644 packages/engine/src/services/videoFrameExtractor.slots.test.ts diff --git a/packages/engine/src/services/extractionCache.test.ts b/packages/engine/src/services/extractionCache.test.ts index 98f7ee5a05..227675143a 100644 --- a/packages/engine/src/services/extractionCache.test.ts +++ b/packages/engine/src/services/extractionCache.test.ts @@ -67,8 +67,8 @@ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: s } describe("extractionCache constants", () => { - it("exposes the v9 schema prefix", () => { - expect(SCHEMA_PREFIX).toBe("hfcache-v9-"); + it("exposes the v10 schema prefix", () => { + expect(SCHEMA_PREFIX).toBe("hfcache-v10-"); }); it("exposes the frame filename prefix shared with the extractor", () => { diff --git a/packages/engine/src/services/extractionCache.ts b/packages/engine/src/services/extractionCache.ts index bcfd966e59..677b7d7005 100644 --- a/packages/engine/src/services/extractionCache.ts +++ b/packages/engine/src/services/extractionCache.ts @@ -62,7 +62,7 @@ export const GC_MARKER = ".hf-last-gc"; * Current schema version. Bump it whenever extraction writes different frames for the same key, * or warm entries keep serving the old frames across a deploy. Each bump's commit says why. */ -export const SCHEMA_PREFIX = "hfcache-v9-"; +export const SCHEMA_PREFIX = "hfcache-v10-"; /** Truncated hex chars of SHA-256 used for the entry directory name. */ const KEY_HEX_CHARS = 16; diff --git a/packages/engine/src/services/videoFrameExtractor.slots.test.ts b/packages/engine/src/services/videoFrameExtractor.slots.test.ts new file mode 100644 index 0000000000..42cb75712e --- /dev/null +++ b/packages/engine/src/services/videoFrameExtractor.slots.test.ts @@ -0,0 +1,73 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { runFfmpeg } from "../utils/runFfmpeg.js"; +import { extractAllVideoFrames, type ExtractedFrames } from "./videoFrameExtractor.js"; + +const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; +const WIDTH = 32; +const HEIGHT = 16; + +describe.skipIf(!HAS_FFMPEG)("a clip's frames match the output frames it is on screen", () => { + const fixtureDir = mkdtempSync(join(tmpdir(), "hf-clip-slots-")); + const source = join(fixtureDir, "index-30fps.mp4"); + + beforeAll(async () => { + // Frame k carries luma 16 + 2k, so each extracted frame names its source index. + const synth = await runFfmpeg([ + ...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i"], + `nullsrc=s=${WIDTH}x${HEIGHT}:r=30:d=1.5,geq=lum='16+2*N':cb=128:cr=128`, + ...["-c:v", "libx264", "-qp", "0", "-pix_fmt", "yuv420p", source], + ]); + if (!synth.success) throw new Error(`index fixture synthesis failed: ${synth.stderr}`); + }, 30_000); + + afterAll(() => { + rmSync(fixtureDir, { recursive: true, force: true }); + }); + + function sourceIndexes(extracted: ExtractedFrames): number[] { + const decoded = spawnSync("ffmpeg", [ + ...["-v", "error", "-i", join(extracted.outputDir, extracted.framePattern)], + ...["-f", "rawvideo", "-pix_fmt", "gray", "pipe:1"], + ]); + if (decoded.status !== 0) throw new Error(decoded.stderr.toString()); + const indexes: number[] = []; + for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) { + indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2)); + } + return indexes; + } + + it.each([ + // On screen at output frames 1..8: 0.01 <= f/30 < 0.28. + { label: "0.01-0.28 s", start: 0.01, end: 0.28, onScreen: [0, 1, 2, 3, 4, 5, 6, 7] }, + // On screen at output frames 0..8: f/30 < 0.27. + { label: "0-0.27 s", start: 0, end: 0.27, onScreen: [0, 1, 2, 3, 4, 5, 6, 7, 8] }, + ])( + "gives a $label clip a new source frame on every output frame it is on screen", + async (c) => { + const id = `slots-${c.start}-${c.end}`; + const result = await extractAllVideoFrames( + [ + { + id, + src: source, + start: c.start, + end: c.end, + mediaStart: 0, + loop: false, + hasAudio: false, + }, + ], + fixtureDir, + { fps: 30, outputDir: join(fixtureDir, id), format: "png" }, + ); + expect(result.errors).toEqual([]); + expect(sourceIndexes(result.extracted[0]!)).toEqual(c.onScreen); + }, + 30_000, + ); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index 5694e4fb1f..8f2587ad47 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -473,11 +473,13 @@ describe("resolveVideoExtractionDuration", () => { }); describe("extractionFrameCountForDuration", () => { - it("uses the same VFR ceil and CFR nearest-boundary rules as FFmpeg", () => { + it("ceils VFR windows and counts every output slot a CFR window covers", () => { expect(extractionFrameCountForDuration(0.466666, 30, true)).toBe(14); expect(extractionFrameCountForDuration(0.466666, 30, false)).toBe(14); expect(extractionFrameCountForDuration(0.616666, 30, true)).toBe(19); - expect(extractionFrameCountForDuration(0.616666, 30, false)).toBe(18); + expect(extractionFrameCountForDuration(0.616666, 30, false)).toBe(19); + expect(extractionFrameCountForDuration(0.2667, 30, false)).toBe(9); + expect(extractionFrameCountForDuration(0.266667, 30, false)).toBe(8); }); it.each([ @@ -3561,7 +3563,7 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { label: "non-half-integer boundary", duration: 0.616666, offset: 0.1, - expectedFrames: 18, + expectedFrames: 19, }, ])( "keeps direct and superset CFR extraction equal at a $label", diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 236f83c80e..acbbf702e4 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -147,14 +147,10 @@ export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat { return typeof value === "string" && (VIDEO_FRAME_FORMATS as readonly string[]).includes(value); } +const SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES = 1e-4; /** - * Resolve the frame count produced for a requested extraction duration. - * - * Both paths sample with FFmpeg's fps filter. A CFR window ends at the nearest - * frame boundary; a VFR window rounds up, keeping the slot a trailing still - * covers. Keep this calculation shared by superset slicing and - * producer coverage accounting so a complete VFR extraction cannot be - * rejected because the two paths disagree by one frame. + * Frames a window gets: one per output slot it covers (CFR), or rounded up (VFR). + * Superset slicing and the coverage gate share it, so they never disagree by a frame. */ export function extractionFrameCountForDuration( durationSeconds: number, @@ -214,9 +210,11 @@ export function extractionFrameCountForDuration( const frameNumerator = microseconds * fpsNumerator; const frameDenominator = 1_000_000n * fpsDenominator; + const slotsPerTolerance = BigInt(Math.round(1 / SLOT_EDGE_TOLERANCE_BELOW_EXPORT_SNAP_FRAMES)); const frameCount = isVFR ? (frameNumerator + frameDenominator - 1n) / frameDenominator - : (2n * frameNumerator + frameDenominator) / (2n * frameDenominator); + : (slotsPerTolerance * (frameNumerator + frameDenominator) - frameDenominator - 1n) / + (slotsPerTolerance * frameDenominator); const frames = Number(frameCount); return Math.max(1, Number.isSafeInteger(frames) ? frames : Number.MAX_SAFE_INTEGER); } @@ -1441,13 +1439,10 @@ async function extractVideoFramesRangeUnredacted( false, ); const totalFrames = metadata.isVFR ? windowFrames : Math.min(windowFrames, framesBeforeSourceEnd); - // A VFR window rounds its frame count up, and only the segments' `-frames:v` bound - // emits that last slot: the fps filter ends a `-t` window at the nearest slot. A - // decimal rate counts zero frames here and keeps the `-t` window. - const boundVfrFrameCount = metadata.isVFR && totalFrames > 0; - const frameCap = sampleAtOutputFps && totalFrames > 0 ? ["-frames:v", String(totalFrames)] : []; + // A `-t` window ends at the nearest slot; segments read a frame past and keep exactly their own. + // A decimal rate counts zero frames here and keeps the `-t` window. const extract = (hwaccel: string[]) => - sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames) + sampleAtOutputFps && (options.frameRanges || totalFrames > 0) ? runSegmentedExtraction({ decodeArgs: [...hwaccel, ...decodeArgs], filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs], @@ -1467,20 +1462,17 @@ async function extractVideoFramesRangeUnredacted( }) : inExtractionSlot( async () => - runFfmpeg( - [...hwaccel, ...args, ...frameCap, ...filterArgsFor(startTime), ...encodeArgs], - { - ...runOptions, - onStderr: framesWrittenReader(options.onProgress), - timeout: stallTimeoutMs( - ffmpegProcessTimeout, - options.finalFrameOnly - ? startTime - : await leadInSecondsUnlessAborted(videoPath, startTime, signal), - metadata.fps, - ), - }, - ), + runFfmpeg([...hwaccel, ...args, ...filterArgsFor(startTime), ...encodeArgs], { + ...runOptions, + onStderr: framesWrittenReader(options.onProgress), + timeout: stallTimeoutMs( + ffmpegProcessTimeout, + options.finalFrameOnly + ? startTime + : await leadInSecondsUnlessAborted(videoPath, startTime, signal), + metadata.fps, + ), + }), signal, ); options.onProgress?.({ diff --git a/packages/producer/src/services/render/videoFrameCoverage.test.ts b/packages/producer/src/services/render/videoFrameCoverage.test.ts index 4332e77f0e..aa0d5dd5ce 100644 --- a/packages/producer/src/services/render/videoFrameCoverage.test.ts +++ b/packages/producer/src/services/render/videoFrameCoverage.test.ts @@ -92,21 +92,21 @@ describe("expectedFramesForClip", () => { expect(expectedFramesForClip(0, 0.316666, 30)).toBe(10); }); - it("matches the CFR fps filter's nearest-boundary rounding when requested", () => { - expect(expectedFramesForClip(0, 0.616666, 30, "nearest")).toBe(18); - expect(expectedFramesForClip(0, 0.316666, 30, "nearest")).toBe(9); - expect(expectedFramesForClip(0, 0.633333, 30, "nearest")).toBe(19); + it("counts every output slot a CFR window covers when requested", () => { + expect(expectedFramesForClip(0, 0.616666, 30, "slots")).toBe(19); + expect(expectedFramesForClip(0, 0.316666, 30, "slots")).toBe(10); + expect(expectedFramesForClip(0, 0.633333, 30, "slots")).toBe(19); }); it("uses exact NTSC rationals for short CFR and VFR boundaries", () => { - expect(expectedFramesForClip(0, 0.25025, { num: 30000, den: 1001 }, "nearest")).toBe(8); + expect(expectedFramesForClip(0, 0.25025, { num: 30000, den: 1001 }, "slots")).toBe(8); expect(expectedFramesForClip(0, 0.125125, { num: 24000, den: 1001 })).toBe(3); expect(expectedFramesForClip(0, 0.5005, { num: 24000, den: 1001 })).toBe(12); }); it("requires one frame for every positive sub-frame clip", () => { expect(expectedFramesForClip(0, 0.001, 30)).toBe(1); - expect(expectedFramesForClip(0, 0.001, 30, "nearest")).toBe(1); + expect(expectedFramesForClip(0, 0.001, 30, "slots")).toBe(1); expect(expectedFramesForClip(1, 1, 30)).toBe(0); }); }); @@ -166,17 +166,24 @@ describe("computeVideoFrameCoverage", () => { }); }); - it("reports full coverage for a short CFR clip matching FFmpeg boundary rounding", () => { + it("expects a short CFR clip to fill every output slot it covers", () => { const videos = [makeVideo({ id: "short", start: 0, end: 0.616666 })]; - const reports = computeVideoFrameCoverage(videos, [makeExtracted("short", 18)], 30); + const reports = computeVideoFrameCoverage(videos, [makeExtracted("short", 19)], 30); expect(reports[0]).toMatchObject({ - expectedFrames: 18, - capturedFrames: 18, + expectedFrames: 19, + capturedFrames: 19, ratio: 1, }); expect(() => assertVideoFrameCoverage(reports, 0.95)).not.toThrow(); }); + it("rejects a short CFR clip missing the last output slot it covers", () => { + const videos = [makeVideo({ id: "aligned", start: 1 / 30, end: 0.28 })]; + const reports = computeVideoFrameCoverage(videos, [makeExtracted("aligned", 7)], 30); + expect(reports[0]).toMatchObject({ expectedFrames: 8, capturedFrames: 7 }); + expect(() => assertVideoFrameCoverage(reports, 0.95)).toThrow(/captured 7 of expected 8/); + }); + it("tolerates a single FFmpeg boundary frame on a short 18/19 VFR extraction", () => { const videos = [makeVideo({ id: "short-vfr", start: 0, end: 0.616666 })]; const reports = computeVideoFrameCoverage( diff --git a/packages/producer/src/services/render/videoFrameCoverage.ts b/packages/producer/src/services/render/videoFrameCoverage.ts index 8bac1c296a..5851dbea94 100644 --- a/packages/producer/src/services/render/videoFrameCoverage.ts +++ b/packages/producer/src/services/render/videoFrameCoverage.ts @@ -135,7 +135,7 @@ export function resolveVideoCoverageThreshold( /** * Count frames in a clip's authored `[start,end)` window. CFR extraction - * rounds to the nearest output-frame boundary and VFR extraction rounds up, + * fills every output slot the window covers and VFR extraction rounds up, * as `extractionFrameCountForDuration` does. Missing entries * retain the fail-closed `ceil` default. Positive sub-frame clips emit one. */ @@ -143,7 +143,7 @@ export function expectedFramesForClip( start: number, end: number, fps: FpsInput, - rounding: "ceil" | "nearest" = "ceil", + rounding: "ceil" | "slots" = "ceil", ): number { const fpsValue = fpsToNumber(toFps(fps)); if (!Number.isFinite(start) || !Number.isFinite(end) || !Number.isFinite(fpsValue)) return 0; @@ -171,7 +171,7 @@ function expectedFramesForVideo( entry: ExtractedFrames | undefined, fps: FpsInput, ): number { - const rounding = entry && !entry.metadata.isVFR ? "nearest" : "ceil"; + const rounding = entry && !entry.metadata.isVFR ? "slots" : "ceil"; const slotSourceDuration = sourceTimeAt( normalizeRateSpec(video.playbackRate), Math.max(0, video.end - video.start),