From 10a4ed56cb09159afec0dba3ea784e3162ee2e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 30 Oct 2026 09:01:21 +0000 Subject: [PATCH] fix(engine): a video clip starting between frames shows its media on the right frame (#6759) * fix(engine): a video clip starting between frames shows its media on the right frame * chore(engine): trim the helper comment to one line * chore(engine): drop the helper comment * test(engine): extract once per start in the off-grid clip test * fix(engine): a looped clip keeps its loop origin when the start is off the frame grid * test(engine): three off-grid starts cover the failure modes and keep the flake gate under its limit --- .../src/services/videoFrameExtractor.test.ts | 115 ++++++++++++++++++ .../src/services/videoFrameExtractor.ts | 14 +++ 2 files changed, 129 insertions(+) diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index c3240ab21f..9694cd52c3 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -4518,3 +4518,118 @@ describe.skipIf(!HAS_FFMPEG)("a clip stored on a non-square pixel grid", () => { expect(probe.stdout.toString().trim()).toBe("32,64"); }); }); + +// Each source frame N is a flat gray of 3N, so the frame a render shows reads straight off a pixel. +describe.skipIf(!HAS_FFMPEG)("a clip whose data-start is off the frame grid", () => { + const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-offgrid-start-")); + const SOURCE = join(FIXTURE_DIR, "numbered.mp4"); + const FPS = 30; + + beforeAll(async () => { + const result = await runFfmpeg([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=black:s=64x64:d=6:r=30", + "-vf", + "format=gray,geq=lum='N*3'", + "-c:v", + "libx264", + "-crf", + "0", + "-g", + "30", + "-pix_fmt", + "yuv420p", + SOURCE, + ]); + if (!result.success) throw new Error(`fixture synthesis failed: ${result.stderr.slice(-400)}`); + }, 30_000); + + afterAll(() => rmSync(FIXTURE_DIR, { recursive: true, force: true }), 30_000); + + async function sourceFramesShown( + start: number, + outputFrames: number[], + ): Promise<(number | null)[]> { + const outDir = join(FIXTURE_DIR, `out-${start}`); + mkdirSync(outDir, { recursive: true }); + const video: VideoElement = { + id: "v", + src: SOURCE, + start, + end: start + 1.9, + mediaStart: start, + loop: false, + hasAudio: false, + }; + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: FPS, + outputDir: outDir, + }); + expect(result.errors).toEqual([]); + const table = createFrameLookupTable([video], result.extracted, FPS); + return outputFrames.map((n) => { + const path = table.getFrame("v", n / FPS); + return path ? Math.round(readFirstFramePixel(path, 0, 0)[0] / 3) : null; + }); + } + + it("keeps a looped clip on the same source frame after every wrap", async () => { + const loopSource = join(FIXTURE_DIR, "loop.mp4"); + const synth = await runFfmpeg([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=black:s=64x64:d=2:r=30", + "-vf", + "format=gray,geq=lum='N*4'", + "-c:v", + "libx264", + "-crf", + "0", + "-pix_fmt", + "yuv420p", + loopSource, + ]); + expect(synth.success).toBe(true); + const outDir = join(FIXTURE_DIR, "out-loop"); + mkdirSync(outDir, { recursive: true }); + const video: VideoElement = { + id: "v", + src: loopSource, + start: 0.35, + end: 6.35, + mediaStart: 1, + loop: true, + hasAudio: false, + }; + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: FPS, + outputDir: outDir, + timelineEnd: 6, + }); + expect(result.errors).toEqual([]); + const table = createFrameLookupTable([video], result.extracted, FPS); + const shown = [0, 1, 2, 3, 4].map((wrap) => { + const path = table.getFrame("v", (12 + wrap * FPS + 1) / FPS); + return path ? Math.round(readFirstFramePixel(path, 0, 0)[0] / 4) : null; + }); + expect(new Set(shown).size).toBe(1); + }, 60_000); + + it.each([1.95, 2.0001, 1.999999])( + "shows source frame N at output frame N for data-start=data-media-start=%s", + async (start) => { + const shown = await sourceFramesShown(start, [61, 62, 63, 64, 65]); + expect(shown).toEqual([61, 62, 63, 64, 65]); + }, + 60_000, + ); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 18a8fa1ec1..ca15cd9e69 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -1472,6 +1472,19 @@ export function isVideoPastTimelineEnd(video: Pick, timel return video.start >= timelineEnd; } +function alignClipStartToFirstOutputFrame(video: VideoElement, fps: number): void { + if (video.loop || !Number.isFinite(video.start) || video.start < 0 || !(fps > 0)) return; + const position = exportClipWindow(video.start, Infinity, fps).start * fps; + const firstFrame = + Math.abs(position - Math.round(position)) < 1e-6 ? Math.round(position) : Math.ceil(position); + const shift = firstFrame / fps - video.start; + if (shift === 0) return; + const playbackRate = normalizeRateSpec(video.playbackRate); + video.mediaStart += sourceTimeAt(playbackRate, shift); + video.playbackRate = shiftRateLane(playbackRate, shift); + video.start += shift; +} + /** Move a clip to its extraction window; a rate lane is shifted so it keeps integrating from the new origin. */ export function rebaseVideoToWindow(video: VideoElement, window: TimelineExtractionWindow): void { if (window.preserveTimelinePhase) return; @@ -2596,6 +2609,7 @@ export async function extractAllVideoFrames( } try { const metadata = videoMetadata[index] ?? (await extractMediaMetadata(videoPath)); + alignClipStartToFirstOutputFrame(video, fps); const initialWindow = resolveVideoExtractionWindow(video, metadata, options.timelineEnd); const window = await resolveFinalFrameExtractionWindow( videoPath,