diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index 5415f1fdae..4cc8eeca80 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -1,5 +1,15 @@ -import { describe, expect, it } from "vitest"; -import { parseVideoElements, parseImageElements } from "./videoFrameExtractor.js"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { + parseVideoElements, + parseImageElements, + extractAllVideoFrames, + type VideoElement, +} from "./videoFrameExtractor.js"; +import { extractVideoMetadata } from "../utils/ffprobe.js"; describe("parseVideoElements", () => { it("parses videos without an id or data-start attribute", () => { @@ -81,3 +91,116 @@ describe("parseImageElements", () => { expect(images[0]!.end).toBe(5); }); }); + +// Regression test for the VFR (variable frame rate) freeze bug. +// Screen recordings and phone videos often have irregular timestamps. +// When such inputs hit `extractVideoFramesRange`'s `-ss -i ... -t +// -vf fps=N` pipeline, the fps filter can emit fewer frames than requested — +// e.g. a 4-second segment at 30fps would produce ~90 frames instead of 120. +// FrameLookupTable.getFrameAtTime then returns null for out-of-range indices +// and the compositor holds the last valid frame, which the user perceives as +// the video freezing. extractAllVideoFrames normalizes VFR sources to CFR +// before extraction to fix this. +const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-vfr-test-")); +const VFR_FIXTURE = join(FIXTURE_DIR, "vfr_screen.mp4"); + +function runFfmpegSync(args: string[]): void { + const r = spawnSync("ffmpeg", args, { encoding: "utf8" }); + if (r.status !== 0) { + throw new Error(`ffmpeg failed (${r.status}): ${r.stderr.slice(-400)}`); + } +} + +beforeAll(() => { + // 10s testsrc2 at 60fps, ~40% of frames dropped via select filter and + // encoded with -vsync vfr so timestamps are irregular. Declared fps 60, + // actual average ~36 — well over the 10% threshold used by isVFR. + runFfmpegSync([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=10:rate=60", + "-vf", + "drawtext=text='n=%{n}':fontsize=24:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.6," + + "select='not(between(n,30,89))*not(between(n,180,239))*not(between(n,330,389))*not(between(n,480,539))'", + "-vsync", + "vfr", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-g", + "600", + "-keyint_min", + "600", + VFR_FIXTURE, + ]); +}, 30_000); + +afterAll(() => { + if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true }); +}); + +describe("extractAllVideoFrames on a VFR source", () => { + it("detects the synthesized fixture as VFR", async () => { + const md = await extractVideoMetadata(VFR_FIXTURE); + expect(md.isVFR).toBe(true); + }); + + it("produces the expected frame count for a mid-file segment", async () => { + const outputDir = join(FIXTURE_DIR, "out-mid-segment"); + mkdirSync(outputDir, { recursive: true }); + + const video: VideoElement = { + id: "v1", + src: VFR_FIXTURE, + start: 0, + end: 4, + mediaStart: 3, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 30, + outputDir, + }); + + expect(result.errors).toEqual([]); + expect(result.extracted).toHaveLength(1); + const frames = readdirSync(join(outputDir, "v1")).filter((f) => f.endsWith(".jpg")); + // Expect 120 frames for 4s at 30fps. Allow ±1 for rounding at the + // boundary; pre-fix behavior produced ~90 frames (a 25% shortfall). + expect(frames.length).toBeGreaterThanOrEqual(119); + expect(frames.length).toBeLessThanOrEqual(121); + }, 60_000); + + it("produces the expected frame count when mediaStart=0 and the full VFR file is used", async () => { + const outputDir = join(FIXTURE_DIR, "out-full"); + mkdirSync(outputDir, { recursive: true }); + + const video: VideoElement = { + id: "vfull", + src: VFR_FIXTURE, + start: 0, + end: 10, + mediaStart: 0, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 30, + outputDir, + }); + + expect(result.errors).toEqual([]); + const frames = readdirSync(join(outputDir, "vfull")).filter((f) => f.endsWith(".jpg")); + expect(frames.length).toBeGreaterThanOrEqual(299); + expect(frames.length).toBeLessThanOrEqual(301); + }, 60_000); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index b59d7112df..8b19c9b6c4 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -296,6 +296,62 @@ async function convertSdrToHdr( } } +/** + * Re-encode a VFR (variable frame rate) video segment to CFR so the downstream + * fps filter can extract frames reliably. Screen recordings, phone videos, and + * some webcams emit irregular timestamps that cause two failure modes: + * 1. Output has fewer frames than expected (e.g. -ss 3 -t 4 produces 90 + * frames instead of 120 @ 30fps). FrameLookupTable.getFrameAtTime then + * returns null for late timestamps and the caller freezes on the last + * valid frame. + * 2. Large duplicate-frame runs where source PTS don't land on target + * timestamps. + * + * Only the [startTime, startTime+duration] window is re-encoded, so long + * recordings aren't fully transcoded when only a short clip is used. + */ +async function convertVfrToCfr( + inputPath: string, + outputPath: string, + targetFps: number, + startTime: number, + duration: number, + signal?: AbortSignal, + config?: Partial>, +): Promise { + const timeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; + + const args = [ + "-ss", + String(startTime), + "-i", + inputPath, + "-t", + String(duration), + "-fps_mode", + "cfr", + "-r", + String(targetFps), + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "18", + "-c:a", + "copy", + "-y", + outputPath, + ]; + + const result = await runFfmpeg(args, { signal, timeout }); + if (!result.success) { + throw new Error( + `VFR→CFR conversion failed (exit ${result.exitCode}): ${result.stderr.slice(-300)}`, + ); + } +} + export async function extractAllVideoFrames( videos: VideoElement[], baseDir: string, @@ -371,6 +427,56 @@ export async function extractAllVideoFrames( } } + // Phase 2b: Normalize VFR (variable frame rate) inputs to CFR. Screen + // recordings and phone videos often have irregular timestamps that cause + // the fps filter in extractVideoFramesRange to emit fewer frames than + // expected, which surfaces downstream as frozen video segments. Only the + // used [mediaStart, mediaStart+duration] window is re-encoded, so long + // recordings don't pay the full-file transcode cost for a short clip. + const vfrNormDir = join(options.outputDir, "_vfr_normalized"); + let vfrNormDirCreated = false; + for (let i = 0; i < resolvedVideos.length; i++) { + if (signal?.aborted) break; + const entry = resolvedVideos[i]; + if (!entry) continue; + const metadata = await extractVideoMetadata(entry.videoPath); + if (!metadata.isVFR) continue; + + let segDuration = entry.video.end - entry.video.start; + if (!Number.isFinite(segDuration) || segDuration <= 0) { + const sourceRemaining = metadata.durationSeconds - entry.video.mediaStart; + segDuration = sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds; + } + + if (!vfrNormDirCreated) { + mkdirSync(vfrNormDir, { recursive: true }); + vfrNormDirCreated = true; + } + + const normalizedPath = join(vfrNormDir, `${entry.video.id}_cfr.mp4`); + try { + await convertVfrToCfr( + entry.videoPath, + normalizedPath, + options.fps, + entry.video.mediaStart, + segDuration, + signal, + config, + ); + entry.videoPath = normalizedPath; + // Segment-scoped re-encode starts the new file at t=0, so downstream + // extraction must seek from 0, not the original mediaStart. Shallow-copy + // to avoid mutating the caller's VideoElement. + entry.video = { ...entry.video, mediaStart: 0 }; + } catch (err) { + errors.push({ + videoId: entry.video.id, + error: `VFR→CFR conversion failed: ${err instanceof Error ? err.message : String(err)}`, + }); + } + } + // Phase 3: Extract frames (parallel) const results = await Promise.all( resolvedVideos.map(async ({ video, videoPath }) => { diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts index 1cf41cc3dd..0750814d11 100644 --- a/packages/producer/src/services/htmlCompiler.ts +++ b/packages/producer/src/services/htmlCompiler.ts @@ -1028,9 +1028,10 @@ export async function compileForRender( ); } if (metadata.isVFR) { - console.warn( - `[Compiler] WARNING: Video "${video.id}" is variable frame rate (VFR). ` + - `Screen recordings and phone videos are often VFR, which causes stuttering and frame skipping in renders. Re-encode with: ${reencode}`, + console.info( + `[Compiler] Video "${video.id}" is variable frame rate (VFR); ` + + `the engine will normalize it to CFR before frame extraction. ` + + `If rendering feels slow on this video, pre-encode once with: ${reencode}`, ); } })