import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; import { createHash } from "node:crypto"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { spawnSync } from "node:child_process"; import { parseVideoElements, parseImageElements, extractAllVideoFrames, createFrameLookupTable, type VideoElement, type ExtractedFrames, } from "./videoFrameExtractor.js"; import { extractVideoMetadata } from "../utils/ffprobe.js"; import { runFfmpeg } from "../utils/runFfmpeg.js"; // ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer // regression test at packages/producer/tests/vfr-screen-recording/ runs inside // Dockerfile.test (which does include ffmpeg) and is the primary CI signal // for this bug. Locally and in any CI job with ffmpeg on PATH, the tests // below run too — they exercise the extractor in isolation against a // synthesized VFR fixture. const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; describe("parseVideoElements", () => { it("parses videos without an id or data-start attribute", () => { const videos = parseVideoElements(''); expect(videos).toHaveLength(1); expect(videos[0]).toMatchObject({ id: "hf-video-0", src: "clip.mp4", start: 0, end: Infinity, mediaStart: 0, loop: false, hasAudio: false, }); }); it("preserves explicit ids and derives end from data-duration", () => { const videos = parseVideoElements( '', ); expect(videos).toHaveLength(1); expect(videos[0]).toEqual({ id: "hero", src: "clip.mp4", start: 2, end: 7, mediaStart: 1.5, loop: false, hasAudio: true, }); }); it("preserves looped timed video semantics for render frame lookup", () => { const videos = parseVideoElements( '', ); expect(videos[0]).toMatchObject({ id: "hero", start: 2, end: 7, loop: true, }); }); }); describe("FrameLookupTable", () => { function fakeExtracted(totalFrames: number, fps: number): ExtractedFrames { const framePaths = new Map(); for (let i = 0; i < totalFrames; i += 1) { framePaths.set(i, `frame-${i}.jpg`); } return { videoId: "hero", srcPath: "clip.webm", outputDir: "/tmp/frames", framePattern: "frame-%05d.jpg", fps, totalFrames, metadata: { durationSeconds: totalFrames / fps, width: 320, height: 180, fps, hasAudio: false, videoCodec: "vp9", colorSpace: { colorTransfer: "bt709", colorPrimaries: "bt709", colorSpace: "bt709", }, isVFR: false, hasAlpha: false, }, framePaths, }; } it("wraps active frame payloads for looped clips whose display window exceeds source frames", () => { const table = createFrameLookupTable( [ { id: "hero", src: "clip.webm", start: 0, end: 5, mediaStart: 0, loop: true, hasAudio: false, }, ], [fakeExtracted(30, 30)], ); expect(table.getActiveFramePayloads(0.5).get("hero")?.frameIndex).toBe(15); expect(table.getActiveFramePayloads(1.5).get("hero")?.frameIndex).toBe(15); expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15); }); it("does not hold stale frames for non-looping clips after extracted frames end", () => { const table = createFrameLookupTable( [ { id: "hero", src: "clip.webm", start: 0, end: 5, mediaStart: 0, loop: false, hasAudio: false, }, ], [fakeExtracted(30, 30)], ); expect(table.getActiveFramePayloads(0.5).has("hero")).toBe(true); expect(table.getActiveFramePayloads(1.5).has("hero")).toBe(false); }); }); describe("parseImageElements", () => { it("parses images with data-start and data-duration", () => { const images = parseImageElements( '', ); expect(images).toHaveLength(1); expect(images[0]).toEqual({ id: "photo", src: "hdr-photo.png", start: 0, end: 3, }); }); it("generates stable IDs for images without one", () => { const images = parseImageElements( '', ); expect(images).toHaveLength(2); expect(images[0]!.id).toBe("hf-img-0"); expect(images[1]!.id).toBe("hf-img-1"); }); it("defaults start to 0 and end to Infinity when attributes missing", () => { const images = parseImageElements(''); expect(images).toHaveLength(1); expect(images[0]).toMatchObject({ src: "photo.png", start: 0, end: Infinity, }); }); it("ignores img elements without src", () => { const images = parseImageElements(''); expect(images).toHaveLength(0); }); it("uses data-end over data-duration when both present", () => { const images = parseImageElements( '', ); expect(images[0]!.end).toBe(5); }); }); // Regression test for the VFR (variable frame rate) freeze bug. // Screen recordings and phone videos often have irregular timestamps. // When such inputs hit `extractVideoFramesRange`'s `-ss -i ... -t // -vf fps=N` pipeline, the fps filter can emit fewer frames than requested — // e.g. a 4-second segment at 30fps would produce ~90 frames instead of 120. // FrameLookupTable.getFrameAtTime then returns null for out-of-range indices // and the compositor holds the last valid frame, which the user perceives as // the video freezing. extractAllVideoFrames normalizes VFR sources to CFR // before extraction to fix this. describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-vfr-test-")); const VFR_FIXTURE = join(FIXTURE_DIR, "vfr_screen.mp4"); beforeAll(async () => { // 10s testsrc2 at 60fps, ~40% of frames dropped via select filter and // encoded with -vsync vfr so timestamps are irregular. Declared fps 60, // actual average ~36 — well over the 10% threshold used by isVFR. // The select expression drops four 1-second windows (frames 30-89, // 180-239, 330-389, 480-539) to simulate static segments in a screen // recording where no pixels changed. // -g/-keyint_min 600 forces a single keyframe so mid-segment seeks in the // mediaStart=3 test don't snap to an intermediate IDR and drift the count. const result = await runFfmpeg([ "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=320x180:d=10:rate=60", "-vf", "select='not(between(n\\,30\\,89))*not(between(n\\,180\\,239))*not(between(n\\,330\\,389))*not(between(n\\,480\\,539))'", "-vsync", "vfr", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-g", "600", "-keyint_min", "600", VFR_FIXTURE, ]); if (!result.success) { throw new Error( `ffmpeg fixture synthesis failed (${result.exitCode}): ${result.stderr.slice(-400)}`, ); } }, 30_000); afterAll(() => { if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true }); }); it("detects the synthesized fixture as VFR", async () => { const md = await extractVideoMetadata(VFR_FIXTURE); expect(md.isVFR).toBe(true); }); it("produces the expected frame count for a mid-file segment", async () => { const outputDir = join(FIXTURE_DIR, "out-mid-segment"); mkdirSync(outputDir, { recursive: true }); const video: VideoElement = { id: "v1", src: VFR_FIXTURE, start: 0, end: 4, mediaStart: 3, loop: false, hasAudio: false, }; const result = await extractAllVideoFrames([video], FIXTURE_DIR, { fps: 30, outputDir, }); expect(result.errors).toEqual([]); expect(result.extracted).toHaveLength(1); const frames = readdirSync(join(outputDir, "v1")).filter((f) => f.endsWith(".jpg")); // Pre-fix behavior produced ~90 frames (a 25% shortfall). expect(frames.length).toBeGreaterThanOrEqual(119); expect(frames.length).toBeLessThanOrEqual(121); expect(result.phaseBreakdown).toBeDefined(); expect(result.phaseBreakdown.extractMs).toBeGreaterThan(0); expect(result.phaseBreakdown.vfrPreflightCount).toBe(1); expect(result.phaseBreakdown.vfrPreflightMs).toBeGreaterThan(0); }, 60_000); it("reuses extracted frames on a warm cache hit", async () => { const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-")); const SRC = join(FIXTURE_DIR, "cache-src.mp4"); // Synthesize a clean CFR SDR clip — bypasses VFR preflight so the cache // key is stable across the two runs. const synth = await runFfmpeg([ "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=320x180:d=2:rate=30", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", SRC, ]); if (!synth.success) { throw new Error(`Cache fixture synthesis failed: ${synth.stderr.slice(-400)}`); } const video: VideoElement = { id: "cv1", src: SRC, start: 0, end: 2, mediaStart: 0, loop: false, hasAudio: false, }; const outDirA = join(FIXTURE_DIR, "out-cache-miss"); mkdirSync(outDirA, { recursive: true }); const miss = await extractAllVideoFrames( [video], FIXTURE_DIR, { fps: 30, outputDir: outDirA }, undefined, { extractCacheDir: CACHE_DIR }, ); expect(miss.errors).toEqual([]); expect(miss.phaseBreakdown.cacheHits).toBe(0); expect(miss.phaseBreakdown.cacheMisses).toBe(1); const outDirB = join(FIXTURE_DIR, "out-cache-hit"); mkdirSync(outDirB, { recursive: true }); const hit = await extractAllVideoFrames( [video], FIXTURE_DIR, { fps: 30, outputDir: outDirB }, undefined, { extractCacheDir: CACHE_DIR }, ); expect(hit.errors).toEqual([]); expect(hit.phaseBreakdown.cacheHits).toBe(1); expect(hit.phaseBreakdown.cacheMisses).toBe(0); // extractMs on a hit is only the cache-lookup bookkeeping; asserting <50ms // is loose enough to survive CI jitter but tight enough to catch a // regression that accidentally triggered ffmpeg again. expect(hit.phaseBreakdown.extractMs).toBeLessThan(50); expect(hit.extracted).toHaveLength(1); expect(hit.extracted[0]!.totalFrames).toBe(miss.extracted[0]!.totalFrames); rmSync(CACHE_DIR, { recursive: true, force: true }); }, 60_000); it("invalidates the cache when fps changes", async () => { const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-")); const SRC = join(FIXTURE_DIR, "cache-fps-src.mp4"); const synth = await runFfmpeg([ "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=320x180:d=1:rate=30", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", SRC, ]); if (!synth.success) { throw new Error(`Cache-fps fixture synthesis failed: ${synth.stderr.slice(-400)}`); } const video: VideoElement = { id: "cv2", src: SRC, start: 0, end: 1, mediaStart: 0, loop: false, hasAudio: false, }; const outA = join(FIXTURE_DIR, "out-cache-fps-30"); mkdirSync(outA, { recursive: true }); const first = await extractAllVideoFrames( [video], FIXTURE_DIR, { fps: 30, outputDir: outA }, undefined, { extractCacheDir: CACHE_DIR }, ); expect(first.phaseBreakdown.cacheMisses).toBe(1); const outB = join(FIXTURE_DIR, "out-cache-fps-60"); mkdirSync(outB, { recursive: true }); const second = await extractAllVideoFrames( [video], FIXTURE_DIR, { fps: 60, outputDir: outB }, undefined, { extractCacheDir: CACHE_DIR }, ); expect(second.phaseBreakdown.cacheMisses).toBe(1); expect(second.phaseBreakdown.cacheHits).toBe(0); rmSync(CACHE_DIR, { recursive: true, force: true }); }, 60_000); // Regression test for the segment-scope HDR preflight fix: pre-fix, // convertSdrToHdr re-encoded the entire source, so a 30-minute SDR source // contributing a 2-second clip took ~200× longer than needed. Post-fix the // converted file's duration matches the used segment. it("bounds the SDR→HDR preflight re-encode to the used segment", async () => { const SDR_LONG = join(FIXTURE_DIR, "sdr-long.mp4"); const HDR_SHORT = join(FIXTURE_DIR, "hdr-short.mp4"); const sdrResult = await runFfmpeg([ "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=320x180:d=10:rate=30", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", SDR_LONG, ]); if (!sdrResult.success) { throw new Error(`SDR fixture synthesis failed: ${sdrResult.stderr.slice(-400)}`); } // Tag as bt2020nc / smpte2084 so the preflight path considers the timeline mixed-HDR. const hdrResult = await runFfmpeg([ "-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=320x180:d=2:rate=30", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-color_primaries", "bt2020", "-color_trc", "smpte2084", "-colorspace", "bt2020nc", HDR_SHORT, ]); if (!hdrResult.success) { throw new Error(`HDR fixture synthesis failed: ${hdrResult.stderr.slice(-400)}`); } const outputDir = join(FIXTURE_DIR, "out-hdr-segment"); mkdirSync(outputDir, { recursive: true }); const videos: VideoElement[] = [ { id: "sdr", src: SDR_LONG, start: 0, end: 2, mediaStart: 0, loop: false, hasAudio: false }, { id: "hdr", src: HDR_SHORT, start: 2, end: 4, mediaStart: 0, loop: false, hasAudio: false, }, ]; const result = await extractAllVideoFrames(videos, FIXTURE_DIR, { fps: 30, outputDir, }); expect(result.errors).toEqual([]); expect(result.phaseBreakdown.hdrPreflightCount).toBe(1); const convertedPath = join(outputDir, "_hdr_normalized", "sdr_hdr.mp4"); expect(existsSync(convertedPath)).toBe(true); const convertedMeta = await extractVideoMetadata(convertedPath); // Pre-fix duration matched the 10s source; post-fix it matches the 2s segment // (±0.2s for encoder keyframe/seek alignment). expect(convertedMeta.durationSeconds).toBeGreaterThan(1.8); expect(convertedMeta.durationSeconds).toBeLessThan(2.5); }, 60_000); // Asserts both frame-count correctness and that we don't emit long runs of // byte-identical "duplicate" frames — the user-visible "frozen screen // recording" symptom. Pre-fix duplicate rate on this fixture is ~38% // (116/300); on the actual reporter's ScreenCaptureKit clip, 18–44% across // segments. <10% threshold leaves margin across ffmpeg versions without // letting a regression slip through. it("produces the full frame count and no duplicate-frame runs on the full VFR file", async () => { const outputDir = join(FIXTURE_DIR, "out-full"); mkdirSync(outputDir, { recursive: true }); const video: VideoElement = { id: "vfull", src: VFR_FIXTURE, start: 0, end: 10, mediaStart: 0, loop: false, hasAudio: false, }; const result = await extractAllVideoFrames([video], FIXTURE_DIR, { fps: 30, outputDir, }); expect(result.errors).toEqual([]); const frameDir = join(outputDir, "vfull"); const frames = readdirSync(frameDir) .filter((f) => f.endsWith(".jpg")) .sort(); expect(frames.length).toBeGreaterThanOrEqual(299); expect(frames.length).toBeLessThanOrEqual(301); let prevHash: string | null = null; let duplicates = 0; for (const f of frames) { const hash = createHash("sha256") .update(readFileSync(join(frameDir, f))) .digest("hex"); if (hash === prevHash) duplicates += 1; prevHash = hash; } const duplicateRate = duplicates / frames.length; expect(duplicateRate).toBeLessThan(0.1); }, 60_000); });