diff --git a/packages/engine/src/services/videoFrameExtractor.format.test.ts b/packages/engine/src/services/videoFrameExtractor.format.test.ts new file mode 100644 index 0000000000..a9933b9bc4 --- /dev/null +++ b/packages/engine/src/services/videoFrameExtractor.format.test.ts @@ -0,0 +1,759 @@ +import { vi, afterAll, beforeAll, describe, expect, it } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { basename, join } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { + extractAllVideoFrames, + extractVideoFramesRange, + type VideoElement, +} from "./videoFrameExtractor.js"; +import { extractVideoMetadata } from "../utils/ffprobe.js"; +import { runFfmpeg } from "../utils/runFfmpeg.js"; + +// Pass-through, so a test can tell whether extraction ran ffmpeg. +vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { + const real = await importOriginal(); + return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) }; +}); + +const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; +const FFMPEG_ENCODERS = HAS_FFMPEG + ? spawnSync("ffmpeg", ["-hide_banner", "-encoders"]).stdout.toString() + : ""; +// Homebrew's ffmpeg has libsvtav1 but not libaom-av1. Either one writes an untagged AV1 stream. +const AV1_ENCODER = ["libaom-av1", "libsvtav1"].find((name) => + new RegExp(`\\s${name}\\s`).test(FFMPEG_ENCODERS), +); + +type Rgb = [number, number, number]; + +const UI_FIXTURE_WIDTH = 240; +const UI_FIXTURE_HEIGHT = 160; +const RED_SAMPLE_PIXELS = [ + [70, 72], + [118, 82], + [178, 92], +] as const; + +function pngChunkTypes(path: string): string[] { + const bytes = readFileSync(path); + const types: string[] = []; + for (let offset = 8; offset + 8 <= bytes.length; offset += 12 + bytes.readUInt32BE(offset)) { + types.push(bytes.toString("latin1", offset + 4, offset + 8)); + } + return types; +} + +function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb { + const result = spawnSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + mediaPath, + "-frames:v", + "1", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "pipe:1", + ], + { maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 }, + ); + if (result.status !== 0) { + throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`); + } + + const offset = (y * UI_FIXTURE_WIDTH + x) * 3; + return [ + result.stdout[offset] ?? 0, + result.stdout[offset + 1] ?? 0, + result.stdout[offset + 2] ?? 0, + ]; +} + +function maxChannelDelta(a: Rgb, b: Rgb): number { + return Math.max(Math.abs(a[0] - b[0]), Math.abs(a[1] - b[1]), Math.abs(a[2] - b[2])); +} + +// Regression for saturated UI recordings: default JPEG extraction can shift +// high-chroma reds before browser capture. Forcing PNG should keep extracted +// source-video frames effectively identical to the decoded source pixels. +describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => { + const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-format-")); + const UI_FIXTURE = join(FIXTURE_DIR, "ui-red.mp4"); + + beforeAll(async () => { + const result = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `color=c=0xffe7ee:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`, + "-vf", + "drawbox=x=44:y=58:w=152:h=44:color=0xdd382e@1:t=fill,drawbox=x=64:y=75:w=112:h=10:color=0xfff0f0@1:t=fill", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "0", + "-pix_fmt", + "yuv420p", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-colorspace", + "bt709", + UI_FIXTURE, + ]); + if (!result.success) { + throw new Error(`UI color fixture synthesis failed: ${result.stderr.slice(-400)}`); + } + }, 30_000); + + afterAll(() => { + if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true }); + }, 30_000); + + function fixtureVideo(): VideoElement { + return { + id: "ui", + src: UI_FIXTURE, + start: 0, + end: 1, + mediaStart: 0, + loop: false, + hasAudio: false, + }; + } + + it("extracts into a work directory whose name holds a printf code", async () => { + const outDir = join(FIXTURE_DIR, "x 12% Up"); + mkdirSync(outDir, { recursive: true }); + const result = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, { + fps: 1, + outputDir: outDir, + }); + expect(result.errors).toEqual([]); + const frame = result.extracted[0]!.framePaths.get(0)!; + expect(frame.startsWith(outDir)).toBe(true); + expect(existsSync(frame)).toBe(true); + }); + + it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) jpg extraction", async () => { + const source = join(FIXTURE_DIR, "white-1080.mp4"); + const synth = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=white:s=1080x16:d=1:r=1", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "0", + "-pix_fmt", + "yuv420p", + source, + ]); + expect(synth.success).toBe(true); + const outDir = join(FIXTURE_DIR, "out-white-1080"); + mkdirSync(outDir, { recursive: true }); + const result = await extractAllVideoFrames( + [{ ...fixtureVideo(), id: "white", src: source }], + FIXTURE_DIR, + { fps: 1, outputDir: outDir }, + ); + expect(result.errors).toEqual([]); + const frame = result.extracted[0]!.framePaths.get(0)!; + expect(frame.endsWith(".jpg")).toBe(true); + const lastColumns = (file: string) => [ + ...spawnSync("ffmpeg", [ + "-v", + "error", + "-i", + file, + "-frames:v", + "1", + "-vf", + "format=gray,crop=8:1:iw-8:8", + "-f", + "rawvideo", + "-", + ]).stdout, + ]; + const delivered = lastColumns(frame); + const expected = lastColumns(source); + expect(delivered).toHaveLength(8); + for (const [i, v] of delivered.entries()) + expect(Math.abs(v - expected[i]!)).toBeLessThanOrEqual(5); + }, 60_000); + + async function proresFrame(name: string, pixFmt: string, profile: string, source: string) { + const clip = join(FIXTURE_DIR, `${name}.mov`); + const synth = await runFfmpeg( + ["-y", "-loglevel", "error", "-f", "lavfi", "-i", source].concat([ + "-c:v", + "prores_ks", + "-profile:v", + profile, + "-pix_fmt", + pixFmt, + clip, + ]), + ); + expect(synth.success).toBe(true); + const outDir = join(FIXTURE_DIR, `out-${name}`); + mkdirSync(outDir, { recursive: true }); + const result = await extractAllVideoFrames( + [{ ...fixtureVideo(), id: name, src: clip }], + FIXTURE_DIR, + { fps: 1, outputDir: outDir }, + ); + expect(result.errors).toEqual([]); + return result.extracted[0]!.framePaths.get(0)!; + } + + it("extracts ProRes without an alpha plane to jpg", async () => { + const frame = await proresFrame("prores422", "yuv422p10le", "2", "testsrc2=s=320x180:d=1:r=1"); + expect(frame.endsWith(".jpg")).toBe(true); + }, 60_000); + + it("keeps the alpha of ProRes 4444 exact in png frames", async () => { + const frame = await proresFrame( + "prores4444", + "yuva444p10le", + "4", + "color=c=0x00ff00@0.5:s=320x180:d=1:r=1,format=yuva444p10le", + ); + expect(frame.endsWith(".png")).toBe(true); + const alpha = spawnSync("ffmpeg", [ + "-v", + "error", + "-i", + frame, + "-vf", + "alphaextract,crop=1:1:10:10", + "-f", + "rawvideo", + "-", + ]); + expect(Math.abs(alpha.stdout[0]! - 128)).toBeLessThanOrEqual(1); + }, 60_000); + + it("keeps color-sensitive UI reds closer to source when extraction is forced to png", async () => { + const defaultOut = join(FIXTURE_DIR, "out-default"); + const pngOut = join(FIXTURE_DIR, "out-png"); + mkdirSync(defaultOut, { recursive: true }); + mkdirSync(pngOut, { recursive: true }); + + const defaultResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, { + fps: 1, + outputDir: defaultOut, + }); + const pngResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, { + fps: 1, + outputDir: pngOut, + format: "png", + }); + + expect(defaultResult.errors).toEqual([]); + expect(pngResult.errors).toEqual([]); + const defaultFrame = defaultResult.extracted[0]!.framePaths.get(0)!; + const pngFrame = pngResult.extracted[0]!.framePaths.get(0)!; + expect(defaultFrame.endsWith(".jpg")).toBe(true); + expect(pngFrame.endsWith(".png")).toBe(true); + + let worstDefaultDelta = 0; + let worstPngDelta = 0; + for (const [x, y] of RED_SAMPLE_PIXELS) { + const sourcePixel = readFirstFramePixel(UI_FIXTURE, x, y); + worstDefaultDelta = Math.max( + worstDefaultDelta, + maxChannelDelta(sourcePixel, readFirstFramePixel(defaultFrame, x, y)), + ); + worstPngDelta = Math.max( + worstPngDelta, + maxChannelDelta(sourcePixel, readFirstFramePixel(pngFrame, x, y)), + ); + } + + expect(worstPngDelta).toBeLessThanOrEqual(5); + expect(worstPngDelta).toBeLessThanOrEqual(worstDefaultDelta); + }, 60_000); + + // Chrome runs with --force-color-profile=srgb and colour-manages any frame not tagged sRGB; + // the SDR encoder writes the canvas code values as they are, so frames must reach it unconverted. + it.each([ + ["BT.709", "bt709"], + ["BT.601", "smpte170m"], + ])( + "declares png frames of a %s source as sRGB so Chrome shows their code values", + async (name, tag) => { + const fixture = join(FIXTURE_DIR, `${tag}-shadow.mp4`); + const synth = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `color=c=0x101010:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`, + "-vf", + `setparams=color_primaries=${tag}:color_trc=${tag}:colorspace=${tag}:range=tv`, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "0", + "-pix_fmt", + "yuv420p", + fixture, + ]); + if (!synth.success) { + throw new Error(`${name} fixture synthesis failed: ${synth.stderr.slice(-400)}`); + } + expect((await extractVideoMetadata(fixture)).colorSpace?.colorTransfer).toBe(tag); + const outputDir = join(FIXTURE_DIR, `out-png-transfer-${tag}`); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractAllVideoFrames( + [{ ...fixtureVideo(), id: tag, src: fixture }], + FIXTURE_DIR, + { fps: 1, outputDir, format: "png" }, + ); + + expect(result.errors).toEqual([]); + const frame = result.extracted[0]!.framePaths.get(0)!; + // Chrome reads cICP before sRGB, and ffmpeg < 6.0 writes no cICP, so only an sRGB chunk + // without cICP means "these are sRGB code values" on every ffmpeg. + const chunks = pngChunkTypes(frame); + expect(chunks).toContain("sRGB"); + expect(chunks).not.toContain("cICP"); + expect(readFirstFramePixel(frame, 10, 10)).toEqual(readFirstFramePixel(fixture, 10, 10)); + }, + 60_000, + ); + + it("extracts jpg frames of a BT.709 source as the BT.601 full range JPEG readers assume", async () => { + const fixture = join(FIXTURE_DIR, "bt709-patch.mp4"); + const synth = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1,format=rgb24,drawbox=x=31:y=0:w=64:h=${UI_FIXTURE_HEIGHT}:c=0x0000FE:t=fill`, + "-vf", + "scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "0", + fixture, + ]); + if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); + const outputDir = join(FIXTURE_DIR, "out-jpg-matrix"); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractAllVideoFrames( + [{ ...fixtureVideo(), id: "bt709-jpg", src: fixture }], + FIXTURE_DIR, + { fps: 1, outputDir, format: "jpg" }, + ); + + expect(result.errors).toEqual([]); + const frame = result.extracted[0]!.framePaths.get(0)!; + // x=10 is flat colour; x=32 sits one pixel inside the blue edge. + for (const x of [10, 32]) { + const source = readFirstFramePixel(fixture, x, 10); + const shown = readFirstFramePixel(frame, x, 10); + const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!))); + expect(worst, `x=${x}: source ${source} jpg ${shown}`).toBeLessThanOrEqual(3); + } + }, 60_000); + + // Measured in Chrome 152: untagged H.264 plays as BT.601 at 1280x718 and BT.709 at 1280x720 or 406x720; + // untagged VP9 and AV1 play as BT.601 at every size. + it.each([ + { codec: "libx264", height: 718, matrix: "bt601", format: "png" }, + { codec: "libx264", height: 718, matrix: "bt601", format: "jpg" }, + { codec: "libx264", height: 720, matrix: "bt709", format: "png" }, + { codec: "libx264", height: 720, matrix: "bt709", format: "jpg" }, + { codec: "libvpx-vp9", height: 720, matrix: "bt601", format: "png" }, + ...(AV1_ENCODER + ? [{ codec: AV1_ENCODER, height: 720, matrix: "bt601", format: "png" } as const] + : []), + ] as const)( + "reads an untagged $height-line $codec source as $matrix like Chrome's own playback ($format)", + async ({ codec, height, matrix, format }) => { + const extension = codec === "libvpx-vp9" ? "webm" : "mp4"; + const fixture = join(FIXTURE_DIR, `untagged-${codec}-${height}.${extension}`); + const synth = await runFfmpeg([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + `color=c=0xC83C28:s=64x${height}:d=1:r=1`, + "-vf", + "scale=out_color_matrix=bt601:out_range=tv,format=yuv420p,setparams=colorspace=unknown:color_primaries=unknown:color_trc=unknown:range=tv", + "-c:v", + codec, + ...(codec === "libx264" ? ["-qp", "0"] : ["-crf", "0", "-b:v", "0"]), + ...(codec === "libaom-av1" ? ["-cpu-used", "8"] : []), + fixture, + ]); + if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); + expect((await extractVideoMetadata(fixture)).colorSpace?.colorSpace ?? "unknown").toBe( + "unknown", + ); + + const result = await extractVideoFramesRange( + fixture, + `${basename(fixture)}-${format}`, + 0, + 1, + { + fps: 1, + outputDir: join(FIXTURE_DIR, "out-untagged"), + format, + }, + ); + + const pixelAt = (file: string, decode: string): number[] => [ + ...spawnSync("ffmpeg", [ + "-v", + "error", + "-i", + file, + "-vf", + `${decode}format=rgb24,crop=1:1:10:10`, + "-frames:v", + "1", + "-f", + "rawvideo", + "-", + ]).stdout, + ]; + const expected = pixelAt(fixture, `scale=in_color_matrix=${matrix}:in_range=tv,`); + const shown = pixelAt(result.framePaths.get(0)!, ""); + expect([expected.length, shown.length]).toEqual([3, 3]); + const worst = Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!))); + expect(worst, `${matrix} decode ${expected}, ${format} ${shown}`).toBeLessThanOrEqual( + format === "png" ? 1 : 3, + ); + }, + 60_000, + ); + + it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => { + // Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks. + const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4"); + const synth = await runFfmpeg([ + ...["-y", "-v", "error", "-f", "lavfi", "-i"], + "color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128", + ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"], + fixture, + ]); + if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); + + const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, { + fps: 30, + outputDir: join(FIXTURE_DIR, "out-ntsc-ms"), + format: "png", + }); + const indexOf = (file: string): number => { + const row = spawnSync("ffmpeg", [ + "-v", + "error", + "-i", + file, + "-vf", + "format=gray,crop=64:1:0:4", + "-f", + "rawvideo", + "-", + ]).stdout; + return [...Array(8).keys()].reduce( + (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0), + 0, + ); + }; + const shown = [...result.framePaths.keys()] + .sort((a, b) => a - b) + .map((i) => indexOf(result.framePaths.get(i)!)); + const steps = shown.slice(1).map((n, i) => n - shown[i]!); + + expect(shown.length).toBeGreaterThanOrEqual(44); + expect( + steps.filter((step) => step > 1), + `source frames ${shown.join(",")}`, + ).toEqual([]); + expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1); + }, 60_000); + + it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => { + const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4"); + const synth = await runFfmpeg([ + ...["-y", "-v", "error", "-f", "lavfi", "-i"], + "color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128", + ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"], + fixture, + ]); + if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); + const indexOf = (file: string): number => { + const row = spawnSync("ffmpeg", [ + "-v", + "error", + "-i", + file, + "-vf", + "format=gray,crop=64:1:0:4", + "-f", + "rawvideo", + "-", + ]).stdout; + return [...Array(8).keys()].reduce( + (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0), + 0, + ); + }; + const options = { + fps: 30, + outputDir: join(FIXTURE_DIR, "out-ntsc-seams"), + format: "png" as const, + }; + const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options); + + const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201]; + const differing: string[] = []; + for (const firstFrame of seams) { + const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, { + ...options, + frameRanges: [{ firstFrame, frames: 12 }], + }); + for (const [slot, path] of segment.framePaths) { + const expected = indexOf(single.framePaths.get(slot)!); + const shown = indexOf(path); + if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`); + } + } + expect(differing).toEqual([]); + }, 120_000); + + // ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour + // filters would cut the still that ends this window short. + it.each([ + { + name: "full range BT.709", + format: "png", + codec: "libx264", + pixFmt: "yuvj420p", + probed: { colorTransfer: "bt709" }, + tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709", + }, + { + name: "full range BT.709", + format: "jpg", + codec: "libx264", + pixFmt: "yuvj420p", + probed: { colorTransfer: "bt709" }, + tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709", + }, + { + name: "BT.470BG gamma 2.8", + format: "png", + codec: "libx264", + pixFmt: "yuv420p", + probed: { colorTransfer: "bt470bg" }, + tags: "color_primaries=bt470bg:color_trc=bt470bg:colorspace=bt470bg", + }, + { + name: "RGB", + format: "png", + codec: "libx264rgb", + pixFmt: "rgb24", + probed: { colorSpace: "gbr" }, + tags: "", + }, + ] as const)( + "keeps every frame and the colours of a $name VFR window that ends on a still ($format)", + async ({ name, format, codec, pixFmt, probed, tags }) => { + const fixture = join(FIXTURE_DIR, `vfr-still-${name.replace(/\W+/g, "-")}-${format}.mp4`); + const synth = await runFfmpeg([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + `color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=2:r=60`, + "-vf", + `select='not(between(n\\,30\\,89))'${tags ? `,setparams=${tags}` : ""}`, + "-fps_mode", + "vfr", + "-c:v", + codec, + "-preset", + "ultrafast", + "-pix_fmt", + pixFmt, + fixture, + ]); + if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); + const window = 0.616666; + const unfilteredDir = join(FIXTURE_DIR, `out-${basename(fixture)}-unfiltered`); + mkdirSync(unfilteredDir, { recursive: true }); + const unfiltered = await runFfmpeg([ + "-v", + "error", + "-ss", + "0", + "-i", + fixture, + "-t", + String(window), + "-fps_mode", + "cfr", + "-r", + "30", + join(unfilteredDir, "f_%05d.png"), + ]); + expect(unfiltered.success).toBe(true); + + const result = await extractVideoFramesRange(fixture, basename(fixture), 0, window, { + fps: 30, + outputDir: join(FIXTURE_DIR, "out-vfr-still"), + format, + }); + + expect(result.metadata.isVFR).toBe(true); + expect(result.metadata.colorSpace).toMatchObject(probed); + expect(result.totalFrames).toBe(readdirSync(unfilteredDir).length); + const last = result.framePaths.get(result.totalFrames - 1)!; + const source = readFirstFramePixel(fixture, 10, 10); + const shown = readFirstFramePixel(last, 10, 10); + const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!))); + expect(worst, `source ${source} ${format} ${shown}`).toBeLessThanOrEqual( + format === "png" ? 0 : 3, + ); + if (format === "png") expect(pngChunkTypes(last)).toContain("sRGB"); + }, + 60_000, + ); + + it("leaves SDR-to-HDR frames in the HDR colours they were converted to", async () => { + const outputDir = join(FIXTURE_DIR, "out-sdr-to-hdr-gate"); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractVideoFramesRange(UI_FIXTURE, "sdr-to-hdr-gate", 0, 1, { + fps: 1, + outputDir, + format: "png", + sdrToHdrTransfer: "pq", + }); + + expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB"); + }, 60_000); + + it("keeps jpg and png extraction caches separate", async () => { + const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-")); + try { + const defaultOut = join(FIXTURE_DIR, "cache-default"); + const pngOut = join(FIXTURE_DIR, "cache-png"); + const pngHitOut = join(FIXTURE_DIR, "cache-png-hit"); + mkdirSync(defaultOut, { recursive: true }); + mkdirSync(pngOut, { recursive: true }); + mkdirSync(pngHitOut, { recursive: true }); + + const defaultResult = await extractAllVideoFrames( + [fixtureVideo()], + FIXTURE_DIR, + { fps: 1, outputDir: defaultOut }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(defaultResult.errors).toEqual([]); + expect(defaultResult.phaseBreakdown.cacheHits).toBe(0); + expect(defaultResult.phaseBreakdown.cacheMisses).toBe(1); + expect(defaultResult.extracted[0]!.framePaths.get(0)!.endsWith(".jpg")).toBe(true); + + const pngMiss = await extractAllVideoFrames( + [fixtureVideo()], + FIXTURE_DIR, + { fps: 1, outputDir: pngOut, format: "png" }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(pngMiss.errors).toEqual([]); + expect(pngMiss.phaseBreakdown.cacheHits).toBe(0); + expect(pngMiss.phaseBreakdown.cacheMisses).toBe(1); + expect(pngMiss.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true); + + const pngHit = await extractAllVideoFrames( + [fixtureVideo()], + FIXTURE_DIR, + { fps: 1, outputDir: pngHitOut, format: "png" }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(pngHit.errors).toEqual([]); + expect(pngHit.phaseBreakdown.cacheHits).toBe(1); + expect(pngHit.phaseBreakdown.cacheMisses).toBe(0); + expect(pngHit.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true); + } finally { + rmSync(cacheDir, { recursive: true, force: true }); + } + }, 60_000); + + it("dedupes identical extractions within one render", async () => { + const outputDir = join(FIXTURE_DIR, "out-dedupe"); + mkdirSync(outputDir, { recursive: true }); + + const videoA: VideoElement = { ...fixtureVideo(), id: "dupe-a" }; + const videoB: VideoElement = { ...fixtureVideo(), id: "dupe-b" }; + + const result = await extractAllVideoFrames([videoA, videoB], FIXTURE_DIR, { + fps: 1, + outputDir, + }); + + expect(result.errors).toEqual([]); + expect(result.extracted).toHaveLength(2); + const first = result.extracted[0]!; + const second = result.extracted[1]!; + expect(first.videoId).toBe("dupe-a"); + expect(second.videoId).toBe("dupe-b"); + expect(second.outputDir).toBe(first.outputDir); + expect(Array.from(second.framePaths.entries())).toEqual(Array.from(first.framePaths.entries())); + + const frameDirs = readdirSync(outputDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name); + expect(frameDirs).toEqual(["dupe-a"]); + expect(readdirSync(first.outputDir).filter((f) => f.endsWith(".jpg"))).toHaveLength( + first.totalFrames, + ); + }, 60_000); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.hdr.test.ts b/packages/engine/src/services/videoFrameExtractor.hdr.test.ts new file mode 100644 index 0000000000..01d2b48ba0 --- /dev/null +++ b/packages/engine/src/services/videoFrameExtractor.hdr.test.ts @@ -0,0 +1,675 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { + extractAllVideoFrames, + extractVideoFramesRange, + type VideoElement, + type ExtractionResult, +} from "./videoFrameExtractor.js"; +import { extractVideoMetadata } from "../utils/ffprobe.js"; +import { runFfmpeg } from "../utils/runFfmpeg.js"; +import { SCHEMA_PREFIX } from "./extractionCache.js"; + +// Pass-through, so a test can tell whether extraction ran ffmpeg. +vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { + const real = await importOriginal(); + return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) }; +}); + +const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; +const HAS_ZSCALE = + HAS_FFMPEG && + /\szscale\s/.test(spawnSync("ffmpeg", ["-hide_banner", "-filters"]).stdout.toString()); + +type Rgb = [number, number, number]; + +const UI_FIXTURE_WIDTH = 240; +const UI_FIXTURE_HEIGHT = 160; + +function pngChunkTypes(path: string): string[] { + const bytes = readFileSync(path); + const types: string[] = []; + for (let offset = 8; offset + 8 <= bytes.length; offset += 12 + bytes.readUInt32BE(offset)) { + types.push(bytes.toString("latin1", offset + 4, offset + 8)); + } + return types; +} + +function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb { + const result = spawnSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + mediaPath, + "-frames:v", + "1", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "pipe:1", + ], + { maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 }, + ); + if (result.status !== 0) { + throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`); + } + + const offset = (y * UI_FIXTURE_WIDTH + x) * 3; + return [ + result.stdout[offset] ?? 0, + result.stdout[offset + 1] ?? 0, + result.stdout[offset + 2] ?? 0, + ]; +} + +// Release builds print "version 8.1.3" or "n8.1.3"; master builds print "N-" and are newer. +const FFPROBE_VERSION = spawnSync("ffprobe", ["-version"]).stdout?.toString() ?? ""; +const FFPROBE_RELEASE = FFPROBE_VERSION.match(/version n?(\d+)\.(\d+)/) + ?.slice(1) + .map(Number); +const FFPROBE_READS_CONTAINER_TRANSFER = + /version N-/.test(FFPROBE_VERSION) || + (FFPROBE_RELEASE !== undefined && + (FFPROBE_RELEASE[0]! > 8 || (FFPROBE_RELEASE[0] === 8 && FFPROBE_RELEASE[1]! >= 1))); + +/** The colour probes run so far: first-frame reads, and SPS transfer reads of an HLG source. */ +function colourProbeRuns() { + const runs = vi + .mocked(runFfmpeg) + .mock.calls.map(([args]) => args) + .filter((args) => args.some((arg) => /showinfo|trace_headers/.test(arg))); + return { + firstFrame: runs.filter((args) => args.includes("-frames:v")).length, + keyframes: runs.filter((args) => args.includes("nokey")).length, + }; +} + +function colourProbesSince(before: ReturnType) { + const now = colourProbeRuns(); + return { + firstFrame: now.firstFrame - before.firstFrame, + keyframes: now.keyframes - before.keyframes, + }; +} + +describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => { + let fixtureDir = ""; + + beforeAll(() => { + fixtureDir = mkdtempSync(join(tmpdir(), "hf-forced-sdr-tonemap-test-")); + }); + + afterAll(() => { + rmSync(fixtureDir, { recursive: true, force: true }); + }, 30_000); + + async function synthesizeHlgClip( + path: string, + vui = "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9", + transfer = "arib-std-b67", + ): Promise { + const synthesized = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=0xe0b080:s=64x64:r=1:d=1", + "-vf", + `zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=${transfer}:m=bt2020nc:r=tv,format=yuv420p`, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-color_primaries", + "bt2020", + "-color_trc", + transfer, + "-colorspace", + "bt2020nc", + "-bsf:v", + `h264_metadata=${vui}`, + path, + ]); + if (!synthesized.success) { + throw new Error(`HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`); + } + } + + it.each([ + [ + "HLG", + "matrix", + "arib-std-b67", + "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=2", + ], + [ + "HLG", + "primaries", + "arib-std-b67", + "colour_primaries=2:transfer_characteristics=18:matrix_coefficients=9", + ], + [ + "HLG", + "matrix and primaries", + "arib-std-b67", + "colour_primaries=2:transfer_characteristics=18:matrix_coefficients=2", + ], + [ + "PQ", + "matrix and primaries", + "smpte2084", + "colour_primaries=2:transfer_characteristics=16:matrix_coefficients=2", + ], + [ + "HLG", + "transfer", + "arib-std-b67", + "colour_primaries=9:transfer_characteristics=2:matrix_coefficients=9", + ], + ])( + "tone-maps %s footage whose video stream has no %s tag like the fully tagged clip", + async (name, missing, transfer, vui) => { + const slug = `${name}-no-${missing}`.replace(/\W+/g, "-"); + const tagged = join(fixtureDir, `${name}-fully-tagged.mp4`); + const untagged = join(fixtureDir, `${slug}.mp4`); + const vuiTransfer = transfer === "smpte2084" ? 16 : 18; + await synthesizeHlgClip( + tagged, + `colour_primaries=9:transfer_characteristics=${vuiTransfer}:matrix_coefficients=9`, + transfer, + ); + await synthesizeHlgClip(untagged, vui, transfer); + const extract = (source: string, id: string) => + extractVideoFramesRange(source, id, 0, 1, { + fps: 1, + outputDir: join(fixtureDir, `out-${id}`), + format: "png", + toneMapHdrToSdr: true, + }); + + if (missing === "transfer") { + // ffprobe before 8.1 reads the stream's unspecified transfer, so the clip is SDR there, as on main. + if (!FFPROBE_READS_CONTAINER_TRANSFER) { + await expect(extract(untagged, slug)).resolves.toBeDefined(); + return; + } + expect((await extractVideoMetadata(untagged)).colorSpace?.colorTransfer).toBe(transfer); + } + const reference = await extract(tagged, `ref-${slug}`); + const result = await extract(untagged, slug); + + expect(readFileSync(result.framePaths.get(0)!)).toEqual( + readFileSync(reference.framePaths.get(0)!), + ); + }, + 60_000, + ); + + it("probes each source's first frame once, however many cuts of it extract together", async () => { + const sources = ["a", "b"].map((name) => join(fixtureDir, `cuts-${name}.mp4`)); + for (const source of sources) await synthesizeHlgClip(source); + const probes = colourProbeRuns; + const before = probes(); + await Promise.all( + Array.from({ length: 12 }, (_, cut) => + extractVideoFramesRange(sources[cut % 2]!, `cut-${cut}`, 0, 1, { + fps: 1, + outputDir: join(fixtureDir, "cuts"), + format: "png", + toneMapHdrToSdr: true, + }), + ), + ); + expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 }); + }, 60_000); + + it("probes a source again after its first-frame probe failed", async () => { + const source = join(fixtureDir, "probe-retry.mp4"); + await synthesizeHlgClip(source); + const probes = colourProbeRuns; + const { runFfmpeg: real } = + await vi.importActual("../utils/runFfmpeg.js"); + vi.mocked(runFfmpeg).mockImplementation((args, opts) => + args.includes("showinfo") + ? Promise.resolve({ + success: false, + exitCode: 1, + stderr: "", + durationMs: 0, + terminationReason: "exit", + }) + : real(args, opts), + ); + const extract = (id: string) => + extractVideoFramesRange(source, id, 0, 1, { + fps: 1, + outputDir: join(fixtureDir, "retry"), + format: "png", + toneMapHdrToSdr: true, + }); + const before = probes(); + try { + await extract("retry-failed"); + } finally { + vi.mocked(runFfmpeg).mockImplementation(real); + } + await extract("retry-again"); + expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 1 }); + }, 60_000); + + it("tone-maps an HLG source at HLG's white when a large header SEI follows its SPS", async () => { + const source = join(fixtureDir, "hlg-x265-header.mp4"); + // x265 stores its settings as an SEI after the SPS in the file header; traced, it runs past 64 KB. + // No B-frames: at 1 fps their reorder gap is a whole second, which Windows ffmpeg's mp4 muxer can refuse. + const synthesized = await runFfmpeg([ + ...["-y", "-hide_banner", "-loglevel", "error"], + ...["-f", "lavfi", "-i", "color=c=0xe0b080:s=64x64:r=1:d=1"], + "-vf", + "zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=arib-std-b67:m=bt2020nc:r=tv,format=yuv420p10le", + ...[ + "-c:v", + "libx265", + "-x265-params", + "colorprim=bt2020:transfer=arib-std-b67:colormatrix=bt2020nc:log-level=error:bframes=0", + ], + ...["-color_primaries", "bt2020", "-color_trc", "arib-std-b67", "-colorspace", "bt2020nc"], + source, + ]); + if (!synthesized.success) { + throw new Error(`x265 HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`); + } + const before = vi.mocked(runFfmpeg).mock.calls.length; + await extractVideoFramesRange(source, "hlg-x265-header", 0, 1, { + fps: 1, + outputDir: join(fixtureDir, "x265-header"), + format: "png", + toneMapHdrToSdr: true, + }); + const toneMaps = vi + .mocked(runFfmpeg) + .mock.calls.slice(before) + .flatMap(([args]) => args.filter((arg) => arg.includes("tonemap="))); + expect(toneMaps).toHaveLength(1); + expect(toneMaps[0]).toContain("npl=203"); + }, 60_000); + + it("reads an HLG source's keyframes again after that read failed", async () => { + const source = join(fixtureDir, "keyframe-retry.mp4"); + await synthesizeHlgClip(source); + const { runFfmpeg: real } = + await vi.importActual("../utils/runFfmpeg.js"); + vi.mocked(runFfmpeg).mockImplementation((args, opts) => + args.includes("nokey") + ? Promise.resolve({ + success: false, + exitCode: 1, + stderr: "", + durationMs: 0, + terminationReason: "exit", + }) + : real(args, opts), + ); + const extract = (id: string) => + extractVideoFramesRange(source, id, 0, 1, { + fps: 1, + outputDir: join(fixtureDir, "keyframe-retry"), + format: "png", + toneMapHdrToSdr: true, + }); + const before = colourProbeRuns(); + try { + await extract("keyframes-failed"); + } finally { + vi.mocked(runFfmpeg).mockImplementation(real); + } + await extract("keyframes-again"); + expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 }); + }, 60_000); + + // One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer. + it("tone-maps each frame of a mixed HLG and PQ stream with its own transfer", async () => { + const segment = async (name: string, transfer: number) => { + const path = join(fixtureDir, `mixed-${name}.h264`); + const result = await runFfmpeg([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=s=160x90:r=25:d=0.2", + "-c:v", + "libx264", + // No B-frames: genpts stamps a raw stream in decode order, so reordered frames get wrong times. + "-bf", + "0", + "-pix_fmt", + "yuv420p", + "-bsf:v", + `h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`, + "-f", + "h264", + path, + ]); + if (!result.success) + throw new Error(`segment synthesis failed: ${result.stderr.slice(-400)}`); + return readFileSync(path); + }; + const mux = async (name: string, stream: Buffer) => { + const raw = join(fixtureDir, `${name}.h264`); + const path = join(fixtureDir, `${name}.mp4`); + writeFileSync(raw, stream); + const result = await runFfmpeg([ + ...["-y", "-v", "error", "-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw], + ...["-c", "copy", path], + ]); + if (!result.success) throw new Error(`mux failed: ${result.stderr.slice(-400)}`); + return path; + }; + const hlg = await segment("hlg", 18); + const pq = await segment("pq", 16); + const mixed = await mux("mixed-hlg-pq", Buffer.concat([hlg, pq])); + const pqOnly = await mux("mixed-pq-only", pq); + const extract = (source: string, id: string, duration: number) => + extractVideoFramesRange(source, id, 0, duration, { + fps: 25, + outputDir: join(fixtureDir, `out-${id}`), + format: "png", + toneMapHdrToSdr: true, + }); + + const mixedFrames = await extract(mixed, "mixed-hlg-pq", 0.4); + const pqFrames = await extract(pqOnly, "mixed-pq-only", 0.2); + + expect(readFileSync(mixedFrames.framePaths.get(5)!)).toEqual( + readFileSync(pqFrames.framePaths.get(0)!), + ); + }, 60_000); + + // A stream-copy trim opens on edit-list pre-roll, which ffprobe counts as packets without frames. + it("keeps per-frame tags in a mixed HLG and PQ file trimmed with a stream copy", async () => { + const run = async (args: string[]) => { + const result = await runFfmpeg(["-y", "-v", "error", ...args]); + if (!result.success) throw new Error(`fixture failed: ${result.stderr.slice(-400)}`); + }; + const segment = async (name: string, transfer: number) => { + const path = join(fixtureDir, `trim-${name}.h264`); + await run([ + ...["-f", "lavfi", "-i", "testsrc2=s=160x90:r=25:d=2"], + ...["-c:v", "libx264", "-g", "50", "-bf", "0"], + ...["-pix_fmt", "yuv420p", "-bsf:v"], + `h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`, + ...["-f", "h264", path], + ]); + return readFileSync(path); + }; + const mux = async (name: string, stream: Buffer) => { + const raw = join(fixtureDir, `${name}.h264`); + const path = join(fixtureDir, `${name}.mp4`); + writeFileSync(raw, stream); + await run(["-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw, "-c", "copy", path]); + return path; + }; + const pq = await segment("pq", 16); + const mixed = await mux("trim-hlg-then-pq", Buffer.concat([await segment("hlg", 18), pq])); + const pqOnly = await mux("trim-pq-only", pq); + const trimmed = join(fixtureDir, "trim-from-hlg.mp4"); + await run(["-ss", "1", "-i", mixed, "-c", "copy", trimmed]); + const extract = (source: string, id: string, start: number) => + extractVideoFramesRange(source, id, start, 0.04, { + fps: 25, + outputDir: join(fixtureDir, `out-${id}`), + format: "png", + toneMapHdrToSdr: true, + }); + + // Trimmed 1.6 s is the mixed file's 2.6 s: the PQ segment's 0.6 s. + const fromTrim = await extract(trimmed, "trim-from-hlg", 1.6); + const fromPq = await extract(pqOnly, "trim-pq-only", 0.6); + + const shown = readFirstFramePixel(fromTrim.framePaths.get(0)!, 40, 40); + const expected = readFirstFramePixel(fromPq.framePaths.get(0)!, 40, 40); + expect(Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)))).toBeLessThanOrEqual(2); + }, 60_000); + + it("matches Studio's HLG tone map and isolates transformed cache entries", async () => { + const source = join(fixtureDir, "hlg-warm.mp4"); + await synthesizeHlgClip(source); + + const reference = join(fixtureDir, "studio-reference.png"); + const referenceResult = await runFfmpeg([ + "-y", + "-ss", + "0", + "-i", + source, + "-t", + "1", + "-vf", + "fps=1,zscale=t=linear:npl=203,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv", + "-q:v", + "0", + "-compression_level", + "1", + reference, + ]); + if (!referenceResult.success) { + throw new Error(`Studio reference extraction failed: ${referenceResult.stderr.slice(-400)}`); + } + + const cacheDir = join(fixtureDir, "cache"); + const video = (id: string): VideoElement => ({ + id, + src: source, + start: 0, + end: 1, + mediaStart: 0, + loop: false, + hasAudio: false, + }); + const extract = (id: string, toneMapHdrToSdr = false) => + extractAllVideoFrames( + [video(id)], + fixtureDir, + { + fps: 1, + outputDir: join(fixtureDir, id), + format: "png", + toneMapHdrToSdr, + }, + undefined, + { extractCacheDir: cacheDir }, + ); + + const plain = await extract("plain"); + const toneMapped = await extract("tone-mapped", true); + const toneMappedAgain = await extract("tone-mapped-again", true); + + expect(plain.errors).toEqual([]); + expect(toneMapped.errors).toEqual([]); + expect(toneMapped.phaseBreakdown.cacheHits).toBe(0); + expect(toneMapped.phaseBreakdown.cacheMisses).toBe(1); + expect(toneMappedAgain.phaseBreakdown.cacheHits).toBe(1); + expect(readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX))).toHaveLength(2); + + const frame = (result: ExtractionResult): Buffer => { + const path = result.extracted[0]?.framePaths.get(0); + if (!path) throw new Error("expected extracted frame"); + return readFileSync(path); + }; + // Same pixels as Studio's tone map, declared as sRGB so Chrome shows them unconverted. + const rgb = (path: string): Buffer => + spawnSync("ffmpeg", ["-v", "error", "-i", path, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"]) + .stdout; + const toneMappedPath = toneMapped.extracted[0]!.framePaths.get(0)!; + expect(rgb(toneMappedPath)).toEqual(rgb(reference)); + expect(pngChunkTypes(toneMappedPath)).toContain("sRGB"); + expect(pngChunkTypes(toneMappedPath)).not.toContain("cICP"); + expect(frame(plain)).not.toEqual(frame(toneMapped)); + + const toneMappedJpg = await extractAllVideoFrames([video("tone-mapped-jpg")], fixtureDir, { + fps: 1, + outputDir: join(fixtureDir, "tone-mapped-jpg"), + format: "jpg", + toneMapHdrToSdr: true, + }); + expect(toneMappedJpg.errors).toEqual([]); + const jpgPixels = rgb(toneMappedJpg.extracted[0]!.framePaths.get(0)!); + const referencePixels = rgb(reference); + const worst = Math.max(...[...jpgPixels].map((v, i) => Math.abs(v - referencePixels[i]!))); + expect(worst, "tone-mapped jpg against Studio's tone map").toBeLessThanOrEqual(3); + expect(frame(toneMappedAgain)).toEqual(frame(toneMapped)); + + // A macOS ffmpeg that has zscale uses the same tone map, not VideoToolbox. + const platform = Object.getOwnPropertyDescriptor(process, "platform")!; + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + const mac = await extractAllVideoFrames([video("tone-mapped-mac")], fixtureDir, { + fps: 1, + outputDir: join(fixtureDir, "tone-mapped-mac"), + format: "png", + toneMapHdrToSdr: true, + }); + expect(mac.errors).toEqual([]); + expect(rgb(mac.extracted[0]!.framePaths.get(0)!)).toEqual(rgb(reference)); + } finally { + Object.defineProperty(process, "platform", platform); + } + }, 60_000); + + it("warns once and keeps VideoToolbox, under its own cache key, when a macOS ffmpeg has no zscale", async () => { + const source = join(fixtureDir, "hlg-no-zscale.mp4"); + await synthesizeHlgClip(source); + const cacheDir = join(fixtureDir, "no-zscale-cache"); + const clip = (id: string): VideoElement => ({ + id, + src: source, + start: 0, + end: 1, + mediaStart: 0, + loop: false, + hasAudio: false, + }); + const options = (id: string) => ({ + fps: 1, + outputDir: join(fixtureDir, id), + format: "png" as const, + toneMapHdrToSdr: true, + }); + const zscale = await extractAllVideoFrames( + [clip("zscale")], + fixtureDir, + options("zscale"), + undefined, + { + extractCacheDir: cacheDir, + }, + ); + expect(zscale.errors).toEqual([]); + + const realPlatform = process.platform; + vi.resetModules(); + vi.doMock("../utils/psnrFilterAvailability.js", () => ({ + isFfmpegFilterAvailable: async () => false, + isPsnrFilterAvailable: async () => false, + })); + const platform = Object.getOwnPropertyDescriptor(process, "platform")!; + Object.defineProperty(process, "platform", { value: "darwin" }); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + try { + const { extractAllVideoFrames: extractOnMac } = await import("./videoFrameExtractor.js"); + for (const [i, id] of ["no-zscale-1", "no-zscale-2"].entries()) { + const result = await extractOnMac([clip(id)], fixtureDir, options(id), undefined, { + extractCacheDir: cacheDir, + }); + // The zscale frames above must not be served for the VideoToolbox path. + if (i === 0) expect(result.phaseBreakdown.cacheHits).toBe(0); + // Off macOS the VideoToolbox decode itself fails, which proves it was attempted. + if (realPlatform !== "darwin") + expect(JSON.stringify(result.errors)).toMatch(/videotoolbox/i); + } + const zscaleWarnings = stderr.mock.calls.filter(([message]) => + String(message).includes("no zscale filter"), + ); + expect(zscaleWarnings).toHaveLength(1); + } finally { + stderr.mockRestore(); + Object.defineProperty(process, "platform", platform); + vi.doUnmock("../utils/psnrFilterAvailability.js"); + vi.resetModules(); + } + }, 60_000); + + // A second process would read the frames back without the HDR10 light-level metadata the tone map uses. + it("tone-maps a VFR HDR10 window in one process, as the reference tone map would", async () => { + const source = join(fixtureDir, "pq-vfr-4000nit.mp4"); + const synthesized = await runFfmpeg([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=0xf0e0c0:s=64x64:r=60:d=2", + "-vf", + "select='not(between(n\\,30\\,89))',zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=smpte2084:m=bt2020nc:r=tv:npl=1000,format=yuv420p10le", + "-fps_mode", + "vfr", + "-c:v", + "libx265", + "-x265-params", + "log-level=error:hdr10=1:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:max-cll=4000,400:master-display=G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(40000000,50)", + source, + ]); + if (!synthesized.success) + throw new Error(`PQ fixture synthesis failed: ${synthesized.stderr.slice(-400)}`); + const referenceDir = join(fixtureDir, "pq-vfr-reference"); + mkdirSync(referenceDir, { recursive: true }); + const reference = await runFfmpeg([ + "-v", + "error", + "-ss", + "0", + "-i", + source, + "-t", + "0.616666", + "-vf", + "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv,setparams=color_primaries=bt709:color_trc=iec61966-2-1", + "-fps_mode", + "cfr", + "-r", + "30", + "-q:v", + "0", + "-compression_level", + "1", + join(referenceDir, "frame_%05d.png"), + ]); + expect(reference.success, reference.stderr).toBe(true); + + const result = await extractVideoFramesRange(source, "pq-vfr", 0, 0.616666, { + fps: 30, + outputDir: join(fixtureDir, "pq-vfr-out"), + format: "png", + toneMapHdrToSdr: true, + }); + + expect(result.metadata.isVFR).toBe(true); + const referenceFrames = readdirSync(referenceDir).sort(); + // One slot per 30fps tick before 0.616666s; x265 packet durations cut the reference's still short. + expect(result.totalFrames).toBe(19); + expect(readFileSync(result.framePaths.get(0)!)).toEqual( + readFileSync(join(referenceDir, referenceFrames[0]!)), + ); + }, 60_000); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.sampling.test.ts b/packages/engine/src/services/videoFrameExtractor.sampling.test.ts new file mode 100644 index 0000000000..aefdcc8be8 --- /dev/null +++ b/packages/engine/src/services/videoFrameExtractor.sampling.test.ts @@ -0,0 +1,386 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { + extractAllVideoFrames, + extractVideoFramesRange, + type VideoElement, + type ExtractedFrames, +} from "./videoFrameExtractor.js"; +import { ffmpegStatsReader, runFfmpeg, type RunFfmpegOptions } from "../utils/runFfmpeg.js"; + +// Pass-through, so a test can tell whether extraction ran ffmpeg. +vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { + const real = await importOriginal(); + return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) }; +}); + +const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; + +// Each output slot shows the frame on screen at its time, as the preview does. +describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => { + const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-sampling-")); + const WIDTH = 32; + const HEIGHT = 16; + // Matroska stores whole-millisecond timestamps (a 30 fps frame at 66.67 ms reads 67 ms); + // a 1/30 timescale stores each frame exactly on a coarse tick. + const SOURCES = { + "mp4-60": { fps: 60, file: "index-60fps.mp4", muxer: [] }, + "mkv-30": { fps: 30, file: "index-30fps.mkv", muxer: [] }, + "mp4-30-timescale-30": { + fps: 30, + file: "index-30fps-ts30.mp4", + muxer: ["-video_track_timescale", "30"], + }, + } as const; + type SourceName = keyof typeof SOURCES; + const sourcePath = (name: SourceName) => join(FIXTURE_DIR, SOURCES[name].file); + + beforeAll(async () => { + for (const name of Object.keys(SOURCES) as SourceName[]) { + // Frame k carries luma 16 + 2k, so each extracted frame names its source index. + const result = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `nullsrc=s=${WIDTH}x${HEIGHT}:r=${SOURCES[name].fps}:d=1.5,geq=lum='16+2*N':cb=128:cr=128`, + "-c:v", + "libx264", + "-qp", + "0", + "-pix_fmt", + "yuv420p", + ...SOURCES[name].muxer, + sourcePath(name), + ]); + if (!result.success) throw new Error(`index fixture synthesis failed: ${result.stderr}`); + } + }, 30_000); + + afterAll(() => { + rmSync(FIXTURE_DIR, { recursive: true, force: true }); + }, 30_000); + + function sourceIndexes(extracted: ExtractedFrames): number[] { + const decoded = spawnSync("ffmpeg", [ + "-v", + "error", + "-i", + join(extracted.outputDir, extracted.framePattern), + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "pipe:1", + ]); + if (decoded.status !== 0) throw new Error(decoded.stderr.toString()); + const indexes: number[] = []; + for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) { + indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2)); + } + return indexes; + } + + it.each([ + { source: "mp4-60", fps: 24, startTime: 0, duration: 0.5 }, + { source: "mp4-60", fps: 10, startTime: 0.37, duration: 0.6 }, + { source: "mkv-30", fps: 30, startTime: 0, duration: 0.5 }, + { source: "mp4-30-timescale-30", fps: 60, startTime: 0, duration: 0.5 }, + ] as const)( + "$source: samples the frame on screen at each $fps fps slot from $startTime s", + async (c) => { + const extracted = await extractVideoFramesRange( + sourcePath(c.source), + `${c.source}-${c.fps}`, + c.startTime, + c.duration, + { fps: c.fps, outputDir: FIXTURE_DIR, format: "png" }, + ); + const onScreen = Array.from({ length: Math.round(c.duration * c.fps) }, (_, i) => + Math.floor((c.startTime + i / c.fps) * SOURCES[c.source].fps + 1e-9), + ); + expect(sourceIndexes(extracted)).toEqual(onScreen); + }, + 30_000, + ); + + // #5260: one ffmpeg process over a 26-minute clip hit ffmpegProcessTimeout. + it.each([ + // 260 frames: one 240-frame segment, then 20 more. + { startTime: 0.5, duration: 130, frames: 260 }, + // 200.3 - 80.3 is 120.00000000000001 s: exactly one segment, not a second near-empty one. + { startTime: 80.3, duration: 200.3 - 80.3, frames: 240 }, + ])( + "extracts $duration s from $startTime s at 2 fps across segments without a gap or repeat", + async (c) => { + const source = join(FIXTURE_DIR, "index-2fps-long.mp4"); + if (!existsSync(source)) { + const synth = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `nullsrc=s=${WIDTH}x${HEIGHT}:r=2:d=205,geq=lum='16+2*mod(N\\,100)':cb=128:cr=128`, + "-c:v", + "libx264", + "-qp", + "0", + "-pix_fmt", + "yuv420p", + source, + ]); + if (!synth.success) throw new Error(`long fixture synthesis failed: ${synth.stderr}`); + } + const extracted = await extractVideoFramesRange( + source, + `long-2fps-${c.startTime}`, + c.startTime, + c.duration, + { fps: 2, outputDir: FIXTURE_DIR, format: "png" }, + ); + const onScreen = Array.from( + { length: c.frames }, + (_, i) => Math.floor((c.startTime + i / 2) * 2 + 1e-9) % 100, + ); + expect(extracted.totalFrames).toBe(c.frames); + expect(sourceIndexes(extracted)).toEqual(onScreen); + }, + 60_000, + ); + + it("stops extraction only on a stall, and its ffmpeg reports each frame it writes", async () => { + const source = join(FIXTURE_DIR, "paced-2s.mp4"); + const synth = await runFfmpeg([ + ...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"], + ...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=2`, "-pix_fmt", "yuv420p", source], + ]); + if (!synth.success) throw new Error(`paced fixture synthesis failed: ${synth.stderr}`); + const { runFfmpeg: real } = + await vi.importActual("../utils/runFfmpeg.js"); + let requested: RunFfmpegOptions | undefined; + const reported: number[] = []; + const readStats = ffmpegStatsReader(({ frames }) => { + if (frames) reported.push(frames); + }); + // Real-time pacing spreads the stats over the run. The stall clock itself is runFfmpeg's to + // test; here it is lifted so a starved runner cannot race it. + vi.mocked(runFfmpeg).mockImplementation((args, opts) => { + requested = opts; + return real(["-stats_period", "0.1", "-re", ...args], { + ...opts, + timeout: 60_000, + onStderr: readStats, + }); + }); + try { + const extracted = await extractVideoFramesRange( + source, + "paced-stall-only", + 0, + 2, + { fps: 10, outputDir: FIXTURE_DIR, format: "png" }, + undefined, + { ffmpegProcessTimeout: 2000 }, + ); + expect(extracted.totalFrames).toBe(20); + expect(requested).toMatchObject({ timeout: 2000 }); + expect(new Set(reported).size).toBeGreaterThan(1); + } finally { + vi.mocked(runFfmpeg).mockImplementation(real); + } + }, 30_000); + + it.each([ + { name: "one keyframe", keyint: "infinite", gapSeconds: 8 }, + { name: "a keyframe every second", keyint: "30", gapSeconds: 0 }, + ])( + "gives the stall clock the time to decode up to the seek point ($name)", + async ({ keyint, gapSeconds }) => { + const source = join(FIXTURE_DIR, `gop-${keyint}.mp4`); + const synth = await runFfmpeg([ + ...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"], + ...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=10`, "-pix_fmt", "yuv420p"], + ...["-x264-params", `keyint=${keyint}:scenecut=0`, source], + ]); + if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`); + const { runFfmpeg: real } = + await vi.importActual("../utils/runFfmpeg.js"); + let requested: RunFfmpegOptions | undefined; + vi.mocked(runFfmpeg).mockImplementation((args, opts) => { + requested = opts; + return real(args, opts); + }); + try { + await extractVideoFramesRange( + source, + `stall-seek-${keyint}`, + 8, + 1, + { fps: 10, outputDir: FIXTURE_DIR, format: "png" }, + undefined, + { ffmpegProcessTimeout: 2000 }, + ); + expect(requested?.timeout).toBe(2000 + gapSeconds * 1000); + } finally { + vi.mocked(runFfmpeg).mockImplementation(real); + } + }, + 30_000, + ); + + it("stops a run that writes no new frame for the stall timeout", async () => { + // stdin stays open and empty, so ffmpeg waits for its first frame forever. + const stalled = await runFfmpeg( + ["-f", "rawvideo", "-pix_fmt", "gray", "-s", "2x2", "-i", "pipe:0", "-f", "null", "-"], + { timeout: 500 }, + ); + expect(stalled.terminationReason).toBe("inactivity"); + }, 30_000); + + // A distributed chunk extracts only the frames it renders; they must be the full extraction's. + describe("partial extraction", () => { + const gopSource = join(FIXTURE_DIR, "gop-30fps.mp4"); + + beforeAll(async () => { + // Real motion and long GOPs, so range seeks land mid-GOP. + const synth = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=s=64x36:r=30:d=20", + "-c:v", + "libx264", + "-g", + "250", + "-pix_fmt", + "yuv420p", + gopSource, + ]); + if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`); + }, 30_000); + + function frameBytes(dir: string): Map { + return new Map(readdirSync(dir).map((name) => [name, readFileSync(join(dir, name))])); + } + + it.each([ + { fps: 24, startTime: 1.3, duration: 17 }, + { fps: 60, startTime: 0, duration: 20 }, + ])( + "frameRanges at $fps fps write the same files as a full extraction", + async (c) => { + const full = await extractVideoFramesRange( + gopSource, + `full-${c.fps}`, + c.startTime, + c.duration, + { fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg" }, + ); + const last = full.totalFrames; + const ranges = [ + { firstFrame: 0, frames: 5 }, + { firstFrame: 97, frames: 40 }, + { firstFrame: last - 3, frames: 3 }, + ]; + const partial = await extractVideoFramesRange( + gopSource, + `partial-${c.fps}`, + c.startTime, + c.duration, + { fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg", frameRanges: ranges }, + ); + const fullFrames = frameBytes(full.outputDir); + const partialFrames = frameBytes(partial.outputDir); + expect(partialFrames.size).toBe(48); + for (const [name, bytes] of partialFrames) { + expect(fullFrames.get(name)?.equals(bytes), name).toBe(true); + } + }, + 60_000, + ); + + it.each([ + { fps: 24, mediaStart: 3.3 }, + { fps: 30, mediaStart: 0 }, + { fps: 60, mediaStart: 7.77 }, + ])( + "a deferred window to the source end at $fps fps reports the full frame count", + async (c) => { + const video = (): VideoElement => ({ + id: `defer-${c.fps}`, + src: gopSource, + start: 0, + end: Infinity, + mediaStart: c.mediaStart, + loop: false, + hasAudio: false, + }); + const full = await extractAllVideoFrames([video()], FIXTURE_DIR, { + fps: c.fps, + outputDir: join(FIXTURE_DIR, `defer-full-${c.fps}`), + }); + const deferred = await extractAllVideoFrames([video()], FIXTURE_DIR, { + fps: c.fps, + outputDir: join(FIXTURE_DIR, `defer-${c.fps}`), + deferRangeExtraction: true, + }); + expect(deferred.errors).toEqual([]); + const result = deferred.extracted[0]!; + expect(result.totalFrames).toBe(full.extracted[0]!.totalFrames); + expect(result.framePaths.size).toBe(0); + expect(readdirSync(result.outputDir)).toEqual([]); + expect(result.deferredRange).toEqual({ + startTime: c.mediaStart, + durationSeconds: expect.any(Number), + format: "jpg", + }); + }, + 60_000, + ); + + it("halves frames while they still cover the box a video is shown in", async () => { + const firstFrameSize = async ( + videoId: string, + shownBox: { width: number; height: number }, + ) => { + const result = await extractVideoFramesRange(gopSource, videoId, 0, 1, { + fps: 30, + outputDir: FIXTURE_DIR, + format: "jpg", + frameRanges: [{ firstFrame: 0, frames: 1 }], + shownBox, + }); + const [frame] = result.framePaths.values(); + return spawnSync("ffprobe", [ + "-v", + "error", + "-show_entries", + "stream=width,height", + "-of", + "csv=p=0", + "--", + frame!, + ]) + .stdout.toString() + .trim(); + }; + // 64x36 source: halved as often as the box allows (even sides round up); never below the box. + expect(await firstFrameSize("shown-quarter", { width: 16, height: 9 })).toBe("16,10"); + expect(await firstFrameSize("shown-third", { width: 20, height: 12 })).toBe("32,18"); + expect(await firstFrameSize("shown-near-full", { width: 40, height: 20 })).toBe("64,36"); + }, 30_000); + }); +}); diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index 8f2587ad47..39e8c390c1 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -1,17 +1,7 @@ // fallow-ignore-file code-duplication -import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - readdirSync, - rmSync, - statSync, - utimesSync, - writeFileSync, -} from "node:fs"; -import { basename, join } from "node:path"; +import { vi, afterAll, beforeAll, describe, expect, it } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; import { tmpdir } from "node:os"; import { spawnSync } from "node:child_process"; import { @@ -19,7 +9,6 @@ import { hasGradedCanvas, parseImageElements, extractAllVideoFrames, - extractVideoFramesRange, extractionFrameCountForDuration, createFrameLookupTable, FrameLookupTable, @@ -38,8 +27,65 @@ import { VideoSourceExtractionError, type VideoElement, type ExtractedFrames, - type ExtractionResult, } from "./videoFrameExtractor.js"; +import { + extractFinalVideoFrameTimestamp, + extractVideoMetadata, + type VideoMetadata, +} from "../utils/ffprobe.js"; +import { runFfmpeg } from "../utils/runFfmpeg.js"; +import { resolveRuntimeMediaClipDuration } from "../../../core/src/runtime/media.js"; +import { compileTimingAttrs, sourceTimeAt } from "@hyperframes/core"; +import { RATE_RANGE } from "@hyperframes/core/audio-automation"; + +// Pass-through, so a test can tell whether extraction ran ffmpeg. +vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { + const real = await importOriginal(); + return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) }; +}); + +// ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer +// regression test at packages/producer/tests/vfr-screen-recording/ runs inside +// Dockerfile.test (which does include ffmpeg) and is the primary CI signal +// for this bug. Locally and in any CI job with ffmpeg on PATH, the tests +// below run too — they exercise the extractor in isolation against a +// synthesized VFR fixture. +const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; + +type Rgb = [number, number, number]; + +const UI_FIXTURE_WIDTH = 240; +const UI_FIXTURE_HEIGHT = 160; + +function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb { + const result = spawnSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + mediaPath, + "-frames:v", + "1", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "pipe:1", + ], + { maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 }, + ); + if (result.status !== 0) { + throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`); + } + + const offset = (y * UI_FIXTURE_WIDTH + x) * 3; + return [ + result.stdout[offset] ?? 0, + result.stdout[offset + 1] ?? 0, + result.stdout[offset + 2] ?? 0, + ]; +} describe("parseVideoElements strict literal timing", () => { it.each(["", " ", "0s", "0abc", "0px", "-1s", "Infinity", "NaN", "0x10"])( @@ -53,40 +99,6 @@ describe("parseVideoElements strict literal timing", () => { }, ); }); -import { - extractFinalVideoFrameTimestamp, - extractVideoMetadata, - type VideoMetadata, -} from "../utils/ffprobe.js"; -import { ffmpegStatsReader, runFfmpeg, type RunFfmpegOptions } from "../utils/runFfmpeg.js"; - -// Pass-through, so a test can tell whether extraction ran ffmpeg. -vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { - const real = await importOriginal(); - return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) }; -}); -import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.js"; -import { resolveRuntimeMediaClipDuration } from "../../../core/src/runtime/media.js"; -import { compileTimingAttrs, sourceTimeAt } from "@hyperframes/core"; -import { RATE_RANGE } from "@hyperframes/core/audio-automation"; - -// ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer -// regression test at packages/producer/tests/vfr-screen-recording/ runs inside -// Dockerfile.test (which does include ffmpeg) and is the primary CI signal -// for this bug. Locally and in any CI job with ffmpeg on PATH, the tests -// below run too — they exercise the extractor in isolation against a -// synthesized VFR fixture. -const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; -const HAS_ZSCALE = - HAS_FFMPEG && - /\szscale\s/.test(spawnSync("ffmpeg", ["-hide_banner", "-filters"]).stdout.toString()); -const FFMPEG_ENCODERS = HAS_FFMPEG - ? spawnSync("ffmpeg", ["-hide_banner", "-encoders"]).stdout.toString() - : ""; -// Homebrew's ffmpeg has libsvtav1 but not libaom-av1. Either one writes an untagged AV1 stream. -const AV1_ENCODER = ["libaom-av1", "libsvtav1"].find((name) => - new RegExp(`\\s${name}\\s`).test(FFMPEG_ENCODERS), -); describe("resolveVideoExtractionDuration", () => { const metadata = ( @@ -1423,1104 +1435,6 @@ describe("parseImageElements", () => { }); }); -type Rgb = [number, number, number]; - -const UI_FIXTURE_WIDTH = 240; -const UI_FIXTURE_HEIGHT = 160; -const RED_SAMPLE_PIXELS = [ - [70, 72], - [118, 82], - [178, 92], -] as const; - -function pngChunkTypes(path: string): string[] { - const bytes = readFileSync(path); - const types: string[] = []; - for (let offset = 8; offset + 8 <= bytes.length; offset += 12 + bytes.readUInt32BE(offset)) { - types.push(bytes.toString("latin1", offset + 4, offset + 8)); - } - return types; -} - -function readFirstFramePixel(mediaPath: string, x: number, y: number): Rgb { - const result = spawnSync( - "ffmpeg", - [ - "-v", - "error", - "-i", - mediaPath, - "-frames:v", - "1", - "-f", - "rawvideo", - "-pix_fmt", - "rgb24", - "pipe:1", - ], - { maxBuffer: UI_FIXTURE_WIDTH * UI_FIXTURE_HEIGHT * 3 + 1024 }, - ); - if (result.status !== 0) { - throw new Error(`ffmpeg pixel decode failed: ${result.stderr.toString().slice(-400)}`); - } - - const offset = (y * UI_FIXTURE_WIDTH + x) * 3; - return [ - result.stdout[offset] ?? 0, - result.stdout[offset + 1] ?? 0, - result.stdout[offset + 2] ?? 0, - ]; -} - -function maxChannelDelta(a: Rgb, b: Rgb): number { - return Math.max(Math.abs(a[0] - b[0]), Math.abs(a[1] - b[1]), Math.abs(a[2] - b[2])); -} - -// Regression for saturated UI recordings: default JPEG extraction can shift -// high-chroma reds before browser capture. Forcing PNG should keep extracted -// source-video frames effectively identical to the decoded source pixels. -describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => { - const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-format-")); - const UI_FIXTURE = join(FIXTURE_DIR, "ui-red.mp4"); - - beforeAll(async () => { - const result = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - `color=c=0xffe7ee:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`, - "-vf", - "drawbox=x=44:y=58:w=152:h=44:color=0xdd382e@1:t=fill,drawbox=x=64:y=75:w=112:h=10:color=0xfff0f0@1:t=fill", - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-crf", - "0", - "-pix_fmt", - "yuv420p", - "-color_primaries", - "bt709", - "-color_trc", - "bt709", - "-colorspace", - "bt709", - UI_FIXTURE, - ]); - if (!result.success) { - throw new Error(`UI color fixture synthesis failed: ${result.stderr.slice(-400)}`); - } - }, 30_000); - - afterAll(() => { - if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true }); - }, 30_000); - - function fixtureVideo(): VideoElement { - return { - id: "ui", - src: UI_FIXTURE, - start: 0, - end: 1, - mediaStart: 0, - loop: false, - hasAudio: false, - }; - } - - it("extracts into a work directory whose name holds a printf code", async () => { - const outDir = join(FIXTURE_DIR, "x 12% Up"); - mkdirSync(outDir, { recursive: true }); - const result = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, { - fps: 1, - outputDir: outDir, - }); - expect(result.errors).toEqual([]); - const frame = result.extracted[0]!.framePaths.get(0)!; - expect(frame.startsWith(outDir)).toBe(true); - expect(existsSync(frame)).toBe(true); - }); - - it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) jpg extraction", async () => { - const source = join(FIXTURE_DIR, "white-1080.mp4"); - const synth = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - "color=c=white:s=1080x16:d=1:r=1", - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-crf", - "0", - "-pix_fmt", - "yuv420p", - source, - ]); - expect(synth.success).toBe(true); - const outDir = join(FIXTURE_DIR, "out-white-1080"); - mkdirSync(outDir, { recursive: true }); - const result = await extractAllVideoFrames( - [{ ...fixtureVideo(), id: "white", src: source }], - FIXTURE_DIR, - { fps: 1, outputDir: outDir }, - ); - expect(result.errors).toEqual([]); - const frame = result.extracted[0]!.framePaths.get(0)!; - expect(frame.endsWith(".jpg")).toBe(true); - const lastColumns = (file: string) => [ - ...spawnSync("ffmpeg", [ - "-v", - "error", - "-i", - file, - "-frames:v", - "1", - "-vf", - "format=gray,crop=8:1:iw-8:8", - "-f", - "rawvideo", - "-", - ]).stdout, - ]; - const delivered = lastColumns(frame); - const expected = lastColumns(source); - expect(delivered).toHaveLength(8); - for (const [i, v] of delivered.entries()) - expect(Math.abs(v - expected[i]!)).toBeLessThanOrEqual(5); - }, 60_000); - - async function proresFrame(name: string, pixFmt: string, profile: string, source: string) { - const clip = join(FIXTURE_DIR, `${name}.mov`); - const synth = await runFfmpeg( - ["-y", "-loglevel", "error", "-f", "lavfi", "-i", source].concat([ - "-c:v", - "prores_ks", - "-profile:v", - profile, - "-pix_fmt", - pixFmt, - clip, - ]), - ); - expect(synth.success).toBe(true); - const outDir = join(FIXTURE_DIR, `out-${name}`); - mkdirSync(outDir, { recursive: true }); - const result = await extractAllVideoFrames( - [{ ...fixtureVideo(), id: name, src: clip }], - FIXTURE_DIR, - { fps: 1, outputDir: outDir }, - ); - expect(result.errors).toEqual([]); - return result.extracted[0]!.framePaths.get(0)!; - } - - it("extracts ProRes without an alpha plane to jpg", async () => { - const frame = await proresFrame("prores422", "yuv422p10le", "2", "testsrc2=s=320x180:d=1:r=1"); - expect(frame.endsWith(".jpg")).toBe(true); - }, 60_000); - - it("keeps the alpha of ProRes 4444 exact in png frames", async () => { - const frame = await proresFrame( - "prores4444", - "yuva444p10le", - "4", - "color=c=0x00ff00@0.5:s=320x180:d=1:r=1,format=yuva444p10le", - ); - expect(frame.endsWith(".png")).toBe(true); - const alpha = spawnSync("ffmpeg", [ - "-v", - "error", - "-i", - frame, - "-vf", - "alphaextract,crop=1:1:10:10", - "-f", - "rawvideo", - "-", - ]); - expect(Math.abs(alpha.stdout[0]! - 128)).toBeLessThanOrEqual(1); - }, 60_000); - - it("keeps color-sensitive UI reds closer to source when extraction is forced to png", async () => { - const defaultOut = join(FIXTURE_DIR, "out-default"); - const pngOut = join(FIXTURE_DIR, "out-png"); - mkdirSync(defaultOut, { recursive: true }); - mkdirSync(pngOut, { recursive: true }); - - const defaultResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, { - fps: 1, - outputDir: defaultOut, - }); - const pngResult = await extractAllVideoFrames([fixtureVideo()], FIXTURE_DIR, { - fps: 1, - outputDir: pngOut, - format: "png", - }); - - expect(defaultResult.errors).toEqual([]); - expect(pngResult.errors).toEqual([]); - const defaultFrame = defaultResult.extracted[0]!.framePaths.get(0)!; - const pngFrame = pngResult.extracted[0]!.framePaths.get(0)!; - expect(defaultFrame.endsWith(".jpg")).toBe(true); - expect(pngFrame.endsWith(".png")).toBe(true); - - let worstDefaultDelta = 0; - let worstPngDelta = 0; - for (const [x, y] of RED_SAMPLE_PIXELS) { - const sourcePixel = readFirstFramePixel(UI_FIXTURE, x, y); - worstDefaultDelta = Math.max( - worstDefaultDelta, - maxChannelDelta(sourcePixel, readFirstFramePixel(defaultFrame, x, y)), - ); - worstPngDelta = Math.max( - worstPngDelta, - maxChannelDelta(sourcePixel, readFirstFramePixel(pngFrame, x, y)), - ); - } - - expect(worstPngDelta).toBeLessThanOrEqual(5); - expect(worstPngDelta).toBeLessThanOrEqual(worstDefaultDelta); - }, 60_000); - - // Chrome runs with --force-color-profile=srgb and colour-manages any frame not tagged sRGB; - // the SDR encoder writes the canvas code values as they are, so frames must reach it unconverted. - it.each([ - ["BT.709", "bt709"], - ["BT.601", "smpte170m"], - ])( - "declares png frames of a %s source as sRGB so Chrome shows their code values", - async (name, tag) => { - const fixture = join(FIXTURE_DIR, `${tag}-shadow.mp4`); - const synth = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - `color=c=0x101010:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1`, - "-vf", - `setparams=color_primaries=${tag}:color_trc=${tag}:colorspace=${tag}:range=tv`, - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-crf", - "0", - "-pix_fmt", - "yuv420p", - fixture, - ]); - if (!synth.success) { - throw new Error(`${name} fixture synthesis failed: ${synth.stderr.slice(-400)}`); - } - expect((await extractVideoMetadata(fixture)).colorSpace?.colorTransfer).toBe(tag); - const outputDir = join(FIXTURE_DIR, `out-png-transfer-${tag}`); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractAllVideoFrames( - [{ ...fixtureVideo(), id: tag, src: fixture }], - FIXTURE_DIR, - { fps: 1, outputDir, format: "png" }, - ); - - expect(result.errors).toEqual([]); - const frame = result.extracted[0]!.framePaths.get(0)!; - // Chrome reads cICP before sRGB, and ffmpeg < 6.0 writes no cICP, so only an sRGB chunk - // without cICP means "these are sRGB code values" on every ffmpeg. - const chunks = pngChunkTypes(frame); - expect(chunks).toContain("sRGB"); - expect(chunks).not.toContain("cICP"); - expect(readFirstFramePixel(frame, 10, 10)).toEqual(readFirstFramePixel(fixture, 10, 10)); - }, - 60_000, - ); - - it("extracts jpg frames of a BT.709 source as the BT.601 full range JPEG readers assume", async () => { - const fixture = join(FIXTURE_DIR, "bt709-patch.mp4"); - const synth = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - `color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=1:r=1,format=rgb24,drawbox=x=31:y=0:w=64:h=${UI_FIXTURE_HEIGHT}:c=0x0000FE:t=fill`, - "-vf", - "scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv", - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-crf", - "0", - fixture, - ]); - if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); - const outputDir = join(FIXTURE_DIR, "out-jpg-matrix"); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractAllVideoFrames( - [{ ...fixtureVideo(), id: "bt709-jpg", src: fixture }], - FIXTURE_DIR, - { fps: 1, outputDir, format: "jpg" }, - ); - - expect(result.errors).toEqual([]); - const frame = result.extracted[0]!.framePaths.get(0)!; - // x=10 is flat colour; x=32 sits one pixel inside the blue edge. - for (const x of [10, 32]) { - const source = readFirstFramePixel(fixture, x, 10); - const shown = readFirstFramePixel(frame, x, 10); - const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!))); - expect(worst, `x=${x}: source ${source} jpg ${shown}`).toBeLessThanOrEqual(3); - } - }, 60_000); - - // Measured in Chrome 152: untagged H.264 plays as BT.601 at 1280x718 and BT.709 at 1280x720 or 406x720; - // untagged VP9 and AV1 play as BT.601 at every size. - it.each([ - { codec: "libx264", height: 718, matrix: "bt601", format: "png" }, - { codec: "libx264", height: 718, matrix: "bt601", format: "jpg" }, - { codec: "libx264", height: 720, matrix: "bt709", format: "png" }, - { codec: "libx264", height: 720, matrix: "bt709", format: "jpg" }, - { codec: "libvpx-vp9", height: 720, matrix: "bt601", format: "png" }, - ...(AV1_ENCODER - ? [{ codec: AV1_ENCODER, height: 720, matrix: "bt601", format: "png" } as const] - : []), - ] as const)( - "reads an untagged $height-line $codec source as $matrix like Chrome's own playback ($format)", - async ({ codec, height, matrix, format }) => { - const extension = codec === "libvpx-vp9" ? "webm" : "mp4"; - const fixture = join(FIXTURE_DIR, `untagged-${codec}-${height}.${extension}`); - const synth = await runFfmpeg([ - "-y", - "-v", - "error", - "-f", - "lavfi", - "-i", - `color=c=0xC83C28:s=64x${height}:d=1:r=1`, - "-vf", - "scale=out_color_matrix=bt601:out_range=tv,format=yuv420p,setparams=colorspace=unknown:color_primaries=unknown:color_trc=unknown:range=tv", - "-c:v", - codec, - ...(codec === "libx264" ? ["-qp", "0"] : ["-crf", "0", "-b:v", "0"]), - ...(codec === "libaom-av1" ? ["-cpu-used", "8"] : []), - fixture, - ]); - if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); - expect((await extractVideoMetadata(fixture)).colorSpace?.colorSpace ?? "unknown").toBe( - "unknown", - ); - - const result = await extractVideoFramesRange( - fixture, - `${basename(fixture)}-${format}`, - 0, - 1, - { - fps: 1, - outputDir: join(FIXTURE_DIR, "out-untagged"), - format, - }, - ); - - const pixelAt = (file: string, decode: string): number[] => [ - ...spawnSync("ffmpeg", [ - "-v", - "error", - "-i", - file, - "-vf", - `${decode}format=rgb24,crop=1:1:10:10`, - "-frames:v", - "1", - "-f", - "rawvideo", - "-", - ]).stdout, - ]; - const expected = pixelAt(fixture, `scale=in_color_matrix=${matrix}:in_range=tv,`); - const shown = pixelAt(result.framePaths.get(0)!, ""); - expect([expected.length, shown.length]).toEqual([3, 3]); - const worst = Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!))); - expect(worst, `${matrix} decode ${expected}, ${format} ${shown}`).toBeLessThanOrEqual( - format === "png" ? 1 : 3, - ); - }, - 60_000, - ); - - it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => { - // Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks. - const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4"); - const synth = await runFfmpeg([ - ...["-y", "-v", "error", "-f", "lavfi", "-i"], - "color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128", - ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"], - fixture, - ]); - if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); - - const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, { - fps: 30, - outputDir: join(FIXTURE_DIR, "out-ntsc-ms"), - format: "png", - }); - const indexOf = (file: string): number => { - const row = spawnSync("ffmpeg", [ - "-v", - "error", - "-i", - file, - "-vf", - "format=gray,crop=64:1:0:4", - "-f", - "rawvideo", - "-", - ]).stdout; - return [...Array(8).keys()].reduce( - (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0), - 0, - ); - }; - const shown = [...result.framePaths.keys()] - .sort((a, b) => a - b) - .map((i) => indexOf(result.framePaths.get(i)!)); - const steps = shown.slice(1).map((n, i) => n - shown[i]!); - - expect(shown.length).toBeGreaterThanOrEqual(44); - expect( - steps.filter((step) => step > 1), - `source frames ${shown.join(",")}`, - ).toEqual([]); - expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1); - }, 60_000); - - it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => { - const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4"); - const synth = await runFfmpeg([ - ...["-y", "-v", "error", "-f", "lavfi", "-i"], - "color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128", - ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"], - fixture, - ]); - if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); - const indexOf = (file: string): number => { - const row = spawnSync("ffmpeg", [ - "-v", - "error", - "-i", - file, - "-vf", - "format=gray,crop=64:1:0:4", - "-f", - "rawvideo", - "-", - ]).stdout; - return [...Array(8).keys()].reduce( - (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0), - 0, - ); - }; - const options = { - fps: 30, - outputDir: join(FIXTURE_DIR, "out-ntsc-seams"), - format: "png" as const, - }; - const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options); - - const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201]; - const differing: string[] = []; - for (const firstFrame of seams) { - const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, { - ...options, - frameRanges: [{ firstFrame, frames: 12 }], - }); - for (const [slot, path] of segment.framePaths) { - const expected = indexOf(single.framePaths.get(slot)!); - const shown = indexOf(path); - if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`); - } - } - expect(differing).toEqual([]); - }, 120_000); - - // ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour - // filters would cut the still that ends this window short. - it.each([ - { - name: "full range BT.709", - format: "png", - codec: "libx264", - pixFmt: "yuvj420p", - probed: { colorTransfer: "bt709" }, - tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709", - }, - { - name: "full range BT.709", - format: "jpg", - codec: "libx264", - pixFmt: "yuvj420p", - probed: { colorTransfer: "bt709" }, - tags: "range=pc:color_primaries=bt709:color_trc=bt709:colorspace=bt709", - }, - { - name: "BT.470BG gamma 2.8", - format: "png", - codec: "libx264", - pixFmt: "yuv420p", - probed: { colorTransfer: "bt470bg" }, - tags: "color_primaries=bt470bg:color_trc=bt470bg:colorspace=bt470bg", - }, - { - name: "RGB", - format: "png", - codec: "libx264rgb", - pixFmt: "rgb24", - probed: { colorSpace: "gbr" }, - tags: "", - }, - ] as const)( - "keeps every frame and the colours of a $name VFR window that ends on a still ($format)", - async ({ name, format, codec, pixFmt, probed, tags }) => { - const fixture = join(FIXTURE_DIR, `vfr-still-${name.replace(/\W+/g, "-")}-${format}.mp4`); - const synth = await runFfmpeg([ - "-y", - "-v", - "error", - "-f", - "lavfi", - "-i", - `color=c=0xC83C28:s=${UI_FIXTURE_WIDTH}x${UI_FIXTURE_HEIGHT}:d=2:r=60`, - "-vf", - `select='not(between(n\\,30\\,89))'${tags ? `,setparams=${tags}` : ""}`, - "-fps_mode", - "vfr", - "-c:v", - codec, - "-preset", - "ultrafast", - "-pix_fmt", - pixFmt, - fixture, - ]); - if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); - const window = 0.616666; - const unfilteredDir = join(FIXTURE_DIR, `out-${basename(fixture)}-unfiltered`); - mkdirSync(unfilteredDir, { recursive: true }); - const unfiltered = await runFfmpeg([ - "-v", - "error", - "-ss", - "0", - "-i", - fixture, - "-t", - String(window), - "-fps_mode", - "cfr", - "-r", - "30", - join(unfilteredDir, "f_%05d.png"), - ]); - expect(unfiltered.success).toBe(true); - - const result = await extractVideoFramesRange(fixture, basename(fixture), 0, window, { - fps: 30, - outputDir: join(FIXTURE_DIR, "out-vfr-still"), - format, - }); - - expect(result.metadata.isVFR).toBe(true); - expect(result.metadata.colorSpace).toMatchObject(probed); - expect(result.totalFrames).toBe(readdirSync(unfilteredDir).length); - const last = result.framePaths.get(result.totalFrames - 1)!; - const source = readFirstFramePixel(fixture, 10, 10); - const shown = readFirstFramePixel(last, 10, 10); - const worst = Math.max(...shown.map((v, i) => Math.abs(v - source[i]!))); - expect(worst, `source ${source} ${format} ${shown}`).toBeLessThanOrEqual( - format === "png" ? 0 : 3, - ); - if (format === "png") expect(pngChunkTypes(last)).toContain("sRGB"); - }, - 60_000, - ); - - it("leaves SDR-to-HDR frames in the HDR colours they were converted to", async () => { - const outputDir = join(FIXTURE_DIR, "out-sdr-to-hdr-gate"); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractVideoFramesRange(UI_FIXTURE, "sdr-to-hdr-gate", 0, 1, { - fps: 1, - outputDir, - format: "png", - sdrToHdrTransfer: "pq", - }); - - expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB"); - }, 60_000); - - it("keeps jpg and png extraction caches separate", async () => { - const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-")); - try { - const defaultOut = join(FIXTURE_DIR, "cache-default"); - const pngOut = join(FIXTURE_DIR, "cache-png"); - const pngHitOut = join(FIXTURE_DIR, "cache-png-hit"); - mkdirSync(defaultOut, { recursive: true }); - mkdirSync(pngOut, { recursive: true }); - mkdirSync(pngHitOut, { recursive: true }); - - const defaultResult = await extractAllVideoFrames( - [fixtureVideo()], - FIXTURE_DIR, - { fps: 1, outputDir: defaultOut }, - undefined, - { extractCacheDir: cacheDir }, - ); - expect(defaultResult.errors).toEqual([]); - expect(defaultResult.phaseBreakdown.cacheHits).toBe(0); - expect(defaultResult.phaseBreakdown.cacheMisses).toBe(1); - expect(defaultResult.extracted[0]!.framePaths.get(0)!.endsWith(".jpg")).toBe(true); - - const pngMiss = await extractAllVideoFrames( - [fixtureVideo()], - FIXTURE_DIR, - { fps: 1, outputDir: pngOut, format: "png" }, - undefined, - { extractCacheDir: cacheDir }, - ); - expect(pngMiss.errors).toEqual([]); - expect(pngMiss.phaseBreakdown.cacheHits).toBe(0); - expect(pngMiss.phaseBreakdown.cacheMisses).toBe(1); - expect(pngMiss.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true); - - const pngHit = await extractAllVideoFrames( - [fixtureVideo()], - FIXTURE_DIR, - { fps: 1, outputDir: pngHitOut, format: "png" }, - undefined, - { extractCacheDir: cacheDir }, - ); - expect(pngHit.errors).toEqual([]); - expect(pngHit.phaseBreakdown.cacheHits).toBe(1); - expect(pngHit.phaseBreakdown.cacheMisses).toBe(0); - expect(pngHit.extracted[0]!.framePaths.get(0)!.endsWith(".png")).toBe(true); - } finally { - rmSync(cacheDir, { recursive: true, force: true }); - } - }, 60_000); - - it("dedupes identical extractions within one render", async () => { - const outputDir = join(FIXTURE_DIR, "out-dedupe"); - mkdirSync(outputDir, { recursive: true }); - - const videoA: VideoElement = { ...fixtureVideo(), id: "dupe-a" }; - const videoB: VideoElement = { ...fixtureVideo(), id: "dupe-b" }; - - const result = await extractAllVideoFrames([videoA, videoB], FIXTURE_DIR, { - fps: 1, - outputDir, - }); - - expect(result.errors).toEqual([]); - expect(result.extracted).toHaveLength(2); - const first = result.extracted[0]!; - const second = result.extracted[1]!; - expect(first.videoId).toBe("dupe-a"); - expect(second.videoId).toBe("dupe-b"); - expect(second.outputDir).toBe(first.outputDir); - expect(Array.from(second.framePaths.entries())).toEqual(Array.from(first.framePaths.entries())); - - const frameDirs = readdirSync(outputDir, { withFileTypes: true }) - .filter((entry) => entry.isDirectory()) - .map((entry) => entry.name); - expect(frameDirs).toEqual(["dupe-a"]); - expect(readdirSync(first.outputDir).filter((f) => f.endsWith(".jpg"))).toHaveLength( - first.totalFrames, - ); - }, 60_000); -}); - -// Each output slot shows the frame on screen at its time, as the preview does. -describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => { - const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-video-frame-sampling-")); - const WIDTH = 32; - const HEIGHT = 16; - // Matroska stores whole-millisecond timestamps (a 30 fps frame at 66.67 ms reads 67 ms); - // a 1/30 timescale stores each frame exactly on a coarse tick. - const SOURCES = { - "mp4-60": { fps: 60, file: "index-60fps.mp4", muxer: [] }, - "mkv-30": { fps: 30, file: "index-30fps.mkv", muxer: [] }, - "mp4-30-timescale-30": { - fps: 30, - file: "index-30fps-ts30.mp4", - muxer: ["-video_track_timescale", "30"], - }, - } as const; - type SourceName = keyof typeof SOURCES; - const sourcePath = (name: SourceName) => join(FIXTURE_DIR, SOURCES[name].file); - - beforeAll(async () => { - for (const name of Object.keys(SOURCES) as SourceName[]) { - // Frame k carries luma 16 + 2k, so each extracted frame names its source index. - const result = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - `nullsrc=s=${WIDTH}x${HEIGHT}:r=${SOURCES[name].fps}:d=1.5,geq=lum='16+2*N':cb=128:cr=128`, - "-c:v", - "libx264", - "-qp", - "0", - "-pix_fmt", - "yuv420p", - ...SOURCES[name].muxer, - sourcePath(name), - ]); - if (!result.success) throw new Error(`index fixture synthesis failed: ${result.stderr}`); - } - }, 30_000); - - afterAll(() => { - rmSync(FIXTURE_DIR, { recursive: true, force: true }); - }, 30_000); - - function sourceIndexes(extracted: ExtractedFrames): number[] { - const decoded = spawnSync("ffmpeg", [ - "-v", - "error", - "-i", - join(extracted.outputDir, extracted.framePattern), - "-f", - "rawvideo", - "-pix_fmt", - "gray", - "pipe:1", - ]); - if (decoded.status !== 0) throw new Error(decoded.stderr.toString()); - const indexes: number[] = []; - for (let offset = 0; offset < decoded.stdout.length; offset += WIDTH * HEIGHT) { - indexes.push(Math.round(((decoded.stdout[offset] ?? 0) * 219) / 255 / 2)); - } - return indexes; - } - - it.each([ - { source: "mp4-60", fps: 24, startTime: 0, duration: 0.5 }, - { source: "mp4-60", fps: 10, startTime: 0.37, duration: 0.6 }, - { source: "mkv-30", fps: 30, startTime: 0, duration: 0.5 }, - { source: "mp4-30-timescale-30", fps: 60, startTime: 0, duration: 0.5 }, - ] as const)( - "$source: samples the frame on screen at each $fps fps slot from $startTime s", - async (c) => { - const extracted = await extractVideoFramesRange( - sourcePath(c.source), - `${c.source}-${c.fps}`, - c.startTime, - c.duration, - { fps: c.fps, outputDir: FIXTURE_DIR, format: "png" }, - ); - const onScreen = Array.from({ length: Math.round(c.duration * c.fps) }, (_, i) => - Math.floor((c.startTime + i / c.fps) * SOURCES[c.source].fps + 1e-9), - ); - expect(sourceIndexes(extracted)).toEqual(onScreen); - }, - 30_000, - ); - - // #5260: one ffmpeg process over a 26-minute clip hit ffmpegProcessTimeout. - it.each([ - // 260 frames: one 240-frame segment, then 20 more. - { startTime: 0.5, duration: 130, frames: 260 }, - // 200.3 - 80.3 is 120.00000000000001 s: exactly one segment, not a second near-empty one. - { startTime: 80.3, duration: 200.3 - 80.3, frames: 240 }, - ])( - "extracts $duration s from $startTime s at 2 fps across segments without a gap or repeat", - async (c) => { - const source = join(FIXTURE_DIR, "index-2fps-long.mp4"); - if (!existsSync(source)) { - const synth = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - `nullsrc=s=${WIDTH}x${HEIGHT}:r=2:d=205,geq=lum='16+2*mod(N\\,100)':cb=128:cr=128`, - "-c:v", - "libx264", - "-qp", - "0", - "-pix_fmt", - "yuv420p", - source, - ]); - if (!synth.success) throw new Error(`long fixture synthesis failed: ${synth.stderr}`); - } - const extracted = await extractVideoFramesRange( - source, - `long-2fps-${c.startTime}`, - c.startTime, - c.duration, - { fps: 2, outputDir: FIXTURE_DIR, format: "png" }, - ); - const onScreen = Array.from( - { length: c.frames }, - (_, i) => Math.floor((c.startTime + i / 2) * 2 + 1e-9) % 100, - ); - expect(extracted.totalFrames).toBe(c.frames); - expect(sourceIndexes(extracted)).toEqual(onScreen); - }, - 60_000, - ); - - it("stops extraction only on a stall, and its ffmpeg reports each frame it writes", async () => { - const source = join(FIXTURE_DIR, "paced-2s.mp4"); - const synth = await runFfmpeg([ - ...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"], - ...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=2`, "-pix_fmt", "yuv420p", source], - ]); - if (!synth.success) throw new Error(`paced fixture synthesis failed: ${synth.stderr}`); - const { runFfmpeg: real } = - await vi.importActual("../utils/runFfmpeg.js"); - let requested: RunFfmpegOptions | undefined; - const reported: number[] = []; - const readStats = ffmpegStatsReader(({ frames }) => { - if (frames) reported.push(frames); - }); - // Real-time pacing spreads the stats over the run. The stall clock itself is runFfmpeg's to - // test; here it is lifted so a starved runner cannot race it. - vi.mocked(runFfmpeg).mockImplementation((args, opts) => { - requested = opts; - return real(["-stats_period", "0.1", "-re", ...args], { - ...opts, - timeout: 60_000, - onStderr: readStats, - }); - }); - try { - const extracted = await extractVideoFramesRange( - source, - "paced-stall-only", - 0, - 2, - { fps: 10, outputDir: FIXTURE_DIR, format: "png" }, - undefined, - { ffmpegProcessTimeout: 2000 }, - ); - expect(extracted.totalFrames).toBe(20); - expect(requested).toMatchObject({ timeout: 2000 }); - expect(new Set(reported).size).toBeGreaterThan(1); - } finally { - vi.mocked(runFfmpeg).mockImplementation(real); - } - }, 30_000); - - it.each([ - { name: "one keyframe", keyint: "infinite", gapSeconds: 8 }, - { name: "a keyframe every second", keyint: "30", gapSeconds: 0 }, - ])( - "gives the stall clock the time to decode up to the seek point ($name)", - async ({ keyint, gapSeconds }) => { - const source = join(FIXTURE_DIR, `gop-${keyint}.mp4`); - const synth = await runFfmpeg([ - ...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"], - ...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=10`, "-pix_fmt", "yuv420p"], - ...["-x264-params", `keyint=${keyint}:scenecut=0`, source], - ]); - if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`); - const { runFfmpeg: real } = - await vi.importActual("../utils/runFfmpeg.js"); - let requested: RunFfmpegOptions | undefined; - vi.mocked(runFfmpeg).mockImplementation((args, opts) => { - requested = opts; - return real(args, opts); - }); - try { - await extractVideoFramesRange( - source, - `stall-seek-${keyint}`, - 8, - 1, - { fps: 10, outputDir: FIXTURE_DIR, format: "png" }, - undefined, - { ffmpegProcessTimeout: 2000 }, - ); - expect(requested?.timeout).toBe(2000 + gapSeconds * 1000); - } finally { - vi.mocked(runFfmpeg).mockImplementation(real); - } - }, - 30_000, - ); - - it("stops a run that writes no new frame for the stall timeout", async () => { - // stdin stays open and empty, so ffmpeg waits for its first frame forever. - const stalled = await runFfmpeg( - ["-f", "rawvideo", "-pix_fmt", "gray", "-s", "2x2", "-i", "pipe:0", "-f", "null", "-"], - { timeout: 500 }, - ); - expect(stalled.terminationReason).toBe("inactivity"); - }, 30_000); - - // A distributed chunk extracts only the frames it renders; they must be the full extraction's. - describe("partial extraction", () => { - const gopSource = join(FIXTURE_DIR, "gop-30fps.mp4"); - - beforeAll(async () => { - // Real motion and long GOPs, so range seeks land mid-GOP. - const synth = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - "testsrc2=s=64x36:r=30:d=20", - "-c:v", - "libx264", - "-g", - "250", - "-pix_fmt", - "yuv420p", - gopSource, - ]); - if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`); - }, 30_000); - - function frameBytes(dir: string): Map { - return new Map(readdirSync(dir).map((name) => [name, readFileSync(join(dir, name))])); - } - - it.each([ - { fps: 24, startTime: 1.3, duration: 17 }, - { fps: 60, startTime: 0, duration: 20 }, - ])( - "frameRanges at $fps fps write the same files as a full extraction", - async (c) => { - const full = await extractVideoFramesRange( - gopSource, - `full-${c.fps}`, - c.startTime, - c.duration, - { fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg" }, - ); - const last = full.totalFrames; - const ranges = [ - { firstFrame: 0, frames: 5 }, - { firstFrame: 97, frames: 40 }, - { firstFrame: last - 3, frames: 3 }, - ]; - const partial = await extractVideoFramesRange( - gopSource, - `partial-${c.fps}`, - c.startTime, - c.duration, - { fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg", frameRanges: ranges }, - ); - const fullFrames = frameBytes(full.outputDir); - const partialFrames = frameBytes(partial.outputDir); - expect(partialFrames.size).toBe(48); - for (const [name, bytes] of partialFrames) { - expect(fullFrames.get(name)?.equals(bytes), name).toBe(true); - } - }, - 60_000, - ); - - it.each([ - { fps: 24, mediaStart: 3.3 }, - { fps: 30, mediaStart: 0 }, - { fps: 60, mediaStart: 7.77 }, - ])( - "a deferred window to the source end at $fps fps reports the full frame count", - async (c) => { - const video = (): VideoElement => ({ - id: `defer-${c.fps}`, - src: gopSource, - start: 0, - end: Infinity, - mediaStart: c.mediaStart, - loop: false, - hasAudio: false, - }); - const full = await extractAllVideoFrames([video()], FIXTURE_DIR, { - fps: c.fps, - outputDir: join(FIXTURE_DIR, `defer-full-${c.fps}`), - }); - const deferred = await extractAllVideoFrames([video()], FIXTURE_DIR, { - fps: c.fps, - outputDir: join(FIXTURE_DIR, `defer-${c.fps}`), - deferRangeExtraction: true, - }); - expect(deferred.errors).toEqual([]); - const result = deferred.extracted[0]!; - expect(result.totalFrames).toBe(full.extracted[0]!.totalFrames); - expect(result.framePaths.size).toBe(0); - expect(readdirSync(result.outputDir)).toEqual([]); - expect(result.deferredRange).toEqual({ - startTime: c.mediaStart, - durationSeconds: expect.any(Number), - format: "jpg", - }); - }, - 60_000, - ); - - it("halves frames while they still cover the box a video is shown in", async () => { - const firstFrameSize = async ( - videoId: string, - shownBox: { width: number; height: number }, - ) => { - const result = await extractVideoFramesRange(gopSource, videoId, 0, 1, { - fps: 30, - outputDir: FIXTURE_DIR, - format: "jpg", - frameRanges: [{ firstFrame: 0, frames: 1 }], - shownBox, - }); - const [frame] = result.framePaths.values(); - return spawnSync("ffprobe", [ - "-v", - "error", - "-show_entries", - "stream=width,height", - "-of", - "csv=p=0", - "--", - frame!, - ]) - .stdout.toString() - .trim(); - }; - // 64x36 source: halved as often as the box allows (even sides round up); never below the box. - expect(await firstFrameSize("shown-quarter", { width: 16, height: 9 })).toBe("16,10"); - expect(await firstFrameSize("shown-third", { width: 20, height: 12 })).toBe("32,18"); - expect(await firstFrameSize("shown-near-full", { width: 40, height: 20 })).toBe("64,36"); - }, 30_000); - }); -}); - describe.skipIf(!HAS_FFMPEG)("held tails on sparse-timestamp sources", () => { const fixtureDir = mkdtempSync(join(tmpdir(), "hf-sparse-held-tail-")); const cfrFixture = join(fixtureDir, "sub-1fps-cfr.mp4"); @@ -2694,1768 +1608,6 @@ describe.skipIf(!HAS_FFMPEG)("held tails on sparse-timestamp sources", () => { }, 30_000); }); -// Regression test for the VFR (variable frame rate) freeze bug. -// Screen recordings and phone videos often have irregular timestamps. -// When such inputs hit `extractVideoFramesRange`'s `-ss -i ... -t -// -vf fps=N` pipeline, the fps filter can emit fewer frames than requested — -// e.g. a 4-second segment at 30fps would produce ~90 frames instead of 120. -// FrameLookupTable.getFrameAtTime then returns null for out-of-range indices -// and the compositor holds the last valid frame, which the user perceives as -// the video freezing. extractAllVideoFrames now routes VFR sources through -// FFmpeg's one-pass `-fps_mode cfr -r` extraction path to fix this without a -// separate normalization encode. -describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { - const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-vfr-test-")); - const VFR_FIXTURE = join(FIXTURE_DIR, "vfr_screen.mp4"); - - beforeAll(async () => { - // 10s testsrc2 at 60fps, ~40% of frames dropped via select filter and - // encoded with -fps_mode vfr so timestamps are irregular. Declared fps 60, - // actual average ~36 — well over the 10% threshold used by isVFR. - // The select expression drops four 1-second windows (frames 30-89, - // 180-239, 330-389, 480-539) to simulate static segments in a screen - // recording where no pixels changed. - // -g/-keyint_min 600 forces a single keyframe so mid-segment seeks in the - // mediaStart=3 test don't snap to an intermediate IDR and drift the count. - const result = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - "testsrc2=s=320x180:d=10:rate=60", - "-vf", - "select='not(between(n\\,30\\,89))*not(between(n\\,180\\,239))*not(between(n\\,330\\,389))*not(between(n\\,480\\,539))'", - "-fps_mode", - "vfr", - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-pix_fmt", - "yuv420p", - "-g", - "600", - "-keyint_min", - "600", - VFR_FIXTURE, - ]); - if (!result.success) { - throw new Error( - `ffmpeg fixture synthesis failed (${result.exitCode}): ${result.stderr.slice(-400)}`, - ); - } - }, 30_000); - - afterAll(() => { - if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true }); - }, 30_000); - - it("skips a clip entirely before time zero without reporting an extraction error", async () => { - const outputDir = join(FIXTURE_DIR, "out-before-timeline"); - mkdirSync(outputDir, { recursive: true }); - const video: VideoElement = { - id: "before-timeline", - src: VFR_FIXTURE, - start: -2, - end: -1, - mediaStart: 0, - loop: false, - hasAudio: false, - }; - - const result = await extractAllVideoFrames([video], FIXTURE_DIR, { - fps: 1, - outputDir, - timelineEnd: 2, - }); - - expect(result).toMatchObject({ success: true, extracted: [], errors: [] }); - }); - - it("preserves loop phase when negative preroll crosses the source boundary", async () => { - const outputDir = join(FIXTURE_DIR, "out-negative-loop"); - mkdirSync(outputDir, { recursive: true }); - const video: VideoElement = { - id: "negative-loop", - src: VFR_FIXTURE, - start: -19, - end: 5, - mediaStart: 0, - loop: true, - hasAudio: false, - }; - - const result = await extractAllVideoFrames([video], FIXTURE_DIR, { - fps: 1, - outputDir, - timelineEnd: 2, - }); - - expect(result.errors).toEqual([]); - const extracted = result.extracted[0]; - if (!extracted) throw new Error("expected loop source frames"); - const lookup = createFrameLookupTable([video], result.extracted); - expect(video).toMatchObject({ start: -19, mediaStart: 0, loop: true }); - expect(lookup.getFrame("negative-loop", 0)).toBe( - extracted.framePaths.get(extracted.totalFrames - 1), - ); - }, 30_000); - - it("preserves the held final frame after negative preroll exhausts a source", async () => { - const outputDir = join(FIXTURE_DIR, "out-negative-held-tail"); - mkdirSync(outputDir, { recursive: true }); - const video: VideoElement = { - id: "negative-held-tail", - src: VFR_FIXTURE, - start: -15, - end: 5, - mediaStart: 0, - loop: false, - hasAudio: false, - }; - - const result = await extractAllVideoFrames([video], FIXTURE_DIR, { - fps: 1, - outputDir, - timelineEnd: 2, - }); - - expect(result.errors).toEqual([]); - const extracted = result.extracted[0]; - if (!extracted) throw new Error("expected held-tail source frames"); - const lookup = createFrameLookupTable([video], result.extracted); - // The authored slot remains active through end=5, but lookup is rebased to - // one exact final frame instead of assuming the last second contains a - // timestamp or materializing the full source. - expect(video).toMatchObject({ start: 0, end: 5, mediaStart: 9.999999, loop: false }); - expect(extracted.totalFrames).toBe(1); - expect(lookup.getFrame("negative-held-tail", 0)).toBe( - extracted.framePaths.get(extracted.totalFrames - 1), - ); - }, 30_000); - - it("detects the synthesized fixture as VFR", async () => { - const md = await extractVideoMetadata(VFR_FIXTURE); - expect(md.isVFR).toBe(true); - }); - - it.each([ - ["compiled EOF", 4, 1], - ["compiled past EOF", 5, 2], - ["ordinary explicit zero", 6, 0], - ])( - "drops a known zero timeline window without extraction or error: %s", - async (label, start, mediaStart) => { - const src = await synthCfrClip(`zero-window-${label.replaceAll(" ", "-")}.mp4`, 1); - const videos = parseVideoElements( - ``, - ); - const outputDir = join(FIXTURE_DIR, `out-zero-window-${label.replaceAll(" ", "-")}`); - const result = await extractAllVideoFrames(videos, FIXTURE_DIR, { fps: 30, outputDir }); - - expect(result.errors).toEqual([]); - expect(result.extracted).toEqual([]); - expect(result.totalFramesExtracted).toBe(0); - expect(videos).toEqual([]); - }, - ); - - it("passes an exact 24000/1001 rate through VFR normalization", async () => { - const outputDir = join(FIXTURE_DIR, "out-vfr-ntsc-boundary"); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractVideoFramesRange(VFR_FIXTURE, "vfr-ntsc-boundary", 0, 0.125125, { - fps: { num: 24000, den: 1001 }, - outputDir, - format: "jpg", - }); - - expect(result.metadata.isVFR).toBe(true); - expect(result.totalFrames).toBe(3); - }, 60_000); - - // With B-frames, mp4 packet durations are decode-order gaps, so a frame can claim 0.5s while - // the next one starts 17ms later. Each slot must still show the frame on screen at its time: - // `-fps_mode cfr` on ffmpeg 8 held such a frame for 14 slots and cut the still before a gap short. - it("samples a B-frame VFR source by timestamp, not by packet duration", async () => { - const source = join(FIXTURE_DIR, "vfr_bframes.mp4"); - const synth = await runFfmpeg([ - "-y", - "-v", - "error", - "-f", - "lavfi", - "-i", - "testsrc2=s=320x180:d=4:rate=60", - "-vf", - "select='not(between(n\\,30\\,89))*not(between(n\\,150\\,179))'", - "-fps_mode", - "vfr", - "-c:v", - "libx264", - "-preset", - "medium", - "-bf", - "3", - "-g", - "600", - "-pix_fmt", - "yuv420p", - source, - ]); - if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); - const distinctFrames = (paths: Iterable) => - new Set([...paths].map((path) => readFileSync(path).toString("base64"))).size; - - // 2.5s to 3s is a still; every 30fps slot after it lands on its own 60fps source frame. - const afterGap = await extractVideoFramesRange(source, "vfr-bframes-after-gap", 2.5, 1.5, { - fps: 30, - outputDir: join(FIXTURE_DIR, "out-vfr-bframes-after-gap"), - format: "png", - }); - expect(afterGap.metadata.isVFR).toBe(true); - expect(afterGap.totalFrames).toBe(45); - const slots = [...afterGap.framePaths.entries()].sort(([a], [b]) => a - b); - expect(distinctFrames(slots.slice(0, 15).map(([, path]) => path))).toBe(1); - expect(distinctFrames(slots.slice(15).map(([, path]) => path))).toBe(30); - - const endsOnStill = await extractVideoFramesRange(source, "vfr-bframes-still", 0, 3, { - fps: 30, - outputDir: join(FIXTURE_DIR, "out-vfr-bframes-still"), - format: "png", - }); - expect(endsOnStill.totalFrames).toBe(90); - }, 60_000); - - it("produces the expected frame count for a mid-file segment", async () => { - const outputDir = join(FIXTURE_DIR, "out-mid-segment"); - mkdirSync(outputDir, { recursive: true }); - - const video: VideoElement = { - id: "v1", - src: VFR_FIXTURE, - start: 0, - end: 4, - mediaStart: 3, - loop: false, - hasAudio: false, - }; - - const result = await extractAllVideoFrames([video], FIXTURE_DIR, { - fps: 30, - outputDir, - }); - - expect(result.errors).toEqual([]); - expect(result.extracted).toHaveLength(1); - const frames = readdirSync(join(outputDir, "v1")).filter((f) => f.endsWith(".jpg")); - // Pre-fix behavior produced ~90 frames (a 25% shortfall). - // ±3 tolerance: FFmpeg's one-pass VFR→CFR extraction yields slightly - // different frame counts across versions (timestamp rounding). - expect(frames.length).toBeGreaterThanOrEqual(117); - expect(frames.length).toBeLessThanOrEqual(123); - - expect(result.phaseBreakdown).toBeDefined(); - expect(result.phaseBreakdown.extractMs).toBeGreaterThan(0); - expect(result.phaseBreakdown.vfrPreflightCount).toBe(1); - expect(result.phaseBreakdown.vfrPreflightMs).toBeGreaterThanOrEqual(0); - }, 60_000); - - // Shared fixture helpers for the cache tests below. All synthesize clean - // CFR SDR clips — keeps VFR preflight count at zero so cache keys are - // stable across runs within a test. - async function synthCfrClip(name: string, durationSeconds: number): Promise { - const src = join(FIXTURE_DIR, name); - const synth = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - `testsrc2=s=320x180:d=${durationSeconds}:rate=30`, - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-pix_fmt", - "yuv420p", - src, - ]); - if (!synth.success) { - throw new Error(`Fixture synthesis failed (${name}): ${synth.stderr.slice(-400)}`); - } - return src; - } - - it("rejects a media start beyond source duration before invoking FFmpeg", async () => { - const src = await synthCfrClip("zero-output-src.mp4", 1); - const outputDir = join(FIXTURE_DIR, "out-zero-output"); - await expect( - extractVideoFramesRange(src, "past-eof", 2, 1, { fps: 30, outputDir }), - ).rejects.toMatchObject({ - kind: "media_start_out_of_range", - retryable: false, - }); - }, 60_000); - - it("preserves legacy metadata rejection unless typed aggregation is explicitly enabled", async () => { - const src = join(FIXTURE_DIR, "invalid-probe.mp4"); - writeFileSync(src, "not a media container"); - const video = cfrClipElement("invalid-probe", src, 1); - - await expect( - extractAllVideoFrames([video], FIXTURE_DIR, { - fps: 30, - outputDir: join(FIXTURE_DIR, "out-invalid-probe-legacy"), - }), - ).rejects.toThrow(); - - const collected = await extractAllVideoFrames([video], FIXTURE_DIR, { - fps: 30, - outputDir: join(FIXTURE_DIR, "out-invalid-probe-typed"), - collectProbeFailures: true, - }); - expect(collected.success).toBe(false); - expect(collected.extracted).toEqual([]); - expect(collected.errors).toEqual([ - expect.objectContaining({ - videoId: "invalid-probe", - kind: "invalid_media", - retryable: false, - }), - ]); - }, 60_000); - - async function synthHdrTaggedClip(name: string, durationSeconds: number): Promise { - const src = join(FIXTURE_DIR, name); - const synth = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - `testsrc2=s=320x180:d=${durationSeconds}:rate=30`, - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-pix_fmt", - "yuv420p", - "-color_primaries", - "bt2020", - "-color_trc", - "smpte2084", - "-colorspace", - "bt2020nc", - // The -color_* flags above only tag the container/encoder context; - // whether they reach the H.264 VUI depends on the ffmpeg build (the - // pinned Windows CI build drops the transfer). Write the VUI directly - // so probing reports smpte2084 on every build (9/16/9 = bt2020/PQ/bt2020nc). - "-bsf:v", - "h264_metadata=colour_primaries=9:transfer_characteristics=16:matrix_coefficients=9", - src, - ]); - if (!synth.success) { - throw new Error(`HDR fixture synthesis failed (${name}): ${synth.stderr.slice(-400)}`); - } - return src; - } - - function cfrClipElement( - id: string, - src: string, - endSeconds: number, - mediaStart = 0, - ): VideoElement { - return { - id, - src, - start: 0, - end: endSeconds, - mediaStart, - loop: false, - hasAudio: false, - }; - } - - async function extractWithCache( - video: VideoElement, - outName: string, - cacheDir: string, - fps = 30, - ): Promise { - const outputDir = join(FIXTURE_DIR, outName); - mkdirSync(outputDir, { recursive: true }); - return extractAllVideoFrames([video], FIXTURE_DIR, { fps, outputDir }, undefined, { - extractCacheDir: cacheDir, - }); - } - - function cacheEntryNames(cacheDir: string): string[] { - return readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX)); - } - - function supersetDirNames(outputDir: string): string[] { - if (!existsSync(outputDir)) return []; - return readdirSync(outputDir).filter((name) => name.startsWith("__superset-")); - } - - function extractedFor(result: ExtractionResult, videoId: string): ExtractedFrames { - const extracted = result.extracted.find((item) => item.videoId === videoId); - if (!extracted) throw new Error(`missing extraction result for ${videoId}`); - return extracted; - } - - function framePath(result: ExtractionResult, videoId: string, frameIndex: number): string { - const extracted = extractedFor(result, videoId); - const frame = extracted?.framePaths.get(frameIndex); - if (!frame) throw new Error(`missing frame ${frameIndex} for ${videoId}`); - return frame; - } - - it("reuses extracted frames on a warm cache hit", async () => { - const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-")); - const SRC = await synthCfrClip("cache-src.mp4", 2); - const video = cfrClipElement("cv1", SRC, 2); - - const miss = await extractWithCache(video, "out-cache-miss", CACHE_DIR); - expect(miss.errors).toEqual([]); - expect(miss.phaseBreakdown.cacheHits).toBe(0); - expect(miss.phaseBreakdown.cacheMisses).toBe(1); - const ffmpegRunsBeforeHit = vi.mocked(runFfmpeg).mock.calls.length; - - const hit = await extractWithCache(video, "out-cache-hit", CACHE_DIR); - expect(hit.errors).toEqual([]); - expect(hit.phaseBreakdown.cacheHits).toBe(1); - expect(hit.phaseBreakdown.cacheMisses).toBe(0); - expect(vi.mocked(runFfmpeg).mock.calls.length).toBe(ffmpegRunsBeforeHit); - expect(hit.extracted).toHaveLength(1); - expect(hit.extracted[0]!.totalFrames).toBe(miss.extracted[0]!.totalFrames); - - rmSync(CACHE_DIR, { recursive: true, force: true }); - }, 60_000); - - it("does not reuse a decimal-rate VFR cache entry for the exact rational rate", async () => { - const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-cache-ntsc-rate-test-")); - const decimalOutputDir = join(FIXTURE_DIR, "out-cache-vfr-ntsc-decimal"); - const rationalOutputDir = join(FIXTURE_DIR, "out-cache-vfr-ntsc-rational"); - mkdirSync(decimalOutputDir, { recursive: true }); - mkdirSync(rationalOutputDir, { recursive: true }); - try { - const decimal = await extractAllVideoFrames( - [cfrClipElement("vfr-ntsc-cache", VFR_FIXTURE, 0.125125)], - FIXTURE_DIR, - { fps: 24000 / 1001, outputDir: decimalOutputDir }, - undefined, - { extractCacheDir: cacheDir }, - ); - expect(decimal.errors).toEqual([]); - expect(decimal.phaseBreakdown.cacheMisses).toBe(1); - expect(decimal.extracted[0]?.totalFrames).toBe(3); - // Model the warm v3 entry from before exact-rate extraction: the - // decimal path could persist one extra frame at this boundary. If the - // rational lookup collides, rehydration below will observe all four. - writeFileSync( - join(decimal.extracted[0]!.outputDir, "frame_00004.jpg"), - "stale-decimal-boundary-frame", - "utf-8", - ); - - const rational = await extractAllVideoFrames( - [cfrClipElement("vfr-ntsc-cache", VFR_FIXTURE, 0.125125)], - FIXTURE_DIR, - { fps: { num: 24000, den: 1001 }, outputDir: rationalOutputDir }, - undefined, - { extractCacheDir: cacheDir }, - ); - expect(rational.errors).toEqual([]); - expect(rational.phaseBreakdown.cacheHits).toBe(0); - expect(rational.phaseBreakdown.cacheMisses).toBe(1); - expect(rational.extracted[0]?.totalFrames).toBe(3); - expect(cacheEntryNames(cacheDir)).toHaveLength(2); - } finally { - rmSync(cacheDir, { recursive: true, force: true }); - } - }, 60_000); - - it("reuses one-cycle loop extraction across different authored starts", async () => { - const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-loop-phase-cache-test-")); - const src = await synthCfrClip("cache-loop-phase-src.mp4", 3); - try { - const firstOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-first"); - const secondOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-second"); - mkdirSync(firstOutputDir, { recursive: true }); - mkdirSync(secondOutputDir, { recursive: true }); - - const first = await extractAllVideoFrames( - [ - { - ...cfrClipElement("loop-cache-first", src, 60), - loop: true, - }, - ], - FIXTURE_DIR, - { fps: 30, outputDir: firstOutputDir, timelineEnd: 60 }, - undefined, - { extractCacheDir: cacheDir }, - ); - expect(first.errors).toEqual([]); - expect(first.phaseBreakdown.cacheMisses).toBe(1); - - const second = await extractAllVideoFrames( - [ - { - ...cfrClipElement("loop-cache-second", src, 66), - start: -6, - loop: true, - }, - ], - FIXTURE_DIR, - { fps: 30, outputDir: secondOutputDir, timelineEnd: 60 }, - undefined, - { extractCacheDir: cacheDir }, - ); - expect(second.errors).toEqual([]); - expect(second.phaseBreakdown.cacheHits).toBe(1); - expect(second.phaseBreakdown.cacheMisses).toBe(0); - expect(second.extracted[0]?.totalFrames).toBe(first.extracted[0]?.totalFrames); - } finally { - rmSync(cacheDir, { recursive: true, force: true }); - } - }, 60_000); - - it("updates the cache sentinel mtime on a hit", async () => { - const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-touch-test-")); - const SRC = await synthCfrClip("cache-touch-src.mp4", 1); - const video = cfrClipElement("touch", SRC, 1); - - const miss = await extractWithCache(video, "out-cache-touch-miss", CACHE_DIR); - expect(miss.errors).toEqual([]); - expect(miss.phaseBreakdown.cacheMisses).toBe(1); - - const cacheEntryNames = readdirSync(CACHE_DIR).filter((name) => name.startsWith(SCHEMA_PREFIX)); - expect(cacheEntryNames).toHaveLength(1); - const sentinel = join(CACHE_DIR, cacheEntryNames[0]!, COMPLETE_SENTINEL); - const old = new Date(Date.now() - 120_000); - utimesSync(sentinel, old, old); - const before = statSync(sentinel).mtimeMs; - - const hit = await extractWithCache(video, "out-cache-touch-hit", CACHE_DIR); - - expect(hit.errors).toEqual([]); - expect(hit.phaseBreakdown.cacheHits).toBe(1); - expect(statSync(sentinel).mtimeMs).toBeGreaterThan(before); - - rmSync(CACHE_DIR, { recursive: true, force: true }); - }, 60_000); - - it("skips cache GC on all-hit renders", async () => { - const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-gc-skip-test-")); - const SRC = await synthCfrClip("cache-gc-skip-src.mp4", 1); - const video = cfrClipElement("gc-skip", SRC, 1); - - const miss = await extractWithCache(video, "out-cache-gc-skip-miss", CACHE_DIR); - expect(miss.errors).toEqual([]); - expect(miss.phaseBreakdown.cacheMisses).toBe(1); - - const agedPartial = join(CACHE_DIR, `${SCHEMA_PREFIX}aged.partial-1234-deadbeef`); - mkdirSync(agedPartial, { recursive: true }); - writeFileSync(join(agedPartial, "frame_00001.jpg"), "stale", "utf-8"); - const old = new Date(Date.now() - 2 * 60 * 60 * 1000); - utimesSync(agedPartial, old, old); - - const hit = await extractWithCache(video, "out-cache-gc-skip-hit", CACHE_DIR); - expect(hit.errors).toEqual([]); - expect(hit.phaseBreakdown.cacheHits).toBe(1); - expect(hit.phaseBreakdown.cacheMisses).toBe(0); - expect(existsSync(agedPartial)).toBe(true); - - rmSync(CACHE_DIR, { recursive: true, force: true }); - }, 60_000); - - it("disables caching for this render when the cache dir is not writable", async () => { - const CACHE_FILE = join(FIXTURE_DIR, "cache-dir-is-a-file"); - writeFileSync(CACHE_FILE, "not a directory", "utf-8"); - const SRC = await synthCfrClip("cache-disabled-src.mp4", 1); - - const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); - try { - const result = await extractWithCache( - cfrClipElement("uncached", SRC, 1), - "out-cache-disabled", - CACHE_FILE, - ); - - expect(result.errors).toEqual([]); - expect(result.extracted).toHaveLength(1); - expect(result.phaseBreakdown.cacheHits).toBe(0); - expect(result.phaseBreakdown.cacheMisses).toBe(0); - expect(stderr).toHaveBeenCalledTimes(1); - expect(String(stderr.mock.calls[0]?.[0])).toContain("extraction cache dir"); - expect(String(stderr.mock.calls[0]?.[0])).toContain("caching disabled for this render"); - } finally { - stderr.mockRestore(); - } - }, 60_000); - - it("invalidates the cache when fps changes", async () => { - const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-")); - const SRC = await synthCfrClip("cache-fps-src.mp4", 1); - const video = cfrClipElement("cv2", SRC, 1); - - const first = await extractWithCache(video, "out-cache-fps-30", CACHE_DIR); - expect(first.phaseBreakdown.cacheMisses).toBe(1); - - const second = await extractWithCache(video, "out-cache-fps-60", CACHE_DIR, 60); - expect(second.phaseBreakdown.cacheMisses).toBe(1); - expect(second.phaseBreakdown.cacheHits).toBe(0); - - rmSync(CACHE_DIR, { recursive: true, force: true }); - }, 60_000); - - it("applies SDR→HDR conversion during extraction without normalized intermediates", async () => { - const SDR_LONG = await synthCfrClip("sdr-long.mp4", 10); - const HDR_SHORT = await synthHdrTaggedClip("hdr-short.mp4", 2); - const outputDir = join(FIXTURE_DIR, "out-hdr-segment"); - const plainOutputDir = join(FIXTURE_DIR, "out-hdr-plain-sdr"); - mkdirSync(outputDir, { recursive: true }); - mkdirSync(plainOutputDir, { recursive: true }); - - const videos: VideoElement[] = [ - { id: "sdr", src: SDR_LONG, start: 0, end: 2, mediaStart: 0, loop: false, hasAudio: false }, - { - id: "hdr", - src: HDR_SHORT, - start: 2, - end: 4, - mediaStart: 0, - loop: false, - hasAudio: false, - }, - ]; - - const result = await extractAllVideoFrames(videos, FIXTURE_DIR, { - fps: 30, - outputDir, - }); - expect(result.errors).toEqual([]); - expect(result.phaseBreakdown.hdrPreflightCount).toBe(1); - expect(existsSync(join(outputDir, "_hdr_normalized"))).toBe(false); - - const sdrFrames = result.extracted.find((item) => item.videoId === "sdr"); - expect(sdrFrames?.totalFrames).toBe(60); - - const plain = await extractVideoFramesRange(SDR_LONG, "plain-sdr", 0, 2, { - fps: 30, - outputDir: plainOutputDir, - format: "jpg", - }); - expect(plain.totalFrames).toBe(60); - expect( - readFileSync(framePath(result, "sdr", 0)).equals(readFileSync(plain.framePaths.get(0)!)), - ).toBe(false); - }, 60_000); - - it("leaves SDR clips alone in a mixed timeline that tone-maps to SDR", async () => { - const SDR = await synthCfrClip("sdr-tonemap-peer.mp4", 1); - const HDR = await synthHdrTaggedClip("hdr-tonemap-peer.mp4", 1); - const outputDir = join(FIXTURE_DIR, "out-hdr-tonemap-mixed"); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractAllVideoFrames( - [cfrClipElement("sdr", SDR, 1), { ...cfrClipElement("hdr", HDR, 1), start: 1, end: 2 }], - FIXTURE_DIR, - { fps: 30, outputDir, toneMapHdrToSdr: true }, - ); - - expect(result.errors).toEqual([]); - expect(result.phaseBreakdown.hdrPreflightCount).toBe(0); - }, 60_000); - - it("keeps a finite SDR past-EOF slot in a mixed HDR timeline", async () => { - const SDR_SHORT = await synthCfrClip("sdr-past-eof.mp4", 1); - const HDR_SHORT = await synthHdrTaggedClip("hdr-past-eof-peer.mp4", 1); - const outputDir = join(FIXTURE_DIR, "out-hdr-past-eof"); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractAllVideoFrames( - [ - cfrClipElement("sdr-past-eof", SDR_SHORT, 4, 5), - { ...cfrClipElement("hdr-peer", HDR_SHORT, 1), start: 1, end: 2 }, - ], - FIXTURE_DIR, - { fps: 30, outputDir }, - ); - - // The SDR slot must survive the mixed-HDR preflight and use the held-tail - // path. Reverting its guard to `mediaStart >= playableDuration` records an - // out-of-range error here and drops the slot before extraction. - expect(result.errors).toEqual([]); - expect(result.phaseBreakdown.hdrPreflightCount).toBe(1); - expect(extractedFor(result, "sdr-past-eof").totalFrames).toBe(1); - expect(extractedFor(result, "hdr-peer").totalFrames).toBeGreaterThan(0); - }, 60_000); - - it("keeps SDR→HDR cache entries distinct from plain SDR entries", async () => { - const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-hdr-cache-test-")); - const SDR = await synthCfrClip("cache-hdr-sdr.mp4", 1); - const HDR = await synthHdrTaggedClip("cache-hdr-hdr.mp4", 1); - try { - const mixedOutputDir = join(FIXTURE_DIR, "out-cache-hdr-mixed"); - mkdirSync(mixedOutputDir, { recursive: true }); - const mixed = await extractAllVideoFrames( - [ - cfrClipElement("sdr-transform", SDR, 1), - { ...cfrClipElement("hdr-peer", HDR, 1), start: 1, end: 2 }, - ], - FIXTURE_DIR, - { fps: 30, outputDir: mixedOutputDir }, - undefined, - { extractCacheDir: CACHE_DIR }, - ); - expect(mixed.errors).toEqual([]); - expect(mixed.phaseBreakdown.hdrPreflightCount).toBe(1); - expect(mixed.phaseBreakdown.cacheHits).toBe(0); - expect(mixed.phaseBreakdown.cacheMisses).toBe(2); - - const plain = await extractWithCache( - cfrClipElement("sdr-plain", SDR, 1), - "out-cache-hdr-plain", - CACHE_DIR, - ); - expect(plain.errors).toEqual([]); - expect(plain.phaseBreakdown.cacheHits).toBe(0); - expect(plain.phaseBreakdown.cacheMisses).toBe(1); - expect(cacheEntryNames(CACHE_DIR)).toHaveLength(3); - - // Cross-render poisoning regression: the plain-SDR render must not be - // served the BT.2020-converted frames the mixed render cached for the - // SAME source+trim. Compare actual frame bytes across the cache - // boundary, not just entry counts. - expect( - readFileSync(framePath(plain, "sdr-plain", 0)).equals( - readFileSync(framePath(mixed, "sdr-transform", 0)), - ), - ).toBe(false); - - // And a repeat plain render must HIT the plain entry and serve - // byte-identical plain frames (proves the hit path keys correctly too). - const plainAgain = await extractWithCache( - cfrClipElement("sdr-plain-again", SDR, 1), - "out-cache-hdr-plain-again", - CACHE_DIR, - ); - expect(plainAgain.phaseBreakdown.cacheHits).toBe(1); - expect( - readFileSync(framePath(plainAgain, "sdr-plain-again", 0)).equals( - readFileSync(framePath(plain, "sdr-plain", 0)), - ), - ).toBe(true); - } finally { - rmSync(CACHE_DIR, { recursive: true, force: true }); - } - }, 60_000); - - it("clusters overlap components so a disjoint outlier does not break the group", async () => { - const SRC = await synthCfrClip("superset-cluster-src.mp4", 12); - const outputDir = join(FIXTURE_DIR, "out-superset-cluster"); - mkdirSync(outputDir, { recursive: true }); - - // Three overlapping trims [0..4], [2..6], [4..8] plus one disjoint trim - // [10..12] of the same source. Pre-clustering, the outlier failed the - // union<=sum check for the whole bucket and ALL FOUR fell back to direct - // extraction; the overlapping three must still share one superset. - const result = await extractAllVideoFrames( - [ - cfrClipElement("cl-a", SRC, 4, 0), - cfrClipElement("cl-b", SRC, 4, 2), - cfrClipElement("cl-c", SRC, 4, 4), - cfrClipElement("cl-out", SRC, 2, 10), - ], - FIXTURE_DIR, - { fps: 30, outputDir }, - ); - - expect(result.errors).toEqual([]); - // Overlap region t=2..4 of the source: cl-a frame 60 and cl-b frame 0 - // must be the SAME inode (shared superset extraction). - expect(statSync(framePath(result, "cl-a", 60)).ino).toBe( - statSync(framePath(result, "cl-b", 0)).ino, - ); - expect(statSync(framePath(result, "cl-b", 60)).ino).toBe( - statSync(framePath(result, "cl-c", 0)).ino, - ); - // The outlier extracted directly: its frames share no inode with the - // cluster (frame at source t=10 exists only in its own extraction). - expect(extractedFor(result, "cl-out").totalFrames).toBe(60); - expect(supersetDirNames(outputDir)).toEqual([]); - }, 60_000); - - it("runs the GC staleness fallback sweep on all-hit renders with a stale marker", async () => { - const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-gc-stale-test-")); - const SRC = await synthCfrClip("cache-gc-stale-src.mp4", 1); - const video = cfrClipElement("gc-stale", SRC, 1); - - const miss = await extractWithCache(video, "out-cache-gc-stale-miss", CACHE_DIR); - expect(miss.phaseBreakdown.cacheMisses).toBe(1); - - const agedPartial = join(CACHE_DIR, `${SCHEMA_PREFIX}aged.partial-1234-cafef00d`); - mkdirSync(agedPartial, { recursive: true }); - writeFileSync(join(agedPartial, "frame_00001.jpg"), "stale", "utf-8"); - const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000); - utimesSync(agedPartial, twoHoursAgo, twoHoursAgo); - - // Age the sweep marker past the 24h staleness window: the next all-hit - // render must sweep anyway and clear the aged partial. - const twoDaysAgo = new Date(Date.now() - 48 * 60 * 60 * 1000); - utimesSync(join(CACHE_DIR, GC_MARKER), twoDaysAgo, twoDaysAgo); - - const hit = await extractWithCache(video, "out-cache-gc-stale-hit", CACHE_DIR); - expect(hit.phaseBreakdown.cacheHits).toBe(1); - expect(hit.phaseBreakdown.cacheMisses).toBe(0); - expect(existsSync(agedPartial)).toBe(false); - expect(hit.phaseBreakdown.cacheAgedPartialsCleared).toBe(1); - - rmSync(CACHE_DIR, { recursive: true, force: true }); - }, 60_000); - - it("hardlinks overlapping aligned trims from one superset extraction", async () => { - const SRC = await synthCfrClip("superset-overlap-src.mp4", 10); - const outputDir = join(FIXTURE_DIR, "out-superset-overlap"); - const directOutputDir = join(FIXTURE_DIR, "out-superset-direct"); - mkdirSync(outputDir, { recursive: true }); - mkdirSync(directOutputDir, { recursive: true }); - - const result = await extractAllVideoFrames( - [cfrClipElement("trim-a", SRC, 4, 0), cfrClipElement("trim-b", SRC, 4, 2)], - FIXTURE_DIR, - { fps: 30, outputDir }, - ); - - expect(result.errors).toEqual([]); - expect(extractedFor(result, "trim-a").totalFrames).toBe(120); - expect(extractedFor(result, "trim-b").totalFrames).toBe(120); - expect(statSync(framePath(result, "trim-a", 60)).ino).toBe( - statSync(framePath(result, "trim-b", 0)).ino, - ); - - const direct = await extractVideoFramesRange(SRC, "direct-trim-b", 2, 4, { - fps: 30, - outputDir: directOutputDir, - format: "jpg", - }); - expect( - readFileSync(framePath(result, "trim-b", 0)).equals(readFileSync(direct.framePaths.get(0)!)), - ).toBe(true); - expect(supersetDirNames(outputDir)).toEqual([]); - }, 60_000); - - it.each([ - { label: "decimal underflow half-frame", duration: 2.05, offset: 1, expectedFrames: 62 }, - { - label: "non-half-integer boundary", - duration: 0.616666, - offset: 0.1, - expectedFrames: 19, - }, - ])( - "keeps direct and superset CFR extraction equal at a $label", - async ({ label, duration, offset, expectedFrames }) => { - const fixtureKey = label.replaceAll(" ", "-"); - const src = await synthCfrClip(`superset-cfr-${fixtureKey}.mp4`, 4); - const groupedOutputDir = join(FIXTURE_DIR, `out-superset-cfr-${fixtureKey}`); - const directOutputDir = join(FIXTURE_DIR, `out-direct-cfr-${fixtureKey}`); - mkdirSync(groupedOutputDir, { recursive: true }); - mkdirSync(directOutputDir, { recursive: true }); - - const grouped = await extractAllVideoFrames( - [ - cfrClipElement(`${fixtureKey}-base`, src, duration, 0), - cfrClipElement(`${fixtureKey}-member`, src, duration, offset), - ], - FIXTURE_DIR, - { fps: 30, outputDir: groupedOutputDir }, - ); - const direct = await extractVideoFramesRange(src, `${fixtureKey}-direct`, offset, duration, { - fps: 30, - outputDir: directOutputDir, - format: "jpg", - }); - - expect(grouped.errors).toEqual([]); - expect(direct.totalFrames).toBe(expectedFrames); - expect(extractedFor(grouped, `${fixtureKey}-base`).totalFrames).toBe(expectedFrames); - expect(extractedFor(grouped, `${fixtureKey}-member`).totalFrames).toBe(expectedFrames); - expect(statSync(framePath(grouped, `${fixtureKey}-base`, Math.round(offset * 30))).ino).toBe( - statSync(framePath(grouped, `${fixtureKey}-member`, 0)).ino, - ); - for (let frame = 0; frame < expectedFrames; frame += 1) { - expect( - readFileSync(framePath(grouped, `${fixtureKey}-member`, frame)).equals( - readFileSync(direct.framePaths.get(frame)!), - ), - ).toBe(true); - } - expect(supersetDirNames(groupedOutputDir)).toEqual([]); - }, - 60_000, - ); - - it("keeps direct and superset CFR extraction equal at 30000/1001", async () => { - const src = await synthCfrClip("superset-cfr-ntsc.mp4", 4); - const groupedOutputDir = join(FIXTURE_DIR, "out-superset-cfr-ntsc"); - const directOutputDir = join(FIXTURE_DIR, "out-direct-cfr-ntsc"); - const fps = { num: 30000, den: 1001 }; - const duration = 0.25025; - mkdirSync(groupedOutputDir, { recursive: true }); - mkdirSync(directOutputDir, { recursive: true }); - - const grouped = await extractAllVideoFrames( - [ - cfrClipElement("ntsc-base", src, 0.5005, 0), - cfrClipElement("ntsc-member", src, duration, 0), - ], - FIXTURE_DIR, - { fps, outputDir: groupedOutputDir }, - ); - const direct = await extractVideoFramesRange(src, "ntsc-direct", 0, duration, { - fps, - outputDir: directOutputDir, - format: "jpg", - }); - - expect(grouped.errors).toEqual([]); - expect(direct.totalFrames).toBe(8); - expect(extractedFor(grouped, "ntsc-base").totalFrames).toBe(15); - expect(extractedFor(grouped, "ntsc-member").totalFrames).toBe(8); - expect(statSync(framePath(grouped, "ntsc-base", 0)).ino).toBe( - statSync(framePath(grouped, "ntsc-member", 0)).ino, - ); - for (let frame = 0; frame < 8; frame += 1) { - expect( - readFileSync(framePath(grouped, "ntsc-member", frame)).equals( - readFileSync(direct.framePaths.get(frame)!), - ), - ).toBe(true); - } - expect(supersetDirNames(groupedOutputDir)).toEqual([]); - }, 60_000); - - it("does not superset disjoint trims", async () => { - const SRC = await synthCfrClip("superset-disjoint-src.mp4", 10); - const outputDir = join(FIXTURE_DIR, "out-superset-disjoint"); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractAllVideoFrames( - [cfrClipElement("trim-a", SRC, 2, 0), cfrClipElement("trim-b", SRC, 2, 8)], - FIXTURE_DIR, - { fps: 30, outputDir }, - ); - - expect(result.errors).toEqual([]); - expect(statSync(framePath(result, "trim-a", 0)).ino).not.toBe( - statSync(framePath(result, "trim-b", 0)).ino, - ); - expect(supersetDirNames(outputDir)).toEqual([]); - }, 60_000); - - it("does not superset trims whose offsets are not frame-aligned", async () => { - const SRC = await synthCfrClip("superset-misaligned-src.mp4", 2); - const outputDir = join(FIXTURE_DIR, "out-superset-misaligned"); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractAllVideoFrames( - [cfrClipElement("trim-a", SRC, 1, 0), cfrClipElement("trim-b", SRC, 1, 0.017)], - FIXTURE_DIR, - { fps: 30, outputDir }, - ); - - expect(result.errors).toEqual([]); - expect(statSync(framePath(result, "trim-a", 0)).ino).not.toBe( - statSync(framePath(result, "trim-b", 0)).ino, - ); - expect(supersetDirNames(outputDir)).toEqual([]); - }, 60_000); - - it("keeps overlapping VFR trims direct because CFR resampling phase resets per seek", async () => { - const outputDir = join(FIXTURE_DIR, "out-vfr-superset-short"); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractAllVideoFrames( - [ - cfrClipElement("vfr-short-a", VFR_FIXTURE, 0.616666, 0), - cfrClipElement("vfr-short-b", VFR_FIXTURE, 0.616666, 0.1), - ], - FIXTURE_DIR, - { fps: 30, outputDir }, - ); - - expect(result.errors).toEqual([]); - expect(extractedFor(result, "vfr-short-a").metadata.isVFR).toBe(true); - expect(extractedFor(result, "vfr-short-a").totalFrames).toBe(19); - expect(extractedFor(result, "vfr-short-b").totalFrames).toBe(19); - expect(statSync(framePath(result, "vfr-short-a", 3)).ino).not.toBe( - statSync(framePath(result, "vfr-short-b", 0)).ino, - ); - expect(supersetDirNames(outputDir)).toEqual([]); - }, 60_000); - - it("keeps batched and direct VFR extraction equal at a floating integral boundary", async () => { - const groupedOutputDir = join(FIXTURE_DIR, "out-vfr-superset-integral-boundary"); - const directOutputDir = join(FIXTURE_DIR, "out-vfr-direct-integral-boundary"); - mkdirSync(groupedOutputDir, { recursive: true }); - mkdirSync(directOutputDir, { recursive: true }); - - const first: VideoElement = { - id: "vfr-integral-a", - src: VFR_FIXTURE, - start: 0.03, - end: 0.33, - mediaStart: 0.03, - loop: false, - hasAudio: false, - }; - const second: VideoElement = { - ...first, - id: "vfr-integral-b", - mediaStart: 0.13, - }; - - const direct = await extractAllVideoFrames([{ ...second }], FIXTURE_DIR, { - fps: 30, - outputDir: directOutputDir, - }); - const grouped = await extractAllVideoFrames([{ ...first }, second], FIXTURE_DIR, { - fps: 30, - outputDir: groupedOutputDir, - }); - - expect(direct.errors).toEqual([]); - expect(grouped.errors).toEqual([]); - expect(extractedFor(direct, second.id).totalFrames).toBe(9); - expect(extractedFor(grouped, first.id).totalFrames).toBe(9); - expect(extractedFor(grouped, second.id).totalFrames).toBe(9); - for (let frame = 0; frame < 9; frame += 1) { - expect( - readFileSync(framePath(grouped, second.id, frame)).equals( - readFileSync(framePath(direct, second.id, frame)), - ), - ).toBe(true); - } - expect(statSync(framePath(grouped, first.id, 3)).ino).not.toBe( - statSync(framePath(grouped, second.id, 0)).ino, - ); - expect(supersetDirNames(groupedOutputDir)).toEqual([]); - }, 60_000); - - it("publishes overlapping superset slices to cache entries and hits them on the next render", async () => { - const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-superset-cache-test-")); - const SRC = await synthCfrClip("superset-cache-src.mp4", 10); - try { - const firstOutputDir = join(FIXTURE_DIR, "out-superset-cache-first"); - const secondOutputDir = join(FIXTURE_DIR, "out-superset-cache-second"); - mkdirSync(firstOutputDir, { recursive: true }); - mkdirSync(secondOutputDir, { recursive: true }); - const videos = [cfrClipElement("trim-a", SRC, 4, 0), cfrClipElement("trim-b", SRC, 4, 2)]; - - const first = await extractAllVideoFrames( - videos, - FIXTURE_DIR, - { fps: 30, outputDir: firstOutputDir }, - undefined, - { extractCacheDir: CACHE_DIR }, - ); - expect(first.errors).toEqual([]); - expect(first.phaseBreakdown.cacheHits).toBe(0); - expect(first.phaseBreakdown.cacheMisses).toBe(2); - expect(cacheEntryNames(CACHE_DIR)).toHaveLength(2); - expect(statSync(framePath(first, "trim-a", 60)).ino).toBe( - statSync(framePath(first, "trim-b", 0)).ino, - ); - - const second = await extractAllVideoFrames( - videos, - FIXTURE_DIR, - { fps: 30, outputDir: secondOutputDir }, - undefined, - { extractCacheDir: CACHE_DIR }, - ); - expect(second.errors).toEqual([]); - expect(second.phaseBreakdown.cacheHits).toBe(2); - expect(second.phaseBreakdown.cacheMisses).toBe(0); - expect(supersetDirNames(secondOutputDir)).toEqual([]); - } finally { - rmSync(CACHE_DIR, { recursive: true, force: true }); - } - }, 60_000); - - it("clamps loop-past-EOF superset slices to available source frames", async () => { - const SRC = await synthCfrClip("superset-eof-src.mp4", 10); - const outputDir = join(FIXTURE_DIR, "out-superset-eof"); - mkdirSync(outputDir, { recursive: true }); - - const result = await extractAllVideoFrames( - [cfrClipElement("covered", SRC, 4, 6), cfrClipElement("past-eof", SRC, 6, 8)], - FIXTURE_DIR, - { fps: 30, outputDir }, - ); - - expect(result.errors).toEqual([]); - expect(extractedFor(result, "covered").totalFrames).toBe(120); - expect(extractedFor(result, "past-eof").totalFrames).toBe(60); - expect(statSync(framePath(result, "covered", 60)).ino).toBe( - statSync(framePath(result, "past-eof", 0)).ino, - ); - expect(supersetDirNames(outputDir)).toEqual([]); - }, 60_000); - - // Asserts frame-count correctness for a full VFR file. One-pass CFR image - // extraction may repeat held source frames across timestamp gaps; the freeze - // regression is missing frames, which leaves late timeline lookups null. - it("produces the full frame count on the full VFR file", async () => { - const outputDir = join(FIXTURE_DIR, "out-full"); - mkdirSync(outputDir, { recursive: true }); - - const video: VideoElement = { - id: "vfull", - src: VFR_FIXTURE, - start: 0, - end: 10, - mediaStart: 0, - loop: false, - hasAudio: false, - }; - - const result = await extractAllVideoFrames([video], FIXTURE_DIR, { - fps: 30, - outputDir, - }); - expect(result.errors).toEqual([]); - - const frameDir = join(outputDir, "vfull"); - const frames = readdirSync(frameDir) - .filter((f) => f.endsWith(".jpg")) - .sort(); - // ±3 tolerance: same FFmpeg one-pass VFR→CFR rounding variance as the - // mid-segment test. - expect(frames.length).toBeGreaterThanOrEqual(297); - expect(frames.length).toBeLessThanOrEqual(303); - }, 60_000); -}); - -// Release builds print "version 8.1.3" or "n8.1.3"; master builds print "N-" and are newer. -const FFPROBE_VERSION = spawnSync("ffprobe", ["-version"]).stdout?.toString() ?? ""; -const FFPROBE_RELEASE = FFPROBE_VERSION.match(/version n?(\d+)\.(\d+)/) - ?.slice(1) - .map(Number); -const FFPROBE_READS_CONTAINER_TRANSFER = - /version N-/.test(FFPROBE_VERSION) || - (FFPROBE_RELEASE !== undefined && - (FFPROBE_RELEASE[0]! > 8 || (FFPROBE_RELEASE[0] === 8 && FFPROBE_RELEASE[1]! >= 1))); - -/** The colour probes run so far: first-frame reads, and SPS transfer reads of an HLG source. */ -function colourProbeRuns() { - const runs = vi - .mocked(runFfmpeg) - .mock.calls.map(([args]) => args) - .filter((args) => args.some((arg) => /showinfo|trace_headers/.test(arg))); - return { - firstFrame: runs.filter((args) => args.includes("-frames:v")).length, - keyframes: runs.filter((args) => args.includes("nokey")).length, - }; -} - -function colourProbesSince(before: ReturnType) { - const now = colourProbeRuns(); - return { - firstFrame: now.firstFrame - before.firstFrame, - keyframes: now.keyframes - before.keyframes, - }; -} - -describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => { - let fixtureDir = ""; - - beforeAll(() => { - fixtureDir = mkdtempSync(join(tmpdir(), "hf-forced-sdr-tonemap-test-")); - }); - - afterAll(() => { - rmSync(fixtureDir, { recursive: true, force: true }); - }, 30_000); - - async function synthesizeHlgClip( - path: string, - vui = "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=9", - transfer = "arib-std-b67", - ): Promise { - const synthesized = await runFfmpeg([ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - "color=c=0xe0b080:s=64x64:r=1:d=1", - "-vf", - `zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=${transfer}:m=bt2020nc:r=tv,format=yuv420p`, - "-c:v", - "libx264", - "-preset", - "ultrafast", - "-color_primaries", - "bt2020", - "-color_trc", - transfer, - "-colorspace", - "bt2020nc", - "-bsf:v", - `h264_metadata=${vui}`, - path, - ]); - if (!synthesized.success) { - throw new Error(`HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`); - } - } - - it.each([ - [ - "HLG", - "matrix", - "arib-std-b67", - "colour_primaries=9:transfer_characteristics=18:matrix_coefficients=2", - ], - [ - "HLG", - "primaries", - "arib-std-b67", - "colour_primaries=2:transfer_characteristics=18:matrix_coefficients=9", - ], - [ - "HLG", - "matrix and primaries", - "arib-std-b67", - "colour_primaries=2:transfer_characteristics=18:matrix_coefficients=2", - ], - [ - "PQ", - "matrix and primaries", - "smpte2084", - "colour_primaries=2:transfer_characteristics=16:matrix_coefficients=2", - ], - [ - "HLG", - "transfer", - "arib-std-b67", - "colour_primaries=9:transfer_characteristics=2:matrix_coefficients=9", - ], - ])( - "tone-maps %s footage whose video stream has no %s tag like the fully tagged clip", - async (name, missing, transfer, vui) => { - const slug = `${name}-no-${missing}`.replace(/\W+/g, "-"); - const tagged = join(fixtureDir, `${name}-fully-tagged.mp4`); - const untagged = join(fixtureDir, `${slug}.mp4`); - const vuiTransfer = transfer === "smpte2084" ? 16 : 18; - await synthesizeHlgClip( - tagged, - `colour_primaries=9:transfer_characteristics=${vuiTransfer}:matrix_coefficients=9`, - transfer, - ); - await synthesizeHlgClip(untagged, vui, transfer); - const extract = (source: string, id: string) => - extractVideoFramesRange(source, id, 0, 1, { - fps: 1, - outputDir: join(fixtureDir, `out-${id}`), - format: "png", - toneMapHdrToSdr: true, - }); - - if (missing === "transfer") { - // ffprobe before 8.1 reads the stream's unspecified transfer, so the clip is SDR there, as on main. - if (!FFPROBE_READS_CONTAINER_TRANSFER) { - await expect(extract(untagged, slug)).resolves.toBeDefined(); - return; - } - expect((await extractVideoMetadata(untagged)).colorSpace?.colorTransfer).toBe(transfer); - } - const reference = await extract(tagged, `ref-${slug}`); - const result = await extract(untagged, slug); - - expect(readFileSync(result.framePaths.get(0)!)).toEqual( - readFileSync(reference.framePaths.get(0)!), - ); - }, - 60_000, - ); - - it("probes each source's first frame once, however many cuts of it extract together", async () => { - const sources = ["a", "b"].map((name) => join(fixtureDir, `cuts-${name}.mp4`)); - for (const source of sources) await synthesizeHlgClip(source); - const probes = colourProbeRuns; - const before = probes(); - await Promise.all( - Array.from({ length: 12 }, (_, cut) => - extractVideoFramesRange(sources[cut % 2]!, `cut-${cut}`, 0, 1, { - fps: 1, - outputDir: join(fixtureDir, "cuts"), - format: "png", - toneMapHdrToSdr: true, - }), - ), - ); - expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 }); - }, 60_000); - - it("probes a source again after its first-frame probe failed", async () => { - const source = join(fixtureDir, "probe-retry.mp4"); - await synthesizeHlgClip(source); - const probes = colourProbeRuns; - const { runFfmpeg: real } = - await vi.importActual("../utils/runFfmpeg.js"); - vi.mocked(runFfmpeg).mockImplementation((args, opts) => - args.includes("showinfo") - ? Promise.resolve({ - success: false, - exitCode: 1, - stderr: "", - durationMs: 0, - terminationReason: "exit", - }) - : real(args, opts), - ); - const extract = (id: string) => - extractVideoFramesRange(source, id, 0, 1, { - fps: 1, - outputDir: join(fixtureDir, "retry"), - format: "png", - toneMapHdrToSdr: true, - }); - const before = probes(); - try { - await extract("retry-failed"); - } finally { - vi.mocked(runFfmpeg).mockImplementation(real); - } - await extract("retry-again"); - expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 1 }); - }, 60_000); - - it("tone-maps an HLG source at HLG's white when a large header SEI follows its SPS", async () => { - const source = join(fixtureDir, "hlg-x265-header.mp4"); - // x265 stores its settings as an SEI after the SPS in the file header; traced, it runs past 64 KB. - // No B-frames: at 1 fps their reorder gap is a whole second, which Windows ffmpeg's mp4 muxer can refuse. - const synthesized = await runFfmpeg([ - ...["-y", "-hide_banner", "-loglevel", "error"], - ...["-f", "lavfi", "-i", "color=c=0xe0b080:s=64x64:r=1:d=1"], - "-vf", - "zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=arib-std-b67:m=bt2020nc:r=tv,format=yuv420p10le", - ...[ - "-c:v", - "libx265", - "-x265-params", - "colorprim=bt2020:transfer=arib-std-b67:colormatrix=bt2020nc:log-level=error:bframes=0", - ], - ...["-color_primaries", "bt2020", "-color_trc", "arib-std-b67", "-colorspace", "bt2020nc"], - source, - ]); - if (!synthesized.success) { - throw new Error(`x265 HLG fixture synthesis failed: ${synthesized.stderr.slice(-400)}`); - } - const before = vi.mocked(runFfmpeg).mock.calls.length; - await extractVideoFramesRange(source, "hlg-x265-header", 0, 1, { - fps: 1, - outputDir: join(fixtureDir, "x265-header"), - format: "png", - toneMapHdrToSdr: true, - }); - const toneMaps = vi - .mocked(runFfmpeg) - .mock.calls.slice(before) - .flatMap(([args]) => args.filter((arg) => arg.includes("tonemap="))); - expect(toneMaps).toHaveLength(1); - expect(toneMaps[0]).toContain("npl=203"); - }, 60_000); - - it("reads an HLG source's keyframes again after that read failed", async () => { - const source = join(fixtureDir, "keyframe-retry.mp4"); - await synthesizeHlgClip(source); - const { runFfmpeg: real } = - await vi.importActual("../utils/runFfmpeg.js"); - vi.mocked(runFfmpeg).mockImplementation((args, opts) => - args.includes("nokey") - ? Promise.resolve({ - success: false, - exitCode: 1, - stderr: "", - durationMs: 0, - terminationReason: "exit", - }) - : real(args, opts), - ); - const extract = (id: string) => - extractVideoFramesRange(source, id, 0, 1, { - fps: 1, - outputDir: join(fixtureDir, "keyframe-retry"), - format: "png", - toneMapHdrToSdr: true, - }); - const before = colourProbeRuns(); - try { - await extract("keyframes-failed"); - } finally { - vi.mocked(runFfmpeg).mockImplementation(real); - } - await extract("keyframes-again"); - expect(colourProbesSince(before)).toEqual({ firstFrame: 2, keyframes: 2 }); - }, 60_000); - - // One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer. - it("tone-maps each frame of a mixed HLG and PQ stream with its own transfer", async () => { - const segment = async (name: string, transfer: number) => { - const path = join(fixtureDir, `mixed-${name}.h264`); - const result = await runFfmpeg([ - "-y", - "-v", - "error", - "-f", - "lavfi", - "-i", - "testsrc2=s=160x90:r=25:d=0.2", - "-c:v", - "libx264", - // No B-frames: genpts stamps a raw stream in decode order, so reordered frames get wrong times. - "-bf", - "0", - "-pix_fmt", - "yuv420p", - "-bsf:v", - `h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`, - "-f", - "h264", - path, - ]); - if (!result.success) - throw new Error(`segment synthesis failed: ${result.stderr.slice(-400)}`); - return readFileSync(path); - }; - const mux = async (name: string, stream: Buffer) => { - const raw = join(fixtureDir, `${name}.h264`); - const path = join(fixtureDir, `${name}.mp4`); - writeFileSync(raw, stream); - const result = await runFfmpeg([ - ...["-y", "-v", "error", "-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw], - ...["-c", "copy", path], - ]); - if (!result.success) throw new Error(`mux failed: ${result.stderr.slice(-400)}`); - return path; - }; - const hlg = await segment("hlg", 18); - const pq = await segment("pq", 16); - const mixed = await mux("mixed-hlg-pq", Buffer.concat([hlg, pq])); - const pqOnly = await mux("mixed-pq-only", pq); - const extract = (source: string, id: string, duration: number) => - extractVideoFramesRange(source, id, 0, duration, { - fps: 25, - outputDir: join(fixtureDir, `out-${id}`), - format: "png", - toneMapHdrToSdr: true, - }); - - const mixedFrames = await extract(mixed, "mixed-hlg-pq", 0.4); - const pqFrames = await extract(pqOnly, "mixed-pq-only", 0.2); - - expect(readFileSync(mixedFrames.framePaths.get(5)!)).toEqual( - readFileSync(pqFrames.framePaths.get(0)!), - ); - }, 60_000); - - // A stream-copy trim opens on edit-list pre-roll, which ffprobe counts as packets without frames. - it("keeps per-frame tags in a mixed HLG and PQ file trimmed with a stream copy", async () => { - const run = async (args: string[]) => { - const result = await runFfmpeg(["-y", "-v", "error", ...args]); - if (!result.success) throw new Error(`fixture failed: ${result.stderr.slice(-400)}`); - }; - const segment = async (name: string, transfer: number) => { - const path = join(fixtureDir, `trim-${name}.h264`); - await run([ - ...["-f", "lavfi", "-i", "testsrc2=s=160x90:r=25:d=2"], - ...["-c:v", "libx264", "-g", "50", "-bf", "0"], - ...["-pix_fmt", "yuv420p", "-bsf:v"], - `h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`, - ...["-f", "h264", path], - ]); - return readFileSync(path); - }; - const mux = async (name: string, stream: Buffer) => { - const raw = join(fixtureDir, `${name}.h264`); - const path = join(fixtureDir, `${name}.mp4`); - writeFileSync(raw, stream); - await run(["-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw, "-c", "copy", path]); - return path; - }; - const pq = await segment("pq", 16); - const mixed = await mux("trim-hlg-then-pq", Buffer.concat([await segment("hlg", 18), pq])); - const pqOnly = await mux("trim-pq-only", pq); - const trimmed = join(fixtureDir, "trim-from-hlg.mp4"); - await run(["-ss", "1", "-i", mixed, "-c", "copy", trimmed]); - const extract = (source: string, id: string, start: number) => - extractVideoFramesRange(source, id, start, 0.04, { - fps: 25, - outputDir: join(fixtureDir, `out-${id}`), - format: "png", - toneMapHdrToSdr: true, - }); - - // Trimmed 1.6 s is the mixed file's 2.6 s: the PQ segment's 0.6 s. - const fromTrim = await extract(trimmed, "trim-from-hlg", 1.6); - const fromPq = await extract(pqOnly, "trim-pq-only", 0.6); - - const shown = readFirstFramePixel(fromTrim.framePaths.get(0)!, 40, 40); - const expected = readFirstFramePixel(fromPq.framePaths.get(0)!, 40, 40); - expect(Math.max(...shown.map((v, i) => Math.abs(v - expected[i]!)))).toBeLessThanOrEqual(2); - }, 60_000); - - it("matches Studio's HLG tone map and isolates transformed cache entries", async () => { - const source = join(fixtureDir, "hlg-warm.mp4"); - await synthesizeHlgClip(source); - - const reference = join(fixtureDir, "studio-reference.png"); - const referenceResult = await runFfmpeg([ - "-y", - "-ss", - "0", - "-i", - source, - "-t", - "1", - "-vf", - "fps=1,zscale=t=linear:npl=203,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv", - "-q:v", - "0", - "-compression_level", - "1", - reference, - ]); - if (!referenceResult.success) { - throw new Error(`Studio reference extraction failed: ${referenceResult.stderr.slice(-400)}`); - } - - const cacheDir = join(fixtureDir, "cache"); - const video = (id: string): VideoElement => ({ - id, - src: source, - start: 0, - end: 1, - mediaStart: 0, - loop: false, - hasAudio: false, - }); - const extract = (id: string, toneMapHdrToSdr = false) => - extractAllVideoFrames( - [video(id)], - fixtureDir, - { - fps: 1, - outputDir: join(fixtureDir, id), - format: "png", - toneMapHdrToSdr, - }, - undefined, - { extractCacheDir: cacheDir }, - ); - - const plain = await extract("plain"); - const toneMapped = await extract("tone-mapped", true); - const toneMappedAgain = await extract("tone-mapped-again", true); - - expect(plain.errors).toEqual([]); - expect(toneMapped.errors).toEqual([]); - expect(toneMapped.phaseBreakdown.cacheHits).toBe(0); - expect(toneMapped.phaseBreakdown.cacheMisses).toBe(1); - expect(toneMappedAgain.phaseBreakdown.cacheHits).toBe(1); - expect(readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX))).toHaveLength(2); - - const frame = (result: ExtractionResult): Buffer => { - const path = result.extracted[0]?.framePaths.get(0); - if (!path) throw new Error("expected extracted frame"); - return readFileSync(path); - }; - // Same pixels as Studio's tone map, declared as sRGB so Chrome shows them unconverted. - const rgb = (path: string): Buffer => - spawnSync("ffmpeg", ["-v", "error", "-i", path, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"]) - .stdout; - const toneMappedPath = toneMapped.extracted[0]!.framePaths.get(0)!; - expect(rgb(toneMappedPath)).toEqual(rgb(reference)); - expect(pngChunkTypes(toneMappedPath)).toContain("sRGB"); - expect(pngChunkTypes(toneMappedPath)).not.toContain("cICP"); - expect(frame(plain)).not.toEqual(frame(toneMapped)); - - const toneMappedJpg = await extractAllVideoFrames([video("tone-mapped-jpg")], fixtureDir, { - fps: 1, - outputDir: join(fixtureDir, "tone-mapped-jpg"), - format: "jpg", - toneMapHdrToSdr: true, - }); - expect(toneMappedJpg.errors).toEqual([]); - const jpgPixels = rgb(toneMappedJpg.extracted[0]!.framePaths.get(0)!); - const referencePixels = rgb(reference); - const worst = Math.max(...[...jpgPixels].map((v, i) => Math.abs(v - referencePixels[i]!))); - expect(worst, "tone-mapped jpg against Studio's tone map").toBeLessThanOrEqual(3); - expect(frame(toneMappedAgain)).toEqual(frame(toneMapped)); - - // A macOS ffmpeg that has zscale uses the same tone map, not VideoToolbox. - const platform = Object.getOwnPropertyDescriptor(process, "platform")!; - Object.defineProperty(process, "platform", { value: "darwin" }); - try { - const mac = await extractAllVideoFrames([video("tone-mapped-mac")], fixtureDir, { - fps: 1, - outputDir: join(fixtureDir, "tone-mapped-mac"), - format: "png", - toneMapHdrToSdr: true, - }); - expect(mac.errors).toEqual([]); - expect(rgb(mac.extracted[0]!.framePaths.get(0)!)).toEqual(rgb(reference)); - } finally { - Object.defineProperty(process, "platform", platform); - } - }, 60_000); - - it("warns once and keeps VideoToolbox, under its own cache key, when a macOS ffmpeg has no zscale", async () => { - const source = join(fixtureDir, "hlg-no-zscale.mp4"); - await synthesizeHlgClip(source); - const cacheDir = join(fixtureDir, "no-zscale-cache"); - const clip = (id: string): VideoElement => ({ - id, - src: source, - start: 0, - end: 1, - mediaStart: 0, - loop: false, - hasAudio: false, - }); - const options = (id: string) => ({ - fps: 1, - outputDir: join(fixtureDir, id), - format: "png" as const, - toneMapHdrToSdr: true, - }); - const zscale = await extractAllVideoFrames( - [clip("zscale")], - fixtureDir, - options("zscale"), - undefined, - { - extractCacheDir: cacheDir, - }, - ); - expect(zscale.errors).toEqual([]); - - const realPlatform = process.platform; - vi.resetModules(); - vi.doMock("../utils/psnrFilterAvailability.js", () => ({ - isFfmpegFilterAvailable: async () => false, - isPsnrFilterAvailable: async () => false, - })); - const platform = Object.getOwnPropertyDescriptor(process, "platform")!; - Object.defineProperty(process, "platform", { value: "darwin" }); - const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); - try { - const { extractAllVideoFrames: extractOnMac } = await import("./videoFrameExtractor.js"); - for (const [i, id] of ["no-zscale-1", "no-zscale-2"].entries()) { - const result = await extractOnMac([clip(id)], fixtureDir, options(id), undefined, { - extractCacheDir: cacheDir, - }); - // The zscale frames above must not be served for the VideoToolbox path. - if (i === 0) expect(result.phaseBreakdown.cacheHits).toBe(0); - // Off macOS the VideoToolbox decode itself fails, which proves it was attempted. - if (realPlatform !== "darwin") - expect(JSON.stringify(result.errors)).toMatch(/videotoolbox/i); - } - const zscaleWarnings = stderr.mock.calls.filter(([message]) => - String(message).includes("no zscale filter"), - ); - expect(zscaleWarnings).toHaveLength(1); - } finally { - stderr.mockRestore(); - Object.defineProperty(process, "platform", platform); - vi.doUnmock("../utils/psnrFilterAvailability.js"); - vi.resetModules(); - } - }, 60_000); - - // A second process would read the frames back without the HDR10 light-level metadata the tone map uses. - it("tone-maps a VFR HDR10 window in one process, as the reference tone map would", async () => { - const source = join(fixtureDir, "pq-vfr-4000nit.mp4"); - const synthesized = await runFfmpeg([ - "-y", - "-v", - "error", - "-f", - "lavfi", - "-i", - "color=c=0xf0e0c0:s=64x64:r=60:d=2", - "-vf", - "select='not(between(n\\,30\\,89))',zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=smpte2084:m=bt2020nc:r=tv:npl=1000,format=yuv420p10le", - "-fps_mode", - "vfr", - "-c:v", - "libx265", - "-x265-params", - "log-level=error:hdr10=1:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc:max-cll=4000,400:master-display=G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(40000000,50)", - source, - ]); - if (!synthesized.success) - throw new Error(`PQ fixture synthesis failed: ${synthesized.stderr.slice(-400)}`); - const referenceDir = join(fixtureDir, "pq-vfr-reference"); - mkdirSync(referenceDir, { recursive: true }); - const reference = await runFfmpeg([ - "-v", - "error", - "-ss", - "0", - "-i", - source, - "-t", - "0.616666", - "-vf", - "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv,setparams=color_primaries=bt709:color_trc=iec61966-2-1", - "-fps_mode", - "cfr", - "-r", - "30", - "-q:v", - "0", - "-compression_level", - "1", - join(referenceDir, "frame_%05d.png"), - ]); - expect(reference.success, reference.stderr).toBe(true); - - const result = await extractVideoFramesRange(source, "pq-vfr", 0, 0.616666, { - fps: 30, - outputDir: join(fixtureDir, "pq-vfr-out"), - format: "png", - toneMapHdrToSdr: true, - }); - - expect(result.metadata.isVFR).toBe(true); - const referenceFrames = readdirSync(referenceDir).sort(); - // One slot per 30fps tick before 0.616666s; x265 packet durations cut the reference's still short. - expect(result.totalFrames).toBe(19); - expect(readFileSync(result.framePaths.get(0)!)).toEqual( - readFileSync(join(referenceDir, referenceFrames[0]!)), - ); - }, 60_000); -}); - describe("getFrameAtTime — IEEE 754 boundary precision", () => { function makeExtracted(fps: number, totalFrames: number): ExtractedFrames { const framePaths = new Map(); diff --git a/packages/engine/src/services/videoFrameExtractor.vfr.test.ts b/packages/engine/src/services/videoFrameExtractor.vfr.test.ts new file mode 100644 index 0000000000..9705d6018b --- /dev/null +++ b/packages/engine/src/services/videoFrameExtractor.vfr.test.ts @@ -0,0 +1,1177 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { + parseVideoElements, + extractAllVideoFrames, + extractVideoFramesRange, + createFrameLookupTable, + type VideoElement, + type ExtractedFrames, + type ExtractionResult, +} from "./videoFrameExtractor.js"; +import { extractVideoMetadata } from "../utils/ffprobe.js"; +import { runFfmpeg } from "../utils/runFfmpeg.js"; +import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.js"; + +// Pass-through, so a test can tell whether extraction ran ffmpeg. +vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { + const real = await importOriginal(); + return { ...real, runFfmpeg: vi.fn(real.runFfmpeg) }; +}); + +const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; + +// Irregular timestamps once made the fps filter emit too few frames, so the video froze; +// VFR sources now extract through one `-fps_mode cfr -r` pass. +describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { + const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-vfr-test-")); + const VFR_FIXTURE = join(FIXTURE_DIR, "vfr_screen.mp4"); + + beforeAll(async () => { + // 10 s at 60 fps minus frames 30-89, 180-239, 330-389, 480-539: about 36 fps on average, past + // isVFR's 10% threshold. One keyframe (-g 600) so the mediaStart=3 seek cannot snap to an IDR. + const result = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=10:rate=60", + "-vf", + "select='not(between(n\\,30\\,89))*not(between(n\\,180\\,239))*not(between(n\\,330\\,389))*not(between(n\\,480\\,539))'", + "-fps_mode", + "vfr", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-g", + "600", + "-keyint_min", + "600", + VFR_FIXTURE, + ]); + if (!result.success) { + throw new Error( + `ffmpeg fixture synthesis failed (${result.exitCode}): ${result.stderr.slice(-400)}`, + ); + } + }, 30_000); + + afterAll(() => { + if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true }); + }, 30_000); + + it("skips a clip entirely before time zero without reporting an extraction error", async () => { + const outputDir = join(FIXTURE_DIR, "out-before-timeline"); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "before-timeline", + src: VFR_FIXTURE, + start: -2, + end: -1, + mediaStart: 0, + loop: false, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 1, + outputDir, + timelineEnd: 2, + }); + + expect(result).toMatchObject({ success: true, extracted: [], errors: [] }); + }); + + it("preserves loop phase when negative preroll crosses the source boundary", async () => { + const outputDir = join(FIXTURE_DIR, "out-negative-loop"); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "negative-loop", + src: VFR_FIXTURE, + start: -19, + end: 5, + mediaStart: 0, + loop: true, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 1, + outputDir, + timelineEnd: 2, + }); + + expect(result.errors).toEqual([]); + const extracted = result.extracted[0]; + if (!extracted) throw new Error("expected loop source frames"); + const lookup = createFrameLookupTable([video], result.extracted); + expect(video).toMatchObject({ start: -19, mediaStart: 0, loop: true }); + expect(lookup.getFrame("negative-loop", 0)).toBe( + extracted.framePaths.get(extracted.totalFrames - 1), + ); + }, 30_000); + + it("preserves the held final frame after negative preroll exhausts a source", async () => { + const outputDir = join(FIXTURE_DIR, "out-negative-held-tail"); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "negative-held-tail", + src: VFR_FIXTURE, + start: -15, + end: 5, + mediaStart: 0, + loop: false, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 1, + outputDir, + timelineEnd: 2, + }); + + expect(result.errors).toEqual([]); + const extracted = result.extracted[0]; + if (!extracted) throw new Error("expected held-tail source frames"); + const lookup = createFrameLookupTable([video], result.extracted); + // The authored slot remains active through end=5, but lookup is rebased to + // one exact final frame instead of assuming the last second contains a + // timestamp or materializing the full source. + expect(video).toMatchObject({ start: 0, end: 5, mediaStart: 9.999999, loop: false }); + expect(extracted.totalFrames).toBe(1); + expect(lookup.getFrame("negative-held-tail", 0)).toBe( + extracted.framePaths.get(extracted.totalFrames - 1), + ); + }, 30_000); + + it("detects the synthesized fixture as VFR", async () => { + const md = await extractVideoMetadata(VFR_FIXTURE); + expect(md.isVFR).toBe(true); + }); + + it.each([ + ["compiled EOF", 4, 1], + ["compiled past EOF", 5, 2], + ["ordinary explicit zero", 6, 0], + ])( + "drops a known zero timeline window without extraction or error: %s", + async (label, start, mediaStart) => { + const src = await synthCfrClip(`zero-window-${label.replaceAll(" ", "-")}.mp4`, 1); + const videos = parseVideoElements( + ``, + ); + const outputDir = join(FIXTURE_DIR, `out-zero-window-${label.replaceAll(" ", "-")}`); + const result = await extractAllVideoFrames(videos, FIXTURE_DIR, { fps: 30, outputDir }); + + expect(result.errors).toEqual([]); + expect(result.extracted).toEqual([]); + expect(result.totalFramesExtracted).toBe(0); + expect(videos).toEqual([]); + }, + ); + + it("passes an exact 24000/1001 rate through VFR normalization", async () => { + const outputDir = join(FIXTURE_DIR, "out-vfr-ntsc-boundary"); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractVideoFramesRange(VFR_FIXTURE, "vfr-ntsc-boundary", 0, 0.125125, { + fps: { num: 24000, den: 1001 }, + outputDir, + format: "jpg", + }); + + expect(result.metadata.isVFR).toBe(true); + expect(result.totalFrames).toBe(3); + }, 60_000); + + // With B-frames, mp4 packet durations are decode-order gaps, so a frame can claim 0.5s while + // the next one starts 17ms later. Each slot must still show the frame on screen at its time: + // `-fps_mode cfr` on ffmpeg 8 held such a frame for 14 slots and cut the still before a gap short. + it("samples a B-frame VFR source by timestamp, not by packet duration", async () => { + const source = join(FIXTURE_DIR, "vfr_bframes.mp4"); + const synth = await runFfmpeg([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=4:rate=60", + "-vf", + "select='not(between(n\\,30\\,89))*not(between(n\\,150\\,179))'", + "-fps_mode", + "vfr", + "-c:v", + "libx264", + "-preset", + "medium", + "-bf", + "3", + "-g", + "600", + "-pix_fmt", + "yuv420p", + source, + ]); + if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`); + const distinctFrames = (paths: Iterable) => + new Set([...paths].map((path) => readFileSync(path).toString("base64"))).size; + + // 2.5s to 3s is a still; every 30fps slot after it lands on its own 60fps source frame. + const afterGap = await extractVideoFramesRange(source, "vfr-bframes-after-gap", 2.5, 1.5, { + fps: 30, + outputDir: join(FIXTURE_DIR, "out-vfr-bframes-after-gap"), + format: "png", + }); + expect(afterGap.metadata.isVFR).toBe(true); + expect(afterGap.totalFrames).toBe(45); + const slots = [...afterGap.framePaths.entries()].sort(([a], [b]) => a - b); + expect(distinctFrames(slots.slice(0, 15).map(([, path]) => path))).toBe(1); + expect(distinctFrames(slots.slice(15).map(([, path]) => path))).toBe(30); + + const endsOnStill = await extractVideoFramesRange(source, "vfr-bframes-still", 0, 3, { + fps: 30, + outputDir: join(FIXTURE_DIR, "out-vfr-bframes-still"), + format: "png", + }); + expect(endsOnStill.totalFrames).toBe(90); + }, 60_000); + + it("produces the expected frame count for a mid-file segment", async () => { + const outputDir = join(FIXTURE_DIR, "out-mid-segment"); + mkdirSync(outputDir, { recursive: true }); + + const video: VideoElement = { + id: "v1", + src: VFR_FIXTURE, + start: 0, + end: 4, + mediaStart: 3, + loop: false, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 30, + outputDir, + }); + + expect(result.errors).toEqual([]); + expect(result.extracted).toHaveLength(1); + const frames = readdirSync(join(outputDir, "v1")).filter((f) => f.endsWith(".jpg")); + // Pre-fix behavior produced ~90 frames (a 25% shortfall). + // ±3 tolerance: FFmpeg's one-pass VFR→CFR extraction yields slightly + // different frame counts across versions (timestamp rounding). + expect(frames.length).toBeGreaterThanOrEqual(117); + expect(frames.length).toBeLessThanOrEqual(123); + + expect(result.phaseBreakdown).toBeDefined(); + expect(result.phaseBreakdown.extractMs).toBeGreaterThan(0); + expect(result.phaseBreakdown.vfrPreflightCount).toBe(1); + expect(result.phaseBreakdown.vfrPreflightMs).toBeGreaterThanOrEqual(0); + }, 60_000); + + // Shared fixture helpers for the cache tests below. All synthesize clean + // CFR SDR clips — keeps VFR preflight count at zero so cache keys are + // stable across runs within a test. + async function synthCfrClip(name: string, durationSeconds: number): Promise { + const src = join(FIXTURE_DIR, name); + const synth = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `testsrc2=s=320x180:d=${durationSeconds}:rate=30`, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + src, + ]); + if (!synth.success) { + throw new Error(`Fixture synthesis failed (${name}): ${synth.stderr.slice(-400)}`); + } + return src; + } + + it("rejects a media start beyond source duration before invoking FFmpeg", async () => { + const src = await synthCfrClip("zero-output-src.mp4", 1); + const outputDir = join(FIXTURE_DIR, "out-zero-output"); + await expect( + extractVideoFramesRange(src, "past-eof", 2, 1, { fps: 30, outputDir }), + ).rejects.toMatchObject({ + kind: "media_start_out_of_range", + retryable: false, + }); + }, 60_000); + + it("preserves legacy metadata rejection unless typed aggregation is explicitly enabled", async () => { + const src = join(FIXTURE_DIR, "invalid-probe.mp4"); + writeFileSync(src, "not a media container"); + const video = cfrClipElement("invalid-probe", src, 1); + + await expect( + extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 30, + outputDir: join(FIXTURE_DIR, "out-invalid-probe-legacy"), + }), + ).rejects.toThrow(); + + const collected = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 30, + outputDir: join(FIXTURE_DIR, "out-invalid-probe-typed"), + collectProbeFailures: true, + }); + expect(collected.success).toBe(false); + expect(collected.extracted).toEqual([]); + expect(collected.errors).toEqual([ + expect.objectContaining({ + videoId: "invalid-probe", + kind: "invalid_media", + retryable: false, + }), + ]); + }, 60_000); + + async function synthHdrTaggedClip(name: string, durationSeconds: number): Promise { + const src = join(FIXTURE_DIR, name); + const synth = await runFfmpeg([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `testsrc2=s=320x180:d=${durationSeconds}:rate=30`, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-color_primaries", + "bt2020", + "-color_trc", + "smpte2084", + "-colorspace", + "bt2020nc", + // The -color_* flags above only tag the container/encoder context; + // whether they reach the H.264 VUI depends on the ffmpeg build (the + // pinned Windows CI build drops the transfer). Write the VUI directly + // so probing reports smpte2084 on every build (9/16/9 = bt2020/PQ/bt2020nc). + "-bsf:v", + "h264_metadata=colour_primaries=9:transfer_characteristics=16:matrix_coefficients=9", + src, + ]); + if (!synth.success) { + throw new Error(`HDR fixture synthesis failed (${name}): ${synth.stderr.slice(-400)}`); + } + return src; + } + + function cfrClipElement( + id: string, + src: string, + endSeconds: number, + mediaStart = 0, + ): VideoElement { + return { + id, + src, + start: 0, + end: endSeconds, + mediaStart, + loop: false, + hasAudio: false, + }; + } + + async function extractWithCache( + video: VideoElement, + outName: string, + cacheDir: string, + fps = 30, + ): Promise { + const outputDir = join(FIXTURE_DIR, outName); + mkdirSync(outputDir, { recursive: true }); + return extractAllVideoFrames([video], FIXTURE_DIR, { fps, outputDir }, undefined, { + extractCacheDir: cacheDir, + }); + } + + function cacheEntryNames(cacheDir: string): string[] { + return readdirSync(cacheDir).filter((name) => name.startsWith(SCHEMA_PREFIX)); + } + + function supersetDirNames(outputDir: string): string[] { + if (!existsSync(outputDir)) return []; + return readdirSync(outputDir).filter((name) => name.startsWith("__superset-")); + } + + function extractedFor(result: ExtractionResult, videoId: string): ExtractedFrames { + const extracted = result.extracted.find((item) => item.videoId === videoId); + if (!extracted) throw new Error(`missing extraction result for ${videoId}`); + return extracted; + } + + function framePath(result: ExtractionResult, videoId: string, frameIndex: number): string { + const extracted = extractedFor(result, videoId); + const frame = extracted?.framePaths.get(frameIndex); + if (!frame) throw new Error(`missing frame ${frameIndex} for ${videoId}`); + return frame; + } + + it("reuses extracted frames on a warm cache hit", async () => { + const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-")); + const SRC = await synthCfrClip("cache-src.mp4", 2); + const video = cfrClipElement("cv1", SRC, 2); + + const miss = await extractWithCache(video, "out-cache-miss", CACHE_DIR); + expect(miss.errors).toEqual([]); + expect(miss.phaseBreakdown.cacheHits).toBe(0); + expect(miss.phaseBreakdown.cacheMisses).toBe(1); + const ffmpegRunsBeforeHit = vi.mocked(runFfmpeg).mock.calls.length; + + const hit = await extractWithCache(video, "out-cache-hit", CACHE_DIR); + expect(hit.errors).toEqual([]); + expect(hit.phaseBreakdown.cacheHits).toBe(1); + expect(hit.phaseBreakdown.cacheMisses).toBe(0); + expect(vi.mocked(runFfmpeg).mock.calls.length).toBe(ffmpegRunsBeforeHit); + expect(hit.extracted).toHaveLength(1); + expect(hit.extracted[0]!.totalFrames).toBe(miss.extracted[0]!.totalFrames); + + rmSync(CACHE_DIR, { recursive: true, force: true }); + }, 60_000); + + it("does not reuse a decimal-rate VFR cache entry for the exact rational rate", async () => { + const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-cache-ntsc-rate-test-")); + const decimalOutputDir = join(FIXTURE_DIR, "out-cache-vfr-ntsc-decimal"); + const rationalOutputDir = join(FIXTURE_DIR, "out-cache-vfr-ntsc-rational"); + mkdirSync(decimalOutputDir, { recursive: true }); + mkdirSync(rationalOutputDir, { recursive: true }); + try { + const decimal = await extractAllVideoFrames( + [cfrClipElement("vfr-ntsc-cache", VFR_FIXTURE, 0.125125)], + FIXTURE_DIR, + { fps: 24000 / 1001, outputDir: decimalOutputDir }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(decimal.errors).toEqual([]); + expect(decimal.phaseBreakdown.cacheMisses).toBe(1); + expect(decimal.extracted[0]?.totalFrames).toBe(3); + // Model the warm v3 entry from before exact-rate extraction: the + // decimal path could persist one extra frame at this boundary. If the + // rational lookup collides, rehydration below will observe all four. + writeFileSync( + join(decimal.extracted[0]!.outputDir, "frame_00004.jpg"), + "stale-decimal-boundary-frame", + "utf-8", + ); + + const rational = await extractAllVideoFrames( + [cfrClipElement("vfr-ntsc-cache", VFR_FIXTURE, 0.125125)], + FIXTURE_DIR, + { fps: { num: 24000, den: 1001 }, outputDir: rationalOutputDir }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(rational.errors).toEqual([]); + expect(rational.phaseBreakdown.cacheHits).toBe(0); + expect(rational.phaseBreakdown.cacheMisses).toBe(1); + expect(rational.extracted[0]?.totalFrames).toBe(3); + expect(cacheEntryNames(cacheDir)).toHaveLength(2); + } finally { + rmSync(cacheDir, { recursive: true, force: true }); + } + }, 60_000); + + it("reuses one-cycle loop extraction across different authored starts", async () => { + const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-loop-phase-cache-test-")); + const src = await synthCfrClip("cache-loop-phase-src.mp4", 3); + try { + const firstOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-first"); + const secondOutputDir = join(FIXTURE_DIR, "out-cache-loop-phase-second"); + mkdirSync(firstOutputDir, { recursive: true }); + mkdirSync(secondOutputDir, { recursive: true }); + + const first = await extractAllVideoFrames( + [ + { + ...cfrClipElement("loop-cache-first", src, 60), + loop: true, + }, + ], + FIXTURE_DIR, + { fps: 30, outputDir: firstOutputDir, timelineEnd: 60 }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(first.errors).toEqual([]); + expect(first.phaseBreakdown.cacheMisses).toBe(1); + + const second = await extractAllVideoFrames( + [ + { + ...cfrClipElement("loop-cache-second", src, 66), + start: -6, + loop: true, + }, + ], + FIXTURE_DIR, + { fps: 30, outputDir: secondOutputDir, timelineEnd: 60 }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(second.errors).toEqual([]); + expect(second.phaseBreakdown.cacheHits).toBe(1); + expect(second.phaseBreakdown.cacheMisses).toBe(0); + expect(second.extracted[0]?.totalFrames).toBe(first.extracted[0]?.totalFrames); + } finally { + rmSync(cacheDir, { recursive: true, force: true }); + } + }, 60_000); + + it("updates the cache sentinel mtime on a hit", async () => { + const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-touch-test-")); + const SRC = await synthCfrClip("cache-touch-src.mp4", 1); + const video = cfrClipElement("touch", SRC, 1); + + const miss = await extractWithCache(video, "out-cache-touch-miss", CACHE_DIR); + expect(miss.errors).toEqual([]); + expect(miss.phaseBreakdown.cacheMisses).toBe(1); + + const cacheEntryNames = readdirSync(CACHE_DIR).filter((name) => name.startsWith(SCHEMA_PREFIX)); + expect(cacheEntryNames).toHaveLength(1); + const sentinel = join(CACHE_DIR, cacheEntryNames[0]!, COMPLETE_SENTINEL); + const old = new Date(Date.now() - 120_000); + utimesSync(sentinel, old, old); + const before = statSync(sentinel).mtimeMs; + + const hit = await extractWithCache(video, "out-cache-touch-hit", CACHE_DIR); + + expect(hit.errors).toEqual([]); + expect(hit.phaseBreakdown.cacheHits).toBe(1); + expect(statSync(sentinel).mtimeMs).toBeGreaterThan(before); + + rmSync(CACHE_DIR, { recursive: true, force: true }); + }, 60_000); + + it("skips cache GC on all-hit renders", async () => { + const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-gc-skip-test-")); + const SRC = await synthCfrClip("cache-gc-skip-src.mp4", 1); + const video = cfrClipElement("gc-skip", SRC, 1); + + const miss = await extractWithCache(video, "out-cache-gc-skip-miss", CACHE_DIR); + expect(miss.errors).toEqual([]); + expect(miss.phaseBreakdown.cacheMisses).toBe(1); + + const agedPartial = join(CACHE_DIR, `${SCHEMA_PREFIX}aged.partial-1234-deadbeef`); + mkdirSync(agedPartial, { recursive: true }); + writeFileSync(join(agedPartial, "frame_00001.jpg"), "stale", "utf-8"); + const old = new Date(Date.now() - 2 * 60 * 60 * 1000); + utimesSync(agedPartial, old, old); + + const hit = await extractWithCache(video, "out-cache-gc-skip-hit", CACHE_DIR); + expect(hit.errors).toEqual([]); + expect(hit.phaseBreakdown.cacheHits).toBe(1); + expect(hit.phaseBreakdown.cacheMisses).toBe(0); + expect(existsSync(agedPartial)).toBe(true); + + rmSync(CACHE_DIR, { recursive: true, force: true }); + }, 60_000); + + it("disables caching for this render when the cache dir is not writable", async () => { + const CACHE_FILE = join(FIXTURE_DIR, "cache-dir-is-a-file"); + writeFileSync(CACHE_FILE, "not a directory", "utf-8"); + const SRC = await synthCfrClip("cache-disabled-src.mp4", 1); + + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + try { + const result = await extractWithCache( + cfrClipElement("uncached", SRC, 1), + "out-cache-disabled", + CACHE_FILE, + ); + + expect(result.errors).toEqual([]); + expect(result.extracted).toHaveLength(1); + expect(result.phaseBreakdown.cacheHits).toBe(0); + expect(result.phaseBreakdown.cacheMisses).toBe(0); + expect(stderr).toHaveBeenCalledTimes(1); + expect(String(stderr.mock.calls[0]?.[0])).toContain("extraction cache dir"); + expect(String(stderr.mock.calls[0]?.[0])).toContain("caching disabled for this render"); + } finally { + stderr.mockRestore(); + } + }, 60_000); + + it("invalidates the cache when fps changes", async () => { + const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-test-")); + const SRC = await synthCfrClip("cache-fps-src.mp4", 1); + const video = cfrClipElement("cv2", SRC, 1); + + const first = await extractWithCache(video, "out-cache-fps-30", CACHE_DIR); + expect(first.phaseBreakdown.cacheMisses).toBe(1); + + const second = await extractWithCache(video, "out-cache-fps-60", CACHE_DIR, 60); + expect(second.phaseBreakdown.cacheMisses).toBe(1); + expect(second.phaseBreakdown.cacheHits).toBe(0); + + rmSync(CACHE_DIR, { recursive: true, force: true }); + }, 60_000); + + it("applies SDR→HDR conversion during extraction without normalized intermediates", async () => { + const SDR_LONG = await synthCfrClip("sdr-long.mp4", 10); + const HDR_SHORT = await synthHdrTaggedClip("hdr-short.mp4", 2); + const outputDir = join(FIXTURE_DIR, "out-hdr-segment"); + const plainOutputDir = join(FIXTURE_DIR, "out-hdr-plain-sdr"); + mkdirSync(outputDir, { recursive: true }); + mkdirSync(plainOutputDir, { recursive: true }); + + const videos: VideoElement[] = [ + { id: "sdr", src: SDR_LONG, start: 0, end: 2, mediaStart: 0, loop: false, hasAudio: false }, + { + id: "hdr", + src: HDR_SHORT, + start: 2, + end: 4, + mediaStart: 0, + loop: false, + hasAudio: false, + }, + ]; + + const result = await extractAllVideoFrames(videos, FIXTURE_DIR, { + fps: 30, + outputDir, + }); + expect(result.errors).toEqual([]); + expect(result.phaseBreakdown.hdrPreflightCount).toBe(1); + expect(existsSync(join(outputDir, "_hdr_normalized"))).toBe(false); + + const sdrFrames = result.extracted.find((item) => item.videoId === "sdr"); + expect(sdrFrames?.totalFrames).toBe(60); + + const plain = await extractVideoFramesRange(SDR_LONG, "plain-sdr", 0, 2, { + fps: 30, + outputDir: plainOutputDir, + format: "jpg", + }); + expect(plain.totalFrames).toBe(60); + expect( + readFileSync(framePath(result, "sdr", 0)).equals(readFileSync(plain.framePaths.get(0)!)), + ).toBe(false); + }, 60_000); + + it("leaves SDR clips alone in a mixed timeline that tone-maps to SDR", async () => { + const SDR = await synthCfrClip("sdr-tonemap-peer.mp4", 1); + const HDR = await synthHdrTaggedClip("hdr-tonemap-peer.mp4", 1); + const outputDir = join(FIXTURE_DIR, "out-hdr-tonemap-mixed"); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractAllVideoFrames( + [cfrClipElement("sdr", SDR, 1), { ...cfrClipElement("hdr", HDR, 1), start: 1, end: 2 }], + FIXTURE_DIR, + { fps: 30, outputDir, toneMapHdrToSdr: true }, + ); + + expect(result.errors).toEqual([]); + expect(result.phaseBreakdown.hdrPreflightCount).toBe(0); + }, 60_000); + + it("keeps a finite SDR past-EOF slot in a mixed HDR timeline", async () => { + const SDR_SHORT = await synthCfrClip("sdr-past-eof.mp4", 1); + const HDR_SHORT = await synthHdrTaggedClip("hdr-past-eof-peer.mp4", 1); + const outputDir = join(FIXTURE_DIR, "out-hdr-past-eof"); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractAllVideoFrames( + [ + cfrClipElement("sdr-past-eof", SDR_SHORT, 4, 5), + { ...cfrClipElement("hdr-peer", HDR_SHORT, 1), start: 1, end: 2 }, + ], + FIXTURE_DIR, + { fps: 30, outputDir }, + ); + + // The SDR slot must survive the mixed-HDR preflight and use the held-tail + // path. Reverting its guard to `mediaStart >= playableDuration` records an + // out-of-range error here and drops the slot before extraction. + expect(result.errors).toEqual([]); + expect(result.phaseBreakdown.hdrPreflightCount).toBe(1); + expect(extractedFor(result, "sdr-past-eof").totalFrames).toBe(1); + expect(extractedFor(result, "hdr-peer").totalFrames).toBeGreaterThan(0); + }, 60_000); + + it("keeps SDR→HDR cache entries distinct from plain SDR entries", async () => { + const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-hdr-cache-test-")); + const SDR = await synthCfrClip("cache-hdr-sdr.mp4", 1); + const HDR = await synthHdrTaggedClip("cache-hdr-hdr.mp4", 1); + try { + const mixedOutputDir = join(FIXTURE_DIR, "out-cache-hdr-mixed"); + mkdirSync(mixedOutputDir, { recursive: true }); + const mixed = await extractAllVideoFrames( + [ + cfrClipElement("sdr-transform", SDR, 1), + { ...cfrClipElement("hdr-peer", HDR, 1), start: 1, end: 2 }, + ], + FIXTURE_DIR, + { fps: 30, outputDir: mixedOutputDir }, + undefined, + { extractCacheDir: CACHE_DIR }, + ); + expect(mixed.errors).toEqual([]); + expect(mixed.phaseBreakdown.hdrPreflightCount).toBe(1); + expect(mixed.phaseBreakdown.cacheHits).toBe(0); + expect(mixed.phaseBreakdown.cacheMisses).toBe(2); + + const plain = await extractWithCache( + cfrClipElement("sdr-plain", SDR, 1), + "out-cache-hdr-plain", + CACHE_DIR, + ); + expect(plain.errors).toEqual([]); + expect(plain.phaseBreakdown.cacheHits).toBe(0); + expect(plain.phaseBreakdown.cacheMisses).toBe(1); + expect(cacheEntryNames(CACHE_DIR)).toHaveLength(3); + + // Cross-render poisoning regression: the plain-SDR render must not be + // served the BT.2020-converted frames the mixed render cached for the + // SAME source+trim. Compare actual frame bytes across the cache + // boundary, not just entry counts. + expect( + readFileSync(framePath(plain, "sdr-plain", 0)).equals( + readFileSync(framePath(mixed, "sdr-transform", 0)), + ), + ).toBe(false); + + // And a repeat plain render must HIT the plain entry and serve + // byte-identical plain frames (proves the hit path keys correctly too). + const plainAgain = await extractWithCache( + cfrClipElement("sdr-plain-again", SDR, 1), + "out-cache-hdr-plain-again", + CACHE_DIR, + ); + expect(plainAgain.phaseBreakdown.cacheHits).toBe(1); + expect( + readFileSync(framePath(plainAgain, "sdr-plain-again", 0)).equals( + readFileSync(framePath(plain, "sdr-plain", 0)), + ), + ).toBe(true); + } finally { + rmSync(CACHE_DIR, { recursive: true, force: true }); + } + }, 60_000); + + it("clusters overlap components so a disjoint outlier does not break the group", async () => { + const SRC = await synthCfrClip("superset-cluster-src.mp4", 12); + const outputDir = join(FIXTURE_DIR, "out-superset-cluster"); + mkdirSync(outputDir, { recursive: true }); + + // Three overlapping trims [0..4], [2..6], [4..8] plus one disjoint trim + // [10..12] of the same source. Pre-clustering, the outlier failed the + // union<=sum check for the whole bucket and ALL FOUR fell back to direct + // extraction; the overlapping three must still share one superset. + const result = await extractAllVideoFrames( + [ + cfrClipElement("cl-a", SRC, 4, 0), + cfrClipElement("cl-b", SRC, 4, 2), + cfrClipElement("cl-c", SRC, 4, 4), + cfrClipElement("cl-out", SRC, 2, 10), + ], + FIXTURE_DIR, + { fps: 30, outputDir }, + ); + + expect(result.errors).toEqual([]); + // Overlap region t=2..4 of the source: cl-a frame 60 and cl-b frame 0 + // must be the SAME inode (shared superset extraction). + expect(statSync(framePath(result, "cl-a", 60)).ino).toBe( + statSync(framePath(result, "cl-b", 0)).ino, + ); + expect(statSync(framePath(result, "cl-b", 60)).ino).toBe( + statSync(framePath(result, "cl-c", 0)).ino, + ); + // The outlier extracted directly: its frames share no inode with the + // cluster (frame at source t=10 exists only in its own extraction). + expect(extractedFor(result, "cl-out").totalFrames).toBe(60); + expect(supersetDirNames(outputDir)).toEqual([]); + }, 60_000); + + it("runs the GC staleness fallback sweep on all-hit renders with a stale marker", async () => { + const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-cache-gc-stale-test-")); + const SRC = await synthCfrClip("cache-gc-stale-src.mp4", 1); + const video = cfrClipElement("gc-stale", SRC, 1); + + const miss = await extractWithCache(video, "out-cache-gc-stale-miss", CACHE_DIR); + expect(miss.phaseBreakdown.cacheMisses).toBe(1); + + const agedPartial = join(CACHE_DIR, `${SCHEMA_PREFIX}aged.partial-1234-cafef00d`); + mkdirSync(agedPartial, { recursive: true }); + writeFileSync(join(agedPartial, "frame_00001.jpg"), "stale", "utf-8"); + const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000); + utimesSync(agedPartial, twoHoursAgo, twoHoursAgo); + + // Age the sweep marker past the 24h staleness window: the next all-hit + // render must sweep anyway and clear the aged partial. + const twoDaysAgo = new Date(Date.now() - 48 * 60 * 60 * 1000); + utimesSync(join(CACHE_DIR, GC_MARKER), twoDaysAgo, twoDaysAgo); + + const hit = await extractWithCache(video, "out-cache-gc-stale-hit", CACHE_DIR); + expect(hit.phaseBreakdown.cacheHits).toBe(1); + expect(hit.phaseBreakdown.cacheMisses).toBe(0); + expect(existsSync(agedPartial)).toBe(false); + expect(hit.phaseBreakdown.cacheAgedPartialsCleared).toBe(1); + + rmSync(CACHE_DIR, { recursive: true, force: true }); + }, 60_000); + + it("hardlinks overlapping aligned trims from one superset extraction", async () => { + const SRC = await synthCfrClip("superset-overlap-src.mp4", 10); + const outputDir = join(FIXTURE_DIR, "out-superset-overlap"); + const directOutputDir = join(FIXTURE_DIR, "out-superset-direct"); + mkdirSync(outputDir, { recursive: true }); + mkdirSync(directOutputDir, { recursive: true }); + + const result = await extractAllVideoFrames( + [cfrClipElement("trim-a", SRC, 4, 0), cfrClipElement("trim-b", SRC, 4, 2)], + FIXTURE_DIR, + { fps: 30, outputDir }, + ); + + expect(result.errors).toEqual([]); + expect(extractedFor(result, "trim-a").totalFrames).toBe(120); + expect(extractedFor(result, "trim-b").totalFrames).toBe(120); + expect(statSync(framePath(result, "trim-a", 60)).ino).toBe( + statSync(framePath(result, "trim-b", 0)).ino, + ); + + const direct = await extractVideoFramesRange(SRC, "direct-trim-b", 2, 4, { + fps: 30, + outputDir: directOutputDir, + format: "jpg", + }); + expect( + readFileSync(framePath(result, "trim-b", 0)).equals(readFileSync(direct.framePaths.get(0)!)), + ).toBe(true); + expect(supersetDirNames(outputDir)).toEqual([]); + }, 60_000); + + it.each([ + { label: "decimal underflow half-frame", duration: 2.05, offset: 1, expectedFrames: 62 }, + { + label: "non-half-integer boundary", + duration: 0.616666, + offset: 0.1, + expectedFrames: 19, + }, + ])( + "keeps direct and superset CFR extraction equal at a $label", + async ({ label, duration, offset, expectedFrames }) => { + const fixtureKey = label.replaceAll(" ", "-"); + const src = await synthCfrClip(`superset-cfr-${fixtureKey}.mp4`, 4); + const groupedOutputDir = join(FIXTURE_DIR, `out-superset-cfr-${fixtureKey}`); + const directOutputDir = join(FIXTURE_DIR, `out-direct-cfr-${fixtureKey}`); + mkdirSync(groupedOutputDir, { recursive: true }); + mkdirSync(directOutputDir, { recursive: true }); + + const grouped = await extractAllVideoFrames( + [ + cfrClipElement(`${fixtureKey}-base`, src, duration, 0), + cfrClipElement(`${fixtureKey}-member`, src, duration, offset), + ], + FIXTURE_DIR, + { fps: 30, outputDir: groupedOutputDir }, + ); + const direct = await extractVideoFramesRange(src, `${fixtureKey}-direct`, offset, duration, { + fps: 30, + outputDir: directOutputDir, + format: "jpg", + }); + + expect(grouped.errors).toEqual([]); + expect(direct.totalFrames).toBe(expectedFrames); + expect(extractedFor(grouped, `${fixtureKey}-base`).totalFrames).toBe(expectedFrames); + expect(extractedFor(grouped, `${fixtureKey}-member`).totalFrames).toBe(expectedFrames); + expect(statSync(framePath(grouped, `${fixtureKey}-base`, Math.round(offset * 30))).ino).toBe( + statSync(framePath(grouped, `${fixtureKey}-member`, 0)).ino, + ); + for (let frame = 0; frame < expectedFrames; frame += 1) { + expect( + readFileSync(framePath(grouped, `${fixtureKey}-member`, frame)).equals( + readFileSync(direct.framePaths.get(frame)!), + ), + ).toBe(true); + } + expect(supersetDirNames(groupedOutputDir)).toEqual([]); + }, + 60_000, + ); + + it("keeps direct and superset CFR extraction equal at 30000/1001", async () => { + const src = await synthCfrClip("superset-cfr-ntsc.mp4", 4); + const groupedOutputDir = join(FIXTURE_DIR, "out-superset-cfr-ntsc"); + const directOutputDir = join(FIXTURE_DIR, "out-direct-cfr-ntsc"); + const fps = { num: 30000, den: 1001 }; + const duration = 0.25025; + mkdirSync(groupedOutputDir, { recursive: true }); + mkdirSync(directOutputDir, { recursive: true }); + + const grouped = await extractAllVideoFrames( + [ + cfrClipElement("ntsc-base", src, 0.5005, 0), + cfrClipElement("ntsc-member", src, duration, 0), + ], + FIXTURE_DIR, + { fps, outputDir: groupedOutputDir }, + ); + const direct = await extractVideoFramesRange(src, "ntsc-direct", 0, duration, { + fps, + outputDir: directOutputDir, + format: "jpg", + }); + + expect(grouped.errors).toEqual([]); + expect(direct.totalFrames).toBe(8); + expect(extractedFor(grouped, "ntsc-base").totalFrames).toBe(15); + expect(extractedFor(grouped, "ntsc-member").totalFrames).toBe(8); + expect(statSync(framePath(grouped, "ntsc-base", 0)).ino).toBe( + statSync(framePath(grouped, "ntsc-member", 0)).ino, + ); + for (let frame = 0; frame < 8; frame += 1) { + expect( + readFileSync(framePath(grouped, "ntsc-member", frame)).equals( + readFileSync(direct.framePaths.get(frame)!), + ), + ).toBe(true); + } + expect(supersetDirNames(groupedOutputDir)).toEqual([]); + }, 60_000); + + it("does not superset disjoint trims", async () => { + const SRC = await synthCfrClip("superset-disjoint-src.mp4", 10); + const outputDir = join(FIXTURE_DIR, "out-superset-disjoint"); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractAllVideoFrames( + [cfrClipElement("trim-a", SRC, 2, 0), cfrClipElement("trim-b", SRC, 2, 8)], + FIXTURE_DIR, + { fps: 30, outputDir }, + ); + + expect(result.errors).toEqual([]); + expect(statSync(framePath(result, "trim-a", 0)).ino).not.toBe( + statSync(framePath(result, "trim-b", 0)).ino, + ); + expect(supersetDirNames(outputDir)).toEqual([]); + }, 60_000); + + it("does not superset trims whose offsets are not frame-aligned", async () => { + const SRC = await synthCfrClip("superset-misaligned-src.mp4", 2); + const outputDir = join(FIXTURE_DIR, "out-superset-misaligned"); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractAllVideoFrames( + [cfrClipElement("trim-a", SRC, 1, 0), cfrClipElement("trim-b", SRC, 1, 0.017)], + FIXTURE_DIR, + { fps: 30, outputDir }, + ); + + expect(result.errors).toEqual([]); + expect(statSync(framePath(result, "trim-a", 0)).ino).not.toBe( + statSync(framePath(result, "trim-b", 0)).ino, + ); + expect(supersetDirNames(outputDir)).toEqual([]); + }, 60_000); + + it("keeps overlapping VFR trims direct because CFR resampling phase resets per seek", async () => { + const outputDir = join(FIXTURE_DIR, "out-vfr-superset-short"); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractAllVideoFrames( + [ + cfrClipElement("vfr-short-a", VFR_FIXTURE, 0.616666, 0), + cfrClipElement("vfr-short-b", VFR_FIXTURE, 0.616666, 0.1), + ], + FIXTURE_DIR, + { fps: 30, outputDir }, + ); + + expect(result.errors).toEqual([]); + expect(extractedFor(result, "vfr-short-a").metadata.isVFR).toBe(true); + expect(extractedFor(result, "vfr-short-a").totalFrames).toBe(19); + expect(extractedFor(result, "vfr-short-b").totalFrames).toBe(19); + expect(statSync(framePath(result, "vfr-short-a", 3)).ino).not.toBe( + statSync(framePath(result, "vfr-short-b", 0)).ino, + ); + expect(supersetDirNames(outputDir)).toEqual([]); + }, 60_000); + + it("keeps batched and direct VFR extraction equal at a floating integral boundary", async () => { + const groupedOutputDir = join(FIXTURE_DIR, "out-vfr-superset-integral-boundary"); + const directOutputDir = join(FIXTURE_DIR, "out-vfr-direct-integral-boundary"); + mkdirSync(groupedOutputDir, { recursive: true }); + mkdirSync(directOutputDir, { recursive: true }); + + const first: VideoElement = { + id: "vfr-integral-a", + src: VFR_FIXTURE, + start: 0.03, + end: 0.33, + mediaStart: 0.03, + loop: false, + hasAudio: false, + }; + const second: VideoElement = { + ...first, + id: "vfr-integral-b", + mediaStart: 0.13, + }; + + const direct = await extractAllVideoFrames([{ ...second }], FIXTURE_DIR, { + fps: 30, + outputDir: directOutputDir, + }); + const grouped = await extractAllVideoFrames([{ ...first }, second], FIXTURE_DIR, { + fps: 30, + outputDir: groupedOutputDir, + }); + + expect(direct.errors).toEqual([]); + expect(grouped.errors).toEqual([]); + expect(extractedFor(direct, second.id).totalFrames).toBe(9); + expect(extractedFor(grouped, first.id).totalFrames).toBe(9); + expect(extractedFor(grouped, second.id).totalFrames).toBe(9); + for (let frame = 0; frame < 9; frame += 1) { + expect( + readFileSync(framePath(grouped, second.id, frame)).equals( + readFileSync(framePath(direct, second.id, frame)), + ), + ).toBe(true); + } + expect(statSync(framePath(grouped, first.id, 3)).ino).not.toBe( + statSync(framePath(grouped, second.id, 0)).ino, + ); + expect(supersetDirNames(groupedOutputDir)).toEqual([]); + }, 60_000); + + it("publishes overlapping superset slices to cache entries and hits them on the next render", async () => { + const CACHE_DIR = mkdtempSync(join(tmpdir(), "hf-extract-superset-cache-test-")); + const SRC = await synthCfrClip("superset-cache-src.mp4", 10); + try { + const firstOutputDir = join(FIXTURE_DIR, "out-superset-cache-first"); + const secondOutputDir = join(FIXTURE_DIR, "out-superset-cache-second"); + mkdirSync(firstOutputDir, { recursive: true }); + mkdirSync(secondOutputDir, { recursive: true }); + const videos = [cfrClipElement("trim-a", SRC, 4, 0), cfrClipElement("trim-b", SRC, 4, 2)]; + + const first = await extractAllVideoFrames( + videos, + FIXTURE_DIR, + { fps: 30, outputDir: firstOutputDir }, + undefined, + { extractCacheDir: CACHE_DIR }, + ); + expect(first.errors).toEqual([]); + expect(first.phaseBreakdown.cacheHits).toBe(0); + expect(first.phaseBreakdown.cacheMisses).toBe(2); + expect(cacheEntryNames(CACHE_DIR)).toHaveLength(2); + expect(statSync(framePath(first, "trim-a", 60)).ino).toBe( + statSync(framePath(first, "trim-b", 0)).ino, + ); + + const second = await extractAllVideoFrames( + videos, + FIXTURE_DIR, + { fps: 30, outputDir: secondOutputDir }, + undefined, + { extractCacheDir: CACHE_DIR }, + ); + expect(second.errors).toEqual([]); + expect(second.phaseBreakdown.cacheHits).toBe(2); + expect(second.phaseBreakdown.cacheMisses).toBe(0); + expect(supersetDirNames(secondOutputDir)).toEqual([]); + } finally { + rmSync(CACHE_DIR, { recursive: true, force: true }); + } + }, 60_000); + + it("clamps loop-past-EOF superset slices to available source frames", async () => { + const SRC = await synthCfrClip("superset-eof-src.mp4", 10); + const outputDir = join(FIXTURE_DIR, "out-superset-eof"); + mkdirSync(outputDir, { recursive: true }); + + const result = await extractAllVideoFrames( + [cfrClipElement("covered", SRC, 4, 6), cfrClipElement("past-eof", SRC, 6, 8)], + FIXTURE_DIR, + { fps: 30, outputDir }, + ); + + expect(result.errors).toEqual([]); + expect(extractedFor(result, "covered").totalFrames).toBe(120); + expect(extractedFor(result, "past-eof").totalFrames).toBe(60); + expect(statSync(framePath(result, "covered", 60)).ino).toBe( + statSync(framePath(result, "past-eof", 0)).ino, + ); + expect(supersetDirNames(outputDir)).toEqual([]); + }, 60_000); + + // Asserts frame-count correctness for a full VFR file. One-pass CFR image + // extraction may repeat held source frames across timestamp gaps; the freeze + // regression is missing frames, which leaves late timeline lookups null. + it("produces the full frame count on the full VFR file", async () => { + const outputDir = join(FIXTURE_DIR, "out-full"); + mkdirSync(outputDir, { recursive: true }); + + const video: VideoElement = { + id: "vfull", + src: VFR_FIXTURE, + start: 0, + end: 10, + mediaStart: 0, + loop: false, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 30, + outputDir, + }); + expect(result.errors).toEqual([]); + + const frameDir = join(outputDir, "vfull"); + const frames = readdirSync(frameDir) + .filter((f) => f.endsWith(".jpg")) + .sort(); + // ±3 tolerance: same FFmpeg one-pass VFR→CFR rounding variance as the + // mid-segment test. + expect(frames.length).toBeGreaterThanOrEqual(297); + expect(frames.length).toBeLessThanOrEqual(303); + }, 60_000); +});