fix(engine): a 29.97 fps camera clip exports smoothly at 30 fps (#5849)

* fix(engine): a 29.97 fps camera clip exports smoothly at 30 fps

Extraction samples each output frame from the last source frame starting at
or before it. Millisecond container ticks move each frame by up to half a
millisecond, while 29.97 fps drifts against 30 fps by 0.033 ms per frame, so
the rounding decided the slot and the export alternated repeats and skips.
Constant-rate frames are now re-spaced on the stream's own frame grid from
the first decoded frame before sampling. The cache schema moves to v8 so
exports stop reusing frames extracted the old way.

* fix(engine): every extraction process snaps frames to one absolute grid

The grid was anchored on each process's first decoded frame, which carries its
own millisecond rounding, so a segment or distributed chunk could place the
29.97 -> 30 repeat elsewhere and skip a frame at its seam. Frames are now
snapped to the stream's own grid in file time (this process's seek plus the
video's lead), the same in every process.

packages/engine/src/services/extractionCache.test.ts

@@ -68,7 +68,7 @@ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: s
 
 describe("extractionCache constants", () => {
   it("exposes the v7 schema prefix", () => {
-    expect(SCHEMA_PREFIX).toBe("hfcache-v7-");
+    expect(SCHEMA_PREFIX).toBe("hfcache-v8-");
   });
 
   it("exposes the frame filename prefix shared with the extractor", () => {

packages/engine/src/services/extractionCache.ts

@@ -62,7 +62,7 @@ export const GC_MARKER = ".hf-last-gc";
  * Current schema version. Bump it whenever extraction writes different frames for the same key,
  * or warm entries keep serving the old frames across a deploy. Each bump's commit says why.
  */
-export const SCHEMA_PREFIX = "hfcache-v7-";
+export const SCHEMA_PREFIX = "hfcache-v8-";
 
 /** Truncated hex chars of SHA-256 used for the entry directory name. */
 const KEY_HEX_CHARS = 16;

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -1653,6 +1653,101 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
     60_000,
   );
 
+  it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => {
+    // Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks.
+    const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4");
+    const synth = await runFfmpeg([
+      ...["-y", "-v", "error", "-f", "lavfi", "-i"],
+      "color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
+      ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
+      fixture,
+    ]);
+    if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
+
+    const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, {
+      fps: 30,
+      outputDir: join(FIXTURE_DIR, "out-ntsc-ms"),
+      format: "png",
+    });
+    const indexOf = (file: string): number => {
+      const row = spawnSync("ffmpeg", [
+        "-v",
+        "error",
+        "-i",
+        file,
+        "-vf",
+        "format=gray,crop=64:1:0:4",
+        "-f",
+        "rawvideo",
+        "-",
+      ]).stdout;
+      return [...Array(8).keys()].reduce(
+        (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
+        0,
+      );
+    };
+    const shown = [...result.framePaths.keys()]
+      .sort((a, b) => a - b)
+      .map((i) => indexOf(result.framePaths.get(i)!));
+    const steps = shown.slice(1).map((n, i) => n - shown[i]!);
+
+    expect(shown.length).toBeGreaterThanOrEqual(44);
+    expect(
+      steps.filter((step) => step > 1),
+      `source frames ${shown.join(",")}`,
+    ).toEqual([]);
+    expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1);
+  }, 60_000);
+
+  it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => {
+    const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4");
+    const synth = await runFfmpeg([
+      ...["-y", "-v", "error", "-f", "lavfi", "-i"],
+      "color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
+      ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
+      fixture,
+    ]);
+    if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
+    const indexOf = (file: string): number => {
+      const row = spawnSync("ffmpeg", [
+        "-v",
+        "error",
+        "-i",
+        file,
+        "-vf",
+        "format=gray,crop=64:1:0:4",
+        "-f",
+        "rawvideo",
+        "-",
+      ]).stdout;
+      return [...Array(8).keys()].reduce(
+        (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
+        0,
+      );
+    };
+    const options = {
+      fps: 30,
+      outputDir: join(FIXTURE_DIR, "out-ntsc-seams"),
+      format: "png" as const,
+    };
+    const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options);
+
+    const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201];
+    const differing: string[] = [];
+    for (const firstFrame of seams) {
+      const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, {
+        ...options,
+        frameRanges: [{ firstFrame, frames: 12 }],
+      });
+      for (const [slot, path] of segment.framePaths) {
+        const expected = indexOf(single.framePaths.get(slot)!);
+        const shown = indexOf(path);
+        if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`);
+      }
+    }
+    expect(differing).toEqual([]);
+  }, 120_000);
+
   // ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour
   // filters would cut the still that ends this window short.
   it.each([

packages/engine/src/services/videoFrameExtractor.ts

@@ -836,6 +836,14 @@ function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {
  * same absolute times as a single pass, so their frames are byte-identical to it.
  */
 const EXTRACTION_SEGMENT_SECONDS = 120;
+/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */
+function undoTickRoundingOnTheSourceFrameGrid(seek: number, lead: number): string[] {
+  const sinceFileStart = `(PTS*TB+${seek}-${lead})`;
+  return [
+    "settb=AVTB",
+    `setpts=if(isnan(FR)\\,PTS\\,(${lead}+round(${sinceFileStart}*FR)/FR-${seek})/TB)`,
+  ];
+}
 
 // Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts.
 const EXTRACTION_SLOTS = Math.max(1, Math.floor(cpus().length / 2));
@@ -843,7 +851,7 @@ const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS);
 
 interface SegmentedExtraction {
   decodeArgs: string[];
-  filterAndEncodeArgs: string[];
+  filterAndEncodeArgs: (seek: number) => string[];
   videoPath: string;
   startTime: number;
   fps: number;
@@ -879,6 +887,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp
     while (next < segmentCount && !signal.aborted) {
       const index = next++;
       const { firstFrame, frames } = segments[index]!;
+      const seek = startTime + firstFrame / fps;
       const result = await inExtractionSlot(
         () =>
           runFfmpeg(
@@ -886,7 +895,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp
               ...decodeArgs,
               "-noaccurate_seek",
               "-ss",
-              String(startTime + firstFrame / fps),
+              String(seek),
               "-i",
               videoPath,
               // Read one frame past the segment and keep exactly its own frames.
@@ -896,7 +905,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp
               String(frames),
               "-start_number",
               String(firstFrame + 1),
-              ...filterAndEncodeArgs,
+              ...filterAndEncodeArgs(seek),
             ],
             { ...job.runOptions, signal },
           ),
@@ -1019,15 +1028,20 @@ export async function extractVideoFramesRange(
   if (decodeWithVideoToolbox) {
     vfFilters.push("format=nv12");
   }
-  if (sampleAtOutputFps) {
-    // Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
-    // first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
-    vfFilters.push(
-      "settb=intb/2",
-      "setpts=PTS-lte(TB\\,0.001)",
-      `fps=${ffmpegFps}:start_time=0:round=up`,
-    );
-  }
+  const samplingAt = vfFilters.length;
+  // Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
+  // first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
+  const samplingFilters = (seek: number): string[] =>
+    sampleAtOutputFps
+      ? [
+          ...(metadata.isVFR
+            ? []
+            : undoTickRoundingOnTheSourceFrameGrid(seek, metadata.videoStreamLeadSeconds ?? 0)),
+          "settb=intb/2",
+          "setpts=PTS-lte(TB\\,0.001)",
+          `fps=${ffmpegFps}:start_time=0:round=up`,
+        ]
+      : [];
   if (options.sdrToHdrTransfer) {
     // Ordering intent: fps sampling runs BEFORE the colorspace remap so only
     // kept frames are converted. The remap is pointwise per-frame, so the
@@ -1055,7 +1069,11 @@ export async function extractVideoFramesRange(
   encodeArgs.push("-y", outputPattern);
 
   const runOptions = { signal, timeout: ffmpegProcessTimeout };
-  const filterArgs = vfFilters.length > 0 ? ["-vf", vfFilters.join(",")] : [];
+  const filterArgsFor = (seek: number): string[] => {
+    const filters = [...vfFilters];
+    filters.splice(samplingAt, 0, ...samplingFilters(seek));
+    return filters.length > 0 ? ["-vf", filters.join(",")] : [];
+  };
   const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);
   const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
   // A VFR window rounds its frame count up, and only the segments' `-frames:v` bound
@@ -1066,7 +1084,7 @@ export async function extractVideoFramesRange(
     sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)
       ? await runSegmentedExtraction({
           decodeArgs,
-          filterAndEncodeArgs: [...filterArgs, ...encodeArgs],
+          filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],
           videoPath,
           startTime,
           fps,
@@ -1077,7 +1095,7 @@ export async function extractVideoFramesRange(
           runOptions,
         })
       : await inExtractionSlot(
-          () => runFfmpeg([...args, ...filterArgs, ...encodeArgs], runOptions),
+          () => runFfmpeg([...args, ...filterArgsFor(startTime), ...encodeArgs], runOptions),
           signal,
         );
   if (processResult.failureReason === "external_interruption") {