fix(engine): export draws a video in exactly the frames it is shown in (#4565)

* fix(engine): export draws a video in exactly the frames export visibility shows it

* test(producer): guard that chunk frame planning snaps clip windows like export

* refactor(engine): set the render fps once per frame table and name its clip window

* chore(engine): trim frame table comments

packages/core/src/index.ts

@@ -269,6 +269,7 @@ export {
 export {
   MEDIA_VISUAL_STYLE_PROPERTIES,
   copyMediaVisualStyles,
+  exportClipWindow,
   quantizeSeekTime,
   quantizeTimeToFrame,
   type MediaVisualStyleProperty,

packages/core/src/inline-scripts/parityContract.ts

@@ -82,6 +82,18 @@ export function snapTimeToFrameBoundary(timeSeconds: number, fps: number): numbe
     : time;
 }
 
+/** The window export shows a clip in: both ends snapped like its seeks, so a hair off a frame counts as on it. */
+export function exportClipWindow(
+  start: number,
+  end: number,
+  fps: number,
+): { start: number; end: number } {
+  return {
+    start: snapTimeToFrameBoundary(start, fps),
+    end: Number.isFinite(end) ? snapTimeToFrameBoundary(end, fps) : end,
+  };
+}
+
 export function copyMediaVisualStyles(
   targetStyle: CSSStyleDeclaration,
   sourceStyle: CSSStyleDeclaration,

packages/core/src/runtime/timeline.ts

@@ -22,7 +22,7 @@ import {
 import { resolveCssStackingContextId } from "./stackingContext";
 import { createRuntimeStartTimeResolver } from "./startResolver";
 import { isClipVisibleAt } from "./clipWindow";
-import { snapTimeToFrameBoundary } from "../inline-scripts/parityContract";
+import { exportClipWindow } from "../inline-scripts/parityContract";
 import { isSceneLikeCompositionId } from "../slideshow/index.js";
 import { COMPOSITION_CONTRACT_VERSION } from "../compositionContract.js";
 import { runtimeProtocolMetadata } from "./protocol.js";
@@ -79,17 +79,13 @@ export function isRuntimeElementVisibleAt(
   const computedEnd =
     duration != null && duration > 0 ? start + duration : Number.POSITIVE_INFINITY;
   // Export seeks snap to frame boundaries; interactive visibility uses authored seconds.
-  const visibilityStart = options.exportRenderSeek
-    ? snapTimeToFrameBoundary(start, options.canonicalFps)
-    : start;
-  const visibilityEnd =
-    options.exportRenderSeek && Number.isFinite(computedEnd)
-      ? snapTimeToFrameBoundary(computedEnd, options.canonicalFps)
-      : computedEnd;
+  const clipWindow = options.exportRenderSeek
+    ? exportClipWindow(start, computedEnd, options.canonicalFps)
+    : { start, end: computedEnd };
   return isClipVisibleAt(
     options.currentTime,
-    visibilityStart,
-    visibilityEnd,
+    clipWindow.start,
+    clipWindow.end,
     options.compositionDuration,
   );
 }

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -1131,6 +1131,35 @@ describe("FrameLookupTable", () => {
     expect(table.getActiveFramePayloads(3.44).get("hero")?.frameIndex).toBe(42);
   });
 
+  it("gives a frame a hair before two clips meet to the clip export shows there", () => {
+    const clip = (id: string, start: number, end: number) => ({
+      id,
+      src: `${id}.webm`,
+      start,
+      end,
+      mediaStart: 0,
+      loop: false,
+      hasAudio: false,
+    });
+    const table = () =>
+      createFrameLookupTable(
+        [clip("a", 0, 1.00001), clip("b", 1.00001, 2)],
+        [
+          { ...fakeExtracted(90, 30), videoId: "a" },
+          { ...fakeExtracted(90, 30), videoId: "b" },
+        ],
+        30,
+      );
+    const stepping = table();
+    stepping.getActiveFramePayloads(29 / 30);
+    const payloads = stepping.getActiveFramePayloads(1);
+
+    expect([...payloads.keys()]).toEqual(["b"]);
+    expect(payloads.get("b")?.frameIndex).toBe(0);
+    expect([...table().getActiveFramePayloads(1).keys()]).toEqual(["b"]);
+    expect(table().getFrame("a", 1)).toBeNull();
+  });
+
   it.each([
     ["an exact", 0.1, 0.3, 0.3],
     ["a float-sum", 0.1, 0.1 + 0.2, 9 / 30],

packages/engine/src/services/videoFrameExtractor.ts

@@ -21,6 +21,7 @@ import {
   sourceTimeAt,
   timeAtSourceTime,
   type RateSpec,
+  exportClipWindow,
   hasClipStarted,
   isInClipWindow,
   parseStrictFiniteTimingNumber,
@@ -2199,7 +2200,6 @@ function getFrameIndexAtTime(
   holdLastFrame = false,
   playbackRate: RateSpec = 1,
 ): number | null {
-  if (!hasClipStarted(globalTime, videoStart)) return null;
   let localTime = Math.max(0, globalTime - videoStart);
   const normalizedPlaybackRate = normalizeRateSpec(playbackRate);
   const loopDuration = timeAtSourceTime(
@@ -2237,6 +2237,7 @@ export function getFrameAtTime(
   loop = false,
   mediaStart = 0,
 ): string | null {
+  if (!hasClipStarted(globalTime, videoStart)) return null;
   const frameIndex = getFrameIndexAtTime(extracted, globalTime, videoStart, loop, mediaStart);
   return frameIndex == null ? null : extracted.framePaths.get(frameIndex) || null;
 }
@@ -2270,7 +2271,7 @@ export class FrameLookupTable {
     {
       extracted: ExtractedFrames;
       start: number;
-      end: number;
+      clipWindow: { start: number; end: number };
       mediaStart: number;
       loop: boolean;
       playbackRate: RateSpec;
@@ -2280,7 +2281,7 @@ export class FrameLookupTable {
     videoId: string;
     extracted: ExtractedFrames;
     start: number;
-    end: number;
+    clipWindow: { start: number; end: number };
     mediaStart: number;
     loop: boolean;
     playbackRate: RateSpec;
@@ -2289,6 +2290,9 @@ export class FrameLookupTable {
   private startCursor = 0;
   private lastTime: number | null = null;
 
+  /** The render's `fps`, so a clip shows in exactly export visibility's frames; omit for authored times. */
+  constructor(private readonly fps?: number) {}
+
   addVideo(
     extracted: ExtractedFrames,
     start: number,
@@ -2300,21 +2304,21 @@ export class FrameLookupTable {
     this.videos.set(extracted.videoId, {
       extracted,
       start,
-      end,
+      clipWindow: this.fps ? exportClipWindow(start, end, this.fps) : { start, end },
       mediaStart,
       loop,
       playbackRate: normalizeRateSpec(playbackRate),
     });
     this.orderedVideos = Array.from(this.videos.entries())
       .map(([videoId, video]) => ({ videoId, ...video }))
-      .sort((a, b) => a.start - b.start);
+      .sort((a, b) => a.clipWindow.start - b.clipWindow.start);
     this.resetActiveState();
   }
 
   getFrame(videoId: string, globalTime: number): string | null {
     const video = this.videos.get(videoId);
     if (!video) return null;
-    if (!isInClipWindow(globalTime, video.start, video.end)) return null;
+    if (!isInClipWindow(globalTime, video.clipWindow.start, video.clipWindow.end)) return null;
     const frameIndex = getFrameIndexAtTime(
       video.extracted,
       globalTime,
@@ -2334,14 +2338,13 @@ export class FrameLookupTable {
   }
 
   private refreshActiveSet(globalTime: number): void {
-    // The runtime's half-open window. Rendered times stay below the composition end, so its
-    // terminal hold (isClipVisibleAt) never applies here.
+    // Half-open exportClipWindow; rendered times stay below the composition end, so no terminal hold here.
     if (this.lastTime == null || globalTime < this.lastTime) {
       this.activeVideoIds.clear();
       this.startCursor = 0;
       for (const entry of this.orderedVideos) {
-        if (!hasClipStarted(globalTime, entry.start)) break;
-        if (isInClipWindow(globalTime, entry.start, entry.end)) {
+        if (!hasClipStarted(globalTime, entry.clipWindow.start)) break;
+        if (isInClipWindow(globalTime, entry.clipWindow.start, entry.clipWindow.end)) {
           this.activeVideoIds.add(entry.videoId);
         }
         this.startCursor += 1;
@@ -2353,10 +2356,10 @@ export class FrameLookupTable {
     while (this.startCursor < this.orderedVideos.length) {
       const candidate = this.orderedVideos[this.startCursor];
       if (!candidate) break;
-      if (!hasClipStarted(globalTime, candidate.start)) {
+      if (!hasClipStarted(globalTime, candidate.clipWindow.start)) {
         break;
       }
-      if (isInClipWindow(globalTime, candidate.start, candidate.end)) {
+      if (isInClipWindow(globalTime, candidate.clipWindow.start, candidate.clipWindow.end)) {
         this.activeVideoIds.add(candidate.videoId);
       }
       this.startCursor += 1;
@@ -2364,7 +2367,7 @@ export class FrameLookupTable {
 
     for (const videoId of Array.from(this.activeVideoIds)) {
       const video = this.videos.get(videoId);
-      if (!video || !isInClipWindow(globalTime, video.start, video.end)) {
+      if (!video || !isInClipWindow(globalTime, video.clipWindow.start, video.clipWindow.end)) {
         this.activeVideoIds.delete(videoId);
       }
     }
@@ -2424,8 +2427,9 @@ export class FrameLookupTable {
 export function createFrameLookupTable(
   videos: VideoElement[],
   extracted: ExtractedFrames[],
+  fps?: number,
 ): FrameLookupTable {
-  const table = new FrameLookupTable();
+  const table = new FrameLookupTable(fps);
   const extractedMap = new Map<string, ExtractedFrames>();
   for (const ext of extracted) extractedMap.set(ext.videoId, ext);
 

packages/producer/src/services/distributed/planV2.test.ts

@@ -409,6 +409,17 @@ describe("Plan v2 manifest", () => {
     );
   });
 
+  it("gives a chunk the frames of a video starting a hair after its frame, as export shows it", () => {
+    const root = tempPath("hf-plan-v2-snapped-start-");
+    const v1 = createV1Plan(root, { video: true, videoStart: 1 / 30 + 1e-6, videoEnd: 1 });
+    const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir);
+    const chunk1 = listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex: 1 });
+
+    expect(chunk1.some((artifact) => artifact.path === "video-frames/hero/frame_00001.jpg")).toBe(
+      true,
+    );
+  });
+
   it("materializes empty video directories for chunks where the video is inactive", () => {
     const root = tempPath("hf-plan-v2-inactive-video-directory-");
     const v1 = createV1Plan(root, {

packages/producer/src/services/distributed/planV2.ts

@@ -46,6 +46,7 @@ import {
 } from "./planV2Publisher.js";
 import { PLAN_V2_INTEGRITY_UNRECOVERABLE, PlanV2IntegrityError } from "./planV2Errors.js";
 import { planV2BlobPath } from "./planV2Layout.js";
+import { resolveRenderFpsConfig } from "../fileServer.js";
 import {
   isPlanAudioArtifactPath,
   PLAN_VIDEOS_META_RELATIVE_PATH,
@@ -407,9 +408,13 @@ function buildVideoChunkDependencies(
   const parsedVideos = parsePlanVideosJson(videos);
   const parsedChunks = parseChunkSlices(chunks);
   const extracted = listVideoFramePaths(executionPlanDir, parsedVideos);
-  const table = createFrameLookupTable(parsedVideos.videos, extracted);
   const fpsNum = readPositiveInteger(dimensions.fpsNum, "dimensions.fpsNum");
   const fpsDen = readPositiveInteger(dimensions.fpsDen, "dimensions.fpsDen");
+  const table = createFrameLookupTable(
+    parsedVideos.videos,
+    extracted,
+    resolveRenderFpsConfig({ num: fpsNum, den: fpsDen }).value,
+  );
   const mutable = new Map<string, Set<number>>();
 
   for (const chunk of parsedChunks) {

packages/producer/src/services/distributed/renderChunk.ts

@@ -78,6 +78,7 @@ import {
   buildVirtualTimeShim,
   closeFileServerSafely,
   createFileServer,
+  resolveRenderFpsConfig,
   type FileServerHandle,
 } from "../fileServer.js";
 import {
@@ -678,6 +679,7 @@ export async function renderChunk(
               planVideos.extracted,
               v2Manifest === null ? "dense-v1" : "sparse-v2",
             ),
+            resolveRenderFpsConfig(job.config.fps).value,
           )
         : null;
     const createChunkVideoFrameInjector = createChunkVideoFrameInjectorFactory(videoFrameLookup);

packages/producer/src/services/fileServer.ts

@@ -410,7 +410,7 @@ const RENDER_SEEK_OFFSET_FRACTION = Math.max(
   Math.min(0.95, Number(process.env.PRODUCER_RUNTIME_RENDER_SEEK_OFFSET_FRACTION || 0.5)),
 );
 
-function resolveRenderFpsConfig(fps: Fps | undefined): {
+export function resolveRenderFpsConfig(fps: Fps | undefined): {
   value: number;
   source: "render-options" | "default";
   fallbackReason?: "missing" | "invalid";

packages/producer/src/services/render/stages/extractVideosStage.ts

@@ -56,6 +56,7 @@ import {
   type RenderJob,
 } from "../../renderOrchestrator.js";
 import { materializeExtractedFramesForCompiledDir, type CompositionMetadata } from "../shared.js";
+import { resolveRenderFpsConfig } from "../../fileServer.js";
 import type { ProducerLogger } from "../../../logger.js";
 import { encoderFailureError } from "../encoderInterruption.js";
 import {
@@ -516,7 +517,11 @@ export async function runExtractVideosStage(
     });
 
     if (extractionResult.extracted.length > 0) {
-      frameLookup = createFrameLookupTable(composition.videos, extractionResult.extracted);
+      frameLookup = createFrameLookupTable(
+        composition.videos,
+        extractionResult.extracted,
+        resolveRenderFpsConfig(job.config.fps).value,
+      );
     }
     videoReadinessSkipIds = collectVideoReadinessSkipIds(
       nativeHdrVideoIds,