perf(producer): cloud render chunks hand Chrome each video near the size it is shown (#6001)

* perf(producer): cloud render chunks hand Chrome each video near the size it is shown

A chunk now measures the largest box each video is drawn in over its own frames,
in device pixels, before extracting that video's frames, and halves the frames
while they still cover that box. Graded videos, pages with a canvas, and
object-fit none/scale-down keep full-size frames.

* docs(engine): shorten the shown-box measurement comment

* refactor(engine): name the halving filter instead of commenting it

* test(producer): end ffprobe options before the frame path in the shown-size tests

packages/engine/src/index.ts

@@ -434,6 +434,8 @@ export {
   showVideoElements,
   queryVideoElementBounds,
   queryElementStacking,
+  measureVideoShownBoxes,
+  type ShownBox,
   type VideoElementBounds,
   type ElementStackingInfo,
 } from "./services/videoFrameInjector.js";

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -2238,6 +2238,38 @@ describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {
       },
       60_000,
     );
+
+    it("halves frames while they still cover the box a video is shown in", async () => {
+      const firstFrameSize = async (
+        videoId: string,
+        shownBox: { width: number; height: number },
+      ) => {
+        const result = await extractVideoFramesRange(gopSource, videoId, 0, 1, {
+          fps: 30,
+          outputDir: FIXTURE_DIR,
+          format: "jpg",
+          frameRanges: [{ firstFrame: 0, frames: 1 }],
+          shownBox,
+        });
+        const [frame] = result.framePaths.values();
+        return spawnSync("ffprobe", [
+          "-v",
+          "error",
+          "-show_entries",
+          "stream=width,height",
+          "-of",
+          "csv=p=0",
+          "--",
+          frame!,
+        ])
+          .stdout.toString()
+          .trim();
+      };
+      // 64x36 source: halved as often as the box allows (even sides round up); never below the box.
+      expect(await firstFrameSize("shown-quarter", { width: 16, height: 9 })).toBe("16,10");
+      expect(await firstFrameSize("shown-third", { width: 20, height: 12 })).toBe("32,18");
+      expect(await firstFrameSize("shown-near-full", { width: 40, height: 20 })).toBe("64,36");
+    }, 30_000);
   });
 });
 

packages/engine/src/services/videoFrameExtractor.ts

@@ -253,6 +253,7 @@ export interface ExtractionOptions {
   /** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */
   contentKeyedDirs?: readonly string[];
   urlByAbsentLocalPath?: ReadonlyMap<string, string>;
+  shownBox?: { width: number; height: number };
 }
 
 const DOWNLOAD_SUBDIR = "_downloads";
@@ -263,6 +264,14 @@ const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv"
 const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";
 const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr";
 const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1";
+function halveToCoverScaleFilter(box: { width: number; height: number }): string {
+  const fit = `max(${Math.ceil(box.width)}/iw\\,${Math.ceil(box.height)}/ih)`;
+  const halvings = `floor(log(1/${fit})/log(2))`;
+  const side = (dim: "iw" | "ih") =>
+    `if(lt(${fit}\\,0.5)\\,2*ceil(${dim}/pow(2\\,${halvings})/2)\\,${dim})`;
+  return `scale=w=${side("iw")}:h=${side("ih")}:flags=area+accurate_rnd`;
+}
+
 const SDR_JPEG_AS_BT601_FULL_RANGE_FILTER =
   "scale=flags=neighbor+accurate_rnd,format=gbrp,scale=out_color_matrix=bt601:out_range=pc:flags=neighbor,format=yuv420p";
 
@@ -1094,7 +1103,10 @@ async function extractVideoFramesRangeUnredacted(
       ...(isHdr ? [] : chromeGuessForUntaggedMatrix(metadata)),
       SDR_CANVAS_PASSTHROUGH_FILTER,
     );
-    if (format === "jpg") vfFilters.push(SDR_JPEG_AS_BT601_FULL_RANGE_FILTER);
+    if (format === "jpg") {
+      if (options.shownBox) vfFilters.push(halveToCoverScaleFilter(options.shownBox));
+      vfFilters.push(SDR_JPEG_AS_BT601_FULL_RANGE_FILTER);
+    }
   }
   const encodeArgs = ["-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"];
   // Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files.

packages/engine/src/services/videoFrameInjector-shownBoxes.browser.test.ts

@@ -0,0 +1,59 @@
+import { spawnSync } from "node:child_process";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { Browser, Page } from "puppeteer-core";
+import { resolveHeadlessShellPath } from "./browserManager.js";
+import { measureVideoShownBoxes } from "./videoFrameInjector.js";
+import { closeTestChrome, launchTestChrome } from "./__fixtures__/loopback.js";
+
+const chromePath = resolveHeadlessShellPath();
+const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
+
+// `#zoom` grows from 400x225 at t=0 to 800x450 at t=1; `#late` is only laid out from t=0.5.
+const page1 = `<body style="margin:0">
+  <video id="zoom" style="position:absolute;width:400px;height:225px;transform-origin:0 0"></video>
+  <video id="late" style="position:absolute;top:500px;width:320px;height:180px;display:none"></video>
+  <video id="graded" data-color-grading="warm" style="width:200px;height:100px"></video>
+  <video id="natural" style="width:200px;height:100px;object-fit:none"></video>
+  <script>
+    window.__hf = { seek(t) {
+      document.getElementById("zoom").style.transform = "scale(" + (1 + t) + ")";
+      document.getElementById("late").style.display = t >= 0.5 ? "block" : "none";
+    } };
+  </script>
+</body>`;
+
+describe.skipIf(!hasBrowser)("measureVideoShownBoxes in Chrome", () => {
+  let browser: Browser;
+
+  beforeAll(async () => {
+    browser = await launchTestChrome(chromePath);
+  });
+  afterAll(() => closeTestChrome(browser));
+
+  async function open(html: string, deviceScaleFactor: number): Promise<Page> {
+    const page = await browser.newPage();
+    await page.setViewport({ width: 1000, height: 800, deviceScaleFactor });
+    await page.setContent(html);
+    return page;
+  }
+
+  it("returns each video's largest box over the seeked times, in device pixels", async () => {
+    const page = await open(page1, 2);
+    const ids = ["zoom", "late", "graded", "natural", "missing"];
+    const boxes = await measureVideoShownBoxes(page, ids, [0, 0.25, 1]);
+    expect(Object.fromEntries(boxes)).toEqual({
+      zoom: { width: 1600, height: 900 },
+      late: { width: 640, height: 360 },
+    });
+    await page.close();
+  });
+
+  it("keeps every video full size when the page has a canvas that could sample it", async () => {
+    const page = await open(
+      page1.replace("<script>", '<canvas width="10" height="10"></canvas><script>'),
+      1,
+    );
+    expect((await measureVideoShownBoxes(page, ["zoom", "late"], [0, 1])).size).toBe(0);
+    await page.close();
+  });
+});

packages/engine/src/services/videoFrameInjector.ts

@@ -15,6 +15,7 @@ import { injectVideoFramesBatch, syncVideoFrameVisibility } from "./screenshotSe
 import { type BeforeCaptureHook } from "./frameCapture.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 import {
+  HF_COLOR_GRADING_ATTR,
   HF_COLOR_GRADING_CANVAS_ID_PREFIX,
   RENDER_FRAME_ID_PREFIX,
   RENDER_FRAME_ID_SUFFIX,
@@ -342,6 +343,74 @@ async function setVideoElementsVisibility(
   );
 }
 
+export interface ShownBox {
+  width: number;
+  height: number;
+}
+
+/**
+ * Seeks to each of `times`; returns each video's largest drawn box in device pixels. Videos that are
+ * graded, sampled by a canvas, `object-fit: none | scale-down` or never laid out get none (full size).
+ */
+export async function measureVideoShownBoxes(
+  page: Page,
+  videoIds: readonly string[],
+  times: readonly number[],
+): Promise<Map<string, ShownBox>> {
+  if (videoIds.length === 0 || times.length === 0) return new Map();
+  const boxes = await page.evaluate(
+    async (
+      ids: string[],
+      seekTimes: number[],
+      gradingAttr: string,
+      gradingCanvasPrefix: string,
+    ) => {
+      const hf = (window as Window & { __hf?: { seek?: (t: number) => void } }).__hf;
+      if (typeof hf?.seek !== "function") return [];
+      const waitForSeek = (window as Window & { __hfWaitForSeekCompletion?: () => Promise<void> })
+        .__hfWaitForSeekCompletion;
+      const largest = new Map<string, ShownBox>();
+      const fullSize = new Set<string>();
+      const keepsFullSize = (video: Element): boolean => {
+        if (video.hasAttribute(gradingAttr)) return true;
+        const fit = window.getComputedStyle(video).objectFit;
+        return fit === "none" || fit === "scale-down";
+      };
+      const measure = (id: string): void => {
+        const video = window.__hfMediaEl?.(id) ?? document.getElementById(id);
+        if (!video || keepsFullSize(video)) {
+          fullSize.add(id);
+          return;
+        }
+        const rect = video.getBoundingClientRect();
+        const box = largest.get(id) ?? { width: 0, height: 0 };
+        box.width = Math.max(box.width, rect.width * window.devicePixelRatio);
+        box.height = Math.max(box.height, rect.height * window.devicePixelRatio);
+        largest.set(id, box);
+      };
+      const pageHasCanvas = (): boolean =>
+        Array.from(document.querySelectorAll("canvas")).some(
+          (canvas) => !canvas.id.startsWith(gradingCanvasPrefix),
+        );
+      for (const t of seekTimes) {
+        hf.seek(t);
+        await waitForSeek?.();
+        // A canvas could sample any video, so every video keeps full size.
+        if (pageHasCanvas()) return [];
+        for (const id of ids) measure(id);
+      }
+      return Array.from(largest).filter(
+        ([id, box]) => !fullSize.has(id) && box.width > 0 && box.height > 0,
+      );
+    },
+    [...videoIds],
+    [...times],
+    HF_COLOR_GRADING_ATTR,
+    HF_COLOR_GRADING_CANVAS_ID_PREFIX,
+  );
+  return new Map(boxes);
+}
+
 /**
  * Query the current bounds, transform, and visibility of video elements.
  * Called after seeking (so GSAP has moved things) but before the screenshot.

packages/producer/src/services/distributed/chunkFrames.ts

@@ -13,6 +13,7 @@ import {
   extractVideoFramesRange,
   type FrameRange,
   type FrameLookupTable,
+  type ShownBox,
 } from "@hyperframes/engine";
 import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
 import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";
@@ -167,6 +168,34 @@ export function buildChunkCaptureOptions(input: {
 }
 
 /** Extracts the given frames of deferred videos under the names a full extraction gives them. */
+export function shownDeferredFrameIndexes(
+  planVideos: PlanVideosJson,
+  frames: Iterable<number>,
+  fps: { num: number; den: number },
+): Map<string, Set<number>> {
+  const shown = new Map<string, Set<number>>();
+  const deferred = planVideos.extracted.filter((video) => video.deferredRange);
+  if (deferred.length === 0) return shown;
+  const table = createFrameLookupTable(
+    planVideos.videos,
+    deferred.map((video) => ({
+      ...video,
+      outputDir: "",
+      framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
+    })),
+    resolveRenderFpsConfig(fps).value,
+  );
+  for (const frame of frames) {
+    const globalTime = (frame * fps.den) / fps.num;
+    for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
+      const indexes = shown.get(videoId) ?? new Set<number>();
+      indexes.add(payload.frameIndex);
+      shown.set(videoId, indexes);
+    }
+  }
+  return shown;
+}
+
 export async function extractDeferredVideoFramesForChunk(input: {
   planDir: string;
   planVideos: PlanVideosJson;
@@ -175,6 +204,7 @@ export async function extractDeferredVideoFramesForChunk(input: {
   outputRoot: string;
   cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;
   remoteSources?: PlanV2RemoteSources;
+  shownBoxes?: ReadonlyMap<string, ShownBox>;
 }): Promise<ExtractedFrames[]> {
   const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);
   const remoteSources = input.remoteSources ?? {};
@@ -192,24 +222,7 @@ export async function extractDeferredVideoFramesForChunk(input: {
     });
   }
   if (deferred.length === 0) return [];
-  const table = createFrameLookupTable(
-    input.planVideos.videos,
-    deferred.map((video) => ({
-      ...video,
-      outputDir: input.outputRoot,
-      framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
-    })),
-    resolveRenderFpsConfig(input.fps).value,
-  );
-  const shown = new Map<string, Set<number>>();
-  for (const frame of input.frames) {
-    const globalTime = (frame * input.fps.den) / input.fps.num;
-    for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
-      const indexes = shown.get(videoId) ?? new Set<number>();
-      indexes.add(payload.frameIndex);
-      shown.set(videoId, indexes);
-    }
-  }
+  const shown = shownDeferredFrameIndexes(input.planVideos, input.frames, input.fps);
   return Promise.all(
     deferred.map(async (video) => {
       const outputDir = join(input.outputRoot, video.videoId);
@@ -236,6 +249,7 @@ export async function extractDeferredVideoFramesForChunk(input: {
           outputDir: input.outputRoot,
           format: range.format,
           frameRanges: contiguousRuns(indexes),
+          shownBox: input.shownBoxes?.get(video.videoId),
         },
         undefined,
         input.cfg,

packages/producer/src/services/distributed/renderChunk.test.ts

@@ -680,6 +680,58 @@ describe("renderChunk() — a video the chunk does not show", () => {
   }, 120_000);
 });
 
+describe("renderChunk() — a video shown smaller than its source", () => {
+  it("measures the box each video is drawn in before extracting its frames", async () => {
+    if (!hasChrome) {
+      console.warn(
+        "[renderChunk.test] skipping shown-size test — chrome-headless-shell not available",
+      );
+      return;
+    }
+    const projectDir = join(runRoot, "project-shown-size");
+    mkdirSync(projectDir, { recursive: true });
+    const synth = spawnSync("ffmpeg", [
+      ...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=128x72:r=30:d=2"],
+      ...["-c:v", "libx264", "-pix_fmt", "yuv420p", join(projectDir, "clip.mp4")],
+    ]);
+    if (synth.status !== 0) throw new Error(synth.stderr.toString());
+    writeFileSync(
+      join(projectDir, "index.html"),
+      // Clips over a second long, so the planner defers them to the chunks.
+      FIXTURE_HTML.replace('data-duration="0.16667"', 'data-duration="1.2"').replace(
+        "</div>",
+        '<video id="pip" src="clip.mp4" muted data-start="0" data-duration="1.2" style="position:absolute;right:0;bottom:0;width:32px;height:18px"></video>' +
+          '<video id="full" src="clip.mp4" muted data-start="0" data-duration="1.2" style="position:absolute;left:0;top:0;width:128px;height:72px"></video></div>',
+      ),
+      "utf-8",
+    );
+    const planDir = join(runRoot, "plan-shown-size");
+    await planV2(
+      projectDir,
+      { fps: 30, width: 160, height: 120, format: "png-sequence", chunkSize: 36 },
+      planDir,
+    );
+    const chunkDir = join(runRoot, "chunk0-shown-size");
+    materializePlanV2Target(planDir, { role: "chunk", chunkIndex: 0 }, chunkDir);
+    let result;
+    try {
+      result = await renderChunk(chunkDir, 0, join(runRoot, "chunk0-shown-size-out"));
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      if (!HOST_CHROME_FAILURE_PATTERNS.test(message)) throw err;
+      console.warn(
+        "[renderChunk.test] skipping shown-size test — host Chrome stack can't render:",
+        message.slice(0, 240),
+      );
+      return;
+    }
+    expect(JSON.parse(readFileSync(result.perfPath, "utf-8")).videoShownBoxes).toEqual({
+      pip: { width: 32, height: 18 },
+      full: { width: 128, height: 72 },
+    });
+  }, 120_000);
+});
+
 describe("renderChunk() — frameRange", () => {
   // A box sliding across the frame, so every frame differs and a misplaced,
   // dropped or repeated frame changes the assembled output.
@@ -1030,6 +1082,30 @@ describe("extractDeferredVideoFramesForChunk", () => {
     }
   });
 
+  it("extracts a video's frames at the size it is shown in when given its box", async () => {
+    const [chunk] = await extractDeferredVideoFramesForChunk({
+      planDir,
+      planVideos: planVideos(false),
+      frames: frameSpan(30, 32),
+      fps: { num: 30, den: 1 },
+      outputRoot: join(root, "chunk-shown"),
+      cfg: {},
+      shownBoxes: new Map([["clip", { width: 16, height: 9 }]]),
+    });
+    const [frame] = chunk!.framePaths.values();
+    const size = spawnSync("ffprobe", [
+      "-v",
+      "error",
+      "-show_entries",
+      "stream=width,height",
+      "-of",
+      "csv=p=0",
+      "--",
+      frame!,
+    ]);
+    expect(size.stdout.toString().trim()).toBe("16,10");
+  });
+
   // Lossless keeps the file large next to what one seek reads.
   const big = join(root, "big.mp4");
   const synthBig = () => {

packages/producer/src/services/distributed/renderChunk.ts

@@ -53,9 +53,11 @@ import {
   type EngineConfig,
   getEncoderPreset,
   initializeSession,
+  measureVideoShownBoxes,
   probeBeginFrameLiveness,
   readWebGlVendorInfoFromCanvas,
   resolveConfig,
+  type ShownBox,
 } from "@hyperframes/engine";
 import { defaultLogger } from "../../logger.js";
 import { applyRenderWarningPolicy, collectVideoMetadataHints } from "../renderOrchestrator.js";
@@ -100,6 +102,7 @@ import {
   MISSING_RUNTIME_ENV_SNAPSHOT,
   rebuildExtractedFramesFromPlanDir,
   RenderChunkValidationError,
+  shownDeferredFrameIndexes,
 } from "./chunkFrames.js";
 
 export {
@@ -623,21 +626,12 @@ export async function renderChunk(
     // Beside workDir, not in it: a screenshot retry wipes workDir.
     const videoFramesDir = `${workDir}.video-frames`;
 
-    // Build the immutable frame lookup once. Each browser session/worker gets
-    // its own injector hook because the hook remembers the last injected frame
-    // per video; sharing that state across a fresh retry page can suppress the
-    // first injection and produce a blank/stale frame.
-    const deferredVideoFrames = planVideos
-      ? await extractDeferredVideoFramesForChunk({
-          planDir,
-          planVideos,
-          frames: Array.from({ length: framesInChunk }, (_, i) => range.startFrame + i),
-          fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
-          outputRoot: videoFramesDir,
-          cfg,
-          remoteSources,
-        })
-      : [];
+    const chunkFrameNumbers = Array.from({ length: framesInChunk }, (_, i) => range.startFrame + i);
+    const chunkFps = { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen };
+    const deferredVideos = planVideos?.extracted.filter((video) => video.deferredRange) ?? [];
+    const shownFrames = planVideos
+      ? shownDeferredFrameIndexes(planVideos, chunkFrameNumbers, chunkFps)
+      : new Map<string, Set<number>>();
     // The browser cannot load a source read by URL; size its <video> from the plan instead.
     const remoteVideoIds = new Set(
       planVideos?.extracted
@@ -647,23 +641,6 @@ export async function renderChunk(
         )
         .map((video) => video.videoId),
     );
-    const remoteVideos = deferredVideoFrames.filter((video) => remoteVideoIds.has(video.videoId));
-    const videoFrameLookup =
-      planVideos && planVideos.extracted.length > 0
-        ? createFrameLookupTable(
-            planVideos.videos,
-            [
-              ...rebuildExtractedFramesFromPlanDir(
-                planDir,
-                planVideos.extracted.filter((video) => !video.deferredRange),
-                v2Manifest === null ? "dense-v1" : "sparse-v2",
-              ),
-              ...deferredVideoFrames,
-            ],
-            resolveRenderFpsConfig(job.config.fps).value,
-          )
-        : null;
-    const createChunkVideoFrameInjector = createChunkVideoFrameInjectorFactory(videoFrameLookup);
 
     // ── File server with the seeded-random shim ──
     // `Math.random` / `crypto.getRandomValues` are seeded from virtual
@@ -684,10 +661,14 @@ export async function renderChunk(
       videoCount: planVideos?.videos.length ?? 0,
       html: readFileSync(join(compiledDir, "index.html"), "utf-8"),
       // The plan ships a deferred source only to chunks that show it; don't wait on the rest.
-      skipReadinessVideoIds: deferredVideoFrames
-        .filter((video) => video.framePaths.size === 0 || remoteVideoIds.has(video.videoId))
+      skipReadinessVideoIds: deferredVideos
+        .filter(
+          (video) => !shownFrames.get(video.videoId)?.size || remoteVideoIds.has(video.videoId),
+        )
         .map((video) => video.videoId),
-      videoMetadataHints: collectVideoMetadataHints(remoteVideos),
+      videoMetadataHints: collectVideoMetadataHints(
+        deferredVideos.filter((video) => remoteVideoIds.has(video.videoId)),
+      ),
     });
 
     // Resolve worker count up-front. Sequential capture reuses the initialized
@@ -717,6 +698,7 @@ export async function renderChunk(
     const capturePerfs: CapturePerfSummary[] = [];
     const captureAttempts: Parameters<typeof runCaptureStage>[0]["captureAttempts"] = [];
     let forceScreenshotForChunk = encoder.forceScreenshot;
+    let shownBoxes: Map<string, ShownBox> | undefined;
     try {
       if (chunkWorkerCount === 1 || !forceScreenshotForChunk) {
         // Sequential capture reuses this session. Parallel BeginFrame capture
@@ -738,6 +720,12 @@ export async function renderChunk(
           cfg,
         );
         sessionBootMs = Date.now() - bootStarted;
+        // Measured before extraction so each video is extracted at the size it is drawn.
+        shownBoxes = await measureVideoShownBoxes(
+          session.page,
+          [...shownFrames.keys()],
+          chunkFrameNumbers.map((frame) => (frame * chunkFps.den) / chunkFps.num),
+        );
         // `discardWarmupCapture` is intentionally NOT called: every frame
         // seeks fresh DOM, so `lastFrameCache` is never read; priming it
         // would deadlock Chrome's compositor by issuing a second beginFrame
@@ -784,6 +772,39 @@ export async function renderChunk(
         }
       }
 
+      // Build the immutable frame lookup once. Each browser session/worker gets
+      // its own injector hook because the hook remembers the last injected frame
+      // per video; sharing that state across a fresh retry page can suppress the
+      // first injection and produce a blank/stale frame.
+      const deferredVideoFrames = planVideos
+        ? await extractDeferredVideoFramesForChunk({
+            planDir,
+            planVideos,
+            frames: chunkFrameNumbers,
+            fps: chunkFps,
+            outputRoot: videoFramesDir,
+            cfg,
+            remoteSources,
+            shownBoxes,
+          })
+        : [];
+      const videoFrameLookup =
+        planVideos && planVideos.extracted.length > 0
+          ? createFrameLookupTable(
+              planVideos.videos,
+              [
+                ...rebuildExtractedFramesFromPlanDir(
+                  planDir,
+                  planVideos.extracted.filter((video) => !video.deferredRange),
+                  v2Manifest === null ? "dense-v1" : "sparse-v2",
+                ),
+                ...deferredVideoFrames,
+              ],
+              resolveRenderFpsConfig(job.config.fps).value,
+            )
+          : null;
+      const createChunkVideoFrameInjector = createChunkVideoFrameInjectorFactory(videoFrameLookup);
+
       const captureStarted = Date.now();
       captureMode = await runCaptureWithScreenshotFallback({
         forceScreenshot: forceScreenshotForChunk,
@@ -1009,6 +1030,8 @@ export async function renderChunk(
       encodeStageMs,
       workers: chunkWorkerCount,
       captureMode,
+      // Device-pixel box each video's frames were extracted to cover; a video not listed kept full size.
+      videoShownBoxes: Object.fromEntries(shownBoxes ?? []),
       sha256,
       outputKind,
       producerVersion: plan.producerVersion,