fix(producer): a local render extracts each video at about the size it is shown (#7609)

This commit is contained in:
Miguel Ángel
2026-11-12 17:16:59 +00:00
committed by GitHub
parent 395c845cd3
commit 54bcdc616f
15 changed files with 453 additions and 8 deletions
+2
View File
@@ -512,7 +512,9 @@ export {
queryVideoElementBounds,
queryElementStacking,
measureVideoShownBoxes,
measureVideoShownBoxRanges,
type ShownBox,
type ShownBoxRange,
type VideoElementBounds,
type ElementStackingInfo,
} from "./services/videoFrameInjector.js";
@@ -692,6 +692,65 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB");
}, 60_000);
it("extracts each video at its own shown box, and caches halved frames apart from full ones", async () => {
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-shown-cache-"));
const size = (path: string) =>
spawnSync("ffprobe", [
"-v",
"error",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0",
"--",
path,
])
.stdout.toString()
.trim();
const run = async (
name: string,
shownBoxes?: Map<string, { width: number; height: number }>,
) => {
const outputDir = join(FIXTURE_DIR, name);
mkdirSync(outputDir, { recursive: true });
const result = await extractAllVideoFrames(
[fixtureVideo()],
FIXTURE_DIR,
{ fps: 1, outputDir, shownBoxes },
undefined,
{ extractCacheDir: cacheDir },
);
expect(result.errors).toEqual([]);
return {
size: size(result.extracted[0]!.framePaths.get(0)!),
hits: result.phaseBreakdown.cacheHits,
};
};
try {
const half = new Map([
["ui", { width: UI_FIXTURE_WIDTH / 2, height: UI_FIXTURE_HEIGHT / 2 }],
]);
expect(await run("shown-full")).toEqual({
size: `${UI_FIXTURE_WIDTH},${UI_FIXTURE_HEIGHT}`,
hits: 0,
});
expect(await run("shown-half", half)).toEqual({
size: `${UI_FIXTURE_WIDTH / 2},${UI_FIXTURE_HEIGHT / 2}`,
hits: 0,
});
expect(await run("shown-half-again", half)).toEqual({
size: `${UI_FIXTURE_WIDTH / 2},${UI_FIXTURE_HEIGHT / 2}`,
hits: 1,
});
expect(await run("shown-full-again")).toEqual({
size: `${UI_FIXTURE_WIDTH},${UI_FIXTURE_HEIGHT}`,
hits: 1,
});
} finally {
rmSync(cacheDir, { recursive: true, force: true });
}
}, 60_000);
it("keeps jpg and png extraction caches separate", async () => {
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-"));
try {
@@ -311,6 +311,7 @@ export interface ExtractionOptions {
contentKeyedDirs?: readonly string[];
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
shownBox?: { width: number; height: number };
shownBoxes?: ReadonlyMap<string, { width: number; height: number }>;
}
const DOWNLOAD_SUBDIR = "_downloads";
@@ -321,6 +322,10 @@ const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv"
const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";
const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr";
const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1";
function shownBoxKey(box: { width: number; height: number } | undefined): string | undefined {
return box && `shown-${Math.ceil(box.width)}x${Math.ceil(box.height)}`;
}
function halveToCoverScaleFilter(box: { width: number; height: number }): string {
const fit = `max(${Math.ceil(box.width)}/iw\\,${Math.ceil(box.height)}/ih)`;
const halvings = `floor(log(1/${fit})/log(2))`;
@@ -2497,6 +2502,7 @@ export async function extractAllVideoFrames(
sdrToHdrTransfer: work.sdrToHdrTransfer,
toneMapHdrToSdr: work.hdrToSdrTransformKey !== undefined,
finalFrameOnly: work.finalFrameOnly,
shownBox: options.shownBoxes?.get(work.video.id) ?? options.shownBox,
};
}
@@ -2539,6 +2545,7 @@ export async function extractAllVideoFrames(
work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined,
work.hdrToSdrTransformKey,
work.finalFrameOnly ? "final-frame" : undefined,
shownBoxKey(options.shownBoxes?.get(work.video.id) ?? options.shownBox),
].filter((part): part is string => part !== undefined);
const transform = transformParts.length > 0 ? transformParts.join("+") : undefined;
@@ -2,7 +2,7 @@ import { spawnSync } from "node:child_process";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import type { Browser, Page } from "puppeteer-core";
import { resolveHeadlessShellPath } from "./browserManager.js";
import { measureVideoShownBoxes } from "./videoFrameInjector.js";
import { measureVideoShownBoxRanges, measureVideoShownBoxes } from "./videoFrameInjector.js";
import { closeTestChrome, launchTestChrome } from "./__fixtures__/loopback.js";
const chromePath = resolveHeadlessShellPath();
@@ -50,6 +50,17 @@ describe.skipIf(!hasBrowser)("measureVideoShownBoxes in Chrome", () => {
await page.close();
});
it("reports each video's smallest laid-out box beside its largest", async () => {
const page = await open(page1, 2);
const ranges = await measureVideoShownBoxRanges(page, ["zoom", "late"], [0, 0.25, 1]);
expect(Object.fromEntries(ranges)).toEqual({
zoom: { largest: { width: 1600, height: 900 }, smallest: { width: 800, height: 450 } },
// not laid out before 0.5, which is no smaller box
late: { largest: { width: 640, height: 360 }, smallest: { width: 640, height: 360 } },
});
await page.close();
});
it("keeps every video full size when the page has a canvas that could sample it", async () => {
const page = await open(
page1.replace("<script>", '<canvas width="10" height="10"></canvas><script>'),
@@ -366,6 +366,22 @@ export async function measureVideoShownBoxes(
videoIds: readonly string[],
times: readonly number[],
): Promise<Map<string, ShownBox>> {
const ranges = await measureVideoShownBoxRanges(page, videoIds, times);
return new Map(Array.from(ranges, ([id, range]) => [id, range.largest]));
}
export interface ShownBoxRange {
largest: ShownBox;
/** The smallest box it is laid out at; equal to `largest` when the video never moves or resizes. */
smallest: ShownBox;
}
/** As `measureVideoShownBoxes`, with each video's smallest laid-out box beside its largest. */
export async function measureVideoShownBoxRanges(
page: Page,
videoIds: readonly string[],
times: readonly number[],
): Promise<Map<string, ShownBoxRange>> {
if (videoIds.length === 0 || times.length === 0) return new Map();
const boxes = await page.evaluate(
async (
@@ -381,6 +397,7 @@ export async function measureVideoShownBoxes(
const waitForSeek = (window as Window & { __hfWaitForSeekCompletion?: () => Promise<void> })
.__hfWaitForSeekCompletion;
const largest = new Map<string, ShownBox>();
const smallest = new Map<string, ShownBox>();
const fullSize = new Set<string>();
const gradedThroughGroup = (video: Element): boolean => {
const groupId = video.getAttribute(groupAttr);
@@ -405,10 +422,18 @@ export async function measureVideoShownBoxes(
return;
}
const rect = video.getBoundingClientRect();
const width = rect.width * window.devicePixelRatio;
const height = rect.height * window.devicePixelRatio;
const box = largest.get(id) ?? { width: 0, height: 0 };
box.width = Math.max(box.width, rect.width * window.devicePixelRatio);
box.height = Math.max(box.height, rect.height * window.devicePixelRatio);
box.width = Math.max(box.width, width);
box.height = Math.max(box.height, height);
largest.set(id, box);
// A video outside its own clip window is not laid out (0x0); that is not a smaller box.
if (width === 0 || height === 0) return;
const least = smallest.get(id) ?? { width, height };
least.width = Math.min(least.width, width);
least.height = Math.min(least.height, height);
smallest.set(id, least);
};
const pageHasCanvas = (): boolean =>
Array.from(document.querySelectorAll("canvas")).some(
@@ -421,9 +446,12 @@ export async function measureVideoShownBoxes(
if (pageHasCanvas()) return [];
for (const id of ids) measure(id);
}
return Array.from(largest).filter(
([id, box]) => !fullSize.has(id) && box.width > 0 && box.height > 0,
);
return Array.from(largest)
.filter(([id, box]) => !fullSize.has(id) && box.width > 0 && box.height > 0)
.map(([id, box]): [string, { largest: ShownBox; smallest: ShownBox }] => [
id,
{ largest: box, smallest: smallest.get(id) ?? box },
]);
},
[...videoIds],
[...times],
@@ -65,5 +65,6 @@
"src/services/distributed/referenceBackground.integration.test.ts",
"src/services/render/planReferenceBackground.polling.test.ts",
"src/services/render/deferredVideoFrames.integration.test.ts",
"src/services/render/stages/extractVideosStage.integration.test.ts"
"src/services/render/stages/extractVideosStage.integration.test.ts",
"src/services/render/videoShownBoxes.integration.test.ts"
]
@@ -111,6 +111,7 @@ export async function extractAllDeferredFrames(
cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;
signal?: AbortSignal;
sourceOf?: (video: ExtractedFrames) => string;
shownBoxes?: ReadonlyMap<string, ShownBox>;
maxTransientRetries?: number;
onProgress?: (framesDone: number, framesTotal: number) => void;
},
@@ -130,7 +131,12 @@ export async function extractAllDeferredFrames(
video.videoId,
range.startTime,
range.durationSeconds,
{ fps: input.fps, outputDir: dirname(video.outputDir), format: range.format },
{
fps: input.fps,
outputDir: dirname(video.outputDir),
format: range.format,
shownBox: input.shownBoxes?.get(video.videoId),
},
input.signal,
input.cfg,
),
@@ -7,6 +7,7 @@ import {
type BeforeCaptureHook,
type EngineConfig,
type ExtractedFrames,
type ShownBox,
type VideoElement,
} from "@hyperframes/engine";
import { resolveRenderFpsConfig } from "../fileServer.js";
@@ -52,6 +53,7 @@ export function createSegmentVideoFrames(input: {
injectorOptions: Parameters<typeof createVideoFrameInjector>[1];
abortSignal?: AbortSignal;
sourceOf?: (video: ExtractedFrames) => string;
shownBoxes?: ReadonlyMap<string, ShownBox>;
extractFrames?: FrameExtractor;
}): SegmentVideoFrames {
const extractFrames = input.extractFrames ?? extractDeferredFrames;
@@ -69,6 +71,7 @@ export function createSegmentVideoFrames(input: {
cfg: input.cfg,
signal: input.abortSignal,
sourceOf,
shownBoxes: input.shownBoxes,
onMissingFrame: (video, index) =>
new Error(
`[Render] video ${JSON.stringify(video.videoId)} has no frame ${index}; ` +
@@ -55,6 +55,7 @@ import {
runVideoExtractionWithRetry,
safeVideoExtractionSourceIdentity,
videoAudioId,
type ShownBox,
} from "@hyperframes/engine";
import {
collectVideoMetadataHints,
@@ -97,6 +98,7 @@ export interface ExtractVideosStageInput {
*/
materializeSymlinks?: boolean;
deferRangeExtraction?: boolean;
shownBoxes?: ReadonlyMap<string, ShownBox>;
skipFrameExtractionIds?: readonly string[];
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
onProgress?: ProgressCallback;
@@ -423,6 +425,7 @@ export async function runExtractVideosStage(
assertNotAborted,
materializeSymlinks,
deferRangeExtraction,
shownBoxes,
urlByAbsentLocalPath,
skipFrameExtractionIds = [],
} = input;
@@ -554,6 +557,7 @@ export async function runExtractVideosStage(
maxTransientRetries: extractionPolicy.maxTransientRetries,
collectProbeFailures: extractionPolicy.failureMode === "enforce",
deferRangeExtraction,
shownBoxes,
contentKeyedDirs: [join(compiledDir, REMOTE_MEDIA_SUBDIR)],
urlByAbsentLocalPath,
onProgress: ({ planned = 0, done = 0 }) => {
@@ -1115,6 +1115,21 @@ describe("runProbeStage — forceScreenshot threading", () => {
expect(capturedCfgs).toHaveLength(0);
});
it("opens a page for a video larger than the canvas, and none for a film without one", async () => {
resetRetryMocks();
const { runProbeStage } = await import("./probeStage.js");
for (const measuresShownBoxes of [false, true]) {
capturedCfgs.length = 0;
const input = makeProbeInput({});
input.composition.duration = 5;
input.compiled.html = `<video id="clip" src="clip.mp4" data-start="0" data-duration="5"></video>`;
await runProbeStage({ ...input, measuresShownBoxes });
expect(capturedCfgs.length > 0, `measuresShownBoxes ${measuresShownBoxes}`).toBe(
measuresShownBoxes,
);
}
});
it("probes when a source child of existing media is mutated", async () => {
resetRetryMocks();
capturedCfgs.length = 0;
@@ -94,6 +94,7 @@ import { computeCompositionObservabilityHash } from "../observability.js";
export interface ProbeStageInput {
projectDir: string;
workDir: string;
measuresShownBoxes?: boolean;
recordReferenceBackground?: CaptureOptions["recordReferenceBackground"];
job: RenderJob;
cfg: EngineConfig;
@@ -409,6 +410,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
const hasInsertedMedia = hasRuntimeMediaChanges(compiled.html);
const needsBrowser =
cfg.videoLayerPassthrough ||
input.measuresShownBoxes === true ||
probeRequiresBrowser({
durationSeconds: composition.duration,
unresolvedCompositionCount: compiled.unresolvedCompositions.length,
@@ -429,6 +431,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
if (hasScriptedAudio) reasons.push("scripted audio volume");
if (hasInsertedMedia) reasons.push("runtime-created or source-mutated media");
if (hasVariableMedia) reasons.push("variable-bound media source(s)");
if (input.measuresShownBoxes) reasons.push("video(s) larger than the canvas");
log.info("Launching browser for composition probe...", {
reasons,
@@ -0,0 +1,96 @@
import { spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { GSAP_CDN } from "@hyperframes/core";
import { resolveHeadlessShellPath } from "@hyperframes/engine";
import type { ProducerLogger } from "../../logger.js";
import { createRenderJob, executeRenderJob, resolveRenderDebugDir } from "../renderOrchestrator.js";
const chromePath = resolveHeadlessShellPath();
const CAN_RUN =
chromePath !== undefined &&
spawnSync(chromePath, ["--version"]).status === 0 &&
spawnSync("ffmpeg", ["-version"]).status === 0;
// A 64x36 film with two 128x72 clips: `still` fills it (4K in 1080p, scaled down); `zoom` grows from 16x9 to 40x22.5.
const FILM = `<!doctype html>
<html><head><meta charset="utf-8"><script src="${GSAP_CDN}"></script></head>
<body style="margin:0;background:#000">
<div id="root" data-composition-id="root" data-width="64" data-height="36" data-start="0" data-duration="0.5">
<video id="still" src="clip.mp4" muted data-start="0" data-duration="0.5" style="position:absolute;left:0;top:0;width:64px;height:36px"></video>
<video id="zoom" src="clip.mp4" muted data-start="0" data-duration="0.5" style="position:absolute;left:0;top:0;width:16px;height:9px"></video>
</div>
<script>
window.__timelines = window.__timelines || {};
const tl = gsap.timeline({ paused: true });
tl.to("#zoom", { width: 40, height: 22.5, duration: 0.4, ease: "none" }, 0);
tl.set({}, {}, 0.5);
window.__timelines["root"] = tl;
</script>
</body></html>`;
let root: string;
const savedRendersDir = process.env.PRODUCER_RENDERS_DIR;
beforeAll(() => {
root = mkdtempSync(join(tmpdir(), "hf-shown-box-"));
process.env.PRODUCER_RENDERS_DIR = join(root, "renders");
});
afterAll(() => {
if (savedRendersDir === undefined) delete process.env.PRODUCER_RENDERS_DIR;
else process.env.PRODUCER_RENDERS_DIR = savedRendersDir;
rmSync(root, { recursive: true, force: true });
});
describe.skipIf(!CAN_RUN)("a local render extracts each video at the size it is shown", () => {
it("halves a static clip onto its box and keeps a zooming clip's frames over its peak", async () => {
const project = join(root, "film");
mkdirSync(project, { recursive: true });
const synth = spawnSync("ffmpeg", [
...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=128x72:r=30:d=1"],
...["-c:v", "libx264", "-pix_fmt", "yuv420p", join(project, "clip.mp4")],
]);
if (synth.status !== 0) throw new Error(synth.stderr.toString());
writeFileSync(join(project, "index.html"), FILM);
let measured: unknown;
const quiet = () => {};
const logger: ProducerLogger = {
error: quiet,
warn: quiet,
debug: quiet,
info: (message, meta) => {
if (message.includes("boxes videos are shown at")) measured = meta;
},
};
const job = createRenderJob({ fps: 30, quality: "draft", debug: true, logger });
await executeRenderJob(job, project, join(root, "film.mp4"));
const framesRoot = join(
resolveRenderDebugDir(),
job.id,
"compiled",
"__hyperframes_video_frames",
);
const frameSize = (id: string): string => {
const dir = join(framesRoot, readdirSync(framesRoot).find((entry) => entry.includes(id))!);
const first = join(dir, readdirSync(dir).find((file) => file.startsWith("frame_"))!);
const probe = spawnSync("ffprobe", [
"-v",
"error",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0",
"--",
first,
]);
return probe.stdout.toString().trim();
};
expect(measured).toMatchObject({ videos: 2, boxed: 2, seeks: 3 });
expect(frameSize("still")).toBe("64,36");
// Sized from its first frame (16x9) it would be halved twice to 32x18, under its 40x23 peak.
expect(frameSize("zoom")).toBe("64,36");
}, 90_000);
});
@@ -0,0 +1,98 @@
import { describe, expect, it } from "vitest";
import type { VideoElement } from "@hyperframes/engine";
import type { Page } from "puppeteer-core";
import type { ProducerLogger } from "../../logger.js";
import { measureLocalShownBoxes, shownBoxTimes } from "./videoShownBoxes.js";
const video = (id: string, start: number, end: number): VideoElement => ({
id,
src: `${id}.mp4`,
start,
end,
mediaStart: 0,
loop: false,
hasAudio: false,
});
const fps = { num: 30, den: 1 };
function logger() {
const lines: Array<{ level: string; message: string; meta?: Record<string, unknown> }> = [];
const at =
(level: string) =>
(message: string, meta?: Record<string, unknown>): void => {
lines.push({ level, message, meta });
};
const log: ProducerLogger = {
error: at("error"),
warn: at("warn"),
info: at("info"),
debug: at("debug"),
};
return { log, lines };
}
/** A page whose measure returns `ranges`, or rejects with `error`. */
function page(ranges: unknown, error?: Error): Page {
let calls = 0;
return {
evaluate: async () => {
calls += 1;
if (calls > 1) return undefined; // the seek back to 0
if (error) throw error;
return ranges;
},
} as unknown as Page;
}
describe("shownBoxTimes", () => {
it("samples each video from its first frame every 0.25 s to its last frame", () => {
expect(shownBoxTimes([video("a", 1, 2)], fps)).toEqual([1, 1.25, 1.5, 1.75, 2 - 1 / 30]);
});
it("merges the times of several videos once each, in order", () => {
expect(shownBoxTimes([video("a", 0, 0.5), video("b", 0.25, 0.5)], fps)).toEqual([
0,
0.25,
0.5 - 1 / 30,
]);
});
});
describe("measureLocalShownBoxes", () => {
it("keeps a static box exact and adds 10% to a box that changes", async () => {
const { log, lines } = logger();
const boxes = await measureLocalShownBoxes(
page([
[
"still",
{ largest: { width: 1920, height: 1080 }, smallest: { width: 1920, height: 1080 } },
],
["zoom", { largest: { width: 400, height: 200 }, smallest: { width: 100, height: 50 } }],
]),
[video("still", 0, 1), video("zoom", 0, 1)],
fps,
log,
);
expect(Object.fromEntries(boxes!)).toEqual({
still: { width: 1920, height: 1080 },
zoom: { width: expect.closeTo(440, 6), height: expect.closeTo(220, 6) },
});
expect(lines.find((l) => l.level === "info")?.meta).toMatchObject({
videos: 2,
boxed: 2,
seeks: 5,
});
});
it("returns no boxes, so every video keeps its full size, when the measure fails", async () => {
const { log, lines } = logger();
const boxes = await measureLocalShownBoxes(
page(undefined, new Error("Target closed")),
[video("clip", 0, 1)],
fps,
log,
);
expect(boxes).toBeUndefined();
expect(lines.find((l) => l.level === "warn")?.meta).toMatchObject({ error: "Target closed" });
});
});
@@ -0,0 +1,95 @@
import {
extractMediaMetadata,
measureVideoShownBoxRanges,
resolveProjectRelativeSrc,
type ShownBox,
type VideoElement,
} from "@hyperframes/engine";
import type { Page } from "puppeteer-core";
import type { ProducerLogger } from "../../logger.js";
/** Seconds between samples of a video's box; a 693 s film with one video is ~2.8k seeks. */
const SHOWN_BOX_STRIDE_S = 0.25;
/** A box that changes may peak between samples, so it gets this much more. A static box is exact. */
const MOVING_BOX_HEADROOM = 1.1;
/** Local videos whose source is larger than the canvas: only these can be extracted smaller. */
export async function oversizedVideos(
videos: readonly VideoElement[],
projectDir: string,
compiledDir: string,
canvas: { width: number; height: number },
): Promise<VideoElement[]> {
const sized = await Promise.all(
videos.map(async (video) => {
if (/^https?:\/\//i.test(video.src)) return false;
try {
const path = resolveProjectRelativeSrc(video.src, projectDir, compiledDir);
const { width, height } = await extractMediaMetadata(path);
return width > canvas.width || height > canvas.height;
} catch {
return false; // extraction reports the unreadable source
}
}),
);
return videos.filter((_, index) => sized[index]);
}
/** Each video's first and last frame and every stride between, in composition seconds, ascending. */
export function shownBoxTimes(
videos: readonly VideoElement[],
fps: { num: number; den: number },
): number[] {
const frame = fps.den / fps.num;
const times = new Set<number>();
for (const { start, end } of videos) {
const last = Math.max(start, end - frame);
for (let t = start; t < last; t += SHOWN_BOX_STRIDE_S) times.add(t);
times.add(last);
}
return [...times].sort((a, b) => a - b);
}
/**
* The box to extract each video at: the largest it is drawn at, plus headroom when it moves. Undefined when
* the measure fails, and every video then keeps its full size.
*/
export async function measureLocalShownBoxes(
page: Page,
videos: readonly VideoElement[],
fps: { num: number; den: number },
log: ProducerLogger,
): Promise<Map<string, ShownBox> | undefined> {
const times = shownBoxTimes(videos, fps);
const startedAt = Date.now();
try {
const ranges = await measureVideoShownBoxRanges(
page,
videos.map((video) => video.id),
times,
);
await page.evaluate(() =>
(window as { __hf?: { seek?: (t: number) => void } }).__hf?.seek?.(0),
);
const boxes = new Map<string, ShownBox>();
for (const [id, { largest, smallest }] of ranges) {
const moves = largest.width !== smallest.width || largest.height !== smallest.height;
const grow = moves ? MOVING_BOX_HEADROOM : 1;
boxes.set(id, { width: largest.width * grow, height: largest.height * grow });
}
log.info("[Render] Measured the boxes videos are shown at", {
videos: videos.length,
boxed: boxes.size,
seeks: times.length,
elapsedMs: Date.now() - startedAt,
});
return boxes;
} catch (error) {
log.warn("[Render] Could not measure the boxes videos are shown at; extracting at full size", {
error: error instanceof Error ? error.message : String(error),
seeks: times.length,
elapsedMs: Date.now() - startedAt,
});
return undefined;
}
}
@@ -185,6 +185,7 @@ import {
import { checkPageNeverReady } from "./render/pageNeverReady.js";
import { resolveSegmentFrames } from "./render/segmentPlan.js";
import { extractAllDeferredFrames } from "./render/deferredVideoFrames.js";
import { measureLocalShownBoxes, oversizedVideos } from "./render/videoShownBoxes.js";
import { createSegmentVideoFrames } from "./render/segmentVideoFrames.js";
import { shouldWindowVideoFrames, trackSegmentFramePeak } from "./render/videoFrameWindow.js";
import { resolveSegmentBrowserRecycle } from "./render/segmentRecycle.js";
@@ -3435,6 +3436,13 @@ async function executeRenderPipeline(input: {
);
const subTimelineWaitMemo: SubTimelineWaitMemo = {};
const staticDedupMemo: StaticDedupMemo = {};
// Only a video larger than the canvas can be extracted smaller; the probe page then measures where it is drawn.
const largerThanCanvas = await oversizedVideos(
composition.videos,
projectDir,
join(workDir, "compiled"),
{ width: outputWidth, height: outputHeight },
);
const probeResult = await observeRenderStage(
observability,
"browser_probe",
@@ -3443,6 +3451,7 @@ async function executeRenderPipeline(input: {
runProbeStage({
projectDir,
workDir,
measuresShownBoxes: largerThanCanvas.length > 0,
job,
cfg,
forceScreenshot: captureForceScreenshot,
@@ -3468,6 +3477,11 @@ async function executeRenderPipeline(input: {
compositionHash = computeCompositionObservabilityHash(compiled.html);
fileServer = probeResult.fileServer;
probeSession = probeResult.probeSession;
// Before the video layer may close the probe page; no box means full size, as before.
const shownBoxes =
largerThanCanvas.length > 0 && probeSession?.isInitialized
? await measureLocalShownBoxes(probeSession.page, largerThanCanvas, job.config.fps, log)
: undefined;
job.strictVariablesChecked = probeResult.strictVariablesChecked;
lastBrowserConsole = probeResult.lastBrowserConsole;
let resolvedCaptureBeyondViewport = probeSession?.options.captureBeyondViewport;
@@ -3605,6 +3619,7 @@ async function executeRenderPipeline(input: {
// Local staging can use links; distributed plan() alone requires real copies.
materializeSymlinks: false,
deferRangeExtraction: windowVideoFrames ? true : undefined,
shownBoxes,
onProgress,
}),
);
@@ -3904,6 +3919,7 @@ async function executeRenderPipeline(input: {
cfg,
injectorOptions: frameCacheOptions(cfg, workerCount),
abortSignal: executionSignal,
shownBoxes,
}),
workDir,
(bytes) => updateCaptureObservability({ videoFrameWindowPeakBytes: bytes }),
@@ -3924,6 +3940,7 @@ async function executeRenderPipeline(input: {
extractAllDeferredFrames(extracted, {
fps: job.config.fps,
cfg,
shownBoxes,
signal: executionSignal,
maxTransientRetries: resolveVideoExtractionPolicy().maxTransientRetries,
onProgress: (done, total) => reportExtractProgress(job, done, total, onProgress),