From 54bcdc616fd59fb324367615684c915480a9c634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 12 Nov 2026 17:16:59 +0000 Subject: [PATCH] fix(producer): a local render extracts each video at about the size it is shown (#7609) --- packages/engine/src/index.ts | 2 + .../videoFrameExtractor.format.test.ts | 59 +++++++++++ .../src/services/videoFrameExtractor.ts | 7 ++ ...eoFrameInjector-shownBoxes.browser.test.ts | 13 ++- .../engine/src/services/videoFrameInjector.ts | 38 ++++++- .../producer/scripts/integration-tests.json | 3 +- .../services/render/deferredVideoFrames.ts | 8 +- .../src/services/render/segmentVideoFrames.ts | 3 + .../render/stages/extractVideosStage.ts | 4 + .../services/render/stages/probeStage.test.ts | 15 +++ .../src/services/render/stages/probeStage.ts | 3 + .../videoShownBoxes.integration.test.ts | 96 ++++++++++++++++++ .../services/render/videoShownBoxes.test.ts | 98 +++++++++++++++++++ .../src/services/render/videoShownBoxes.ts | 95 ++++++++++++++++++ .../src/services/renderOrchestrator.ts | 17 ++++ 15 files changed, 453 insertions(+), 8 deletions(-) create mode 100644 packages/producer/src/services/render/videoShownBoxes.integration.test.ts create mode 100644 packages/producer/src/services/render/videoShownBoxes.test.ts create mode 100644 packages/producer/src/services/render/videoShownBoxes.ts diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index a1b5d7bfa1..39c69c7cad 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -512,7 +512,9 @@ export { queryVideoElementBounds, queryElementStacking, measureVideoShownBoxes, + measureVideoShownBoxRanges, type ShownBox, + type ShownBoxRange, type VideoElementBounds, type ElementStackingInfo, } from "./services/videoFrameInjector.js"; diff --git a/packages/engine/src/services/videoFrameExtractor.format.test.ts b/packages/engine/src/services/videoFrameExtractor.format.test.ts index 7529d7bb3c..3ed2cbf8e6 100644 --- a/packages/engine/src/services/videoFrameExtractor.format.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.format.test.ts @@ -692,6 +692,65 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => { expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB"); }, 60_000); + it("extracts each video at its own shown box, and caches halved frames apart from full ones", async () => { + const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-shown-cache-")); + const size = (path: string) => + spawnSync("ffprobe", [ + "-v", + "error", + "-show_entries", + "stream=width,height", + "-of", + "csv=p=0", + "--", + path, + ]) + .stdout.toString() + .trim(); + const run = async ( + name: string, + shownBoxes?: Map, + ) => { + const outputDir = join(FIXTURE_DIR, name); + mkdirSync(outputDir, { recursive: true }); + const result = await extractAllVideoFrames( + [fixtureVideo()], + FIXTURE_DIR, + { fps: 1, outputDir, shownBoxes }, + undefined, + { extractCacheDir: cacheDir }, + ); + expect(result.errors).toEqual([]); + return { + size: size(result.extracted[0]!.framePaths.get(0)!), + hits: result.phaseBreakdown.cacheHits, + }; + }; + try { + const half = new Map([ + ["ui", { width: UI_FIXTURE_WIDTH / 2, height: UI_FIXTURE_HEIGHT / 2 }], + ]); + expect(await run("shown-full")).toEqual({ + size: `${UI_FIXTURE_WIDTH},${UI_FIXTURE_HEIGHT}`, + hits: 0, + }); + expect(await run("shown-half", half)).toEqual({ + size: `${UI_FIXTURE_WIDTH / 2},${UI_FIXTURE_HEIGHT / 2}`, + hits: 0, + }); + expect(await run("shown-half-again", half)).toEqual({ + size: `${UI_FIXTURE_WIDTH / 2},${UI_FIXTURE_HEIGHT / 2}`, + hits: 1, + }); + expect(await run("shown-full-again")).toEqual({ + size: `${UI_FIXTURE_WIDTH},${UI_FIXTURE_HEIGHT}`, + hits: 1, + }); + } finally { + rmSync(cacheDir, { recursive: true, force: true }); + } + }, 60_000); + it("keeps jpg and png extraction caches separate", async () => { const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-")); try { diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 243a28c866..542d89d302 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -311,6 +311,7 @@ export interface ExtractionOptions { contentKeyedDirs?: readonly string[]; urlByAbsentLocalPath?: ReadonlyMap; shownBox?: { width: number; height: number }; + shownBoxes?: ReadonlyMap; } const DOWNLOAD_SUBDIR = "_downloads"; @@ -321,6 +322,10 @@ const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv" const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb"; const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr"; const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1"; +function shownBoxKey(box: { width: number; height: number } | undefined): string | undefined { + return box && `shown-${Math.ceil(box.width)}x${Math.ceil(box.height)}`; +} + function halveToCoverScaleFilter(box: { width: number; height: number }): string { const fit = `max(${Math.ceil(box.width)}/iw\\,${Math.ceil(box.height)}/ih)`; const halvings = `floor(log(1/${fit})/log(2))`; @@ -2497,6 +2502,7 @@ export async function extractAllVideoFrames( sdrToHdrTransfer: work.sdrToHdrTransfer, toneMapHdrToSdr: work.hdrToSdrTransformKey !== undefined, finalFrameOnly: work.finalFrameOnly, + shownBox: options.shownBoxes?.get(work.video.id) ?? options.shownBox, }; } @@ -2539,6 +2545,7 @@ export async function extractAllVideoFrames( work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined, work.hdrToSdrTransformKey, work.finalFrameOnly ? "final-frame" : undefined, + shownBoxKey(options.shownBoxes?.get(work.video.id) ?? options.shownBox), ].filter((part): part is string => part !== undefined); const transform = transformParts.length > 0 ? transformParts.join("+") : undefined; diff --git a/packages/engine/src/services/videoFrameInjector-shownBoxes.browser.test.ts b/packages/engine/src/services/videoFrameInjector-shownBoxes.browser.test.ts index 33d0d29f76..513f85bf26 100644 --- a/packages/engine/src/services/videoFrameInjector-shownBoxes.browser.test.ts +++ b/packages/engine/src/services/videoFrameInjector-shownBoxes.browser.test.ts @@ -2,7 +2,7 @@ import { spawnSync } from "node:child_process"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import type { Browser, Page } from "puppeteer-core"; import { resolveHeadlessShellPath } from "./browserManager.js"; -import { measureVideoShownBoxes } from "./videoFrameInjector.js"; +import { measureVideoShownBoxRanges, measureVideoShownBoxes } from "./videoFrameInjector.js"; import { closeTestChrome, launchTestChrome } from "./__fixtures__/loopback.js"; const chromePath = resolveHeadlessShellPath(); @@ -50,6 +50,17 @@ describe.skipIf(!hasBrowser)("measureVideoShownBoxes in Chrome", () => { await page.close(); }); + it("reports each video's smallest laid-out box beside its largest", async () => { + const page = await open(page1, 2); + const ranges = await measureVideoShownBoxRanges(page, ["zoom", "late"], [0, 0.25, 1]); + expect(Object.fromEntries(ranges)).toEqual({ + zoom: { largest: { width: 1600, height: 900 }, smallest: { width: 800, height: 450 } }, + // not laid out before 0.5, which is no smaller box + late: { largest: { width: 640, height: 360 }, smallest: { width: 640, height: 360 } }, + }); + await page.close(); + }); + it("keeps every video full size when the page has a canvas that could sample it", async () => { const page = await open( page1.replace(" + +
+ + +
+ +`; + +let root: string; +const savedRendersDir = process.env.PRODUCER_RENDERS_DIR; + +beforeAll(() => { + root = mkdtempSync(join(tmpdir(), "hf-shown-box-")); + process.env.PRODUCER_RENDERS_DIR = join(root, "renders"); +}); +afterAll(() => { + if (savedRendersDir === undefined) delete process.env.PRODUCER_RENDERS_DIR; + else process.env.PRODUCER_RENDERS_DIR = savedRendersDir; + rmSync(root, { recursive: true, force: true }); +}); + +describe.skipIf(!CAN_RUN)("a local render extracts each video at the size it is shown", () => { + it("halves a static clip onto its box and keeps a zooming clip's frames over its peak", async () => { + const project = join(root, "film"); + mkdirSync(project, { recursive: true }); + const synth = spawnSync("ffmpeg", [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=128x72:r=30:d=1"], + ...["-c:v", "libx264", "-pix_fmt", "yuv420p", join(project, "clip.mp4")], + ]); + if (synth.status !== 0) throw new Error(synth.stderr.toString()); + writeFileSync(join(project, "index.html"), FILM); + let measured: unknown; + const quiet = () => {}; + const logger: ProducerLogger = { + error: quiet, + warn: quiet, + debug: quiet, + info: (message, meta) => { + if (message.includes("boxes videos are shown at")) measured = meta; + }, + }; + const job = createRenderJob({ fps: 30, quality: "draft", debug: true, logger }); + await executeRenderJob(job, project, join(root, "film.mp4")); + + const framesRoot = join( + resolveRenderDebugDir(), + job.id, + "compiled", + "__hyperframes_video_frames", + ); + const frameSize = (id: string): string => { + const dir = join(framesRoot, readdirSync(framesRoot).find((entry) => entry.includes(id))!); + const first = join(dir, readdirSync(dir).find((file) => file.startsWith("frame_"))!); + const probe = spawnSync("ffprobe", [ + "-v", + "error", + "-show_entries", + "stream=width,height", + "-of", + "csv=p=0", + "--", + first, + ]); + return probe.stdout.toString().trim(); + }; + expect(measured).toMatchObject({ videos: 2, boxed: 2, seeks: 3 }); + expect(frameSize("still")).toBe("64,36"); + // Sized from its first frame (16x9) it would be halved twice to 32x18, under its 40x23 peak. + expect(frameSize("zoom")).toBe("64,36"); + }, 90_000); +}); diff --git a/packages/producer/src/services/render/videoShownBoxes.test.ts b/packages/producer/src/services/render/videoShownBoxes.test.ts new file mode 100644 index 0000000000..0b902445f3 --- /dev/null +++ b/packages/producer/src/services/render/videoShownBoxes.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import type { VideoElement } from "@hyperframes/engine"; +import type { Page } from "puppeteer-core"; +import type { ProducerLogger } from "../../logger.js"; +import { measureLocalShownBoxes, shownBoxTimes } from "./videoShownBoxes.js"; + +const video = (id: string, start: number, end: number): VideoElement => ({ + id, + src: `${id}.mp4`, + start, + end, + mediaStart: 0, + loop: false, + hasAudio: false, +}); +const fps = { num: 30, den: 1 }; + +function logger() { + const lines: Array<{ level: string; message: string; meta?: Record }> = []; + const at = + (level: string) => + (message: string, meta?: Record): void => { + lines.push({ level, message, meta }); + }; + const log: ProducerLogger = { + error: at("error"), + warn: at("warn"), + info: at("info"), + debug: at("debug"), + }; + return { log, lines }; +} + +/** A page whose measure returns `ranges`, or rejects with `error`. */ +function page(ranges: unknown, error?: Error): Page { + let calls = 0; + return { + evaluate: async () => { + calls += 1; + if (calls > 1) return undefined; // the seek back to 0 + if (error) throw error; + return ranges; + }, + } as unknown as Page; +} + +describe("shownBoxTimes", () => { + it("samples each video from its first frame every 0.25 s to its last frame", () => { + expect(shownBoxTimes([video("a", 1, 2)], fps)).toEqual([1, 1.25, 1.5, 1.75, 2 - 1 / 30]); + }); + + it("merges the times of several videos once each, in order", () => { + expect(shownBoxTimes([video("a", 0, 0.5), video("b", 0.25, 0.5)], fps)).toEqual([ + 0, + 0.25, + 0.5 - 1 / 30, + ]); + }); +}); + +describe("measureLocalShownBoxes", () => { + it("keeps a static box exact and adds 10% to a box that changes", async () => { + const { log, lines } = logger(); + const boxes = await measureLocalShownBoxes( + page([ + [ + "still", + { largest: { width: 1920, height: 1080 }, smallest: { width: 1920, height: 1080 } }, + ], + ["zoom", { largest: { width: 400, height: 200 }, smallest: { width: 100, height: 50 } }], + ]), + [video("still", 0, 1), video("zoom", 0, 1)], + fps, + log, + ); + expect(Object.fromEntries(boxes!)).toEqual({ + still: { width: 1920, height: 1080 }, + zoom: { width: expect.closeTo(440, 6), height: expect.closeTo(220, 6) }, + }); + expect(lines.find((l) => l.level === "info")?.meta).toMatchObject({ + videos: 2, + boxed: 2, + seeks: 5, + }); + }); + + it("returns no boxes, so every video keeps its full size, when the measure fails", async () => { + const { log, lines } = logger(); + const boxes = await measureLocalShownBoxes( + page(undefined, new Error("Target closed")), + [video("clip", 0, 1)], + fps, + log, + ); + expect(boxes).toBeUndefined(); + expect(lines.find((l) => l.level === "warn")?.meta).toMatchObject({ error: "Target closed" }); + }); +}); diff --git a/packages/producer/src/services/render/videoShownBoxes.ts b/packages/producer/src/services/render/videoShownBoxes.ts new file mode 100644 index 0000000000..f1944be1ef --- /dev/null +++ b/packages/producer/src/services/render/videoShownBoxes.ts @@ -0,0 +1,95 @@ +import { + extractMediaMetadata, + measureVideoShownBoxRanges, + resolveProjectRelativeSrc, + type ShownBox, + type VideoElement, +} from "@hyperframes/engine"; +import type { Page } from "puppeteer-core"; +import type { ProducerLogger } from "../../logger.js"; + +/** Seconds between samples of a video's box; a 693 s film with one video is ~2.8k seeks. */ +const SHOWN_BOX_STRIDE_S = 0.25; +/** A box that changes may peak between samples, so it gets this much more. A static box is exact. */ +const MOVING_BOX_HEADROOM = 1.1; + +/** Local videos whose source is larger than the canvas: only these can be extracted smaller. */ +export async function oversizedVideos( + videos: readonly VideoElement[], + projectDir: string, + compiledDir: string, + canvas: { width: number; height: number }, +): Promise { + const sized = await Promise.all( + videos.map(async (video) => { + if (/^https?:\/\//i.test(video.src)) return false; + try { + const path = resolveProjectRelativeSrc(video.src, projectDir, compiledDir); + const { width, height } = await extractMediaMetadata(path); + return width > canvas.width || height > canvas.height; + } catch { + return false; // extraction reports the unreadable source + } + }), + ); + return videos.filter((_, index) => sized[index]); +} + +/** Each video's first and last frame and every stride between, in composition seconds, ascending. */ +export function shownBoxTimes( + videos: readonly VideoElement[], + fps: { num: number; den: number }, +): number[] { + const frame = fps.den / fps.num; + const times = new Set(); + for (const { start, end } of videos) { + const last = Math.max(start, end - frame); + for (let t = start; t < last; t += SHOWN_BOX_STRIDE_S) times.add(t); + times.add(last); + } + return [...times].sort((a, b) => a - b); +} + +/** + * The box to extract each video at: the largest it is drawn at, plus headroom when it moves. Undefined when + * the measure fails, and every video then keeps its full size. + */ +export async function measureLocalShownBoxes( + page: Page, + videos: readonly VideoElement[], + fps: { num: number; den: number }, + log: ProducerLogger, +): Promise | undefined> { + const times = shownBoxTimes(videos, fps); + const startedAt = Date.now(); + try { + const ranges = await measureVideoShownBoxRanges( + page, + videos.map((video) => video.id), + times, + ); + await page.evaluate(() => + (window as { __hf?: { seek?: (t: number) => void } }).__hf?.seek?.(0), + ); + const boxes = new Map(); + for (const [id, { largest, smallest }] of ranges) { + const moves = largest.width !== smallest.width || largest.height !== smallest.height; + const grow = moves ? MOVING_BOX_HEADROOM : 1; + boxes.set(id, { width: largest.width * grow, height: largest.height * grow }); + } + log.info("[Render] Measured the boxes videos are shown at", { + videos: videos.length, + boxed: boxes.size, + seeks: times.length, + elapsedMs: Date.now() - startedAt, + }); + return boxes; + } catch (error) { + log.warn("[Render] Could not measure the boxes videos are shown at; extracting at full size", { + error: error instanceof Error ? error.message : String(error), + seeks: times.length, + elapsedMs: Date.now() - startedAt, + }); + return undefined; + } +} diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 31c98805d5..3721fbbf3b 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -185,6 +185,7 @@ import { import { checkPageNeverReady } from "./render/pageNeverReady.js"; import { resolveSegmentFrames } from "./render/segmentPlan.js"; import { extractAllDeferredFrames } from "./render/deferredVideoFrames.js"; +import { measureLocalShownBoxes, oversizedVideos } from "./render/videoShownBoxes.js"; import { createSegmentVideoFrames } from "./render/segmentVideoFrames.js"; import { shouldWindowVideoFrames, trackSegmentFramePeak } from "./render/videoFrameWindow.js"; import { resolveSegmentBrowserRecycle } from "./render/segmentRecycle.js"; @@ -3435,6 +3436,13 @@ async function executeRenderPipeline(input: { ); const subTimelineWaitMemo: SubTimelineWaitMemo = {}; const staticDedupMemo: StaticDedupMemo = {}; + // Only a video larger than the canvas can be extracted smaller; the probe page then measures where it is drawn. + const largerThanCanvas = await oversizedVideos( + composition.videos, + projectDir, + join(workDir, "compiled"), + { width: outputWidth, height: outputHeight }, + ); const probeResult = await observeRenderStage( observability, "browser_probe", @@ -3443,6 +3451,7 @@ async function executeRenderPipeline(input: { runProbeStage({ projectDir, workDir, + measuresShownBoxes: largerThanCanvas.length > 0, job, cfg, forceScreenshot: captureForceScreenshot, @@ -3468,6 +3477,11 @@ async function executeRenderPipeline(input: { compositionHash = computeCompositionObservabilityHash(compiled.html); fileServer = probeResult.fileServer; probeSession = probeResult.probeSession; + // Before the video layer may close the probe page; no box means full size, as before. + const shownBoxes = + largerThanCanvas.length > 0 && probeSession?.isInitialized + ? await measureLocalShownBoxes(probeSession.page, largerThanCanvas, job.config.fps, log) + : undefined; job.strictVariablesChecked = probeResult.strictVariablesChecked; lastBrowserConsole = probeResult.lastBrowserConsole; let resolvedCaptureBeyondViewport = probeSession?.options.captureBeyondViewport; @@ -3605,6 +3619,7 @@ async function executeRenderPipeline(input: { // Local staging can use links; distributed plan() alone requires real copies. materializeSymlinks: false, deferRangeExtraction: windowVideoFrames ? true : undefined, + shownBoxes, onProgress, }), ); @@ -3904,6 +3919,7 @@ async function executeRenderPipeline(input: { cfg, injectorOptions: frameCacheOptions(cfg, workerCount), abortSignal: executionSignal, + shownBoxes, }), workDir, (bytes) => updateCaptureObservability({ videoFrameWindowPeakBytes: bytes }), @@ -3924,6 +3940,7 @@ async function executeRenderPipeline(input: { extractAllDeferredFrames(extracted, { fps: job.config.fps, cfg, + shownBoxes, signal: executionSignal, maxTransientRetries: resolveVideoExtractionPolicy().maxTransientRetries, onProgress: (done, total) => reportExtractProgress(job, done, total, onProgress),