mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
fix(producer): a local render extracts each video at about the size it is shown (#7609)
This commit is contained in:
@@ -512,7 +512,9 @@ export {
|
||||
queryVideoElementBounds,
|
||||
queryElementStacking,
|
||||
measureVideoShownBoxes,
|
||||
measureVideoShownBoxRanges,
|
||||
type ShownBox,
|
||||
type ShownBoxRange,
|
||||
type VideoElementBounds,
|
||||
type ElementStackingInfo,
|
||||
} from "./services/videoFrameInjector.js";
|
||||
|
||||
@@ -692,6 +692,65 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
|
||||
expect(pngChunkTypes(result.framePaths.get(0)!)).not.toContain("sRGB");
|
||||
}, 60_000);
|
||||
|
||||
it("extracts each video at its own shown box, and caches halved frames apart from full ones", async () => {
|
||||
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-shown-cache-"));
|
||||
const size = (path: string) =>
|
||||
spawnSync("ffprobe", [
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"stream=width,height",
|
||||
"-of",
|
||||
"csv=p=0",
|
||||
"--",
|
||||
path,
|
||||
])
|
||||
.stdout.toString()
|
||||
.trim();
|
||||
const run = async (
|
||||
name: string,
|
||||
shownBoxes?: Map<string, { width: number; height: number }>,
|
||||
) => {
|
||||
const outputDir = join(FIXTURE_DIR, name);
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
const result = await extractAllVideoFrames(
|
||||
[fixtureVideo()],
|
||||
FIXTURE_DIR,
|
||||
{ fps: 1, outputDir, shownBoxes },
|
||||
undefined,
|
||||
{ extractCacheDir: cacheDir },
|
||||
);
|
||||
expect(result.errors).toEqual([]);
|
||||
return {
|
||||
size: size(result.extracted[0]!.framePaths.get(0)!),
|
||||
hits: result.phaseBreakdown.cacheHits,
|
||||
};
|
||||
};
|
||||
try {
|
||||
const half = new Map([
|
||||
["ui", { width: UI_FIXTURE_WIDTH / 2, height: UI_FIXTURE_HEIGHT / 2 }],
|
||||
]);
|
||||
expect(await run("shown-full")).toEqual({
|
||||
size: `${UI_FIXTURE_WIDTH},${UI_FIXTURE_HEIGHT}`,
|
||||
hits: 0,
|
||||
});
|
||||
expect(await run("shown-half", half)).toEqual({
|
||||
size: `${UI_FIXTURE_WIDTH / 2},${UI_FIXTURE_HEIGHT / 2}`,
|
||||
hits: 0,
|
||||
});
|
||||
expect(await run("shown-half-again", half)).toEqual({
|
||||
size: `${UI_FIXTURE_WIDTH / 2},${UI_FIXTURE_HEIGHT / 2}`,
|
||||
hits: 1,
|
||||
});
|
||||
expect(await run("shown-full-again")).toEqual({
|
||||
size: `${UI_FIXTURE_WIDTH},${UI_FIXTURE_HEIGHT}`,
|
||||
hits: 1,
|
||||
});
|
||||
} finally {
|
||||
rmSync(cacheDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it("keeps jpg and png extraction caches separate", async () => {
|
||||
const cacheDir = mkdtempSync(join(tmpdir(), "hf-extract-format-cache-"));
|
||||
try {
|
||||
|
||||
@@ -311,6 +311,7 @@ export interface ExtractionOptions {
|
||||
contentKeyedDirs?: readonly string[];
|
||||
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
|
||||
shownBox?: { width: number; height: number };
|
||||
shownBoxes?: ReadonlyMap<string, { width: number; height: number }>;
|
||||
}
|
||||
|
||||
const DOWNLOAD_SUBDIR = "_downloads";
|
||||
@@ -321,6 +322,10 @@ const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv"
|
||||
const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";
|
||||
const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr";
|
||||
const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1";
|
||||
function shownBoxKey(box: { width: number; height: number } | undefined): string | undefined {
|
||||
return box && `shown-${Math.ceil(box.width)}x${Math.ceil(box.height)}`;
|
||||
}
|
||||
|
||||
function halveToCoverScaleFilter(box: { width: number; height: number }): string {
|
||||
const fit = `max(${Math.ceil(box.width)}/iw\\,${Math.ceil(box.height)}/ih)`;
|
||||
const halvings = `floor(log(1/${fit})/log(2))`;
|
||||
@@ -2497,6 +2502,7 @@ export async function extractAllVideoFrames(
|
||||
sdrToHdrTransfer: work.sdrToHdrTransfer,
|
||||
toneMapHdrToSdr: work.hdrToSdrTransformKey !== undefined,
|
||||
finalFrameOnly: work.finalFrameOnly,
|
||||
shownBox: options.shownBoxes?.get(work.video.id) ?? options.shownBox,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2539,6 +2545,7 @@ export async function extractAllVideoFrames(
|
||||
work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined,
|
||||
work.hdrToSdrTransformKey,
|
||||
work.finalFrameOnly ? "final-frame" : undefined,
|
||||
shownBoxKey(options.shownBoxes?.get(work.video.id) ?? options.shownBox),
|
||||
].filter((part): part is string => part !== undefined);
|
||||
const transform = transformParts.length > 0 ? transformParts.join("+") : undefined;
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import { spawnSync } from "node:child_process";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { Browser, Page } from "puppeteer-core";
|
||||
import { resolveHeadlessShellPath } from "./browserManager.js";
|
||||
import { measureVideoShownBoxes } from "./videoFrameInjector.js";
|
||||
import { measureVideoShownBoxRanges, measureVideoShownBoxes } from "./videoFrameInjector.js";
|
||||
import { closeTestChrome, launchTestChrome } from "./__fixtures__/loopback.js";
|
||||
|
||||
const chromePath = resolveHeadlessShellPath();
|
||||
@@ -50,6 +50,17 @@ describe.skipIf(!hasBrowser)("measureVideoShownBoxes in Chrome", () => {
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it("reports each video's smallest laid-out box beside its largest", async () => {
|
||||
const page = await open(page1, 2);
|
||||
const ranges = await measureVideoShownBoxRanges(page, ["zoom", "late"], [0, 0.25, 1]);
|
||||
expect(Object.fromEntries(ranges)).toEqual({
|
||||
zoom: { largest: { width: 1600, height: 900 }, smallest: { width: 800, height: 450 } },
|
||||
// not laid out before 0.5, which is no smaller box
|
||||
late: { largest: { width: 640, height: 360 }, smallest: { width: 640, height: 360 } },
|
||||
});
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it("keeps every video full size when the page has a canvas that could sample it", async () => {
|
||||
const page = await open(
|
||||
page1.replace("<script>", '<canvas width="10" height="10"></canvas><script>'),
|
||||
|
||||
@@ -366,6 +366,22 @@ export async function measureVideoShownBoxes(
|
||||
videoIds: readonly string[],
|
||||
times: readonly number[],
|
||||
): Promise<Map<string, ShownBox>> {
|
||||
const ranges = await measureVideoShownBoxRanges(page, videoIds, times);
|
||||
return new Map(Array.from(ranges, ([id, range]) => [id, range.largest]));
|
||||
}
|
||||
|
||||
export interface ShownBoxRange {
|
||||
largest: ShownBox;
|
||||
/** The smallest box it is laid out at; equal to `largest` when the video never moves or resizes. */
|
||||
smallest: ShownBox;
|
||||
}
|
||||
|
||||
/** As `measureVideoShownBoxes`, with each video's smallest laid-out box beside its largest. */
|
||||
export async function measureVideoShownBoxRanges(
|
||||
page: Page,
|
||||
videoIds: readonly string[],
|
||||
times: readonly number[],
|
||||
): Promise<Map<string, ShownBoxRange>> {
|
||||
if (videoIds.length === 0 || times.length === 0) return new Map();
|
||||
const boxes = await page.evaluate(
|
||||
async (
|
||||
@@ -381,6 +397,7 @@ export async function measureVideoShownBoxes(
|
||||
const waitForSeek = (window as Window & { __hfWaitForSeekCompletion?: () => Promise<void> })
|
||||
.__hfWaitForSeekCompletion;
|
||||
const largest = new Map<string, ShownBox>();
|
||||
const smallest = new Map<string, ShownBox>();
|
||||
const fullSize = new Set<string>();
|
||||
const gradedThroughGroup = (video: Element): boolean => {
|
||||
const groupId = video.getAttribute(groupAttr);
|
||||
@@ -405,10 +422,18 @@ export async function measureVideoShownBoxes(
|
||||
return;
|
||||
}
|
||||
const rect = video.getBoundingClientRect();
|
||||
const width = rect.width * window.devicePixelRatio;
|
||||
const height = rect.height * window.devicePixelRatio;
|
||||
const box = largest.get(id) ?? { width: 0, height: 0 };
|
||||
box.width = Math.max(box.width, rect.width * window.devicePixelRatio);
|
||||
box.height = Math.max(box.height, rect.height * window.devicePixelRatio);
|
||||
box.width = Math.max(box.width, width);
|
||||
box.height = Math.max(box.height, height);
|
||||
largest.set(id, box);
|
||||
// A video outside its own clip window is not laid out (0x0); that is not a smaller box.
|
||||
if (width === 0 || height === 0) return;
|
||||
const least = smallest.get(id) ?? { width, height };
|
||||
least.width = Math.min(least.width, width);
|
||||
least.height = Math.min(least.height, height);
|
||||
smallest.set(id, least);
|
||||
};
|
||||
const pageHasCanvas = (): boolean =>
|
||||
Array.from(document.querySelectorAll("canvas")).some(
|
||||
@@ -421,9 +446,12 @@ export async function measureVideoShownBoxes(
|
||||
if (pageHasCanvas()) return [];
|
||||
for (const id of ids) measure(id);
|
||||
}
|
||||
return Array.from(largest).filter(
|
||||
([id, box]) => !fullSize.has(id) && box.width > 0 && box.height > 0,
|
||||
);
|
||||
return Array.from(largest)
|
||||
.filter(([id, box]) => !fullSize.has(id) && box.width > 0 && box.height > 0)
|
||||
.map(([id, box]): [string, { largest: ShownBox; smallest: ShownBox }] => [
|
||||
id,
|
||||
{ largest: box, smallest: smallest.get(id) ?? box },
|
||||
]);
|
||||
},
|
||||
[...videoIds],
|
||||
[...times],
|
||||
|
||||
@@ -65,5 +65,6 @@
|
||||
"src/services/distributed/referenceBackground.integration.test.ts",
|
||||
"src/services/render/planReferenceBackground.polling.test.ts",
|
||||
"src/services/render/deferredVideoFrames.integration.test.ts",
|
||||
"src/services/render/stages/extractVideosStage.integration.test.ts"
|
||||
"src/services/render/stages/extractVideosStage.integration.test.ts",
|
||||
"src/services/render/videoShownBoxes.integration.test.ts"
|
||||
]
|
||||
|
||||
@@ -111,6 +111,7 @@ export async function extractAllDeferredFrames(
|
||||
cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;
|
||||
signal?: AbortSignal;
|
||||
sourceOf?: (video: ExtractedFrames) => string;
|
||||
shownBoxes?: ReadonlyMap<string, ShownBox>;
|
||||
maxTransientRetries?: number;
|
||||
onProgress?: (framesDone: number, framesTotal: number) => void;
|
||||
},
|
||||
@@ -130,7 +131,12 @@ export async function extractAllDeferredFrames(
|
||||
video.videoId,
|
||||
range.startTime,
|
||||
range.durationSeconds,
|
||||
{ fps: input.fps, outputDir: dirname(video.outputDir), format: range.format },
|
||||
{
|
||||
fps: input.fps,
|
||||
outputDir: dirname(video.outputDir),
|
||||
format: range.format,
|
||||
shownBox: input.shownBoxes?.get(video.videoId),
|
||||
},
|
||||
input.signal,
|
||||
input.cfg,
|
||||
),
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
type BeforeCaptureHook,
|
||||
type EngineConfig,
|
||||
type ExtractedFrames,
|
||||
type ShownBox,
|
||||
type VideoElement,
|
||||
} from "@hyperframes/engine";
|
||||
import { resolveRenderFpsConfig } from "../fileServer.js";
|
||||
@@ -52,6 +53,7 @@ export function createSegmentVideoFrames(input: {
|
||||
injectorOptions: Parameters<typeof createVideoFrameInjector>[1];
|
||||
abortSignal?: AbortSignal;
|
||||
sourceOf?: (video: ExtractedFrames) => string;
|
||||
shownBoxes?: ReadonlyMap<string, ShownBox>;
|
||||
extractFrames?: FrameExtractor;
|
||||
}): SegmentVideoFrames {
|
||||
const extractFrames = input.extractFrames ?? extractDeferredFrames;
|
||||
@@ -69,6 +71,7 @@ export function createSegmentVideoFrames(input: {
|
||||
cfg: input.cfg,
|
||||
signal: input.abortSignal,
|
||||
sourceOf,
|
||||
shownBoxes: input.shownBoxes,
|
||||
onMissingFrame: (video, index) =>
|
||||
new Error(
|
||||
`[Render] video ${JSON.stringify(video.videoId)} has no frame ${index}; ` +
|
||||
|
||||
@@ -55,6 +55,7 @@ import {
|
||||
runVideoExtractionWithRetry,
|
||||
safeVideoExtractionSourceIdentity,
|
||||
videoAudioId,
|
||||
type ShownBox,
|
||||
} from "@hyperframes/engine";
|
||||
import {
|
||||
collectVideoMetadataHints,
|
||||
@@ -97,6 +98,7 @@ export interface ExtractVideosStageInput {
|
||||
*/
|
||||
materializeSymlinks?: boolean;
|
||||
deferRangeExtraction?: boolean;
|
||||
shownBoxes?: ReadonlyMap<string, ShownBox>;
|
||||
skipFrameExtractionIds?: readonly string[];
|
||||
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
|
||||
onProgress?: ProgressCallback;
|
||||
@@ -423,6 +425,7 @@ export async function runExtractVideosStage(
|
||||
assertNotAborted,
|
||||
materializeSymlinks,
|
||||
deferRangeExtraction,
|
||||
shownBoxes,
|
||||
urlByAbsentLocalPath,
|
||||
skipFrameExtractionIds = [],
|
||||
} = input;
|
||||
@@ -554,6 +557,7 @@ export async function runExtractVideosStage(
|
||||
maxTransientRetries: extractionPolicy.maxTransientRetries,
|
||||
collectProbeFailures: extractionPolicy.failureMode === "enforce",
|
||||
deferRangeExtraction,
|
||||
shownBoxes,
|
||||
contentKeyedDirs: [join(compiledDir, REMOTE_MEDIA_SUBDIR)],
|
||||
urlByAbsentLocalPath,
|
||||
onProgress: ({ planned = 0, done = 0 }) => {
|
||||
|
||||
@@ -1115,6 +1115,21 @@ describe("runProbeStage — forceScreenshot threading", () => {
|
||||
expect(capturedCfgs).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("opens a page for a video larger than the canvas, and none for a film without one", async () => {
|
||||
resetRetryMocks();
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
for (const measuresShownBoxes of [false, true]) {
|
||||
capturedCfgs.length = 0;
|
||||
const input = makeProbeInput({});
|
||||
input.composition.duration = 5;
|
||||
input.compiled.html = `<video id="clip" src="clip.mp4" data-start="0" data-duration="5"></video>`;
|
||||
await runProbeStage({ ...input, measuresShownBoxes });
|
||||
expect(capturedCfgs.length > 0, `measuresShownBoxes ${measuresShownBoxes}`).toBe(
|
||||
measuresShownBoxes,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("probes when a source child of existing media is mutated", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
|
||||
@@ -94,6 +94,7 @@ import { computeCompositionObservabilityHash } from "../observability.js";
|
||||
export interface ProbeStageInput {
|
||||
projectDir: string;
|
||||
workDir: string;
|
||||
measuresShownBoxes?: boolean;
|
||||
recordReferenceBackground?: CaptureOptions["recordReferenceBackground"];
|
||||
job: RenderJob;
|
||||
cfg: EngineConfig;
|
||||
@@ -409,6 +410,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
const hasInsertedMedia = hasRuntimeMediaChanges(compiled.html);
|
||||
const needsBrowser =
|
||||
cfg.videoLayerPassthrough ||
|
||||
input.measuresShownBoxes === true ||
|
||||
probeRequiresBrowser({
|
||||
durationSeconds: composition.duration,
|
||||
unresolvedCompositionCount: compiled.unresolvedCompositions.length,
|
||||
@@ -429,6 +431,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
if (hasScriptedAudio) reasons.push("scripted audio volume");
|
||||
if (hasInsertedMedia) reasons.push("runtime-created or source-mutated media");
|
||||
if (hasVariableMedia) reasons.push("variable-bound media source(s)");
|
||||
if (input.measuresShownBoxes) reasons.push("video(s) larger than the canvas");
|
||||
|
||||
log.info("Launching browser for composition probe...", {
|
||||
reasons,
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { GSAP_CDN } from "@hyperframes/core";
|
||||
import { resolveHeadlessShellPath } from "@hyperframes/engine";
|
||||
import type { ProducerLogger } from "../../logger.js";
|
||||
import { createRenderJob, executeRenderJob, resolveRenderDebugDir } from "../renderOrchestrator.js";
|
||||
|
||||
const chromePath = resolveHeadlessShellPath();
|
||||
const CAN_RUN =
|
||||
chromePath !== undefined &&
|
||||
spawnSync(chromePath, ["--version"]).status === 0 &&
|
||||
spawnSync("ffmpeg", ["-version"]).status === 0;
|
||||
|
||||
// A 64x36 film with two 128x72 clips: `still` fills it (4K in 1080p, scaled down); `zoom` grows from 16x9 to 40x22.5.
|
||||
const FILM = `<!doctype html>
|
||||
<html><head><meta charset="utf-8"><script src="${GSAP_CDN}"></script></head>
|
||||
<body style="margin:0;background:#000">
|
||||
<div id="root" data-composition-id="root" data-width="64" data-height="36" data-start="0" data-duration="0.5">
|
||||
<video id="still" src="clip.mp4" muted data-start="0" data-duration="0.5" style="position:absolute;left:0;top:0;width:64px;height:36px"></video>
|
||||
<video id="zoom" src="clip.mp4" muted data-start="0" data-duration="0.5" style="position:absolute;left:0;top:0;width:16px;height:9px"></video>
|
||||
</div>
|
||||
<script>
|
||||
window.__timelines = window.__timelines || {};
|
||||
const tl = gsap.timeline({ paused: true });
|
||||
tl.to("#zoom", { width: 40, height: 22.5, duration: 0.4, ease: "none" }, 0);
|
||||
tl.set({}, {}, 0.5);
|
||||
window.__timelines["root"] = tl;
|
||||
</script>
|
||||
</body></html>`;
|
||||
|
||||
let root: string;
|
||||
const savedRendersDir = process.env.PRODUCER_RENDERS_DIR;
|
||||
|
||||
beforeAll(() => {
|
||||
root = mkdtempSync(join(tmpdir(), "hf-shown-box-"));
|
||||
process.env.PRODUCER_RENDERS_DIR = join(root, "renders");
|
||||
});
|
||||
afterAll(() => {
|
||||
if (savedRendersDir === undefined) delete process.env.PRODUCER_RENDERS_DIR;
|
||||
else process.env.PRODUCER_RENDERS_DIR = savedRendersDir;
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe.skipIf(!CAN_RUN)("a local render extracts each video at the size it is shown", () => {
|
||||
it("halves a static clip onto its box and keeps a zooming clip's frames over its peak", async () => {
|
||||
const project = join(root, "film");
|
||||
mkdirSync(project, { recursive: true });
|
||||
const synth = spawnSync("ffmpeg", [
|
||||
...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=128x72:r=30:d=1"],
|
||||
...["-c:v", "libx264", "-pix_fmt", "yuv420p", join(project, "clip.mp4")],
|
||||
]);
|
||||
if (synth.status !== 0) throw new Error(synth.stderr.toString());
|
||||
writeFileSync(join(project, "index.html"), FILM);
|
||||
let measured: unknown;
|
||||
const quiet = () => {};
|
||||
const logger: ProducerLogger = {
|
||||
error: quiet,
|
||||
warn: quiet,
|
||||
debug: quiet,
|
||||
info: (message, meta) => {
|
||||
if (message.includes("boxes videos are shown at")) measured = meta;
|
||||
},
|
||||
};
|
||||
const job = createRenderJob({ fps: 30, quality: "draft", debug: true, logger });
|
||||
await executeRenderJob(job, project, join(root, "film.mp4"));
|
||||
|
||||
const framesRoot = join(
|
||||
resolveRenderDebugDir(),
|
||||
job.id,
|
||||
"compiled",
|
||||
"__hyperframes_video_frames",
|
||||
);
|
||||
const frameSize = (id: string): string => {
|
||||
const dir = join(framesRoot, readdirSync(framesRoot).find((entry) => entry.includes(id))!);
|
||||
const first = join(dir, readdirSync(dir).find((file) => file.startsWith("frame_"))!);
|
||||
const probe = spawnSync("ffprobe", [
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"stream=width,height",
|
||||
"-of",
|
||||
"csv=p=0",
|
||||
"--",
|
||||
first,
|
||||
]);
|
||||
return probe.stdout.toString().trim();
|
||||
};
|
||||
expect(measured).toMatchObject({ videos: 2, boxed: 2, seeks: 3 });
|
||||
expect(frameSize("still")).toBe("64,36");
|
||||
// Sized from its first frame (16x9) it would be halved twice to 32x18, under its 40x23 peak.
|
||||
expect(frameSize("zoom")).toBe("64,36");
|
||||
}, 90_000);
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { VideoElement } from "@hyperframes/engine";
|
||||
import type { Page } from "puppeteer-core";
|
||||
import type { ProducerLogger } from "../../logger.js";
|
||||
import { measureLocalShownBoxes, shownBoxTimes } from "./videoShownBoxes.js";
|
||||
|
||||
const video = (id: string, start: number, end: number): VideoElement => ({
|
||||
id,
|
||||
src: `${id}.mp4`,
|
||||
start,
|
||||
end,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
});
|
||||
const fps = { num: 30, den: 1 };
|
||||
|
||||
function logger() {
|
||||
const lines: Array<{ level: string; message: string; meta?: Record<string, unknown> }> = [];
|
||||
const at =
|
||||
(level: string) =>
|
||||
(message: string, meta?: Record<string, unknown>): void => {
|
||||
lines.push({ level, message, meta });
|
||||
};
|
||||
const log: ProducerLogger = {
|
||||
error: at("error"),
|
||||
warn: at("warn"),
|
||||
info: at("info"),
|
||||
debug: at("debug"),
|
||||
};
|
||||
return { log, lines };
|
||||
}
|
||||
|
||||
/** A page whose measure returns `ranges`, or rejects with `error`. */
|
||||
function page(ranges: unknown, error?: Error): Page {
|
||||
let calls = 0;
|
||||
return {
|
||||
evaluate: async () => {
|
||||
calls += 1;
|
||||
if (calls > 1) return undefined; // the seek back to 0
|
||||
if (error) throw error;
|
||||
return ranges;
|
||||
},
|
||||
} as unknown as Page;
|
||||
}
|
||||
|
||||
describe("shownBoxTimes", () => {
|
||||
it("samples each video from its first frame every 0.25 s to its last frame", () => {
|
||||
expect(shownBoxTimes([video("a", 1, 2)], fps)).toEqual([1, 1.25, 1.5, 1.75, 2 - 1 / 30]);
|
||||
});
|
||||
|
||||
it("merges the times of several videos once each, in order", () => {
|
||||
expect(shownBoxTimes([video("a", 0, 0.5), video("b", 0.25, 0.5)], fps)).toEqual([
|
||||
0,
|
||||
0.25,
|
||||
0.5 - 1 / 30,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("measureLocalShownBoxes", () => {
|
||||
it("keeps a static box exact and adds 10% to a box that changes", async () => {
|
||||
const { log, lines } = logger();
|
||||
const boxes = await measureLocalShownBoxes(
|
||||
page([
|
||||
[
|
||||
"still",
|
||||
{ largest: { width: 1920, height: 1080 }, smallest: { width: 1920, height: 1080 } },
|
||||
],
|
||||
["zoom", { largest: { width: 400, height: 200 }, smallest: { width: 100, height: 50 } }],
|
||||
]),
|
||||
[video("still", 0, 1), video("zoom", 0, 1)],
|
||||
fps,
|
||||
log,
|
||||
);
|
||||
expect(Object.fromEntries(boxes!)).toEqual({
|
||||
still: { width: 1920, height: 1080 },
|
||||
zoom: { width: expect.closeTo(440, 6), height: expect.closeTo(220, 6) },
|
||||
});
|
||||
expect(lines.find((l) => l.level === "info")?.meta).toMatchObject({
|
||||
videos: 2,
|
||||
boxed: 2,
|
||||
seeks: 5,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns no boxes, so every video keeps its full size, when the measure fails", async () => {
|
||||
const { log, lines } = logger();
|
||||
const boxes = await measureLocalShownBoxes(
|
||||
page(undefined, new Error("Target closed")),
|
||||
[video("clip", 0, 1)],
|
||||
fps,
|
||||
log,
|
||||
);
|
||||
expect(boxes).toBeUndefined();
|
||||
expect(lines.find((l) => l.level === "warn")?.meta).toMatchObject({ error: "Target closed" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,95 @@
|
||||
import {
|
||||
extractMediaMetadata,
|
||||
measureVideoShownBoxRanges,
|
||||
resolveProjectRelativeSrc,
|
||||
type ShownBox,
|
||||
type VideoElement,
|
||||
} from "@hyperframes/engine";
|
||||
import type { Page } from "puppeteer-core";
|
||||
import type { ProducerLogger } from "../../logger.js";
|
||||
|
||||
/** Seconds between samples of a video's box; a 693 s film with one video is ~2.8k seeks. */
|
||||
const SHOWN_BOX_STRIDE_S = 0.25;
|
||||
/** A box that changes may peak between samples, so it gets this much more. A static box is exact. */
|
||||
const MOVING_BOX_HEADROOM = 1.1;
|
||||
|
||||
/** Local videos whose source is larger than the canvas: only these can be extracted smaller. */
|
||||
export async function oversizedVideos(
|
||||
videos: readonly VideoElement[],
|
||||
projectDir: string,
|
||||
compiledDir: string,
|
||||
canvas: { width: number; height: number },
|
||||
): Promise<VideoElement[]> {
|
||||
const sized = await Promise.all(
|
||||
videos.map(async (video) => {
|
||||
if (/^https?:\/\//i.test(video.src)) return false;
|
||||
try {
|
||||
const path = resolveProjectRelativeSrc(video.src, projectDir, compiledDir);
|
||||
const { width, height } = await extractMediaMetadata(path);
|
||||
return width > canvas.width || height > canvas.height;
|
||||
} catch {
|
||||
return false; // extraction reports the unreadable source
|
||||
}
|
||||
}),
|
||||
);
|
||||
return videos.filter((_, index) => sized[index]);
|
||||
}
|
||||
|
||||
/** Each video's first and last frame and every stride between, in composition seconds, ascending. */
|
||||
export function shownBoxTimes(
|
||||
videos: readonly VideoElement[],
|
||||
fps: { num: number; den: number },
|
||||
): number[] {
|
||||
const frame = fps.den / fps.num;
|
||||
const times = new Set<number>();
|
||||
for (const { start, end } of videos) {
|
||||
const last = Math.max(start, end - frame);
|
||||
for (let t = start; t < last; t += SHOWN_BOX_STRIDE_S) times.add(t);
|
||||
times.add(last);
|
||||
}
|
||||
return [...times].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
/**
|
||||
* The box to extract each video at: the largest it is drawn at, plus headroom when it moves. Undefined when
|
||||
* the measure fails, and every video then keeps its full size.
|
||||
*/
|
||||
export async function measureLocalShownBoxes(
|
||||
page: Page,
|
||||
videos: readonly VideoElement[],
|
||||
fps: { num: number; den: number },
|
||||
log: ProducerLogger,
|
||||
): Promise<Map<string, ShownBox> | undefined> {
|
||||
const times = shownBoxTimes(videos, fps);
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const ranges = await measureVideoShownBoxRanges(
|
||||
page,
|
||||
videos.map((video) => video.id),
|
||||
times,
|
||||
);
|
||||
await page.evaluate(() =>
|
||||
(window as { __hf?: { seek?: (t: number) => void } }).__hf?.seek?.(0),
|
||||
);
|
||||
const boxes = new Map<string, ShownBox>();
|
||||
for (const [id, { largest, smallest }] of ranges) {
|
||||
const moves = largest.width !== smallest.width || largest.height !== smallest.height;
|
||||
const grow = moves ? MOVING_BOX_HEADROOM : 1;
|
||||
boxes.set(id, { width: largest.width * grow, height: largest.height * grow });
|
||||
}
|
||||
log.info("[Render] Measured the boxes videos are shown at", {
|
||||
videos: videos.length,
|
||||
boxed: boxes.size,
|
||||
seeks: times.length,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
});
|
||||
return boxes;
|
||||
} catch (error) {
|
||||
log.warn("[Render] Could not measure the boxes videos are shown at; extracting at full size", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
seeks: times.length,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -185,6 +185,7 @@ import {
|
||||
import { checkPageNeverReady } from "./render/pageNeverReady.js";
|
||||
import { resolveSegmentFrames } from "./render/segmentPlan.js";
|
||||
import { extractAllDeferredFrames } from "./render/deferredVideoFrames.js";
|
||||
import { measureLocalShownBoxes, oversizedVideos } from "./render/videoShownBoxes.js";
|
||||
import { createSegmentVideoFrames } from "./render/segmentVideoFrames.js";
|
||||
import { shouldWindowVideoFrames, trackSegmentFramePeak } from "./render/videoFrameWindow.js";
|
||||
import { resolveSegmentBrowserRecycle } from "./render/segmentRecycle.js";
|
||||
@@ -3435,6 +3436,13 @@ async function executeRenderPipeline(input: {
|
||||
);
|
||||
const subTimelineWaitMemo: SubTimelineWaitMemo = {};
|
||||
const staticDedupMemo: StaticDedupMemo = {};
|
||||
// Only a video larger than the canvas can be extracted smaller; the probe page then measures where it is drawn.
|
||||
const largerThanCanvas = await oversizedVideos(
|
||||
composition.videos,
|
||||
projectDir,
|
||||
join(workDir, "compiled"),
|
||||
{ width: outputWidth, height: outputHeight },
|
||||
);
|
||||
const probeResult = await observeRenderStage(
|
||||
observability,
|
||||
"browser_probe",
|
||||
@@ -3443,6 +3451,7 @@ async function executeRenderPipeline(input: {
|
||||
runProbeStage({
|
||||
projectDir,
|
||||
workDir,
|
||||
measuresShownBoxes: largerThanCanvas.length > 0,
|
||||
job,
|
||||
cfg,
|
||||
forceScreenshot: captureForceScreenshot,
|
||||
@@ -3468,6 +3477,11 @@ async function executeRenderPipeline(input: {
|
||||
compositionHash = computeCompositionObservabilityHash(compiled.html);
|
||||
fileServer = probeResult.fileServer;
|
||||
probeSession = probeResult.probeSession;
|
||||
// Before the video layer may close the probe page; no box means full size, as before.
|
||||
const shownBoxes =
|
||||
largerThanCanvas.length > 0 && probeSession?.isInitialized
|
||||
? await measureLocalShownBoxes(probeSession.page, largerThanCanvas, job.config.fps, log)
|
||||
: undefined;
|
||||
job.strictVariablesChecked = probeResult.strictVariablesChecked;
|
||||
lastBrowserConsole = probeResult.lastBrowserConsole;
|
||||
let resolvedCaptureBeyondViewport = probeSession?.options.captureBeyondViewport;
|
||||
@@ -3605,6 +3619,7 @@ async function executeRenderPipeline(input: {
|
||||
// Local staging can use links; distributed plan() alone requires real copies.
|
||||
materializeSymlinks: false,
|
||||
deferRangeExtraction: windowVideoFrames ? true : undefined,
|
||||
shownBoxes,
|
||||
onProgress,
|
||||
}),
|
||||
);
|
||||
@@ -3904,6 +3919,7 @@ async function executeRenderPipeline(input: {
|
||||
cfg,
|
||||
injectorOptions: frameCacheOptions(cfg, workerCount),
|
||||
abortSignal: executionSignal,
|
||||
shownBoxes,
|
||||
}),
|
||||
workDir,
|
||||
(bytes) => updateCaptureObservability({ videoFrameWindowPeakBytes: bytes }),
|
||||
@@ -3924,6 +3940,7 @@ async function executeRenderPipeline(input: {
|
||||
extractAllDeferredFrames(extracted, {
|
||||
fps: job.config.fps,
|
||||
cfg,
|
||||
shownBoxes,
|
||||
signal: executionSignal,
|
||||
maxTransientRetries: resolveVideoExtractionPolicy().maxTransientRetries,
|
||||
onProgress: (done, total) => reportExtractProgress(job, done, total, onProgress),
|
||||
|
||||
Reference in New Issue
Block a user