Files
hyperframes/packages/engine/src/services/screenshotService.ts
T
Miguel Ángel 3bb0a9a5b5 perf(engine): skip the screenshot on frames the page reports unchanged in beginframe capture (#7027)
* perf(engine): skip the screenshot on frames the page reports unchanged in beginframe capture

* fix(engine): beginframe reuse never returns a frame older than a cross-frame page change

* fix(engine): the BeginFrame reuse test gets a headless shell in the flake gate

* fix(ci): run the headless shell installer through its package

* fix(ci): install the headless shell with bunx, npx finds no bin in the workspace
2026-11-02 19:17:31 +00:00

971 lines
42 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// fallow-ignore-file code-duplication complexity
/**
* Screenshot Service
*
* BeginFrame-based deterministic screenshot capture and video frame injection.
*/
// fallow-ignore-file code-duplication
import { type Page } from "puppeteer-core";
import { withVideoFrameSources, type VideoFrameUpdate } from "./videoFrameTransport.js";
import { type CaptureOptions } from "../types.js";
import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "@hyperframes/core/color-grading";
import {
HF_COLOR_GRADING_CANVAS_ID_PREFIX,
MEDIA_RENDER_ID_ATTR,
MEDIA_VISUAL_STYLE_PROPERTIES,
RENDER_FRAME_ID_PREFIX,
RENDER_FRAME_ID_SUFFIX,
renderFrameIdForRenderId,
} from "@hyperframes/core";
export const cdpSessionCache = new WeakMap<Page, import("puppeteer-core").CDPSession>();
export async function getCdpSession(page: Page): Promise<import("puppeteer-core").CDPSession> {
let client = cdpSessionCache.get(page);
if (!client) {
client = await page.createCDPSession();
cdpSessionCache.set(page, client);
}
return client;
}
export function shouldDefaultCaptureBeyondViewport(
browserVersion: string,
platform: NodeJS.Platform = process.platform,
): boolean {
// Regular Chrome's viewport-bound screenshot path can expose a compositor
// surface shorter than the page viewport on affected macOS builds. In that
// case Chrome fills the clipped area with the page background. Headless shell
// reports as HeadlessChrome and keeps the faster viewport-bound path.
return platform === "darwin" && browserVersion.startsWith("Chrome/");
}
/**
* BeginFrame result with screenshot data and damage detection.
*/
export interface BeginFrameResult {
buffer: Buffer;
hasDamage: boolean;
}
/**
* Issue a single no-output BeginFrame and race it against `timeoutMs`.
*
* On SwiftShader, compositions with many promoted layers (multi-group nested
* opacity caption animations) can stall the FIRST BeginFrame indefinitely —
* tested to 30 minutes without completion (style-7/8/10/15-prod). The
* auto-worker calibration path catches this with its own capped protocol
* timeout, but renders with an explicit `--workers N` skip calibration and
* would hang for the full protocol timeout (and never succeed). This probe
* gives the producer a cheap liveness signal right after session init:
* `false` means route the render through screenshot capture instead.
*
* Healthy comps complete the probe in well under a second on GPU and within
* a few seconds on SwiftShader. A protocol error also resolves `false` —
* the safe direction (screenshot capture always works).
*/
export async function probeBeginFrameLiveness(
page: Page,
timeoutMs: number,
// BeginFrame frameTimeTicks must be monotonic per session. The capture loop
// sends `session.beginFrameTimeTicks + frameIndex * interval`, where the
// base carries a 10-interval cushion above the warmup loop's last tick —
// callers probing an initialized session should pass a tick INSIDE that
// cushion (e.g. base − 5·interval) so warmup < probe < first capture stays
// monotonic. Omit both params only for a session that will not issue
// further BeginFrames.
frameTimeTicks?: number,
intervalMs?: number,
): Promise<boolean> {
const client = await getCdpSession(page);
const params: { frameTimeTicks?: number; interval?: number } = {};
if (typeof frameTimeTicks === "number") params.frameTimeTicks = frameTimeTicks;
if (typeof intervalMs === "number") params.interval = intervalMs;
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
client
.send("HeadlessExperimental.beginFrame", params)
.then(() => true)
.catch(() => false),
new Promise<boolean>((resolve) => {
timer = setTimeout(() => resolve(false), timeoutMs);
}),
]);
} finally {
if (timer) clearTimeout(timer);
}
}
/**
* Capture a frame using HeadlessExperimental.beginFrame.
*
* This is an atomic operation: one CDP call runs a single layout-paint-composite
* cycle and returns the screenshot + hasDamage boolean. Replaces the separate
* settle → screenshot pipeline with a single deterministic render cycle.
*
* Requires chrome-headless-shell with --enable-begin-frame-control and
* --deterministic-mode flags.
*/
// The last buffer beginFrameCapture returned per page. A screenshot BeginFrame always
// reports damage, so a frame is reused only when a screenshot-less BeginFrame right after
// the seek reports none. Any BeginFrame sent elsewhere moves the display past this buffer:
// call forgetBeginFrameCapture first.
const lastFrameCache = new WeakMap<Page, Buffer>();
export function forgetBeginFrameCapture(page: Page): void {
lastFrameCache.delete(page);
}
const PENDING_FRAME_RETRIES = 5;
async function sendBeginFrame(
client: import("puppeteer-core").CDPSession,
params: Parameters<typeof client.send<"HeadlessExperimental.beginFrame">>[1],
) {
for (let attempt = 0; ; attempt++) {
try {
return await client.send("HeadlessExperimental.beginFrame", params);
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
const isPending = msg.includes("Another frame is pending");
if (isPending && attempt < PENDING_FRAME_RETRIES) {
await new Promise((r) => setTimeout(r, 50 * 2 ** attempt));
continue;
}
if (isPending) {
throw new Error(
`[BeginFrame] Frame still pending after ${PENDING_FRAME_RETRIES} retries: CPU overloaded by parallel renders. ` +
`Reduce concurrent renders or use --docker for isolation.`,
);
}
throw err;
}
}
}
export async function beginFrameCapture(
page: Page,
options: CaptureOptions,
frameTimeTicks: number,
interval: number,
): Promise<BeginFrameResult> {
const client = await getCdpSession(page);
const isPng = options.format === "png";
const screenshot = {
format: isPng ? "png" : "jpeg",
quality: isPng ? undefined : (options.quality ?? 80),
optimizeForSpeed: true,
} as const;
// With a cached buffer, a screenshot-less BeginFrame first draws the seek and reports
// whether it changed anything. The screenshot after it is not displayed: a displayed
// screenshot frame leaves damage the next probe would report, and a settle frame to
// absorb it would draw page changes that nothing captures.
const cached = lastFrameCache.get(page);
let screenshotTicks = frameTimeTicks;
if (cached) {
const probe = await sendBeginFrame(client, { frameTimeTicks, interval });
if (!probe.hasDamage) return { buffer: cached, hasDamage: false };
screenshotTicks = frameTimeTicks + interval / 2;
}
const result = await sendBeginFrame(client, {
frameTimeTicks: screenshotTicks,
interval,
screenshot,
noDisplayUpdates: cached !== undefined,
});
let buffer: Buffer;
if (result.screenshotData) {
buffer = Buffer.from(result.screenshotData, "base64");
lastFrameCache.set(page, buffer);
} else {
if (cached) {
buffer = cached;
} else {
// Frame 0 always has damage, so this path is near-unreachable.
// Force a composite with a tiny time advance.
const fallback = await sendBeginFrame(client, {
frameTimeTicks: frameTimeTicks + 0.001,
interval,
screenshot,
});
buffer = fallback.screenshotData
? Buffer.from(fallback.screenshotData, "base64")
: Buffer.alloc(0);
if (buffer.length > 0) lastFrameCache.set(page, buffer);
}
}
return {
buffer,
hasDamage: result.hasDamage,
};
}
/**
* True if the page's actual rendered content is taller than the requested
* capture height. `captureBeyondViewport` exists for exactly one reason
* (#1094): a native `<video>` surface whose content genuinely overflows the
* viewport-bound capture path clips its bottom edge to black. A video that
* fits entirely inside its composition's declared viewport doesn't have that
* problem — ground-truth measurement beats the coarser "has a video, so
* always request beyond-viewport" heuristic, which also unnecessarily routes
* every video render through a CDP capture path prone to producing phantom
* duplicate content on SwiftShader (#2550).
*
* Callers measure once after page settle. Hyperframes compositions have a
* fixed-height, overflow-clipped render surface; timeline animation may move
* pixels within that surface but must not grow document flow during capture.
*/
export async function pageContentExceedsCaptureHeight(
page: Page,
requestedHeight: number,
): Promise<boolean> {
const scrollHeight = await page.evaluate(() => document.documentElement.scrollHeight);
// Small tolerance for subpixel layout rounding, not a real overflow signal.
return scrollHeight > requestedHeight + 1;
}
/**
* Capture a screenshot using standard Page.captureScreenshot CDP call.
* Fallback for environments where BeginFrame is unavailable (macOS, Windows).
*
* For `format: "png"` captures we disable Chrome's `optimizeForSpeed` fast
* path. The fast path uses a zero-alpha-aware codec that crushes real alpha
* values to 0 or 255 (verified empirically; CDP docs don't document this) —
* exactly the same caveat called out on `captureScreenshotWithAlpha` /
* `captureAlphaPng`. Keeping the fast path for opaque jpeg captures is fine.
*/
export async function pageScreenshotCapture(page: Page, options: CaptureOptions): Promise<Buffer> {
const client = await getCdpSession(page);
const isPng = options.format === "png";
const dpr = options.deviceScaleFactor ?? 1;
const clip = { x: 0, y: 0, width: options.width, height: options.height, scale: dpr };
const result = await client.send("Page.captureScreenshot", {
format: isPng ? "png" : "jpeg",
quality: isPng ? undefined : (options.quality ?? 80),
fromSurface: true,
// Use Chrome's faster viewport-bound screenshot path by default. Callers
// opt into the beyond-viewport path only for known compositor edge cases,
// such as native video surfaces in tall portrait renders.
captureBeyondViewport: options.captureBeyondViewport ?? false,
optimizeForSpeed: !isPng,
clip,
});
return Buffer.from(result.data, "base64");
}
/**
* Capture a screenshot with transparent background (PNG + alpha channel).
*
* Used in the two-pass HDR compositing pipeline — captures DOM content
* (text, graphics, SDR overlays) with transparency where the background shows,
* so it can be overlaid on top of native HDR video frames in FFmpeg.
*
* Sets and restores the background color override on every call. For sessions
* that capture many frames, prefer calling initTransparentBackground() once
* at session init, then captureAlphaPng() per frame to avoid the 2× CDP
* round-trip overhead.
*/
export async function captureScreenshotWithAlpha(
page: Page,
width: number,
height: number,
): Promise<Buffer> {
const client = await getCdpSession(page);
// Force transparent background so the screenshot has a real alpha channel
await client.send("Emulation.setDefaultBackgroundColorOverride", {
color: { r: 0, g: 0, b: 0, a: 0 },
});
try {
const result = await client.send("Page.captureScreenshot", {
format: "png",
fromSurface: true,
// Preserve the #1094 tall-portrait edge-clipping guard on HDR alpha captures.
captureBeyondViewport: true,
optimizeForSpeed: false, // `true` uses a zero-alpha-aware fast path that crushes real alpha values — observed empirically, CDP docs don't spell it out
clip: { x: 0, y: 0, width, height, scale: 1 },
});
return Buffer.from(result.data, "base64");
} finally {
// Restore opaque background even if captureScreenshot throws, otherwise
// subsequent opaque captures keep a transparent background.
await client.send("Emulation.setDefaultBackgroundColorOverride", {}).catch(() => {});
}
}
/**
* Make the page transparent for alpha capture, then use captureAlphaPng() per frame. html/body always clear;
* [data-composition-id] roots (nested too) keep their background unless clearCompositionRoot (HDR layered pass).
* A repeat call rewrites the rule, so the last call wins.
*/
/** Whether frames must carry alpha. PNG alone does not imply it: motion blur captures opaque output as PNG. */
export function capturesAlpha(options: Pick<CaptureOptions, "format" | "transparent">): boolean {
return options.transparent ?? options.format === "png";
}
const TRANSPARENT_BG_STYLE_ID = "__hf_transparent_bg__";
export async function initTransparentBackground(
page: Page,
{
clearCompositionRoot = false,
hideElementIds = [],
}: { clearCompositionRoot?: boolean; hideElementIds?: readonly string[] } = {},
): Promise<void> {
const client = await getCdpSession(page);
await client.send("Emulation.setDefaultBackgroundColorOverride", {
color: { r: 0, g: 0, b: 0, a: 0 },
});
await page.evaluate(
(styleId: string, clearCompositionRoot: boolean, hideIds: string[]) => {
const selector = clearCompositionRoot ? "html,body,[data-composition-id]" : "html,body";
let style = document.getElementById(styleId);
if (!style) {
style = document.createElement("style");
style.id = styleId;
document.head.appendChild(style);
}
const hidden = hideIds.map((id) => `#${CSS.escape(id)}`).join(",");
style.textContent =
`${selector}{background:transparent !important;background-color:transparent !important;background-image:none !important;}` +
(hidden ? `${hidden}{visibility:hidden !important;}` : "");
},
TRANSPARENT_BG_STYLE_ID,
clearCompositionRoot,
[...hideElementIds],
);
}
/**
* Capture a transparent-background PNG screenshot without setting the
* background color override. Requires initTransparentBackground() to have
* been called once on this session.
*
* Faster than captureScreenshotWithAlpha() for per-frame use in the HDR
* two-pass compositing loop.
*/
export async function captureAlphaPng(page: Page, width: number, height: number): Promise<Buffer> {
const client = await getCdpSession(page);
const result = await client.send("Page.captureScreenshot", {
format: "png",
fromSurface: true,
// Preserve the #1094 tall-portrait edge-clipping guard on HDR alpha captures.
captureBeyondViewport: true,
optimizeForSpeed: false, // must be false to preserve alpha
clip: { x: 0, y: 0, width, height, scale: 1 },
});
return Buffer.from(result.data, "base64");
}
/**
* Stylesheet ID used by applyDomLayerMask / removeDomLayerMask. Exposed so
* tests can assert presence/absence of the mask between captures.
*/
export const DOM_LAYER_MASK_STYLE_ID = "__hf_dom_layer_mask__";
const DOM_LAYER_MASK_HIDDEN_ATTR = "data-hf-dom-layer-mask-hidden";
const DOM_LAYER_MASK_PREV_VISIBILITY_ATTR = "data-hf-dom-layer-mask-prev-visibility";
const DOM_LAYER_MASK_PREV_PRIORITY_ATTR = "data-hf-dom-layer-mask-prev-priority";
/**
* Mask the DOM so a single layer screenshot captures ONLY the layer's pixels.
*
* The HDR layered compositor walks z-ordered layers and blits each one over a
* shared canvas. DOM layers are full-page screenshots — a naive screenshot
* captures every painted pixel on the page, which means root background +
* static overlays + sibling-scene content all overwrite previously composited
* HDR content beneath. The mask narrows each screenshot to the elements that
* actually belong to this layer.
*
* Strategy:
*
* 1. Inject a stylesheet that hides every body descendant
* (`body * { visibility: hidden !important }`) and re-shows the layer's
* elements (and their descendants, injected `__render_frame_*` siblings,
* and media color-grading canvases) via `visibility: visible !important`. CSS `visibility: visible`
* on a descendant overrides an ancestor's `visibility: hidden`, so deep
* layer elements remain visible even though intermediate parents are
* hidden by the mass-hide rule.
* 2. Inline-hide each `extraHideId` (and its render-frame/color-grading siblings) with
* `visibility: hidden !important`, while first recording its previous
* inline visibility. Inline `!important` beats stylesheet `!important`,
* so this overrides the show rule for elements that fall under a show
* selector but should NOT paint — typically other-layer elements that are
* descendants of a container layer (for example HDR videos and other-layer
* SDR videos are descendants of `#root` when we capture the root DOM layer).
* 3. Inline-hide timed descendants of shown elements that were hidden before
* the mask was installed. This covers idless child clips and same-layer
* descendants that the `extraHideIds` id list cannot represent.
*
* Only `visibility` is set on extraHideIds — never `opacity`. CSS opacity is
* multiplicative through the descendant chain and a descendant cannot escape
* an ancestor's `opacity: 0`. If `#root` is in `extraHideIds` and we set
* `opacity: 0` on it, every descendant — including `#vid-5-b` and its
* `__render_frame_vid-5-b__` IMG — becomes invisible even with
* `visibility: visible !important`. `visibility` does NOT have this problem:
* a descendant with `visibility: visible` overrides an ancestor's
* `visibility: hidden`.
*
* Layout is preserved (visibility doesn't trigger reflow), so border-radius
* clipping, overflow:hidden, and absolute positioning continue to apply to
* the visible layer elements. Opacity is also preserved — an ancestor at
* `opacity: 0` (e.g. an inactive scene during a transition) still
* propagates to its descendants, which is the desired behavior during
* cross-scene blends.
*
* Idempotent across calls: an existing mask stylesheet is removed before a
* new one is installed, so consecutive `applyDomLayerMask` invocations leave
* exactly one stylesheet attached.
*/
export async function applyDomLayerMask(
page: Page,
showIds: string[],
extraHideIds: string[],
): Promise<void> {
await page.evaluate(
// fallow-ignore-next-line complexity
(args: {
show: string[];
hide: string[];
renderIdAttr: string;
renderFramePrefix: string;
renderFrameSuffix: string;
styleId: string;
hiddenAttr: string;
prevVisibilityAttr: string;
prevPriorityAttr: string;
canvasIdPrefix: string;
}) => {
const existing = document.getElementById(args.styleId);
if (existing) existing.remove();
// Affixes come from core's renderFrameSibling, so the runtime readers and
// this lookup cannot drift apart on the id format.
const renderFrameId = (id: string) =>
`${args.renderFramePrefix}${id}${args.renderFrameSuffix}`;
const restoreMaskedElements = () => {
const masked = document.querySelectorAll(`[${args.hiddenAttr}="1"]`);
for (const node of masked) {
if (!(node instanceof HTMLElement)) continue;
const prevVisibility = node.getAttribute(args.prevVisibilityAttr);
const prevPriority = node.getAttribute(args.prevPriorityAttr);
if (prevVisibility === null) {
node.style.removeProperty("visibility");
} else {
node.style.setProperty("visibility", prevVisibility, prevPriority ?? "");
}
node.removeAttribute(args.hiddenAttr);
node.removeAttribute(args.prevVisibilityAttr);
node.removeAttribute(args.prevPriorityAttr);
}
};
restoreMaskedElements();
const rememberAndHideElement = (el: HTMLElement) => {
if (el.getAttribute(args.hiddenAttr) !== "1") {
const prevVisibility = el.style.getPropertyValue("visibility");
const prevPriority =
typeof el.style.getPropertyPriority === "function"
? el.style.getPropertyPriority("visibility")
: "";
if (prevVisibility) {
el.setAttribute(args.prevVisibilityAttr, prevVisibility);
} else {
el.removeAttribute(args.prevVisibilityAttr);
}
if (prevPriority) {
el.setAttribute(args.prevPriorityAttr, prevPriority);
} else {
el.removeAttribute(args.prevPriorityAttr);
}
el.setAttribute(args.hiddenAttr, "1");
}
el.style.setProperty("visibility", "hidden", "important");
};
const hiddenTimedDescendants: HTMLElement[] = [];
const rememberHiddenTimedDescendants = (root: Element) => {
for (const node of root.querySelectorAll("[data-start]")) {
if (!(node instanceof HTMLElement)) continue;
const computed = window.getComputedStyle(node);
if (computed.visibility !== "hidden" && computed.display !== "none") continue;
hiddenTimedDescendants.push(node);
}
};
const showSelectors: string[] = [];
for (const id of args.show) {
const el = window.__hfMediaEl?.(id) ?? document.getElementById(id);
if (el) rememberHiddenTimedDescendants(el);
// Address the element by its render id when it has one. `#id` must not
// be used as an extra fallback here: an id is duplicated exactly when
// two compositions share it, so `#id` would also unhide the other
// scene's element — the collision this render id exists to resolve.
if (el?.hasAttribute(args.renderIdAttr)) {
const attrEscaped = id.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
const byRenderId = `[${args.renderIdAttr}="${attrEscaped}"]`;
showSelectors.push(byRenderId, `${byRenderId} *`);
} else {
const escaped = CSS.escape(id);
showSelectors.push(`#${escaped}`, `#${escaped} *`);
}
const renderEscaped = CSS.escape(renderFrameId(id));
showSelectors.push(`#${renderEscaped}`, `#${renderEscaped} *`);
const colorGradingEscaped = CSS.escape(`${args.canvasIdPrefix}${id}`);
showSelectors.push(`#${colorGradingEscaped}`, `#${colorGradingEscaped} *`);
}
const massHideRule = "body *{visibility:hidden !important;}";
const showRule =
showSelectors.length === 0
? ""
: `${showSelectors.join(",")}{visibility:visible !important;}`;
const style = document.createElement("style");
style.id = args.styleId;
style.textContent = `${massHideRule}\n${showRule}`;
document.head.appendChild(style);
for (const el of hiddenTimedDescendants) {
rememberAndHideElement(el);
}
for (const id of args.hide) {
const el = window.__hfMediaEl?.(id) ?? document.getElementById(id);
if (el instanceof HTMLElement) {
rememberAndHideElement(el);
}
const img = document.getElementById(renderFrameId(id));
if (img) {
rememberAndHideElement(img);
}
const colorGradingCanvas = document.getElementById(`${args.canvasIdPrefix}${id}`);
if (colorGradingCanvas instanceof HTMLElement) {
rememberAndHideElement(colorGradingCanvas);
}
}
},
{
show: showIds,
hide: extraHideIds,
renderIdAttr: MEDIA_RENDER_ID_ATTR,
renderFramePrefix: RENDER_FRAME_ID_PREFIX,
renderFrameSuffix: RENDER_FRAME_ID_SUFFIX,
styleId: DOM_LAYER_MASK_STYLE_ID,
hiddenAttr: DOM_LAYER_MASK_HIDDEN_ATTR,
prevVisibilityAttr: DOM_LAYER_MASK_PREV_VISIBILITY_ATTR,
prevPriorityAttr: DOM_LAYER_MASK_PREV_PRIORITY_ATTR,
canvasIdPrefix: HF_COLOR_GRADING_CANVAS_ID_PREFIX,
},
);
}
/**
* Tear down the mask installed by applyDomLayerMask.
*
* Removes the mask stylesheet and restores the inline `visibility` values
* temporarily overwritten for hidden timed descendants, `extraHideIds`, and
* their render-frame/color-grading siblings.
*
* IMPORTANT: We do NOT strip inline `opacity` here. applyDomLayerMask only
* ever sets `visibility` (never `opacity`), so any inline opacity present on
* a wrapper was put there by user animation code (typically GSAP) and must
* survive across per-layer captures. GSAP's seek with suppress-events does
* not re-apply tweens when the timeline is already at the target time, so if
* we strip opacity here and then seek to the same time for the next layer,
* GSAP won't put it back and the wrapper will render fully opaque.
*/
export async function removeDomLayerMask(page: Page, _extraHideIds: string[]): Promise<void> {
await page.evaluate(
(args: {
styleId: string;
hiddenAttr: string;
prevVisibilityAttr: string;
prevPriorityAttr: string;
}) => {
const style = document.getElementById(args.styleId);
if (style) style.remove();
const masked = document.querySelectorAll(`[${args.hiddenAttr}="1"]`);
for (const node of masked) {
if (!(node instanceof HTMLElement)) continue;
const prevVisibility = node.getAttribute(args.prevVisibilityAttr);
const prevPriority = node.getAttribute(args.prevPriorityAttr);
if (prevVisibility === null) {
node.style.removeProperty("visibility");
} else {
node.style.setProperty("visibility", prevVisibility, prevPriority ?? "");
}
node.removeAttribute(args.hiddenAttr);
node.removeAttribute(args.prevVisibilityAttr);
node.removeAttribute(args.prevPriorityAttr);
}
},
{
styleId: DOM_LAYER_MASK_STYLE_ID,
hiddenAttr: DOM_LAYER_MASK_HIDDEN_ATTR,
prevVisibilityAttr: DOM_LAYER_MASK_PREV_VISIBILITY_ATTR,
prevPriorityAttr: DOM_LAYER_MASK_PREV_PRIORITY_ATTR,
},
);
}
/**
* Pre-create hidden `__render_frame__` sibling `<img>`s for every
* `video[data-start]` in the page. Idempotent — videos that already
* have a sibling are skipped.
*
* `injectVideoFramesBatch` creates the sibling on the fly the first time
* it paints a given videoId (the `isNewImage = !hasImg` branch below).
* Under chrome-headless-shell's deterministic + `HeadlessExperimental.
* BeginFrame` mode, the immediately-next BeginFrame captures before the
* freshly-inserted `<img>` layer lands in the compositor's layer tree;
* the layer arrives a frame later. That single frame paints only the
* body background + previously-composed overlays.
*
* Called from `initializeSession`: in the screenshot path at the end (that
* capture path flushes paint, so timing doesn't matter), and in the BeginFrame
* path followed by one explicit visual `HeadlessExperimental.beginFrame`
* (`noDisplayUpdates: false`) that composites the new layers before the first
* capture — the warmup ticks are `noDisplayUpdates: true` and don't paint.
* Every subsequent `injectVideoFramesBatch` then takes the `hasImg = true` path
* (just an `img.src` update). The `isNewImage` branch stays as a fallback for
* callers that don't run through `initializeSession`.
*/
export async function ensureRenderFrameSiblings(page: Page): Promise<void> {
await page.evaluate(
(prefix: string, suffix: string) => {
for (const video of Array.from(
document.querySelectorAll<HTMLVideoElement>("video[data-start]"),
)) {
const next = video.nextElementSibling;
if (next !== null && next.classList.contains("__render_frame__")) continue;
const img = document.createElement("img");
img.classList.add("__render_frame__");
// Derive from the render id, not `video.id` — two scenes can share an
// element id, and two siblings sharing an id would collide in turn.
// `||`, not `??`: `__hfMediaId` returns "" for an element with neither
// id, and core's reader treats that as "no id" rather than a key.
img.id = `${prefix}${window.__hfMediaId?.(video) || video.id}${suffix}`;
img.style.pointerEvents = "none";
img.style.position = "absolute";
img.style.visibility = "hidden";
video.parentNode?.insertBefore(img, video.nextSibling);
}
},
RENDER_FRAME_ID_PREFIX,
RENDER_FRAME_ID_SUFFIX,
);
}
/**
* Returns the subset of `updates.videoId`s that were actually painted in
* this call. Videos skipped because of a hidden visual ancestor are NOT
* included — the caller relies on this to avoid recording a `lastInjected`
* cache entry for a frame that never reached the page, which would otherwise
* short-circuit the next inject at the same frameIndex and leave the host's
* first visible frame blank.
*
* `restyleIds` keep their current picture and only re-copy the video's style.
*/
export async function injectVideoFramesBatch(
page: Page,
updates: Array<{ videoId: string; dataUri: string }>,
restyleIds: readonly string[] = [],
): Promise<string[]> {
if (updates.length === 0 && restyleIds.length === 0) return [];
return withVideoFrameSources(page, updates, (resolved) =>
injectResolvedVideoFramesBatch(page, resolved, restyleIds),
);
}
async function injectResolvedVideoFramesBatch(
page: Page,
updates: VideoFrameUpdate[],
restyleIds: readonly string[],
): Promise<string[]> {
// Build the sibling id with core's function rather than a template here,
// so the id the readers look up has exactly one definition.
const items: Array<Partial<VideoFrameUpdate> & { videoId: string; frameId: string }> = [
...updates,
...restyleIds.map((videoId) => ({ videoId })),
].map((item) => ({ ...item, frameId: renderFrameIdForRenderId(item.videoId) }));
return await page.evaluate(
// fallow-ignore-next-line complexity
async (
items: Array<Partial<VideoFrameUpdate> & { videoId: string; frameId: string }>,
visualProperties: string[],
colorGradingSourceHiddenAttr: string,
) => {
const injectedIds: string[] = [];
const pendingDecodes: Array<Promise<void>> = [];
const imagesToPublish: HTMLImageElement[] = [];
const replacementLayoutProperties = new Set([
"width",
"height",
"top",
"left",
"right",
"bottom",
"inset",
]);
// Walk ancestors looking for a host that the page has hidden. The
// runtime hides `[data-composition-src]` and `[data-start]` hosts that
// fall outside their time window; a nested `<video data-start>` inside
// such a host still appears "active" in the raw time-window check (its
// own `data-start`/`data-end` cover the whole clip), so without this
// guard we would paint a full-bleed replacement frame over a sibling
// host that *is* visible.
//
// `display: none` is always a skip signal — a `display: none` ancestor
// takes its whole subtree out of layout, and a child `<img>` cannot
// escape that. `visibility: hidden`, by contrast, is escapable: a
// descendant with `visibility: visible` overrides an ancestor's
// `visibility: hidden` per the CSS spec, and the replacement `<img>`
// intentionally sets `visibility: visible`. We therefore only treat
// `visibility: hidden` as a skip signal on sub-composition hosts
// (`[data-composition-src]` / `[data-composition-file]`), which is the
// scenario this guard exists for. Plain `[data-start]` containers may
// be hidden with `visibility: hidden` while still wanting their inner
// video's final-state frame to paint through (e.g. a GSAP timeline
// shorter than the host's authored data-duration, where the runtime
// truncates visibility but the replacement <img> must hold its last
// frame) — those must NOT be skipped here.
// fallow-ignore-next-line code-duplication
const isVisualAncestorHidden = (el: HTMLElement): boolean => {
let parent = el.parentElement;
while (parent !== null && parent !== document.documentElement) {
const computed = window.getComputedStyle(parent);
if (computed.display === "none") return true;
if (
computed.visibility === "hidden" &&
(parent.hasAttribute("data-composition-src") ||
parent.hasAttribute("data-composition-file"))
) {
return true;
}
parent = parent.parentElement;
}
return false;
};
for (const item of items) {
const video = (window.__hfMediaEl?.(item.videoId) ??
document.getElementById(item.videoId)) as HTMLVideoElement | null;
if (!video) continue;
let img = video.nextElementSibling as HTMLImageElement | null;
const hasImg = img !== null && img.classList.contains("__render_frame__");
const restyleOnly = item.dataUri === undefined;
if (restyleOnly && !hasImg) continue;
if (isVisualAncestorHidden(video)) {
// Don't paint a frame over a hidden host — if an existing replacement
// <img> is still around from when the host was visible, hide it so it
// doesn't bleed through a sibling host that *is* visible on this seek.
//
// Use `!important` so the inline hide survives `applyDomLayerMask`'s
// stylesheet `#${showId} *{visibility:visible !important}` when the
// sub-comp host happens to land in the active layer's `show` set —
// important stylesheet beats non-important inline, but important
// inline beats important stylesheet.
if (hasImg && img) img.style.setProperty("visibility", "hidden", "important");
continue;
}
const isNewImage = !hasImg;
const computedStyle = window.getComputedStyle(video);
// Read the GSAP-controlled opacity directly from the native <video>.
// We hide the <video> below with `visibility: hidden` only (never
// `opacity: 0`), so its computed opacity is preserved across seeks
// and accurately reflects the user's intent on every frame.
const opacityParsed = parseFloat(computedStyle.opacity);
const computedOpacity = video.hasAttribute(colorGradingSourceHiddenAttr)
? 1
: Number.isNaN(opacityParsed)
? 1
: opacityParsed;
// Measure first: an in-flow bordered <img> sibling would shrink the video's flex box.
const videoRect = video.getBoundingClientRect();
const videoBox = {
left: Number.isFinite(video.offsetLeft) ? video.offsetLeft : 0,
top: Number.isFinite(video.offsetTop) ? video.offsetTop : 0,
width: video.offsetWidth > 0 ? video.offsetWidth : videoRect.width,
height: video.offsetHeight > 0 ? video.offsetHeight : videoRect.height,
};
if (isNewImage) {
img = document.createElement("img");
img.classList.add("__render_frame__");
img.id = item.frameId;
img.style.pointerEvents = "none";
video.parentNode?.insertBefore(img, video.nextSibling);
}
if (!img) continue;
for (const property of visualProperties) {
// Opacity is handled explicitly via `computedOpacity` below — copying
// via the generic loop would race against the opacity:0 hide applied
// to the <video> at the end of this function. GSAP may animate
// opacity either on a wrapper (the <img> inherits via the stacking
// context) or directly on the <video> (we must copy it to the <img>
// since they are siblings). Reading computedStyle.opacity before
// hiding the <video> handles both cases correctly.
if (property === "opacity") continue;
// Layout is set from the video's used box below. Copying authored
// opposing constraints such as `inset: 0` / `right: 0` onto the
// replacement <img> can overconstrain replaced-image sizing and make
// some Chrome capture paths resample the frame anisotropically.
if (replacementLayoutProperties.has(property)) {
continue;
}
const value = computedStyle.getPropertyValue(property);
if (value) {
img.style.setProperty(property, value);
}
}
// Always use absolute positioning so the <img> overlays the <video>
// instead of flowing below it. With position:relative, both elements
// stack vertically — the <img> lands below the video and gets clipped
// by any overflow:hidden ancestor (e.g., border-radius wrappers).
img.style.position = "absolute";
img.style.inset = "auto";
img.style.left = `${videoBox.left}px`;
img.style.top = `${videoBox.top}px`;
img.style.right = "auto";
img.style.bottom = "auto";
img.style.width = `${videoBox.width}px`;
img.style.height = `${videoBox.height}px`;
// `videoBox` is a border-box even when the video is content-box.
img.style.boxSizing = "border-box";
img.style.objectFit = computedStyle.objectFit;
img.style.objectPosition = computedStyle.objectPosition;
img.style.zIndex = computedStyle.zIndex;
// Publish after full decode so raster cannot race it with a reduced JPEG decode.
img.style.setProperty("visibility", "hidden", "important");
img.decoding = "sync";
if (item.dataUri !== undefined && img.getAttribute("src") !== item.dataUri) {
if (item.crossOrigin) img.crossOrigin = item.crossOrigin;
else img.removeAttribute("crossorigin");
img.src = item.dataUri;
const source =
item.sourceLabel ?? (item.dataUri.startsWith("data:") ? "inline frame" : item.dataUri);
pendingDecodes.push(
img.decode().catch((error: unknown) => {
throw new Error(
`Video frame for "${item.videoId}" failed to load (${source}): ${
error instanceof Error ? error.message : String(error)
}`,
);
}),
);
}
img.style.opacity = String(computedOpacity);
imagesToPublish.push(img);
// Hide the native <video> with visibility only — never clobber inline
// opacity, so subsequent reads (and queryElementStacking) see the real
// GSAP-controlled value.
video.style.setProperty("visibility", "hidden", "important");
video.style.setProperty("pointer-events", "none", "important");
if (!restyleOnly) injectedIds.push(item.videoId);
}
if (pendingDecodes.length > 0) {
await Promise.all(pendingDecodes);
}
for (const img of imagesToPublish) img.style.visibility = "visible";
if (imagesToPublish.length > 0) {
const redraw = (window as Window & { __hf?: { colorGrading?: { redraw?: () => void } } })
.__hf?.colorGrading?.redraw;
redraw?.();
}
return injectedIds;
},
items,
[...MEDIA_VISUAL_STYLE_PROPERTIES],
COLOR_GRADING_SOURCE_HIDDEN_ATTR,
);
}
export async function syncVideoFrameVisibility(
page: Page,
activeVideoIds: string[],
): Promise<void> {
await page.evaluate(
// fallow-ignore-next-line complexity
(ids: string[], colorGradingSourceHiddenAttr: string) => {
// Mirror the ancestor-visibility guard from `injectVideoFramesBatch`.
// See that copy for the full rationale on why `visibility: hidden` is
// narrowed to sub-composition hosts only — keep these two functions in
// sync so the inactive-arm decision matches the inject-time decision.
const isVisualAncestorHidden = (el: HTMLElement): boolean => {
let parent = el.parentElement;
while (parent !== null && parent !== document.documentElement) {
const computed = window.getComputedStyle(parent);
if (computed.display === "none") return true;
if (
computed.visibility === "hidden" &&
(parent.hasAttribute("data-composition-src") ||
parent.hasAttribute("data-composition-file"))
) {
return true;
}
parent = parent.parentElement;
}
return false;
};
const active = new Set(ids);
const setColorGradingVisibility = (
window as Window & {
__hf?: {
colorGrading?: { setSourceVisibility?: (target: Element, visible: boolean) => boolean };
};
}
).__hf?.colorGrading?.setSourceVisibility;
const videos = Array.from(
document.querySelectorAll("video[data-start]"),
) as HTMLVideoElement[];
for (const video of videos) {
const img = video.nextElementSibling as HTMLElement | null;
const hasImg = img && img.classList.contains("__render_frame__");
const ancestorHidden = isVisualAncestorHidden(video);
const visible = active.has(window.__hfMediaId?.(video) || video.id) && !ancestorHidden;
if (visible) {
// Active video: show injected <img>, hide native <video>.
// Do NOT clobber inline opacity here — GSAP-controlled opacity must
// survive until injectVideoFramesBatch reads it via getComputedStyle.
// visibility:hidden alone hides the native element without affecting
// its computed opacity.
video.style.setProperty("visibility", "hidden", "important");
video.style.setProperty("pointer-events", "none", "important");
if (hasImg) {
if (video.hasAttribute(colorGradingSourceHiddenAttr)) img.style.opacity = "1";
img.style.visibility = "visible";
}
} else {
// Inactive (or ancestor-hidden) video: hide both. Use visibility only
// (never opacity) so we never clobber GSAP-controlled inline opacity.
// Use `!important` on the <img> hide so `applyDomLayerMask`'s
// important stylesheet rule (`#${showId} *{visibility:visible !important}`)
// cannot revive a stale frame when the sub-comp host lands in the
// active layer's `show` set — same mask-defense reasoning as the
// `isVisualAncestorHidden` branch in `injectVideoFramesBatch`.
video.style.setProperty("visibility", "hidden", "important");
video.style.setProperty("pointer-events", "none", "important");
if (hasImg) {
img.style.setProperty("visibility", "hidden", "important");
}
}
setColorGradingVisibility?.(video, visible);
}
},
activeVideoIds,
COLOR_GRADING_SOURCE_HIDDEN_ATTR,
);
}