mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 23:03:09 +00:00
feat(engine): add HDR two-pass compositing — DOM layer + native HLG video (#288)
## Summary Compositions with HDR video AND DOM overlays (text, graphics, SDR video) couldn't render both correctly — either HDR data was lost (Chrome captures sRGB only) or DOM overlays were missing (FFmpeg pass-through skips Chrome). This PR adds in-memory alpha compositing that combines both. ## What it does **Per-frame two-pass capture:** 1. **DOM pass** — Chrome screenshots the page with a transparent background (CDP alpha). HDR videos are hidden, leaving transparent holes where they go. 2. **HDR pass** — Pre-extracted native HLG/PQ frames (16-bit PNG from FFmpeg) are read from disk. 3. **Composite** — DOM pixels (sRGB RGBA8) are alpha-composited over HDR pixels (rgb48le) in Node.js memory, with sRGB→HLG/PQ conversion via a 256-entry lookup table. **Key components:** - `decodePng()` / `decodePngToRgb48le()` — Pure Node.js PNG decoders (no native dependencies). Support all 5 PNG filter types. - `blitRgba8OverRgb48le()` — Alpha composite with per-pixel sRGB→HDR LUT conversion. Fast paths for alpha=0 (skip) and alpha=255 (overwrite). - `initTransparentBackground()` + `captureAlphaPng()` — Split CDP transparent background setup (once) from per-frame screenshot capture (eliminates 2 CDP round-trips per frame). - Single-pass FFmpeg extraction — All HDR frames extracted in one sequential FFmpeg run (avoids duplicate frames from per-frame `-ss` fast seek). ## Key design decisions | Decision | Why | |----------|-----| | In-memory compositing (not FFmpeg overlay) | Eliminates ~2400 process spawns + temp files per render. Pure pixel math is 10x faster. | | 16-bit PNG intermediate | Raw `-f rawvideo` loses color metadata, causing moiré artifacts. PNG is self-describing. | | sRGB→HLG LUT (256 entries) | DOM content is sRGB. Without conversion, it appears orange-shifted in HLG stream. | | Native HDR detection before extraction | `extractAllVideoFrames` converts SDR→HDR. Pre-extraction probe identifies original HDR sources so only truly-HDR videos get native extraction. | ## Files changed | File | What changed | |------|-------------| | `packages/engine/src/utils/alphaBlit.ts` | **NEW** — PNG decode, sRGB→HDR LUT, alpha compositing (14 tests) | | `packages/engine/src/services/screenshotService.ts` | Transparent background CDP, `captureAlphaPng()` | | `packages/engine/src/services/videoFrameInjector.ts` | `hideVideoElements()` / `showVideoElements()` | | `packages/engine/src/services/streamingEncoder.ts` | Input color space tags for rgb48le | | `packages/producer/src/services/renderOrchestrator.ts` | Two-pass HDR capture loop, native HDR detection | ## How to test Render a composition with an HDR video background and text overlays. Both should be visible — HDR video at full quality, text crisp with correct colors (not orange-shifted). ## Stack position **3 of 6** — Stacked on #265 (HDR output pipeline). This is the foundation for all layered compositing that follows. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
This commit is contained in:
Vance Ingalls
@@ -133,6 +133,82 @@ export async function pageScreenshotCapture(page: Page, options: CaptureOptions)
|
||||
return Buffer.from(result.data, "base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture a screenshot with transparent background (PNG + alpha channel).
|
||||
*
|
||||
* Used in the two-pass HDR compositing pipeline — captures DOM content
|
||||
* (text, graphics, SDR overlays) with transparency where the background shows,
|
||||
* so it can be overlaid on top of native HDR video frames in FFmpeg.
|
||||
*
|
||||
* Sets and restores the background color override on every call. For sessions
|
||||
* that capture many frames, prefer calling initTransparentBackground() once
|
||||
* at session init, then captureAlphaPng() per frame to avoid the 2× CDP
|
||||
* round-trip overhead.
|
||||
*/
|
||||
export async function captureScreenshotWithAlpha(
|
||||
page: Page,
|
||||
width: number,
|
||||
height: number,
|
||||
): Promise<Buffer> {
|
||||
const client = await getCdpSession(page);
|
||||
// Force transparent background so the screenshot has a real alpha channel
|
||||
await client.send("Emulation.setDefaultBackgroundColorOverride", {
|
||||
color: { r: 0, g: 0, b: 0, a: 0 },
|
||||
});
|
||||
try {
|
||||
const result = await client.send("Page.captureScreenshot", {
|
||||
format: "png",
|
||||
fromSurface: true,
|
||||
captureBeyondViewport: false,
|
||||
optimizeForSpeed: false, // `true` uses a zero-alpha-aware fast path that crushes real alpha values — observed empirically, CDP docs don't spell it out
|
||||
clip: { x: 0, y: 0, width, height, scale: 1 },
|
||||
});
|
||||
return Buffer.from(result.data, "base64");
|
||||
} finally {
|
||||
// Restore opaque background even if captureScreenshot throws, otherwise
|
||||
// subsequent opaque captures keep a transparent background.
|
||||
await client.send("Emulation.setDefaultBackgroundColorOverride", {}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the page background to transparent once for a dedicated HDR DOM session.
|
||||
*
|
||||
* Call this once after session initialization. Then use captureAlphaPng() per
|
||||
* frame instead of captureScreenshotWithAlpha() to skip the per-frame CDP
|
||||
* background override round-trips.
|
||||
*
|
||||
* Only use on sessions that are exclusively dedicated to transparent capture
|
||||
* (e.g., the HDR two-pass DOM layer session) — the background will stay
|
||||
* transparent for the lifetime of the session.
|
||||
*/
|
||||
export async function initTransparentBackground(page: Page): Promise<void> {
|
||||
const client = await getCdpSession(page);
|
||||
await client.send("Emulation.setDefaultBackgroundColorOverride", {
|
||||
color: { r: 0, g: 0, b: 0, a: 0 },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture a transparent-background PNG screenshot without setting the
|
||||
* background color override. Requires initTransparentBackground() to have
|
||||
* been called once on this session.
|
||||
*
|
||||
* Faster than captureScreenshotWithAlpha() for per-frame use in the HDR
|
||||
* two-pass compositing loop.
|
||||
*/
|
||||
export async function captureAlphaPng(page: Page, width: number, height: number): Promise<Buffer> {
|
||||
const client = await getCdpSession(page);
|
||||
const result = await client.send("Page.captureScreenshot", {
|
||||
format: "png",
|
||||
fromSurface: true,
|
||||
captureBeyondViewport: false,
|
||||
optimizeForSpeed: false, // must be false to preserve alpha
|
||||
clip: { x: 0, y: 0, width, height, scale: 1 },
|
||||
});
|
||||
return Buffer.from(result.data, "base64");
|
||||
}
|
||||
|
||||
export async function injectVideoFramesBatch(
|
||||
page: Page,
|
||||
updates: Array<{ videoId: string; dataUri: string }>,
|
||||
@@ -160,16 +236,11 @@ export async function injectVideoFramesBatch(
|
||||
}
|
||||
if (!img) continue;
|
||||
|
||||
if (!sourceIsStatic) {
|
||||
img.style.position = computedStyle.position;
|
||||
img.style.width = computedStyle.width;
|
||||
img.style.height = computedStyle.height;
|
||||
img.style.top = computedStyle.top;
|
||||
img.style.left = computedStyle.left;
|
||||
img.style.right = computedStyle.right;
|
||||
img.style.bottom = computedStyle.bottom;
|
||||
img.style.inset = computedStyle.inset;
|
||||
} else {
|
||||
// Always use absolute positioning so the <img> overlays the <video>
|
||||
// instead of flowing below it. With position:relative, both elements
|
||||
// stack vertically — the <img> lands below the video and gets clipped
|
||||
// by any overflow:hidden ancestor (e.g., border-radius wrappers).
|
||||
{
|
||||
const videoRect = video.getBoundingClientRect();
|
||||
const offsetLeft = Number.isFinite(video.offsetLeft) ? video.offsetLeft : 0;
|
||||
const offsetTop = Number.isFinite(video.offsetTop) ? video.offsetTop : 0;
|
||||
@@ -235,14 +306,28 @@ export async function syncVideoFrameVisibility(
|
||||
const active = new Set(ids);
|
||||
const videos = Array.from(document.querySelectorAll("video[data-start]")) as HTMLVideoElement[];
|
||||
for (const video of videos) {
|
||||
if (active.has(video.id)) continue;
|
||||
video.style.removeProperty("display");
|
||||
video.style.setProperty("visibility", "hidden", "important");
|
||||
video.style.setProperty("opacity", "0", "important");
|
||||
video.style.setProperty("pointer-events", "none", "important");
|
||||
const img = video.nextElementSibling as HTMLElement | null;
|
||||
if (img && img.classList.contains("__render_frame__")) {
|
||||
img.style.visibility = "hidden";
|
||||
const hasImg = img && img.classList.contains("__render_frame__");
|
||||
if (active.has(video.id)) {
|
||||
// Active video: show injected <img>, hide native <video>.
|
||||
// Do NOT clobber inline opacity here — GSAP-controlled opacity must
|
||||
// survive until injectVideoFramesBatch reads it via getComputedStyle.
|
||||
// visibility:hidden alone hides the native element without affecting
|
||||
// its computed opacity.
|
||||
video.style.setProperty("visibility", "hidden", "important");
|
||||
video.style.setProperty("pointer-events", "none", "important");
|
||||
if (hasImg) {
|
||||
img.style.visibility = "visible";
|
||||
}
|
||||
} else {
|
||||
// Inactive video: hide both
|
||||
video.style.removeProperty("display");
|
||||
video.style.setProperty("visibility", "hidden", "important");
|
||||
video.style.setProperty("opacity", "0", "important");
|
||||
video.style.setProperty("pointer-events", "none", "important");
|
||||
if (hasImg) {
|
||||
img.style.visibility = "hidden";
|
||||
}
|
||||
}
|
||||
}
|
||||
}, activeVideoIds);
|
||||
|
||||
@@ -117,3 +117,111 @@ export function createVideoFrameInjector(
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// ── HDR compositing utilities ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Bounds and transform of a video element, queried from Chrome each frame.
|
||||
* Used by the two-pass HDR compositing pipeline to position native HDR frames.
|
||||
*/
|
||||
export interface VideoElementBounds {
|
||||
videoId: string;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
opacity: number;
|
||||
/** CSS transform matrix as a DOMMatrix-compatible string, e.g. "matrix(1,0,0,1,0,0)" */
|
||||
transform: string;
|
||||
zIndex: number;
|
||||
visible: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hide specific video elements by ID. Used in Pass 1 of the HDR pipeline so
|
||||
* Chrome screenshots only contain DOM content (text, overlays) with transparent
|
||||
* holes where the HDR videos go.
|
||||
*/
|
||||
export async function hideVideoElements(page: Page, videoIds: string[]): Promise<void> {
|
||||
if (videoIds.length === 0) return;
|
||||
await page.evaluate((ids: string[]) => {
|
||||
for (const id of ids) {
|
||||
const el = document.getElementById(id) as HTMLVideoElement | null;
|
||||
if (el) {
|
||||
el.style.setProperty("visibility", "hidden", "important");
|
||||
el.style.setProperty("opacity", "0", "important");
|
||||
// Also hide the injected render frame image if present
|
||||
const img = document.getElementById(`__render_frame_${id}__`);
|
||||
if (img) img.style.setProperty("visibility", "hidden", "important");
|
||||
}
|
||||
}
|
||||
}, videoIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore visibility of video elements after a DOM screenshot.
|
||||
*/
|
||||
export async function showVideoElements(page: Page, videoIds: string[]): Promise<void> {
|
||||
if (videoIds.length === 0) return;
|
||||
await page.evaluate((ids: string[]) => {
|
||||
for (const id of ids) {
|
||||
const el = document.getElementById(id) as HTMLVideoElement | null;
|
||||
if (el) {
|
||||
el.style.removeProperty("visibility");
|
||||
el.style.removeProperty("opacity");
|
||||
const img = document.getElementById(`__render_frame_${id}__`);
|
||||
if (img) img.style.removeProperty("visibility");
|
||||
}
|
||||
}
|
||||
}, videoIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the current bounds, transform, and visibility of video elements.
|
||||
* Called after seeking (so GSAP has moved things) but before the screenshot.
|
||||
*/
|
||||
export async function queryVideoElementBounds(
|
||||
page: Page,
|
||||
videoIds: string[],
|
||||
): Promise<VideoElementBounds[]> {
|
||||
if (videoIds.length === 0) return [];
|
||||
return page.evaluate((ids: string[]): VideoElementBounds[] => {
|
||||
return ids.map((id) => {
|
||||
const el = document.getElementById(id) as HTMLVideoElement | null;
|
||||
if (!el) {
|
||||
return {
|
||||
videoId: id,
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 0,
|
||||
height: 0,
|
||||
opacity: 0,
|
||||
transform: "none",
|
||||
zIndex: 0,
|
||||
visible: false,
|
||||
};
|
||||
}
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(el);
|
||||
const zIndex = parseInt(style.zIndex) || 0;
|
||||
const opacity = parseFloat(style.opacity) || 1;
|
||||
const transform = style.transform || "none";
|
||||
const visible =
|
||||
style.visibility !== "hidden" &&
|
||||
style.display !== "none" &&
|
||||
rect.width > 0 &&
|
||||
rect.height > 0;
|
||||
return {
|
||||
videoId: id,
|
||||
x: Math.round(rect.x),
|
||||
y: Math.round(rect.y),
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
opacity,
|
||||
transform,
|
||||
zIndex,
|
||||
visible,
|
||||
};
|
||||
});
|
||||
}, videoIds);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user