mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-12 15:20:13 +00:00
* fix: bound HDR and video extraction resources * fix: trim negative video extraction preroll * fix: skip invisible video extraction windows * fix: preserve negative-start loop and held tails * fix: cap finite video slots to source duration * fix: bound held-tail frame extraction * fix: plan from playable video duration * fix: preserve open-ended held video tails * fix: resolve held tails from decoded frames * fix: normalize final-frame probe timestamps * fix: handle unseekable final-frame sources * fix: dedupe final-frame probes per render * refactor: clarify output dynamic range contract
740 lines
25 KiB
TypeScript
740 lines
25 KiB
TypeScript
/**
|
|
* HDR Compositor — pixel-level compositing primitives for the HDR
|
|
* layered render path.
|
|
*
|
|
* Extracted from `renderOrchestrator.ts` so the ~600 LOC of HDR-specific
|
|
* buffer manipulation, video/image blit logic, and per-frame compositor
|
|
* live in a focused module that can be tested and evolved independently.
|
|
*
|
|
* Consumers: `captureHdrStage.ts`, `captureHdrSequentialLoop.ts`,
|
|
* `captureHdrHybridLoop.ts`, `captureHdrFrameShared.ts`,
|
|
* `captureHdrResources.ts`.
|
|
*/
|
|
|
|
import { readSync, closeSync } from "fs";
|
|
import { join } from "path";
|
|
import {
|
|
type CaptureSession,
|
|
type BeforeCaptureHook,
|
|
type HdrTransfer,
|
|
type ElementStackingInfo,
|
|
type HfTransitionMeta,
|
|
captureAlphaPng,
|
|
applyDomLayerMask,
|
|
removeDomLayerMask,
|
|
decodePng,
|
|
blitRgba8OverRgb48le,
|
|
blitRgb48leRegion,
|
|
groupIntoLayers,
|
|
blitRgb48leAffine,
|
|
parseTransformMatrix,
|
|
convertTransfer,
|
|
} from "@hyperframes/engine";
|
|
import type { ProducerLogger } from "../logger.js";
|
|
import { type HdrImageTransferCache } from "./hdrImageTransferCache.js";
|
|
import { writeFileExclusiveSync } from "./render/shared.js";
|
|
import { type HdrPerfCollector, timeHdrPhase, timeHdrPhaseAsync } from "./render/hdrPerf.js";
|
|
|
|
// ─── Diagnostic helpers ────────────────────────────────────────────────────
|
|
|
|
// Diagnostic helpers used by the HDR layered compositor when KEEP_TEMP=1
|
|
// is set. They are pure (capture no state), so we keep them at module scope
|
|
// to avoid re-creating closures per frame and to make them callable from
|
|
// any future composite path that needs to log non-zero pixel counts.
|
|
function countNonZeroAlpha(rgba: Uint8Array): number {
|
|
let n = 0;
|
|
for (let p = 3; p < rgba.length; p += 4) {
|
|
if (rgba[p] !== 0) n++;
|
|
}
|
|
return n;
|
|
}
|
|
|
|
function countNonZeroRgb48(buf: Uint8Array): number {
|
|
let n = 0;
|
|
for (let p = 0; p < buf.length; p += 6) {
|
|
if (
|
|
buf[p] !== 0 ||
|
|
buf[p + 1] !== 0 ||
|
|
buf[p + 2] !== 0 ||
|
|
buf[p + 3] !== 0 ||
|
|
buf[p + 4] !== 0 ||
|
|
buf[p + 5] !== 0
|
|
)
|
|
n++;
|
|
}
|
|
return n;
|
|
}
|
|
|
|
// ─── Constants ────────────────────────────────────────────────────────────
|
|
|
|
const TRANSFORM_IDENTITY_EPSILON = 0.001;
|
|
const OPAQUE_ALPHA_THRESHOLD = 0.999;
|
|
const RGB48_BYTES_PER_PIXEL = 6;
|
|
|
|
type AffineMatrix = [number, number, number, number, number, number];
|
|
|
|
function isAffineMatrix(m: number[]): m is AffineMatrix {
|
|
return m.length === 6;
|
|
}
|
|
|
|
function resolveBlitOpacity(opacity: number): number | undefined {
|
|
return opacity < OPAQUE_ALPHA_THRESHOLD ? opacity : undefined;
|
|
}
|
|
|
|
// ─── Types ─────────────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Metadata for a shader transition between two scenes, extracted from
|
|
* `window.__hf.transitions`. Re-exported from the engine so the producer
|
|
* shares the contract with composition runtime code.
|
|
*/
|
|
export type HdrTransitionMeta = HfTransitionMeta;
|
|
|
|
/** Pre-computed frame range for an active transition. */
|
|
export interface TransitionRange extends HdrTransitionMeta {
|
|
startFrame: number;
|
|
endFrame: number;
|
|
}
|
|
|
|
// ─── Video frame source ────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Crop an rgb48le buffer to a sub-region. Returns a new Buffer containing
|
|
* only the cropped pixels.
|
|
*/
|
|
function cropRgb48le(
|
|
src: Buffer,
|
|
srcW: number,
|
|
srcH: number,
|
|
cropX: number,
|
|
cropY: number,
|
|
cropW: number,
|
|
cropH: number,
|
|
): Buffer {
|
|
const dst = Buffer.alloc(cropW * cropH * RGB48_BYTES_PER_PIXEL);
|
|
for (let row = 0; row < cropH; row++) {
|
|
const srcRow = cropY + row;
|
|
if (srcRow < 0 || srcRow >= srcH) continue;
|
|
const srcOff = (srcRow * srcW + cropX) * RGB48_BYTES_PER_PIXEL;
|
|
const dstOff = row * cropW * RGB48_BYTES_PER_PIXEL;
|
|
const copyLen = Math.min(cropW, srcW - cropX) * RGB48_BYTES_PER_PIXEL;
|
|
if (copyLen > 0) src.copy(dst, dstOff, srcOff, srcOff + copyLen);
|
|
}
|
|
return dst;
|
|
}
|
|
|
|
/**
|
|
* Blit a single HDR video layer onto an rgb48le canvas.
|
|
*
|
|
* Shared between the normal-frame compositing path (compositeToBuffer)
|
|
* and the transition dual-scene compositing loop to avoid duplicating
|
|
* the frame lookup, raw read, transfer, transform, and blit logic.
|
|
*/
|
|
export interface HdrVideoFrameSource {
|
|
dir: string;
|
|
rawPath: string;
|
|
fd: number;
|
|
width: number;
|
|
height: number;
|
|
frameSize: number;
|
|
frameCount: number;
|
|
scratch: Buffer;
|
|
/** The raw file contains one playable source cycle and must wrap at EOF. */
|
|
loop?: boolean;
|
|
}
|
|
|
|
export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: ProducerLogger): void {
|
|
try {
|
|
closeSync(source.fd);
|
|
} catch (err) {
|
|
log?.warn("Failed to close HDR raw frame file", {
|
|
rawPath: source.rawPath,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
});
|
|
}
|
|
}
|
|
|
|
export function resolveHdrVideoFrameIndex(
|
|
time: number,
|
|
startTime: number,
|
|
fps: number,
|
|
frameCount: number,
|
|
loop = false,
|
|
): number | null {
|
|
const frameIndex = Math.round((time - startTime) * fps);
|
|
if (frameIndex < 0 || frameCount < 1) return null;
|
|
return loop ? frameIndex % frameCount : Math.min(frameIndex, frameCount - 1);
|
|
}
|
|
|
|
// fallow-ignore-next-line complexity
|
|
export function blitHdrVideoLayer(
|
|
canvas: Buffer,
|
|
el: ElementStackingInfo,
|
|
time: number,
|
|
fps: number,
|
|
hdrVideoFrameSources: Map<string, HdrVideoFrameSource>,
|
|
hdrStartTimes: Map<string, number>,
|
|
width: number,
|
|
height: number,
|
|
log?: ProducerLogger,
|
|
sourceTransfer?: HdrTransfer,
|
|
targetTransfer?: HdrTransfer,
|
|
hdrPerf?: HdrPerfCollector,
|
|
): void {
|
|
const frameSource = hdrVideoFrameSources.get(el.id);
|
|
const startTime = hdrStartTimes.get(el.id);
|
|
if (!frameSource || startTime === undefined || el.opacity <= 0) {
|
|
return;
|
|
}
|
|
|
|
// Frame index within the extracted playable source range. Loops wrap one
|
|
// extracted cycle; non-loops clamp to its final frame, matching Chrome's
|
|
// held-tail behavior for authored slots that outlive the source.
|
|
const effectiveIndex = resolveHdrVideoFrameIndex(
|
|
time,
|
|
startTime,
|
|
fps,
|
|
frameSource.frameCount,
|
|
frameSource.loop,
|
|
);
|
|
if (effectiveIndex === null) return;
|
|
const frameOffset = effectiveIndex * frameSource.frameSize;
|
|
|
|
try {
|
|
if (hdrPerf) hdrPerf.hdrVideoLayerBlits += 1;
|
|
const bytesRead = timeHdrPhase(hdrPerf, "hdrVideoReadDecodeMs", () =>
|
|
readSync(frameSource.fd, frameSource.scratch, 0, frameSource.frameSize, frameOffset),
|
|
);
|
|
if (bytesRead !== frameSource.frameSize) return;
|
|
const hdrRgb = frameSource.scratch;
|
|
const srcW = frameSource.width;
|
|
const srcH = frameSource.height;
|
|
|
|
// Convert between HDR transfer functions if source doesn't match output
|
|
if (sourceTransfer && targetTransfer && sourceTransfer !== targetTransfer) {
|
|
timeHdrPhase(hdrPerf, "hdrVideoTransferMs", () =>
|
|
convertTransfer(hdrRgb, sourceTransfer, targetTransfer),
|
|
);
|
|
}
|
|
|
|
const rawMatrix = parseTransformMatrix(el.transform);
|
|
const matrix = rawMatrix && isAffineMatrix(rawMatrix) ? rawMatrix : null;
|
|
|
|
const br = el.borderRadius;
|
|
const hasBorderRadius = br[0] > 0 || br[1] > 0 || br[2] > 0 || br[3] > 0;
|
|
const borderRadiusParam = hasBorderRadius ? br : undefined;
|
|
|
|
let blitX = el.x;
|
|
let blitY = el.y;
|
|
let blitSrcX = 0;
|
|
let blitSrcY = 0;
|
|
let blitW = srcW;
|
|
let blitH = srcH;
|
|
let clipped = false;
|
|
|
|
if (el.clipRect) {
|
|
const cr = el.clipRect;
|
|
const cx1 = Math.max(blitX, cr.x);
|
|
const cy1 = Math.max(blitY, cr.y);
|
|
const cx2 = Math.min(blitX + blitW, cr.x + cr.width);
|
|
const cy2 = Math.min(blitY + blitH, cr.y + cr.height);
|
|
if (cx2 <= cx1 || cy2 <= cy1) return;
|
|
blitSrcX = cx1 - blitX;
|
|
blitSrcY = cy1 - blitY;
|
|
blitW = cx2 - cx1;
|
|
blitH = cy2 - cy1;
|
|
blitX = cx1;
|
|
blitY = cy1;
|
|
clipped = true;
|
|
}
|
|
|
|
const isTranslationOnly = !!(
|
|
matrix &&
|
|
Math.abs(matrix[0] - 1) < TRANSFORM_IDENTITY_EPSILON &&
|
|
Math.abs(matrix[1]) < TRANSFORM_IDENTITY_EPSILON &&
|
|
Math.abs(matrix[2]) < TRANSFORM_IDENTITY_EPSILON &&
|
|
Math.abs(matrix[3] - 1) < TRANSFORM_IDENTITY_EPSILON
|
|
);
|
|
|
|
timeHdrPhase(hdrPerf, "hdrVideoBlitMs", () => {
|
|
if (matrix && !isTranslationOnly) {
|
|
if (clipped && log) {
|
|
log.debug(
|
|
`HDR clip rect on affine-transformed element ${el.id} — clip not applied (affine scissor not yet supported)`,
|
|
);
|
|
}
|
|
blitRgb48leAffine(
|
|
canvas,
|
|
hdrRgb,
|
|
matrix,
|
|
srcW,
|
|
srcH,
|
|
width,
|
|
height,
|
|
resolveBlitOpacity(el.opacity),
|
|
borderRadiusParam,
|
|
);
|
|
} else if (clipped) {
|
|
const croppedBuf = cropRgb48le(hdrRgb, srcW, srcH, blitSrcX, blitSrcY, blitW, blitH);
|
|
blitRgb48leRegion(
|
|
canvas,
|
|
croppedBuf,
|
|
blitX,
|
|
blitY,
|
|
blitW,
|
|
blitH,
|
|
width,
|
|
height,
|
|
resolveBlitOpacity(el.opacity),
|
|
borderRadiusParam,
|
|
);
|
|
} else {
|
|
blitRgb48leRegion(
|
|
canvas,
|
|
hdrRgb,
|
|
el.x,
|
|
el.y,
|
|
srcW,
|
|
srcH,
|
|
width,
|
|
height,
|
|
resolveBlitOpacity(el.opacity),
|
|
borderRadiusParam,
|
|
);
|
|
}
|
|
});
|
|
} catch (err) {
|
|
if (log) {
|
|
log.debug(`HDR blit failed for ${el.id}`, {
|
|
error: err instanceof Error ? err.message : String(err),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Image buffer ──────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Pre-decoded HDR image buffer with its native pixel dimensions.
|
|
*
|
|
* Static images decode exactly once at setup time and are blitted on every
|
|
* visible frame, unlike video frames which are read fresh per timestamp.
|
|
*/
|
|
export interface HdrImageBuffer {
|
|
data: Buffer;
|
|
width: number;
|
|
height: number;
|
|
}
|
|
|
|
/**
|
|
* Blit a single HDR image layer onto an rgb48le canvas.
|
|
*
|
|
* Image-equivalent of `blitHdrVideoLayer` — the buffer is pre-decoded and
|
|
* static, so there's no time-based frame lookup or per-frame PNG read.
|
|
*/
|
|
export function blitHdrImageLayer(
|
|
canvas: Buffer,
|
|
el: ElementStackingInfo,
|
|
hdrImageBuffers: Map<string, HdrImageBuffer>,
|
|
hdrImageTransferCache: HdrImageTransferCache,
|
|
width: number,
|
|
height: number,
|
|
log?: ProducerLogger,
|
|
sourceTransfer?: HdrTransfer,
|
|
targetTransfer?: HdrTransfer,
|
|
hdrPerf?: HdrPerfCollector,
|
|
): void {
|
|
const buf = hdrImageBuffers.get(el.id);
|
|
if (!buf || el.opacity <= 0) {
|
|
return;
|
|
}
|
|
if (el.clipRect && log) {
|
|
log.debug(`HDR clip rect on image element ${el.id} — clip not yet supported for images`);
|
|
}
|
|
|
|
try {
|
|
if (hdrPerf) hdrPerf.hdrImageLayerBlits += 1;
|
|
// The cache returns `buf.data` unchanged when no conversion is needed,
|
|
// and otherwise returns a per-(imageId, targetTransfer) buffer that was
|
|
// converted exactly once and reused across every subsequent frame.
|
|
const hdrRgb = timeHdrPhase(hdrPerf, "hdrImageTransferMs", () =>
|
|
sourceTransfer && targetTransfer
|
|
? hdrImageTransferCache.getConverted(el.id, sourceTransfer, targetTransfer, buf.data)
|
|
: buf.data,
|
|
);
|
|
|
|
const rawMatrix = parseTransformMatrix(el.transform);
|
|
const matrix = rawMatrix && isAffineMatrix(rawMatrix) ? rawMatrix : null;
|
|
|
|
const br = el.borderRadius;
|
|
const hasBorderRadius = br[0] > 0 || br[1] > 0 || br[2] > 0 || br[3] > 0;
|
|
const borderRadiusParam = hasBorderRadius ? br : undefined;
|
|
|
|
timeHdrPhase(hdrPerf, "hdrImageBlitMs", () => {
|
|
if (matrix) {
|
|
blitRgb48leAffine(
|
|
canvas,
|
|
hdrRgb,
|
|
matrix,
|
|
buf.width,
|
|
buf.height,
|
|
width,
|
|
height,
|
|
resolveBlitOpacity(el.opacity),
|
|
borderRadiusParam,
|
|
);
|
|
} else {
|
|
blitRgb48leRegion(
|
|
canvas,
|
|
hdrRgb,
|
|
el.x,
|
|
el.y,
|
|
buf.width,
|
|
buf.height,
|
|
width,
|
|
height,
|
|
resolveBlitOpacity(el.opacity),
|
|
borderRadiusParam,
|
|
);
|
|
}
|
|
});
|
|
} catch (err) {
|
|
if (log) {
|
|
log.debug(`HDR image blit failed for ${el.id}`, {
|
|
error: err instanceof Error ? err.message : String(err),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Composite transfer + strategy ─────────────────────────────────────────
|
|
|
|
/**
|
|
* Dependencies passed to `compositeHdrFrame`.
|
|
*
|
|
* Every field except the per-frame arguments is captured once when the HDR
|
|
* render path opens its `try { ... }` block and reused across every frame —
|
|
* extracting them into an explicit struct lets the helper live at module
|
|
* scope (no closure-over-renderJob) and keeps the per-call signature small.
|
|
*/
|
|
export type CompositeTransfer = HdrTransfer | "srgb";
|
|
|
|
export function shouldUseLayeredComposite(options: {
|
|
hasHdrContent: boolean;
|
|
hasShaderTransitions: boolean;
|
|
isPngSequence: boolean;
|
|
}): boolean {
|
|
return options.hasHdrContent || (options.hasShaderTransitions && !options.isPngSequence);
|
|
}
|
|
|
|
export function resolveCompositeTransfer(
|
|
hasHdrContent: boolean,
|
|
effectiveHdr: { transfer: HdrTransfer } | undefined,
|
|
): CompositeTransfer {
|
|
return hasHdrContent && effectiveHdr ? effectiveHdr.transfer : "srgb";
|
|
}
|
|
|
|
export function selectDomLayerShowIds(
|
|
layerElementIds: string[],
|
|
fullStacking: ElementStackingInfo[],
|
|
): string[] {
|
|
const byId = new Map(fullStacking.map((el) => [el.id, el]));
|
|
return layerElementIds.filter((id) => {
|
|
const el = byId.get(id);
|
|
return !!el && el.opacity > 0 && (el.visible || el.renderFrameVisible === true);
|
|
});
|
|
}
|
|
|
|
export interface HdrCompositeContext {
|
|
log: ProducerLogger;
|
|
domSession: CaptureSession;
|
|
beforeCaptureHook: BeforeCaptureHook | null;
|
|
width: number;
|
|
height: number;
|
|
fps: number;
|
|
compositeTransfer: CompositeTransfer;
|
|
nativeHdrImageIds: Set<string>;
|
|
hdrImageBuffers: Map<string, HdrImageBuffer>;
|
|
hdrImageTransferCache: HdrImageTransferCache;
|
|
hdrVideoFrameSources: Map<string, HdrVideoFrameSource>;
|
|
hdrVideoStartTimes: Map<string, number>;
|
|
imageTransfers: Map<string, HdrTransfer>;
|
|
videoTransfers: Map<string, HdrTransfer>;
|
|
debugDumpEnabled: boolean;
|
|
debugDumpDir: string | null;
|
|
hdrPerf?: HdrPerfCollector;
|
|
}
|
|
|
|
// ─── Per-frame compositor ──────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Composite a single HDR frame into a pre-allocated `rgb48le` canvas.
|
|
*
|
|
* Bottom-to-top z-order: HDR layers are blitted directly from cached image
|
|
* buffers / extracted video frames; DOM layers are screenshotted with a
|
|
* mass-hide mask (so each layer paints only its own elements) and then
|
|
* blended into the canvas via `blitRgba8OverRgb48le` in the active HDR
|
|
* transfer space.
|
|
*
|
|
* The `elementFilter` parameter exists so the transition path can composite
|
|
* each scene independently; pass `undefined` for whole-stack rendering.
|
|
*
|
|
* @param ctx - Long-lived dependencies (logger, browser session, dimensions,
|
|
* HDR layer maps). Captured once per render — see
|
|
* {@link HdrCompositeContext}.
|
|
* @param canvas - Pre-allocated `width * height * 6` byte buffer. Caller must
|
|
* zero-fill before every frame (this helper does not).
|
|
* @param time - Seek time in seconds.
|
|
* @param fullStacking - Stacking info for ALL elements at this time. Even when
|
|
* filtering, every other element id is needed to build
|
|
* the DOM-layer hide-list.
|
|
* @param elementFilter - When set, only elements whose id is in the set are
|
|
* composited.
|
|
* @param debugFrameIndex - Frame index used to label per-layer diagnostic
|
|
* dumps. Pass `-1` to disable per-layer dumps even
|
|
* when `KEEP_TEMP=1` (e.g. for warmup frames).
|
|
*/
|
|
// fallow-ignore-next-line complexity
|
|
export async function compositeHdrFrame(
|
|
ctx: HdrCompositeContext,
|
|
canvas: Buffer,
|
|
time: number,
|
|
fullStacking: ElementStackingInfo[],
|
|
elementFilter?: Set<string>,
|
|
debugFrameIndex: number = -1,
|
|
): Promise<void> {
|
|
const {
|
|
log,
|
|
domSession,
|
|
beforeCaptureHook,
|
|
width,
|
|
height,
|
|
fps,
|
|
compositeTransfer,
|
|
nativeHdrImageIds,
|
|
hdrImageBuffers,
|
|
hdrImageTransferCache,
|
|
hdrVideoFrameSources,
|
|
hdrVideoStartTimes,
|
|
imageTransfers,
|
|
videoTransfers,
|
|
debugDumpEnabled,
|
|
debugDumpDir,
|
|
hdrPerf,
|
|
} = ctx;
|
|
|
|
const filteredStacking = elementFilter
|
|
? fullStacking.filter((e) => elementFilter.has(e.id))
|
|
: fullStacking;
|
|
|
|
// Zero-opacity elements stay in the stacking for correct hide-list
|
|
// generation (their <img> replacements must be hidden from sibling
|
|
// screenshots). The actual blit is skipped in the compositing loop below.
|
|
const layers = groupIntoLayers(filteredStacking);
|
|
const allElementIds = fullStacking.map((e) => e.id);
|
|
|
|
const shouldLog = debugDumpEnabled && debugFrameIndex >= 0;
|
|
if (shouldLog) {
|
|
log.info("[diag] compositeToBuffer plan", {
|
|
frame: debugFrameIndex,
|
|
time: time.toFixed(3),
|
|
filterSize: elementFilter?.size,
|
|
fullStackingCount: fullStacking.length,
|
|
filteredCount: filteredStacking.length,
|
|
layerCount: layers.length,
|
|
layers: layers.map((l) =>
|
|
l.type === "hdr"
|
|
? {
|
|
type: "hdr",
|
|
id: l.element.id,
|
|
z: l.element.zIndex,
|
|
visible: l.element.visible,
|
|
opacity: l.element.opacity,
|
|
bounds: `${Math.round(l.element.x)},${Math.round(l.element.y)} ${Math.round(l.element.width)}x${Math.round(l.element.height)}`,
|
|
}
|
|
: { type: "dom", ids: l.elementIds },
|
|
),
|
|
});
|
|
}
|
|
|
|
for (const [layerIdx, layer] of layers.entries()) {
|
|
if (layer.type === "hdr") {
|
|
// Skip zero-opacity HDR elements — their parent scene may have faded out.
|
|
if (layer.element.opacity <= 0) continue;
|
|
const before = shouldLog ? countNonZeroRgb48(canvas) : 0;
|
|
const isHdrImage = nativeHdrImageIds.has(layer.element.id);
|
|
const hdrTargetTransfer = compositeTransfer === "srgb" ? undefined : compositeTransfer;
|
|
if (isHdrImage) {
|
|
blitHdrImageLayer(
|
|
canvas,
|
|
layer.element,
|
|
hdrImageBuffers,
|
|
hdrImageTransferCache,
|
|
width,
|
|
height,
|
|
log,
|
|
imageTransfers.get(layer.element.id),
|
|
hdrTargetTransfer,
|
|
hdrPerf,
|
|
);
|
|
} else {
|
|
blitHdrVideoLayer(
|
|
canvas,
|
|
layer.element,
|
|
time,
|
|
fps,
|
|
hdrVideoFrameSources,
|
|
hdrVideoStartTimes,
|
|
width,
|
|
height,
|
|
log,
|
|
videoTransfers.get(layer.element.id),
|
|
hdrTargetTransfer,
|
|
hdrPerf,
|
|
);
|
|
}
|
|
if (shouldLog) {
|
|
const after = countNonZeroRgb48(canvas);
|
|
if (isHdrImage) {
|
|
const buf = hdrImageBuffers.get(layer.element.id);
|
|
log.info("[diag] hdr layer blit", {
|
|
frame: debugFrameIndex,
|
|
layerIdx,
|
|
id: layer.element.id,
|
|
kind: "image",
|
|
pixelsAdded: after - before,
|
|
totalNonZero: after,
|
|
bufferDecoded: !!buf,
|
|
bufferDims: buf ? `${buf.width}x${buf.height}` : null,
|
|
});
|
|
} else {
|
|
const frameSource = hdrVideoFrameSources.get(layer.element.id);
|
|
const startTime = hdrVideoStartTimes.get(layer.element.id) ?? 0;
|
|
const localTime = time - startTime;
|
|
const frameNum = Math.floor(localTime * fps) + 1;
|
|
log.info("[diag] hdr layer blit", {
|
|
frame: debugFrameIndex,
|
|
layerIdx,
|
|
id: layer.element.id,
|
|
kind: "video",
|
|
pixelsAdded: after - before,
|
|
totalNonZero: after,
|
|
startTime,
|
|
localTime: localTime.toFixed(3),
|
|
hdrFrameNum: frameNum,
|
|
rawPath: frameSource?.rawPath ?? null,
|
|
frameCount: frameSource?.frameCount ?? null,
|
|
});
|
|
}
|
|
}
|
|
} else {
|
|
// DOM layer: capture only elements in this layer.
|
|
//
|
|
// Each layer gets a fresh seek + inject cycle to guarantee correct
|
|
// visibility state — avoids fragile interactions between the frame
|
|
// injector, applyDomLayerMask, removeDomLayerMask, and GSAP re-seek.
|
|
//
|
|
// The mask:
|
|
// - mass-hides every body descendant via stylesheet
|
|
// - re-shows the layer's currently paintable elements (and their
|
|
// descendants and injected `__render_frame_*` siblings) so
|
|
// deep-nested content stays visible even though intermediate
|
|
// ancestors are hidden
|
|
// - inline-hides every other data-start element so they don't
|
|
// paint when they happen to be descendants of a layer element
|
|
// (most importantly: HDR videos and other-layer SDR videos
|
|
// that live inside `#root` when capturing the root DOM layer)
|
|
//
|
|
// Without the mask, every DOM screenshot captures the full page
|
|
// (root background, sibling scenes' static content, the painted
|
|
// border/box-shadow of cards, etc.) and the resulting opaque
|
|
// pixels overwrite previously composited HDR content beneath.
|
|
const layerIds = new Set(layer.elementIds);
|
|
const showIds = selectDomLayerShowIds(layer.elementIds, fullStacking);
|
|
if (showIds.length === 0) {
|
|
if (shouldLog) {
|
|
log.info("[diag] dom layer skipped, all elements not paintable", {
|
|
frame: debugFrameIndex,
|
|
layerIdx,
|
|
layerIds: layer.elementIds,
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
const hideIds = allElementIds.filter((id) => !layerIds.has(id));
|
|
if (hdrPerf) hdrPerf.domLayerCaptures += 1;
|
|
|
|
// 1. Seek GSAP to restore all animated properties from clean state
|
|
await timeHdrPhaseAsync(hdrPerf, "domLayerSeekMs", () =>
|
|
domSession.page.evaluate((t: number) => {
|
|
if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t);
|
|
}, time),
|
|
);
|
|
|
|
// 2. Run frame injector to set correct SDR video visibility
|
|
if (beforeCaptureHook) {
|
|
await timeHdrPhaseAsync(hdrPerf, "domLayerInjectMs", () =>
|
|
beforeCaptureHook(domSession.page, time),
|
|
);
|
|
}
|
|
|
|
// 3. Install the mask (mass-hide stylesheet + inline-hide non-layer ids)
|
|
await timeHdrPhaseAsync(hdrPerf, "domMaskApplyMs", () =>
|
|
applyDomLayerMask(domSession.page, showIds, hideIds),
|
|
);
|
|
|
|
// 4. Screenshot
|
|
const domPng = await timeHdrPhaseAsync(hdrPerf, "domScreenshotMs", () =>
|
|
captureAlphaPng(domSession.page, width, height),
|
|
);
|
|
|
|
// 5. Tear down the mask
|
|
await timeHdrPhaseAsync(hdrPerf, "domMaskRemoveMs", () =>
|
|
removeDomLayerMask(domSession.page, hideIds),
|
|
);
|
|
|
|
try {
|
|
const { data: domRgba } = timeHdrPhase(hdrPerf, "domPngDecodeMs", () => decodePng(domPng));
|
|
const before = shouldLog ? countNonZeroRgb48(canvas) : 0;
|
|
const alphaPixels = shouldLog ? countNonZeroAlpha(domRgba) : 0;
|
|
timeHdrPhase(hdrPerf, "domBlitMs", () =>
|
|
blitRgba8OverRgb48le(domRgba, canvas, width, height, compositeTransfer),
|
|
);
|
|
if (shouldLog && debugDumpDir) {
|
|
const after = countNonZeroRgb48(canvas);
|
|
const dumpName = `frame_${String(debugFrameIndex).padStart(4, "0")}_layer_${String(layerIdx).padStart(2, "0")}_dom.png`;
|
|
const dumpPath = join(debugDumpDir, dumpName);
|
|
writeFileExclusiveSync(dumpPath, domPng);
|
|
log.info("[diag] dom layer blit", {
|
|
frame: debugFrameIndex,
|
|
layerIdx,
|
|
layerIds: layer.elementIds,
|
|
showIds,
|
|
hideCount: hideIds.length,
|
|
pngBytes: domPng.length,
|
|
alphaPixels,
|
|
pixelsAdded: after - before,
|
|
totalNonZero: after,
|
|
dumpPath,
|
|
});
|
|
}
|
|
} catch (err) {
|
|
log.warn("DOM layer decode/blit failed; skipping overlay", {
|
|
layerIds: layer.elementIds,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (shouldLog && debugDumpDir) {
|
|
const finalNonZero = countNonZeroRgb48(canvas);
|
|
log.info("[diag] compositeToBuffer end", {
|
|
frame: debugFrameIndex,
|
|
finalNonZeroPixels: finalNonZero,
|
|
totalPixels: width * height,
|
|
coverage: ((finalNonZero / (width * height)) * 100).toFixed(1) + "%",
|
|
});
|
|
}
|
|
}
|