ext/hyperframes
2
0
Fork 0
mirror of https://github.com/heygen-com/hyperframes.git synced 2026-09-01 19:42:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: render shader transitions for SDR compositions (#640)

Browse Source 
* feat: cache shader transition preview frames

* fix: move shader transition loading to player

* fix: render shader transitions for sdr compositions
This commit is contained in:
Vance Ingalls
2026-05-06 01:29:44 -07:00
committed by GitHub
parent a7b308b667
commit b0fb664873
4 changed files with 134 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/engine/src/utils/alphaBlit.test.ts
+10
View File
@@ -511,6 +511,16 @@ describe("blitRgba8OverRgb48le", () => {
expect(canvas.readUInt16LE(4)).toBe(0);
});
it("fully opaque DOM with srgb transfer expands 8-bit channels to 16-bit SDR", () => {
const canvas = makeHdrFrame(1, 1, 10000, 20000, 30000);
const dom = makeDomRgba(1, 1, 255, 128, 1, 255);
blitRgba8OverRgb48le(dom, canvas, 1, 1, "srgb");
expect(canvas.readUInt16LE(0)).toBe(65535);
expect(canvas.readUInt16LE(2)).toBe(128 * 257);
expect(canvas.readUInt16LE(4)).toBe(257);
});
it("sRGB→HLG: black stays black, white stays white", () => {
const canvasBlack = makeHdrFrame(1, 1, 0, 0, 0);
const domBlack = makeDomRgba(1, 1, 0, 0, 0, 255);
packages/engine/src/utils/alphaBlit.ts
+15 -7
View File
@@ -249,7 +249,7 @@ export function decodePngToRgb48le(buf: Buffer): { width: number; height: number
* bt2020). For neutral/near-neutral content (text, UI) the gamut difference
* is negligible.
*/
function buildSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
function buildSrgbToSignalLut(transfer: "hlg" | "pq" | "srgb"): Uint16Array {
const lut = new Uint16Array(256);
// HLG OETF constants (Rec. 2100)
@@ -267,6 +267,11 @@ function buildSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
const sdrNits = 203.0;
for (let i = 0; i < 256; i++) {
if (transfer === "srgb") {
lut[i] = i * 257;
continue;
}
// sRGB EOTF: signal → linear (range 0–1, relative to SDR white)
const v = i / 255;
const linear = v <= 0.04045 ? v / 12.92 : Math.pow((v + 0.055) / 1.055, 2.4);
@@ -288,12 +293,15 @@ function buildSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
return lut;
}
const SRGB_TO_HLG = buildSrgbToHdrLut("hlg");
const SRGB_TO_PQ = buildSrgbToHdrLut("pq");
const SRGB_TO_SRGB_16 = buildSrgbToSignalLut("srgb");
const SRGB_TO_HLG = buildSrgbToSignalLut("hlg");
const SRGB_TO_PQ = buildSrgbToSignalLut("pq");
/** Select the correct sRGB→HDR LUT for the given transfer function. */
function getSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
return transfer === "pq" ? SRGB_TO_PQ : SRGB_TO_HLG;
function getSrgbToSignalLut(transfer: "hlg" | "pq" | "srgb"): Uint16Array {
if (transfer === "pq") return SRGB_TO_PQ;
if (transfer === "hlg") return SRGB_TO_HLG;
return SRGB_TO_SRGB_16;
}
// ── Alpha compositing ─────────────────────────────────────────────────────────
@@ -317,10 +325,10 @@ export function blitRgba8OverRgb48le(
canvas: Buffer,
width: number,
height: number,
transfer: "hlg" | "pq" = "hlg",
transfer: "hlg" | "pq" | "srgb" = "hlg",
): void {
const pixelCount = width * height;
const lut = getSrgbToHdrLut(transfer);
const lut = getSrgbToSignalLut(transfer);
for (let i = 0; i < pixelCount; i++) {
const da = domRgba[i * 4 + 3] ?? 0;
packages/producer/src/services/renderOrchestrator.test.ts
+44
View File
@@ -21,8 +21,10 @@ import {
materializeExtractedFramesForCompiledDir,
projectBrowserEndToCompositionTimeline,
resolveRenderWorkerCount,
resolveCompositeTransfer,
selectCaptureCalibrationFrames,
shouldFallbackToScreenshotAfterCalibrationError,
shouldUseLayeredComposite,
shouldUseStreamingEncode,
writeCompiledArtifacts,
} from "./renderOrchestrator.js";
@@ -543,6 +545,48 @@ describe("estimateCaptureCostMultiplier", () => {
});
});
describe("shouldUseLayeredComposite", () => {
it("uses the layered compositor for SDR shader transition renders", () => {
expect(
shouldUseLayeredComposite({
hasHdrContent: false,
hasShaderTransitions: true,
isPngSequence: false,
}),
).toBe(true);
});
it("does not route PNG sequence shader renders through the streaming layered compositor", () => {
expect(
shouldUseLayeredComposite({
hasHdrContent: false,
hasShaderTransitions: true,
isPngSequence: true,
}),
).toBe(false);
});
it("keeps HDR content on the layered compositor even without shader transitions", () => {
expect(
shouldUseLayeredComposite({
hasHdrContent: true,
hasShaderTransitions: false,
isPngSequence: false,
}),
).toBe(true);
});
});
describe("resolveCompositeTransfer", () => {
it("uses 16-bit-expanded sRGB for SDR layered shader transition renders", () => {
expect(resolveCompositeTransfer(false, undefined)).toBe("srgb");
});
it("uses the active HDR transfer when HDR content is being preserved", () => {
expect(resolveCompositeTransfer(true, { transfer: "hlg" })).toBe("hlg");
});
});
describe("estimateMeasuredCaptureCostMultiplier", () => {
it("turns slow calibration samples into a capture cost multiplier", () => {
const estimate = estimateMeasuredCaptureCostMultiplier([
packages/producer/src/services/renderOrchestrator.ts
+65 -43
View File
@@ -1534,6 +1534,23 @@ function blitHdrImageLayer(
* extracting them into an explicit struct lets the helper live at module
* scope (no closure-over-renderJob) and keeps the per-call signature small.
*/
type CompositeTransfer = HdrTransfer | "srgb";
export function shouldUseLayeredComposite(options: {
hasHdrContent: boolean;
hasShaderTransitions: boolean;
isPngSequence: boolean;
}): boolean {
return options.hasHdrContent || (options.hasShaderTransitions && !options.isPngSequence);
}
export function resolveCompositeTransfer(
hasHdrContent: boolean,
effectiveHdr: { transfer: HdrTransfer } | undefined,
): CompositeTransfer {
return hasHdrContent && effectiveHdr ? effectiveHdr.transfer : "srgb";
}
interface HdrCompositeContext {
log: ProducerLogger;
domSession: CaptureSession;
@@ -1541,7 +1558,7 @@ interface HdrCompositeContext {
width: number;
height: number;
fps: number;
effectiveHdr: { transfer: HdrTransfer };
compositeTransfer: CompositeTransfer;
nativeHdrImageIds: Set<string>;
hdrImageBuffers: Map<string, HdrImageBuffer>;
hdrImageTransferCache: HdrImageTransferCache;
@@ -1596,7 +1613,7 @@ async function compositeHdrFrame(
width,
height,
fps,
effectiveHdr,
compositeTransfer,
nativeHdrImageIds,
hdrImageBuffers,
hdrImageTransferCache,
@@ -1648,6 +1665,7 @@ async function compositeHdrFrame(
if (layer.element.opacity <= 0) continue;
const before = shouldLog ? countNonZeroRgb48(canvas) : 0;
const isHdrImage = nativeHdrImageIds.has(layer.element.id);
const hdrTargetTransfer = compositeTransfer === "srgb" ? undefined : compositeTransfer;
if (isHdrImage) {
blitHdrImageLayer(
canvas,
@@ -1658,7 +1676,7 @@ async function compositeHdrFrame(
height,
log,
imageTransfers.get(layer.element.id),
effectiveHdr.transfer,
hdrTargetTransfer,
hdrPerf,
);
} else {
@@ -1673,7 +1691,7 @@ async function compositeHdrFrame(
height,
log,
videoTransfers.get(layer.element.id),
effectiveHdr.transfer,
hdrTargetTransfer,
hdrPerf,
);
}
@@ -1774,7 +1792,7 @@ async function compositeHdrFrame(
const before = shouldLog ? countNonZeroRgb48(canvas) : 0;
const alphaPixels = shouldLog ? countNonZeroAlpha(domRgba) : 0;
timingStart = Date.now();
blitRgba8OverRgb48le(domRgba, canvas, width, height, effectiveHdr.transfer);
blitRgba8OverRgb48le(domRgba, canvas, width, height, compositeTransfer);
addHdrTiming(hdrPerf, "domBlitMs", timingStart);
if (shouldLog && debugDumpDir) {
const after = countNonZeroRgb48(canvas);
@@ -2699,7 +2717,12 @@ export async function executeRenderJob(
// auto mode stays SDR since H.265 10-bit causes browser color management
// issues (orange shift) with no quality benefit.
const nativeHdrIds = new Set([...nativeHdrVideoIds, ...nativeHdrImageIds]);
const hasHdrContent = effectiveHdr && nativeHdrIds.size > 0;
const hasHdrContent = Boolean(effectiveHdr && nativeHdrIds.size > 0);
const useLayeredComposite = shouldUseLayeredComposite({
hasHdrContent,
hasShaderTransitions: compiled.hasShaderTransitions,
isPngSequence,
});
const encoderHdr = hasHdrContent ? effectiveHdr : undefined;
// png-sequence has no encoder, but the rest of the orchestrator still
// reads `preset.quality` for `effectiveQuality` and `preset.codec` for
@@ -2730,21 +2753,27 @@ export async function executeRenderJob(
job.framesRendered = 0;
// ── HDR z-ordered multi-layer compositing ──────────────────────────────
// ── Z-ordered multi-layer compositing ─────────────────────────────────
// Per frame: query all elements' z-order, group into layers (DOM or HDR),
// composite bottom-to-top in Node.js memory. HDR layers use native
// pre-extracted HLG pixels; DOM layers use Chrome alpha screenshots
// with sRGB→HLG conversion. Video position/opacity applied via queried bounds.
if (hasHdrContent) {
log.info("[Render] HDR layered composite: z-ordered DOM + native HLG video layers");
// pre-extracted pixels; DOM layers use Chrome alpha screenshots converted
// into the active rgb48le signal space. Shader transitions use this same
// path for SDR compositions so the engine can apply transition math to
// isolated scene buffers instead of recording plain DOM screenshots.
if (useLayeredComposite) {
log.info(
hasHdrContent
? "[Render] HDR layered composite: z-ordered DOM + native HDR video/image layers"
: "[Render] Shader transition composite: z-ordered SDR DOM layers",
);
hdrPerf = createHdrPerfCollector();
// HDR layered compositing relies on captureAlphaPng (Page.captureScreenshot
// with a transparent background) for the SDR DOM overlay layer. That CDP
// call hangs indefinitely when Chrome is launched with --enable-begin-frame-control
// Layered compositing relies on captureAlphaPng (Page.captureScreenshot
// with a transparent background) for DOM layers. That CDP call hangs
// indefinitely when Chrome is launched with --enable-begin-frame-control
// (the default on Linux/headless-shell), because the compositor is paused
// and never produces a frame to capture. Force screenshot mode for the
// entire HDR path — same constraint as alpha output formats above.
// entire layered path — same constraint as alpha output formats above.
cfg.forceScreenshot = true;
// Use NATIVE HDR IDs (probed before SDR→HDR conversion) so only originally-HDR
@@ -2823,8 +2852,13 @@ export async function executeRenderJob(
const scenes = document.querySelectorAll(".scene");
const map: Record<string, string[]> = {};
for (const scene of scenes) {
const els = scene.querySelectorAll("[data-start]");
map[scene.id] = Array.from(els).map((e) => e.id);
if (!scene.id) continue;
const ids = new Set<string>([scene.id]);
const els = scene.querySelectorAll("[id]");
for (const el of els) {
if (el.id) ids.add(el.id);
}
map[scene.id] = Array.from(ids);
}
return map;
});
@@ -2836,7 +2870,7 @@ export async function executeRenderJob(
}));
if (transitionRanges.length > 0) {
log.info("[Render] Detected shader transitions for HDR compositing", {
log.info("[Render] Detected shader transitions for layered compositing", {
count: transitionRanges.length,
transitions: transitionRanges.map((t) => ({
shader: t.shader,
@@ -3109,15 +3143,8 @@ export async function executeRenderJob(
if (debugDumpDir && !existsSync(debugDumpDir)) {
mkdirSync(debugDumpDir, { recursive: true });
}
// INVARIANT: this entire `try` block is reachable only when HDR
// output is enabled (`if (effectiveHdr) { ... try { ... } }`), so
// narrowing here is safe even though `effectiveHdr` is typed as
// `... | undefined` at the outer scope.
if (!effectiveHdr) {
throw new Error(
"Internal: HDR render path entered without effectiveHdr — this is a bug.",
);
}
const compositeTransfer = resolveCompositeTransfer(hasHdrContent, effectiveHdr);
const hdrTargetTransfer = compositeTransfer === "srgb" ? undefined : compositeTransfer;
// Per-job LRU cache for transfer-converted HDR image buffers. Static HDR
// images that need PQ↔HLG conversion are converted exactly once per
// (imageId, targetTransfer) and then reused for every subsequent frame
@@ -3137,7 +3164,7 @@ export async function executeRenderJob(
width,
height,
fps: job.config.fps,
effectiveHdr,
compositeTransfer,
nativeHdrImageIds,
hdrImageBuffers,
hdrImageTransferCache,
@@ -3281,7 +3308,7 @@ export async function executeRenderJob(
height,
log,
imageTransfers.get(el.id),
effectiveHdr?.transfer,
hdrTargetTransfer,
hdrPerf,
);
} else {
@@ -3296,7 +3323,7 @@ export async function executeRenderJob(
height,
log,
videoTransfers.get(el.id),
effectiveHdr?.transfer,
hdrTargetTransfer,
hdrPerf,
);
}
@@ -3328,19 +3355,13 @@ export async function executeRenderJob(
timingStart = Date.now();
const { data: domRgba } = decodePng(domPng);
addHdrTiming(hdrPerf, "domPngDecodeMs", timingStart);
// Invariant: `hasHdrVideo` requires `effectiveHdr` to be set (see line ~919).
if (!effectiveHdr) {
throw new Error(
"Invariant violation: effectiveHdr is undefined inside hasHdrVideo branch",
);
}
timingStart = Date.now();
blitRgba8OverRgb48le(
domRgba,
sceneBuf as Buffer,
width,
height,
effectiveHdr.transfer,
compositeTransfer,
);
addHdrTiming(hdrPerf, "domBlitMs", timingStart);
} catch (err) {
@@ -3352,11 +3373,12 @@ export async function executeRenderJob(
}
}
// Apply shader transition blend directly in PQ/HLG signal space.
// Linearization was attempted but destroys dark PQ content — values below
// PQ ~5000 quantize to zero in 16-bit linear, wiping out the bottom portion
// of dark video content. PQ space is perceptual and works well enough
// for shader math since the shaders were designed for perceptual (sRGB) space.
// Apply shader transition blend directly in the active rgb48le
// signal space. Linearizing HDR was attempted but destroys dark
// PQ content — values below PQ ~5000 quantize to zero in 16-bit
// linear, wiping out the bottom portion of dark video content.
// SDR compositions use 16-bit-expanded sRGB, which matches the
// shader design space.
const transitionFn: TransitionFn = TRANSITIONS[activeTransition.shader] ?? crossfade;
transitionFn(transBufferA, transBufferB, transOutput, width, height, progress);
addHdrTiming(hdrPerf, "transitionCompositeMs", transitionTimingStart);
@@ -3431,7 +3453,7 @@ export async function executeRenderJob(
updateJobStatus(
job,
"rendering",
`HDR composite frame ${i + 1}/${job.totalFrames}`,
`Layered composite frame ${i + 1}/${job.totalFrames}`,
Math.round(25 + frameProgress * 55),
onProgress,
);