From 8548a1777138e369cb340626f5c068cb740d625b Mon Sep 17 00:00:00 2001
From: Vance Ingalls <vance@heygen.com>
Date: Sun, 19 Apr 2026 17:03:29 -0700
Subject: [PATCH] feat(hdr): GSAP transforms and border-radius masks on HDR
 video (#290)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Summary

HDR video elements with GSAP animations (position, scale, rotation, opacity) and CSS border-radius rendered without any transforms applied — the video just sat at (0,0) full-size. This PR adds affine transform support and rounded-corner masking for natively-composited HDR video.

## What it does

**Affine blit with bilinear interpolation:**
- `blitRgb48leAffine()` — Takes a 4x4 DOMMatrix and maps each destination pixel back to source coordinates via the inverse transform. Bilinear interpolation between the 4 nearest source pixels produces smooth edges under rotation and non-integer scaling. Optional opacity and border-radius parameters.
- `parseTransformMatrix()` — Parses CSS `matrix(a,b,c,d,e,f)` strings into `[a,b,c,d,e,f]` tuples.

**Accumulated viewport matrix:**
- `getViewportMatrix()` — Walks the `offsetParent` chain from element to viewport, accumulating position offsets and CSS transforms at each level. Correctly handles `transform-origin` using the CSS sandwich: `translate(origin) × M × translate(-origin)`. This is critical because GSAP animates transforms on wrapper divs, not directly on the video element.

**Effective opacity:**
- `getEffectiveOpacity()` — Multiplies opacity values walking up the ancestor chain. Uses `Number.isNaN()` (not `|| 1`) so opacity:0 isn't incorrectly treated as 1.

**Border-radius masks:**
- `roundedRectAlpha()` — Per-pixel anti-aliased rounded-rectangle mask with support for independent corner radii.
- `getEffectiveBorderRadius()` — Walks ancestors for `overflow:hidden` + border-radius. Resolves percentage values (e.g., `50%` for circles) via `offsetWidth`/`offsetHeight`.

**Layout dimensions for extraction:**
- Uses `offsetWidth`/`offsetHeight` (unaffected by CSS transforms) instead of `getBoundingClientRect()` (which returns the transformed bounding box and wobbles under rotation).

## Files changed

| File | What changed |
|------|-------------|
| `packages/engine/src/utils/alphaBlit.ts` | `blitRgb48leAffine()`, `parseTransformMatrix()`, `roundedRectAlpha()`, `cornerAlpha()` |
| `packages/engine/src/services/videoFrameInjector.ts` | `getViewportMatrix()`, `getEffectiveOpacity()`, `getEffectiveBorderRadius()`, `layoutWidth`/`layoutHeight` on `ElementStackingInfo` |
| `packages/producer/src/services/renderOrchestrator.ts` | Affine blit path, extraction at layout dimensions, border-radius parameter passing |

## How to test

Render a composition with an HDR video that has GSAP scale + rotation animation and a `border-radius: 50%` wrapper (circle mask). The video should rotate smoothly with round edges — no wobble, no sharp corners.

## Stack position

**5 of 6** — Stacked on #289 (z-ordered layers). Adds transform and masking support to the HDR blit that the layer compositor uses.

🤖 Generated with [Claude Code](https://claude.com/claude-code)
---
 packages/engine/src/index.ts                  |   2 +
 .../engine/src/services/videoFrameInjector.ts | 144 +++++++++++++++++-
 .../src/services/renderOrchestrator.ts        | 105 ++++++++++---
 .../shader-transitions/src/hyper-shader.ts    |  59 +++++++
 4 files changed, 284 insertions(+), 26 deletions(-)

diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts
index 6050ac5cc..741c42efd 100644
--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -161,6 +161,8 @@ export {
   decodePngToRgb48le,
   blitRgba8OverRgb48le,
   blitRgb48leRegion,
+  blitRgb48leAffine,
+  parseTransformMatrix,
   getSrgbToHdrLut,
 } from "./utils/alphaBlit.js";
 
diff --git a/packages/engine/src/services/videoFrameInjector.ts b/packages/engine/src/services/videoFrameInjector.ts
index 3eb62a159..288b8903a 100644
--- a/packages/engine/src/services/videoFrameInjector.ts
+++ b/packages/engine/src/services/videoFrameInjector.ts
@@ -236,9 +236,14 @@ export interface ElementStackingInfo {
   y: number;
   width: number;
   height: number;
+  /** Layout dimensions before CSS transforms (offsetWidth/offsetHeight). */
+  layoutWidth: number;
+  layoutHeight: number;
   opacity: number;
   visible: boolean;
   isHdr: boolean;
+  transform: string; // CSS transform matrix string, e.g. "matrix(1,0,0,1,0,0)" or "none"
+  borderRadius: [number, number, number, number]; // [tl, tr, br, bl] in CSS px from nearest clipping ancestor
 }
 
 /**
@@ -290,18 +295,148 @@ export async function queryElementStacking(
       return 0;
     }
 
+    // Find border-radius that clips the element. Replaced elements like <video>
+    // clip to their own border-radius; ancestors need overflow !== visible.
+    function getEffectiveBorderRadius(node: Element): [number, number, number, number] {
+      // Resolve a CSS border-radius value to pixels. Chrome's getComputedStyle
+      // returns percentages as-is (e.g. "50%"), not resolved to px.
+      // Uses offsetWidth/offsetHeight (layout dimensions before CSS transforms)
+      // because CSS resolves percentages against the padding box, not the
+      // transformed bounding box.
+      function resolveRadius(value: string, el: Element): number {
+        if (value.includes("%")) {
+          const pct = parseFloat(value) / 100;
+          const htmlEl = el as HTMLElement;
+          const w = htmlEl.offsetWidth || 0;
+          const h = htmlEl.offsetHeight || 0;
+          return pct * Math.min(w, h);
+        }
+        return parseFloat(value) || 0;
+      }
+
+      // Check element itself (replaced elements clip to own border-radius)
+      const selfCs = window.getComputedStyle(node);
+      const selfRadii: [number, number, number, number] = [
+        resolveRadius(selfCs.borderTopLeftRadius, node),
+        resolveRadius(selfCs.borderTopRightRadius, node),
+        resolveRadius(selfCs.borderBottomRightRadius, node),
+        resolveRadius(selfCs.borderBottomLeftRadius, node),
+      ];
+      if (selfRadii[0] > 0 || selfRadii[1] > 0 || selfRadii[2] > 0 || selfRadii[3] > 0) {
+        return selfRadii;
+      }
+
+      // Walk ancestors looking for clipping container
+      let current: Element | null = node.parentElement;
+      while (current) {
+        const cs = window.getComputedStyle(current);
+        if (cs.overflow !== "visible") {
+          const tl = resolveRadius(cs.borderTopLeftRadius, current);
+          const tr = resolveRadius(cs.borderTopRightRadius, current);
+          const brr = resolveRadius(cs.borderBottomRightRadius, current);
+          const bl = resolveRadius(cs.borderBottomLeftRadius, current);
+          if (tl > 0 || tr > 0 || brr > 0 || bl > 0) {
+            return [tl, tr, brr, bl];
+          }
+        }
+        current = current.parentElement;
+      }
+      return [0, 0, 0, 0];
+    }
+
+    // Walk up the DOM multiplying each ancestor's opacity. GSAP animates
+    // opacity on wrapper divs, not directly on the video element, so the
+    // element's own opacity is often 1.0. Multiplying ancestors gives the
+    // true effective opacity.
+    function getEffectiveOpacity(node: Element): number {
+      let opacity = 1;
+      let current: Element | null = node;
+      while (current) {
+        const cs = window.getComputedStyle(current);
+        const val = parseFloat(cs.opacity);
+        // Note: `val || 1` would turn opacity:0 into 1 (0 is falsy)
+        opacity *= Number.isNaN(val) ? 1 : val;
+        current = current.parentElement;
+      }
+      return opacity;
+    }
+
+    // Compute the full CSS transform matrix from element-local coords to
+    // viewport coords by walking the offsetParent chain and accumulating
+    // position offsets + CSS transforms. This correctly handles GSAP
+    // animations on wrapper divs (rotation, scale) that getBoundingClientRect
+    // conflates into an axis-aligned bounding box.
+    function getViewportMatrix(node: Element): string {
+      const chain: HTMLElement[] = [];
+      let current: Element | null = node;
+      while (current instanceof HTMLElement) {
+        chain.push(current);
+        const next: Element | null =
+          (current.offsetParent as Element | null) ?? current.parentElement;
+        if (next === current) break;
+        current = next;
+      }
+      let mat = new DOMMatrix();
+      for (let i = chain.length - 1; i >= 0; i--) {
+        const htmlEl = chain[i];
+        if (!htmlEl) continue;
+        mat = mat.translate(htmlEl.offsetLeft, htmlEl.offsetTop);
+        const cs = window.getComputedStyle(htmlEl);
+        if (cs.transform && cs.transform !== "none") {
+          const origin = cs.transformOrigin.split(" ");
+          const ox = resolveLength(origin[0] ?? "0", htmlEl.offsetWidth);
+          const oy = resolveLength(origin[1] ?? "0", htmlEl.offsetHeight);
+          try {
+            const t = new DOMMatrix(cs.transform);
+            if (
+              Number.isFinite(t.a) &&
+              Number.isFinite(t.b) &&
+              Number.isFinite(t.c) &&
+              Number.isFinite(t.d) &&
+              Number.isFinite(t.e) &&
+              Number.isFinite(t.f)
+            ) {
+              mat = mat.translate(ox, oy).multiply(t).translate(-ox, -oy);
+            }
+          } catch {
+            // DOMMatrix constructor throws on malformed input — skip ancestor.
+          }
+        }
+      }
+      return mat.toString();
+    }
+
+    function resolveLength(value: string, basis: number): number {
+      if (value.endsWith("%")) {
+        const pct = parseFloat(value) / 100;
+        return Number.isFinite(pct) ? pct * basis : 0;
+      }
+      const n = parseFloat(value);
+      return Number.isFinite(n) ? n : 0;
+    }
+
     for (const el of elements) {
       const id = el.id;
       if (!id) continue;
       const rect = el.getBoundingClientRect();
       const style = window.getComputedStyle(el);
       const zIndex = getEffectiveZIndex(el);
-      const opacity = parseFloat(style.opacity) || 1;
+      // For HDR video elements, the frame injector sets `opacity: 0 !important`
+      // on the element itself. Start the opacity walk from the parent to get the
+      // real GSAP-animated opacity from wrapper divs.
+      const isHdrEl = hdrSet.has(id);
+      const opacityStartNode = isHdrEl ? el.parentElement : el;
+      const opacity = opacityStartNode ? getEffectiveOpacity(opacityStartNode) : 1;
       const visible =
         style.visibility !== "hidden" &&
         style.display !== "none" &&
         rect.width > 0 &&
         rect.height > 0;
+      // offsetWidth/offsetHeight only exist on HTMLElement (not on
+      // SVGElement, MathMLElement, etc.). Fall back to the bounding rect
+      // dimensions for non-HTML elements so callers always get sensible
+      // layout numbers.
+      const htmlEl = el instanceof HTMLElement ? el : null;
       results.push({
         id,
         zIndex,
@@ -309,9 +444,16 @@ export async function queryElementStacking(
         y: Math.round(rect.y),
         width: Math.round(rect.width),
         height: Math.round(rect.height),
+        layoutWidth: htmlEl?.offsetWidth || Math.round(rect.width),
+        layoutHeight: htmlEl?.offsetHeight || Math.round(rect.height),
         opacity,
         visible,
         isHdr: hdrSet.has(id),
+        // For HDR elements, use the full accumulated viewport matrix so the
+        // affine blit can apply rotation/scale/translate properly. For DOM
+        // elements, the element-level transform is sufficient for reference.
+        transform: isHdrEl ? getViewportMatrix(el) : style.transform || "none",
+        borderRadius: isHdrEl ? getEffectiveBorderRadius(el) : [0, 0, 0, 0],
       });
     }
     return results;
diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts
index bd34e6b54..4f6c16949 100644
--- a/packages/producer/src/services/renderOrchestrator.ts
+++ b/packages/producer/src/services/renderOrchestrator.ts
@@ -69,6 +69,7 @@ import {
   showVideoElements,
   queryElementStacking,
   groupIntoLayers,
+  blitRgb48leAffine,
 } from "@hyperframes/engine";
 import { join, dirname, resolve } from "path";
 import { randomUUID } from "crypto";
@@ -992,9 +993,25 @@ export async function executeRenderJob(
 
       const { execSync } = await import("child_process");
 
+      // ── Query initial element bounds for HDR extraction dimensions ──────
+      // Extract at each HDR video's display dimensions (not composition dimensions)
+      // so the source stride matches the blit dimensions. Without this, a 1200x900
+      // video element would have stride mismatch against a 1920x1080 extraction.
+      await domSession.page.evaluate((t: number) => {
+        if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t);
+      }, 0);
+      if (domSession.onBeforeCapture) {
+        await domSession.onBeforeCapture(domSession.page, 0);
+      }
+      const initialStacking = await queryElementStacking(domSession.page, nativeHdrVideoIds);
+      const hdrExtractionDims = new Map<string, { width: number; height: number }>();
+      for (const el of initialStacking) {
+        if (el.isHdr && el.width > 0 && el.height > 0) {
+          hdrExtractionDims.set(el.id, { width: el.width, height: el.height });
+        }
+      }
+
       // ── Pre-extract all HDR video frames in a single FFmpeg pass ──────
-      // Per-frame `-ss` fast seek causes duplicate frames at keyframe boundaries.
-      // A single extraction pass decodes sequentially — every frame is unique.
       const hdrFrameDirs = new Map<string, string>();
       for (const [videoId, srcPath] of hdrVideoSrcPaths) {
         const video = composition.videos.find((v) => v.id === videoId);
@@ -1002,10 +1019,11 @@ export async function executeRenderJob(
         const frameDir = join(framesDir, `hdr_${videoId}`);
         mkdirSync(frameDir, { recursive: true });
         const duration = video.end - video.start;
+        const dims = hdrExtractionDims.get(videoId) ?? { width, height };
         try {
           execSync(
             `ffmpeg -ss ${video.mediaStart} -i "${srcPath}" -t ${duration} -r ${job.config.fps} ` +
-              `-vf "scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}" ` +
+              `-vf "scale=${dims.width}:${dims.height}:force_original_aspect_ratio=increase,crop=${dims.width}:${dims.height}" ` +
               `-pix_fmt rgb48le -c:v png "${join(frameDir, "frame_%04d.png")}"`,
             { maxBuffer: 1024 * 1024, stdio: ["pipe", "pipe", "pipe"] },
           );
@@ -1073,29 +1091,63 @@ export async function executeRenderJob(
               // Frame index within the video (1-based for FFmpeg image2 output).
               // Clamp against the highest extracted frame in the directory to
               // avoid issuing an existsSync per requested time when the
-              // composition outlives the source clip.
+              // composition outlives the source clip. If the requested frame
+              // is past the end of the source, fall back to the last available
+              // frame (freeze on last frame, matching Chrome's <video> behavior).
               const videoFrameIndex = Math.round((time - video.start) * job.config.fps) + 1;
               const maxIndex = getMaxFrameIndex(frameDir);
-              const inBounds =
-                videoFrameIndex >= 1 && (maxIndex === 0 || videoFrameIndex <= maxIndex);
-              const framePath = inBounds
-                ? join(frameDir, `frame_${String(videoFrameIndex).padStart(4, "0")}.png`)
-                : null;
+              const effectiveIndex =
+                videoFrameIndex >= 1
+                  ? maxIndex > 0
+                    ? Math.min(videoFrameIndex, maxIndex)
+                    : videoFrameIndex
+                  : 0;
+              const framePath =
+                effectiveIndex >= 1
+                  ? join(frameDir, `frame_${String(effectiveIndex).padStart(4, "0")}.png`)
+                  : null;
 
               if (framePath !== null && existsSync(framePath)) {
                 try {
-                  const hdrRgb = decodePngToRgb48le(readFileSync(framePath)).data;
-                  blitRgb48leRegion(
-                    canvas,
-                    hdrRgb,
-                    el.x,
-                    el.y,
-                    el.width,
-                    el.height,
-                    width,
-                    height,
-                    el.opacity < 0.999 ? el.opacity : undefined,
-                  );
+                  const {
+                    data: hdrRgb,
+                    width: srcW,
+                    height: srcH,
+                  } = decodePngToRgb48le(readFileSync(framePath));
+
+                  // Derive the effective transform from the bounding rect.
+                  // getBoundingClientRect() already reflects all ancestor transforms
+                  // (GSAP sets transforms on wrapper divs, not video elements).
+                  const scaleX = el.width / srcW;
+                  const scaleY = el.height / srcH;
+                  const needsAffine = Math.abs(scaleX - 1) > 0.001 || Math.abs(scaleY - 1) > 0.001;
+
+                  if (needsAffine) {
+                    // Element bounds differ from extraction dimensions — scale via affine blit
+                    blitRgb48leAffine(
+                      canvas,
+                      hdrRgb,
+                      [scaleX, 0, 0, scaleY, el.x, el.y],
+                      srcW,
+                      srcH,
+                      width,
+                      height,
+                      el.opacity < 0.999 ? el.opacity : undefined,
+                    );
+                  } else {
+                    // Same dimensions — fast path with row copy
+                    blitRgb48leRegion(
+                      canvas,
+                      hdrRgb,
+                      el.x,
+                      el.y,
+                      srcW,
+                      srcH,
+                      width,
+                      height,
+                      el.opacity < 0.999 ? el.opacity : undefined,
+                    );
+                  }
                 } catch (err) {
                   log.warn("HDR layer decode/blit failed; skipping layer for frame", {
                     frameIndex: i,
@@ -1127,10 +1179,13 @@ export async function executeRenderJob(
 
               try {
                 const { data: domRgba } = decodePng(domPng);
-                // We're inside `if (hasHdrVideo)` which already required `effectiveHdr` to be set,
-                // but be defensive: fall back to HLG so we always feed the encoder a valid transfer.
-                const hdrTransfer: HdrTransfer = effectiveHdr ? effectiveHdr.transfer : "hlg";
-                blitRgba8OverRgb48le(domRgba, canvas, width, height, hdrTransfer);
+                // Invariant: `hasHdrVideo` requires `effectiveHdr` to be set (see line ~870).
+                if (!effectiveHdr) {
+                  throw new Error(
+                    "Invariant violation: effectiveHdr is undefined inside hasHdrVideo branch",
+                  );
+                }
+                blitRgba8OverRgb48le(domRgba, canvas, width, height, effectiveHdr.transfer);
               } catch (err) {
                 log.warn("DOM layer decode/blit failed; skipping overlay for frame", {
                   frameIndex: i,
diff --git a/packages/shader-transitions/src/hyper-shader.ts b/packages/shader-transitions/src/hyper-shader.ts
index 0d077ef9e..6d3b30e0e 100644
--- a/packages/shader-transitions/src/hyper-shader.ts
+++ b/packages/shader-transitions/src/hyper-shader.ts
@@ -90,6 +90,21 @@ export function init(config: HyperShaderConfig): GsapTimeline {
   const root = document.querySelector<HTMLElement>("[data-composition-id]");
   const compId = config.compositionId || root?.getAttribute("data-composition-id") || "main";
 
+  // The Hyperframes engine injects a virtual-time shim (window.__HF_VIRTUAL_TIME__)
+  // during render mode and composites every transition itself from the
+  // window.__hf.transitions metadata above. Doing GL work or html2canvas captures
+  // here would (a) waste cycles and (b) leave .scene elements stuck at opacity:0
+  // because captureScene resolves asynchronously, after the engine has already
+  // sampled the DOM. In that mode we only need to keep each scene's effective
+  // opacity correct so queryElementStacking() reports the right visibility.
+  const isEngineRenderMode =
+    typeof window !== "undefined" &&
+    Boolean((window as unknown as { __HF_VIRTUAL_TIME__?: unknown }).__HF_VIRTUAL_TIME__);
+
+  if (isEngineRenderMode) {
+    return initEngineMode(config, scenes, transitions, compId, root);
+  }
+
   const state: TransState = {
     active: false,
     prog: null,
@@ -259,3 +274,47 @@ function registerTimeline(
     w.__timelines[compId] = tl;
   }
 }
+
+// Engine-mode initialization: skip every GL/canvas/html2canvas branch and only
+// schedule deterministic opacity flips so the producer can read each scene's
+// effective opacity at any seek time. tl.set() (zero-duration tweens) is used
+// instead of tl.call() because tl.call only fires in the direction of motion —
+// the engine's warmup loop seeks forward through transition start times and
+// then the main render loop seeks back to t=0, which would leave callback-set
+// state stuck. tl.set tweens revert correctly on backward seeks.
+function initEngineMode(
+  config: HyperShaderConfig,
+  scenes: string[],
+  transitions: TransitionConfig[],
+  compId: string,
+  root: HTMLElement | null,
+): GsapTimeline {
+  const tl: GsapTimeline = config.timeline || gsap.timeline({ paused: true });
+
+  // Match the user-facing branch: when the user supplies a timeline, we
+  // anchor a no-op duration tween at 0 so the timeline length covers the
+  // composition. Without it a brand-new injected timeline would be empty.
+  if (config.timeline) {
+    const duration = Number(root?.getAttribute("data-duration") || "40");
+    tl.to({ t: 0 }, { t: 1, duration, ease: "none" }, 0);
+  }
+
+  for (let i = 0; i < transitions.length; i++) {
+    const t = transitions[i];
+    const fromId = scenes[i];
+    const toId = scenes[i + 1];
+    if (!fromId || !toId) continue;
+
+    const dur = t.duration ?? 0.7;
+    const T = t.time;
+
+    // During the transition both scenes need to be visible so the engine
+    // can composite each side; afterwards the outgoing scene must drop out
+    // so it stops contributing to the normal-frame layer composite.
+    tl.set(`#${toId}`, { opacity: 1 }, T);
+    tl.set(`#${fromId}`, { opacity: 0 }, T + dur);
+  }
+
+  registerTimeline(compId, tl, config.timeline);
+  return tl;
+}