mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
feat(hdr): GSAP transforms and border-radius masks on HDR video (#290)
## Summary HDR video elements with GSAP animations (position, scale, rotation, opacity) and CSS border-radius rendered without any transforms applied — the video just sat at (0,0) full-size. This PR adds affine transform support and rounded-corner masking for natively-composited HDR video. ## What it does **Affine blit with bilinear interpolation:** - `blitRgb48leAffine()` — Takes a 4x4 DOMMatrix and maps each destination pixel back to source coordinates via the inverse transform. Bilinear interpolation between the 4 nearest source pixels produces smooth edges under rotation and non-integer scaling. Optional opacity and border-radius parameters. - `parseTransformMatrix()` — Parses CSS `matrix(a,b,c,d,e,f)` strings into `[a,b,c,d,e,f]` tuples. **Accumulated viewport matrix:** - `getViewportMatrix()` — Walks the `offsetParent` chain from element to viewport, accumulating position offsets and CSS transforms at each level. Correctly handles `transform-origin` using the CSS sandwich: `translate(origin) × M × translate(-origin)`. This is critical because GSAP animates transforms on wrapper divs, not directly on the video element. **Effective opacity:** - `getEffectiveOpacity()` — Multiplies opacity values walking up the ancestor chain. Uses `Number.isNaN()` (not `|| 1`) so opacity:0 isn't incorrectly treated as 1. **Border-radius masks:** - `roundedRectAlpha()` — Per-pixel anti-aliased rounded-rectangle mask with support for independent corner radii. - `getEffectiveBorderRadius()` — Walks ancestors for `overflow:hidden` + border-radius. Resolves percentage values (e.g., `50%` for circles) via `offsetWidth`/`offsetHeight`. **Layout dimensions for extraction:** - Uses `offsetWidth`/`offsetHeight` (unaffected by CSS transforms) instead of `getBoundingClientRect()` (which returns the transformed bounding box and wobbles under rotation). ## Files changed | File | What changed | |------|-------------| | `packages/engine/src/utils/alphaBlit.ts` | `blitRgb48leAffine()`, `parseTransformMatrix()`, `roundedRectAlpha()`, `cornerAlpha()` | | `packages/engine/src/services/videoFrameInjector.ts` | `getViewportMatrix()`, `getEffectiveOpacity()`, `getEffectiveBorderRadius()`, `layoutWidth`/`layoutHeight` on `ElementStackingInfo` | | `packages/producer/src/services/renderOrchestrator.ts` | Affine blit path, extraction at layout dimensions, border-radius parameter passing | ## How to test Render a composition with an HDR video that has GSAP scale + rotation animation and a `border-radius: 50%` wrapper (circle mask). The video should rotate smoothly with round edges — no wobble, no sharp corners. ## Stack position **5 of 6** — Stacked on #289 (z-ordered layers). Adds transform and masking support to the HDR blit that the layer compositor uses. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
This commit is contained in:
@@ -69,6 +69,7 @@ import {
|
||||
showVideoElements,
|
||||
queryElementStacking,
|
||||
groupIntoLayers,
|
||||
blitRgb48leAffine,
|
||||
} from "@hyperframes/engine";
|
||||
import { join, dirname, resolve } from "path";
|
||||
import { randomUUID } from "crypto";
|
||||
@@ -992,9 +993,25 @@ export async function executeRenderJob(
|
||||
|
||||
const { execSync } = await import("child_process");
|
||||
|
||||
// ── Query initial element bounds for HDR extraction dimensions ──────
|
||||
// Extract at each HDR video's display dimensions (not composition dimensions)
|
||||
// so the source stride matches the blit dimensions. Without this, a 1200x900
|
||||
// video element would have stride mismatch against a 1920x1080 extraction.
|
||||
await domSession.page.evaluate((t: number) => {
|
||||
if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t);
|
||||
}, 0);
|
||||
if (domSession.onBeforeCapture) {
|
||||
await domSession.onBeforeCapture(domSession.page, 0);
|
||||
}
|
||||
const initialStacking = await queryElementStacking(domSession.page, nativeHdrVideoIds);
|
||||
const hdrExtractionDims = new Map<string, { width: number; height: number }>();
|
||||
for (const el of initialStacking) {
|
||||
if (el.isHdr && el.width > 0 && el.height > 0) {
|
||||
hdrExtractionDims.set(el.id, { width: el.width, height: el.height });
|
||||
}
|
||||
}
|
||||
|
||||
// ── Pre-extract all HDR video frames in a single FFmpeg pass ──────
|
||||
// Per-frame `-ss` fast seek causes duplicate frames at keyframe boundaries.
|
||||
// A single extraction pass decodes sequentially — every frame is unique.
|
||||
const hdrFrameDirs = new Map<string, string>();
|
||||
for (const [videoId, srcPath] of hdrVideoSrcPaths) {
|
||||
const video = composition.videos.find((v) => v.id === videoId);
|
||||
@@ -1002,10 +1019,11 @@ export async function executeRenderJob(
|
||||
const frameDir = join(framesDir, `hdr_${videoId}`);
|
||||
mkdirSync(frameDir, { recursive: true });
|
||||
const duration = video.end - video.start;
|
||||
const dims = hdrExtractionDims.get(videoId) ?? { width, height };
|
||||
try {
|
||||
execSync(
|
||||
`ffmpeg -ss ${video.mediaStart} -i "${srcPath}" -t ${duration} -r ${job.config.fps} ` +
|
||||
`-vf "scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}" ` +
|
||||
`-vf "scale=${dims.width}:${dims.height}:force_original_aspect_ratio=increase,crop=${dims.width}:${dims.height}" ` +
|
||||
`-pix_fmt rgb48le -c:v png "${join(frameDir, "frame_%04d.png")}"`,
|
||||
{ maxBuffer: 1024 * 1024, stdio: ["pipe", "pipe", "pipe"] },
|
||||
);
|
||||
@@ -1073,29 +1091,63 @@ export async function executeRenderJob(
|
||||
// Frame index within the video (1-based for FFmpeg image2 output).
|
||||
// Clamp against the highest extracted frame in the directory to
|
||||
// avoid issuing an existsSync per requested time when the
|
||||
// composition outlives the source clip.
|
||||
// composition outlives the source clip. If the requested frame
|
||||
// is past the end of the source, fall back to the last available
|
||||
// frame (freeze on last frame, matching Chrome's <video> behavior).
|
||||
const videoFrameIndex = Math.round((time - video.start) * job.config.fps) + 1;
|
||||
const maxIndex = getMaxFrameIndex(frameDir);
|
||||
const inBounds =
|
||||
videoFrameIndex >= 1 && (maxIndex === 0 || videoFrameIndex <= maxIndex);
|
||||
const framePath = inBounds
|
||||
? join(frameDir, `frame_${String(videoFrameIndex).padStart(4, "0")}.png`)
|
||||
: null;
|
||||
const effectiveIndex =
|
||||
videoFrameIndex >= 1
|
||||
? maxIndex > 0
|
||||
? Math.min(videoFrameIndex, maxIndex)
|
||||
: videoFrameIndex
|
||||
: 0;
|
||||
const framePath =
|
||||
effectiveIndex >= 1
|
||||
? join(frameDir, `frame_${String(effectiveIndex).padStart(4, "0")}.png`)
|
||||
: null;
|
||||
|
||||
if (framePath !== null && existsSync(framePath)) {
|
||||
try {
|
||||
const hdrRgb = decodePngToRgb48le(readFileSync(framePath)).data;
|
||||
blitRgb48leRegion(
|
||||
canvas,
|
||||
hdrRgb,
|
||||
el.x,
|
||||
el.y,
|
||||
el.width,
|
||||
el.height,
|
||||
width,
|
||||
height,
|
||||
el.opacity < 0.999 ? el.opacity : undefined,
|
||||
);
|
||||
const {
|
||||
data: hdrRgb,
|
||||
width: srcW,
|
||||
height: srcH,
|
||||
} = decodePngToRgb48le(readFileSync(framePath));
|
||||
|
||||
// Derive the effective transform from the bounding rect.
|
||||
// getBoundingClientRect() already reflects all ancestor transforms
|
||||
// (GSAP sets transforms on wrapper divs, not video elements).
|
||||
const scaleX = el.width / srcW;
|
||||
const scaleY = el.height / srcH;
|
||||
const needsAffine = Math.abs(scaleX - 1) > 0.001 || Math.abs(scaleY - 1) > 0.001;
|
||||
|
||||
if (needsAffine) {
|
||||
// Element bounds differ from extraction dimensions — scale via affine blit
|
||||
blitRgb48leAffine(
|
||||
canvas,
|
||||
hdrRgb,
|
||||
[scaleX, 0, 0, scaleY, el.x, el.y],
|
||||
srcW,
|
||||
srcH,
|
||||
width,
|
||||
height,
|
||||
el.opacity < 0.999 ? el.opacity : undefined,
|
||||
);
|
||||
} else {
|
||||
// Same dimensions — fast path with row copy
|
||||
blitRgb48leRegion(
|
||||
canvas,
|
||||
hdrRgb,
|
||||
el.x,
|
||||
el.y,
|
||||
srcW,
|
||||
srcH,
|
||||
width,
|
||||
height,
|
||||
el.opacity < 0.999 ? el.opacity : undefined,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn("HDR layer decode/blit failed; skipping layer for frame", {
|
||||
frameIndex: i,
|
||||
@@ -1127,10 +1179,13 @@ export async function executeRenderJob(
|
||||
|
||||
try {
|
||||
const { data: domRgba } = decodePng(domPng);
|
||||
// We're inside `if (hasHdrVideo)` which already required `effectiveHdr` to be set,
|
||||
// but be defensive: fall back to HLG so we always feed the encoder a valid transfer.
|
||||
const hdrTransfer: HdrTransfer = effectiveHdr ? effectiveHdr.transfer : "hlg";
|
||||
blitRgba8OverRgb48le(domRgba, canvas, width, height, hdrTransfer);
|
||||
// Invariant: `hasHdrVideo` requires `effectiveHdr` to be set (see line ~870).
|
||||
if (!effectiveHdr) {
|
||||
throw new Error(
|
||||
"Invariant violation: effectiveHdr is undefined inside hasHdrVideo branch",
|
||||
);
|
||||
}
|
||||
blitRgba8OverRgb48le(domRgba, canvas, width, height, effectiveHdr.transfer);
|
||||
} catch (err) {
|
||||
log.warn("DOM layer decode/blit failed; skipping overlay for frame", {
|
||||
frameIndex: i,
|
||||
|
||||
Reference in New Issue
Block a user