Files
hyperframes/packages/core/src/compiler/timingCompiler.ts
T
Miguel Ángel 31e8144304 fix: render parity for transparent looped videos (#478)
## Summary
- preserve alpha for render-injected video frames by detecting alpha streams with ffprobe and extracting alpha video frames as PNG
- keep `<video loop>` semantics through static parsing, compiler duration resolution, browser media discovery, and render frame lookup
- fail embedded preview startup before opening a broken browser page when the Studio bundle is missing
- align snapshot frame injection with looped media timing and VP9 alpha extraction

## Why
The Studio preview and rendered MP4 could disagree for timed transparent looped videos. The Comfy funding composition exposed two separate parity bugs: render-injected frames needed alpha-preserving PNG extraction, and the compiler was clamping a looped `data-duration="4"` video down to the 3.125s source duration. After the first source cycle, render lookup treated the video as inactive, hid the native video, and produced the blank polygon/glow the user saw around the rounded `0:03` mark.

`hyperframes lint` and `hyperframes validate` did not catch this because they check syntax/load/console/accessibility, not preview-vs-render visual parity. This PR adds regression coverage for the compiler loop-duration path and frame lookup path.

## Verification
- `bun run --filter @hyperframes/core test -- src/compiler/timingCompiler.test.ts src/compiler/htmlCompiler.test.ts`
- `bun test packages/producer/src/services/htmlCompiler.test.ts`
- `bun run --filter @hyperframes/engine test -- videoFrameExtractor ffprobe`
- `bun run --filter @hyperframes/core typecheck`
- `bun run --filter @hyperframes/engine typecheck`
- `bun run --filter @hyperframes/producer typecheck`
- `bun run --filter @hyperframes/cli typecheck`
- `bun run lint`
- `bun run format:check ...` on touched files
- Comfy project: `node packages/cli/dist/cli.js validate` -> no console errors, 44 text elements pass WCAG AA
- Comfy project patched render from source: `/tmp/comfy-render-compare/fixed6-comfy.mp4`, 1920x1080, 30fps, 21.8s, 654 frames
- 3.00s-3.97s render contact sheet: `/tmp/comfy-render-compare/fixed6-window-contact.png`
- targeted fixed render capture at 3.733s: `/tmp/comfy-render-compare/probe-capture-fixed/captured/frame_000112.jpg`
- agent-browser Studio proof screenshot at 3.7s: `/tmp/comfy-render-compare/agent-browser-studio-3_7-fixed.png`
- agent-browser-driven recording of 3s seek pass: `/tmp/comfy-render-compare/agent-browser-wysiwyg-3s-fixed.webm`

Note: `bun run --filter @hyperframes/cli dev -- validate` is blocked in source mode by the existing `contrast-audit.browser.js` default-export loader issue; packaged `node packages/cli/dist/cli.js validate` passes for this project.
2026-04-24 23:18:20 +02:00

264 lines
8.4 KiB
TypeScript

/**
* Timing Compiler
*
* Shared, pure HTML compilation that normalizes timing attributes.
* Works in both Node.js and browser (no dependencies, regex-based).
*
* Guarantees every timed element gets:
* - id on media elements when missing
* - data-end (computed from data-start + data-duration when possible)
* - data-has-audio="true" on <video> elements
*
* For elements without data-duration (e.g. videos relying on source duration),
* this compiler identifies them as "unresolved" so the caller can provide
* durations via an environment-specific resolver (ffprobe, el.duration, etc.)
* and call injectDurations() to complete the compilation.
*/
// ── Types ────────────────────────────────────────────────────────────────
export interface UnresolvedElement {
id: string;
tagName: string;
src?: string;
start: number;
end?: number;
duration?: number;
mediaStart: number;
compositionSrc?: string;
}
export interface ResolvedDuration {
id: string;
duration: number;
}
export interface ResolvedMediaElement {
id: string;
tagName: string;
src?: string;
start: number;
duration: number;
mediaStart: number;
loop: boolean;
}
export interface CompilationResult {
html: string;
unresolved: UnresolvedElement[];
}
// ── Helpers ──────────────────────────────────────────────────────────────
function getAttr(tag: string, attr: string): string | null {
const match = tag.match(new RegExp(`${attr}=["']([^"']+)["']`));
return match ? (match[1] ?? null) : null;
}
function hasAttr(tag: string, attr: string): boolean {
return new RegExp(`\\s${attr}(?:\\s|=|>|/)`).test(tag);
}
function injectAttr(tag: string, attr: string, value: string): string {
return tag.replace(/>$/, ` ${attr}="${value}">`);
}
// ── Core compilation ─────────────────────────────────────────────────────
function compileTag(
tag: string,
isVideo: boolean,
generateId: () => number,
): { tag: string; unresolved: UnresolvedElement | null } {
let result = tag;
let unresolved: UnresolvedElement | null = null;
let id = getAttr(result, "id");
if (!id) {
id = `${isVideo ? "hf-video" : "hf-audio"}-${generateId()}`;
result = injectAttr(result, "id", id);
}
const startStr = getAttr(result, "data-start");
const start = startStr !== null ? parseFloat(startStr) : 0;
const mediaStartStr = getAttr(result, "data-media-start");
const mediaStart = mediaStartStr ? parseFloat(mediaStartStr) : 0;
// 1. Compute data-end from data-start + data-duration
if (!hasAttr(result, "data-end")) {
const durationStr = getAttr(result, "data-duration");
if (durationStr !== null) {
const end = start + parseFloat(durationStr);
result = injectAttr(result, "data-end", String(end));
} else if (id) {
// No data-duration: mark as unresolved so caller can provide it
unresolved = {
id,
tagName: isVideo ? "video" : "audio",
src: getAttr(result, "src") ?? undefined,
start,
mediaStart,
};
}
}
// 2. Add data-has-audio="true" to <video> elements
if (isVideo && !hasAttr(result, "data-has-audio")) {
result = injectAttr(result, "data-has-audio", "true");
}
return { tag: result, unresolved };
}
/**
* Compile timing attributes in HTML.
*
* Phase 1 (static): Adds data-end where data-duration exists,
* adds data-has-audio on videos.
*
* Returns the compiled HTML and a list of elements that could not be
* resolved statically (missing data-duration). The caller should resolve
* these via ffprobe / el.duration and call injectDurations().
*/
export function compileTimingAttrs(html: string): CompilationResult {
const unresolved: UnresolvedElement[] = [];
let nextVideoId = 0;
let nextAudioId = 0;
// Process <video ...> tags
html = html.replace(/<video[^>]*>/gi, (match) => {
const { tag, unresolved: u } = compileTag(match, true, () => nextVideoId++);
if (u) unresolved.push(u);
return tag;
});
// Process <audio ...> tags
html = html.replace(/<audio[^>]*>/gi, (match) => {
const { tag, unresolved: u } = compileTag(match, false, () => nextAudioId++);
if (u) unresolved.push(u);
return tag;
});
// Identify unresolved timed elements (divs with data-start but no data-end/data-duration)
// These are typically compositions whose duration depends on GSAP timelines
html.replace(/<(?:div|section)[^>]*>/gi, (match) => {
if (!hasAttr(match, "data-start")) return match;
if (hasAttr(match, "data-end") || hasAttr(match, "data-duration")) return match;
const id = getAttr(match, "id");
const compositionSrc = getAttr(match, "data-composition-src");
if (id) {
const startStr = getAttr(match, "data-start");
unresolved.push({
id,
tagName: "div",
start: startStr ? parseFloat(startStr) : 0,
mediaStart: 0,
compositionSrc: compositionSrc ?? undefined,
});
}
return match;
});
return { html, unresolved };
}
/**
* Inject resolved durations into compiled HTML.
*
* For each resolved element, adds data-duration and data-end attributes.
* Call this after resolving durations via ffprobe, el.duration, or
* GSAP timeline queries.
*/
export function injectDurations(html: string, resolutions: ResolvedDuration[]): string {
for (const { id, duration } of resolutions) {
// Match the element's opening tag by id
const idPattern = new RegExp(`(<[^>]*id=["']${escapeRegex(id)}["'][^>]*>)`, "gi");
html = html.replace(idPattern, (tag) => {
let result = tag;
// Add data-duration if missing
if (!hasAttr(result, "data-duration")) {
result = injectAttr(result, "data-duration", String(duration));
}
// Add data-end if missing
if (!hasAttr(result, "data-end")) {
const startStr = getAttr(result, "data-start");
const start = startStr ? parseFloat(startStr) : 0;
result = injectAttr(result, "data-end", String(start + duration));
}
return result;
});
}
return html;
}
function escapeRegex(str: string): string {
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
/**
* Extract video/audio elements that already have data-duration set.
* Used by callers to validate declared durations against actual source durations.
*/
export function extractResolvedMedia(html: string): ResolvedMediaElement[] {
const resolved: ResolvedMediaElement[] = [];
const mediaRegex = /<(?:video|audio)[^>]*>/gi;
let match: RegExpExecArray | null;
while ((match = mediaRegex.exec(html)) !== null) {
const tag = match[0];
const id = getAttr(tag, "id");
const durationStr = getAttr(tag, "data-duration");
if (!id || durationStr === null) continue;
const duration = parseFloat(durationStr);
if (!Number.isFinite(duration) || duration <= 0) continue;
const isVideo = /^<video/i.test(tag);
const startStr = getAttr(tag, "data-start");
const mediaStartStr = getAttr(tag, "data-media-start");
resolved.push({
id,
tagName: isVideo ? "video" : "audio",
src: getAttr(tag, "src") ?? undefined,
start: startStr !== null ? parseFloat(startStr) : 0,
duration,
mediaStart: mediaStartStr ? parseFloat(mediaStartStr) : 0,
loop: hasAttr(tag, "loop"),
});
}
return resolved;
}
/**
* Clamp existing data-duration and data-end on media elements.
* For each resolution, replaces the declared duration with the clamped value
* and recomputes data-end accordingly.
*/
export function clampDurations(html: string, clamps: ResolvedDuration[]): string {
for (const { id, duration } of clamps) {
const idPattern = new RegExp(`(<[^>]*id=["']${escapeRegex(id)}["'][^>]*>)`, "gi");
html = html.replace(idPattern, (tag) => {
// Replace data-duration value
tag = tag.replace(/data-duration=["'][^"']*["']/, `data-duration="${duration}"`);
// Recompute data-end from data-start + clamped duration
const startStr = getAttr(tag, "data-start");
const start = startStr ? parseFloat(startStr) : 0;
tag = tag.replace(/data-end=["'][^"']*["']/, `data-end="${start + duration}"`);
return tag;
});
}
return html;
}