fix: render parity for transparent looped videos (#478)

## Summary
- preserve alpha for render-injected video frames by detecting alpha streams with ffprobe and extracting alpha video frames as PNG
- keep `<video loop>` semantics through static parsing, compiler duration resolution, browser media discovery, and render frame lookup
- fail embedded preview startup before opening a broken browser page when the Studio bundle is missing
- align snapshot frame injection with looped media timing and VP9 alpha extraction

## Why
The Studio preview and rendered MP4 could disagree for timed transparent looped videos. The Comfy funding composition exposed two separate parity bugs: render-injected frames needed alpha-preserving PNG extraction, and the compiler was clamping a looped `data-duration="4"` video down to the 3.125s source duration. After the first source cycle, render lookup treated the video as inactive, hid the native video, and produced the blank polygon/glow the user saw around the rounded `0:03` mark.

`hyperframes lint` and `hyperframes validate` did not catch this because they check syntax/load/console/accessibility, not preview-vs-render visual parity. This PR adds regression coverage for the compiler loop-duration path and frame lookup path.

## Verification
- `bun run --filter @hyperframes/core test -- src/compiler/timingCompiler.test.ts src/compiler/htmlCompiler.test.ts`
- `bun test packages/producer/src/services/htmlCompiler.test.ts`
- `bun run --filter @hyperframes/engine test -- videoFrameExtractor ffprobe`
- `bun run --filter @hyperframes/core typecheck`
- `bun run --filter @hyperframes/engine typecheck`
- `bun run --filter @hyperframes/producer typecheck`
- `bun run --filter @hyperframes/cli typecheck`
- `bun run lint`
- `bun run format:check ...` on touched files
- Comfy project: `node packages/cli/dist/cli.js validate` -> no console errors, 44 text elements pass WCAG AA
- Comfy project patched render from source: `/tmp/comfy-render-compare/fixed6-comfy.mp4`, 1920x1080, 30fps, 21.8s, 654 frames
- 3.00s-3.97s render contact sheet: `/tmp/comfy-render-compare/fixed6-window-contact.png`
- targeted fixed render capture at 3.733s: `/tmp/comfy-render-compare/probe-capture-fixed/captured/frame_000112.jpg`
- agent-browser Studio proof screenshot at 3.7s: `/tmp/comfy-render-compare/agent-browser-studio-3_7-fixed.png`
- agent-browser-driven recording of 3s seek pass: `/tmp/comfy-render-compare/agent-browser-wysiwyg-3s-fixed.webm`

Note: `bun run --filter @hyperframes/cli dev -- validate` is blocked in source mode by the existing `contrast-audit.browser.js` default-export loader issue; packaged `node packages/cli/dist/cli.js validate` passes for this project.
This commit is contained in:
Miguel Ángel
2026-04-24 23:18:20 +02:00
committed by GitHub
parent e8c43f0889
commit 31e8144304
13 changed files with 384 additions and 27 deletions
+34 -7
View File
@@ -25,6 +25,7 @@ const FFMPEG_EXTRACT_TIMEOUT_MS = 30_000;
async function extractVideoFrameToBuffer(
videoPath: string,
timeSeconds: number,
useVp9AlphaDecoder = false,
): Promise<Buffer | null> {
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
const outPath = join(tmp, "frame.png");
@@ -33,10 +34,11 @@ async function extractVideoFrameToBuffer(
(resolvePromise) => {
// `-ss` before `-i` performs a fast keyframe seek; adequate for snapshot accuracy
// (±1 frame) and orders of magnitude faster than the decode-and-scan alternative.
const ff = spawn("ffmpeg", [
"-hide_banner",
"-loglevel",
"error",
const args = ["-hide_banner", "-loglevel", "error"];
if (useVp9AlphaDecoder) {
args.push("-c:v", "libvpx-vp9");
}
args.push(
"-ss",
String(Math.max(0, timeSeconds)),
"-i",
@@ -47,7 +49,8 @@ async function extractVideoFrameToBuffer(
"2",
"-y",
outPath,
]);
);
const ff = spawn("ffmpeg", args);
let stderr = "";
let timedOut = false;
const timer = setTimeout(() => {
@@ -252,19 +255,36 @@ async function captureSnapshots(
updates: Array<{ videoId: string; dataUri: string }>,
) => Promise<void>;
type SyncVisibilityFn = (page: unknown, activeVideoIds: string[]) => Promise<void>;
type ExtractMediaMetadataFn = (
filePath: string,
) => Promise<{ videoCodec: string; hasAlpha: boolean }>;
let injectVideoFramesBatch: InjectFn | null = null;
let syncVideoFrameVisibility: SyncVisibilityFn | null = null;
let extractMediaMetadata: ExtractMediaMetadataFn | null = null;
try {
const engine = (await import("@hyperframes/engine")) as {
injectVideoFramesBatch: InjectFn;
syncVideoFrameVisibility: SyncVisibilityFn;
extractMediaMetadata: ExtractMediaMetadataFn;
};
injectVideoFramesBatch = engine.injectVideoFramesBatch;
syncVideoFrameVisibility = engine.syncVideoFrameVisibility;
extractMediaMetadata = engine.extractMediaMetadata;
} catch {
// Engine unavailable in this install — snapshot will still run, and
// compositions without <video data-start> get exactly the old behaviour.
}
const alphaDecoderCache = new Map<string, Promise<boolean>>();
const shouldUseVp9AlphaDecoder = (filePath: string): Promise<boolean> => {
if (!extractMediaMetadata) return Promise.resolve(false);
const cached = alphaDecoderCache.get(filePath);
if (cached) return cached;
const pending = extractMediaMetadata(filePath)
.then((meta) => meta.hasAlpha && meta.videoCodec === "vp9")
.catch(() => false);
alphaDecoderCache.set(filePath, pending);
return pending;
};
// Seek and capture each frame
for (let i = 0; i < positions.length; i++) {
@@ -324,7 +344,10 @@ async function captureSnapshots(
: srcDur > 0
? Math.max(0, (srcDur - mediaStart) / playbackRate)
: Number.POSITIVE_INFINITY;
const relTime = (t - start) * playbackRate + mediaStart;
let relTime = (t - start) * playbackRate + mediaStart;
if (v.loop && srcDur > mediaStart && relTime >= srcDur) {
relTime = mediaStart + ((relTime - mediaStart) % (srcDur - mediaStart));
}
const activeNow = t >= start && t < start + duration && relTime >= 0 && !!v.id;
return {
id: v.id,
@@ -356,7 +379,11 @@ async function captureSnapshots(
/* unresolvable src (e.g. blob:, data:) — skip */
}
if (!filePath) continue;
const png = await extractVideoFrameToBuffer(filePath, Math.max(0, v.relTime));
const png = await extractVideoFrameToBuffer(
filePath,
Math.max(0, v.relTime),
await shouldUseVp9AlphaDecoder(filePath),
);
if (!png) continue;
updates.push({
videoId: v.id,