mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-10 22:20:14 +00:00
fix: render parity for transparent looped videos (#478)
## Summary - preserve alpha for render-injected video frames by detecting alpha streams with ffprobe and extracting alpha video frames as PNG - keep `<video loop>` semantics through static parsing, compiler duration resolution, browser media discovery, and render frame lookup - fail embedded preview startup before opening a broken browser page when the Studio bundle is missing - align snapshot frame injection with looped media timing and VP9 alpha extraction ## Why The Studio preview and rendered MP4 could disagree for timed transparent looped videos. The Comfy funding composition exposed two separate parity bugs: render-injected frames needed alpha-preserving PNG extraction, and the compiler was clamping a looped `data-duration="4"` video down to the 3.125s source duration. After the first source cycle, render lookup treated the video as inactive, hid the native video, and produced the blank polygon/glow the user saw around the rounded `0:03` mark. `hyperframes lint` and `hyperframes validate` did not catch this because they check syntax/load/console/accessibility, not preview-vs-render visual parity. This PR adds regression coverage for the compiler loop-duration path and frame lookup path. ## Verification - `bun run --filter @hyperframes/core test -- src/compiler/timingCompiler.test.ts src/compiler/htmlCompiler.test.ts` - `bun test packages/producer/src/services/htmlCompiler.test.ts` - `bun run --filter @hyperframes/engine test -- videoFrameExtractor ffprobe` - `bun run --filter @hyperframes/core typecheck` - `bun run --filter @hyperframes/engine typecheck` - `bun run --filter @hyperframes/producer typecheck` - `bun run --filter @hyperframes/cli typecheck` - `bun run lint` - `bun run format:check ...` on touched files - Comfy project: `node packages/cli/dist/cli.js validate` -> no console errors, 44 text elements pass WCAG AA - Comfy project patched render from source: `/tmp/comfy-render-compare/fixed6-comfy.mp4`, 1920x1080, 30fps, 21.8s, 654 frames - 3.00s-3.97s render contact sheet: `/tmp/comfy-render-compare/fixed6-window-contact.png` - targeted fixed render capture at 3.733s: `/tmp/comfy-render-compare/probe-capture-fixed/captured/frame_000112.jpg` - agent-browser Studio proof screenshot at 3.7s: `/tmp/comfy-render-compare/agent-browser-studio-3_7-fixed.png` - agent-browser-driven recording of 3s seek pass: `/tmp/comfy-render-compare/agent-browser-wysiwyg-3s-fixed.webm` Note: `bun run --filter @hyperframes/cli dev -- validate` is blocked in source mode by the existing `contrast-audit.browser.js` default-export loader issue; packaged `node packages/cli/dist/cli.js validate` passes for this project.
This commit is contained in:
@@ -25,6 +25,7 @@ const FFMPEG_EXTRACT_TIMEOUT_MS = 30_000;
|
||||
async function extractVideoFrameToBuffer(
|
||||
videoPath: string,
|
||||
timeSeconds: number,
|
||||
useVp9AlphaDecoder = false,
|
||||
): Promise<Buffer | null> {
|
||||
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
|
||||
const outPath = join(tmp, "frame.png");
|
||||
@@ -33,10 +34,11 @@ async function extractVideoFrameToBuffer(
|
||||
(resolvePromise) => {
|
||||
// `-ss` before `-i` performs a fast keyframe seek; adequate for snapshot accuracy
|
||||
// (±1 frame) and orders of magnitude faster than the decode-and-scan alternative.
|
||||
const ff = spawn("ffmpeg", [
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
const args = ["-hide_banner", "-loglevel", "error"];
|
||||
if (useVp9AlphaDecoder) {
|
||||
args.push("-c:v", "libvpx-vp9");
|
||||
}
|
||||
args.push(
|
||||
"-ss",
|
||||
String(Math.max(0, timeSeconds)),
|
||||
"-i",
|
||||
@@ -47,7 +49,8 @@ async function extractVideoFrameToBuffer(
|
||||
"2",
|
||||
"-y",
|
||||
outPath,
|
||||
]);
|
||||
);
|
||||
const ff = spawn("ffmpeg", args);
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
@@ -252,19 +255,36 @@ async function captureSnapshots(
|
||||
updates: Array<{ videoId: string; dataUri: string }>,
|
||||
) => Promise<void>;
|
||||
type SyncVisibilityFn = (page: unknown, activeVideoIds: string[]) => Promise<void>;
|
||||
type ExtractMediaMetadataFn = (
|
||||
filePath: string,
|
||||
) => Promise<{ videoCodec: string; hasAlpha: boolean }>;
|
||||
let injectVideoFramesBatch: InjectFn | null = null;
|
||||
let syncVideoFrameVisibility: SyncVisibilityFn | null = null;
|
||||
let extractMediaMetadata: ExtractMediaMetadataFn | null = null;
|
||||
try {
|
||||
const engine = (await import("@hyperframes/engine")) as {
|
||||
injectVideoFramesBatch: InjectFn;
|
||||
syncVideoFrameVisibility: SyncVisibilityFn;
|
||||
extractMediaMetadata: ExtractMediaMetadataFn;
|
||||
};
|
||||
injectVideoFramesBatch = engine.injectVideoFramesBatch;
|
||||
syncVideoFrameVisibility = engine.syncVideoFrameVisibility;
|
||||
extractMediaMetadata = engine.extractMediaMetadata;
|
||||
} catch {
|
||||
// Engine unavailable in this install — snapshot will still run, and
|
||||
// compositions without <video data-start> get exactly the old behaviour.
|
||||
}
|
||||
const alphaDecoderCache = new Map<string, Promise<boolean>>();
|
||||
const shouldUseVp9AlphaDecoder = (filePath: string): Promise<boolean> => {
|
||||
if (!extractMediaMetadata) return Promise.resolve(false);
|
||||
const cached = alphaDecoderCache.get(filePath);
|
||||
if (cached) return cached;
|
||||
const pending = extractMediaMetadata(filePath)
|
||||
.then((meta) => meta.hasAlpha && meta.videoCodec === "vp9")
|
||||
.catch(() => false);
|
||||
alphaDecoderCache.set(filePath, pending);
|
||||
return pending;
|
||||
};
|
||||
|
||||
// Seek and capture each frame
|
||||
for (let i = 0; i < positions.length; i++) {
|
||||
@@ -324,7 +344,10 @@ async function captureSnapshots(
|
||||
: srcDur > 0
|
||||
? Math.max(0, (srcDur - mediaStart) / playbackRate)
|
||||
: Number.POSITIVE_INFINITY;
|
||||
const relTime = (t - start) * playbackRate + mediaStart;
|
||||
let relTime = (t - start) * playbackRate + mediaStart;
|
||||
if (v.loop && srcDur > mediaStart && relTime >= srcDur) {
|
||||
relTime = mediaStart + ((relTime - mediaStart) % (srcDur - mediaStart));
|
||||
}
|
||||
const activeNow = t >= start && t < start + duration && relTime >= 0 && !!v.id;
|
||||
return {
|
||||
id: v.id,
|
||||
@@ -356,7 +379,11 @@ async function captureSnapshots(
|
||||
/* unresolvable src (e.g. blob:, data:) — skip */
|
||||
}
|
||||
if (!filePath) continue;
|
||||
const png = await extractVideoFrameToBuffer(filePath, Math.max(0, v.relTime));
|
||||
const png = await extractVideoFrameToBuffer(
|
||||
filePath,
|
||||
Math.max(0, v.relTime),
|
||||
await shouldUseVp9AlphaDecoder(filePath),
|
||||
);
|
||||
if (!png) continue;
|
||||
updates.push({
|
||||
videoId: v.id,
|
||||
|
||||
Reference in New Issue
Block a user