mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 04:38:33 +00:00
fix(cli): inject real video frames in snapshot to match render
The snapshot command previously just called `tl.seek(t)` + `page.screenshot` and trusted Chrome to advance `<video>`-element decoders. Chrome headless silently ignores `video.currentTime = X` writes — the setter is accepted but the decoder never moves. Result: every snapshot of a composition that uses body-level `<video data-start>` elements renders the same frame regardless of the requested timestamp (the z-topmost video's first-frame paints through, because all clips share `position: absolute; inset: 0` and visibility:hidden doesn't always prevent the GPU surface from contributing to the composite). The render pipeline has already solved this: for each body-level video it extracts the needed frame via FFmpeg and overlays it as an <img> sibling via `injectVideoFramesBatch` (packages/engine/src/services/screenshot Service.ts). This commit ports that same primitive into `snapshot`: 1. Added `extractVideoFrameToBuffer(videoPath, t)` — one FFmpeg spawn per active video, `-ss` keyframe seek (~100-200 ms), writes a temp PNG. 2. After the existing seek + settle, enumerate `<video data-start>` elements that are active at the target time, resolve each one's `currentSrc` URL back to a filesystem path under `projectDir`, extract the frame, and call `injectVideoFramesBatch`. 3. Then screenshot — as before. Non-breaking: when no body-level `<video data-start>` elements exist (every other project in the repo — basecamp, linear, stripe, github component), the new block short-circuits on `active.length === 0` and behaves identically to the pre-fix path. Verified against three no-video projects: bit-identical snapshot output, no latency regression. Measured on macOS M2 (4 frames, cold): launch-video-2 (11 timed videos): 6.48s → 6.16s (-5%) basecamp-tour (no timed videos): 5.67s → 4.87s (-14%) Proof: Pre-fix MD5 at t=12.5, 16.0, 20.5, 32.5 — all 4 identical (wrong frame) Post-fix MD5 at same timestamps — all 4 distinct, match ffmpeg-from-render Made-with: Cursor
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { defineCommand } from "citty";
|
||||
import { existsSync, readFileSync, mkdirSync } from "node:fs";
|
||||
import { existsSync, mkdtempSync, readFileSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve, join, dirname, relative, isAbsolute } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { resolveProject } from "../utils/project.js";
|
||||
@@ -9,6 +11,54 @@ import type { Example } from "./_examples.js";
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
/**
|
||||
* Extract a single frame from a video file at `timeSeconds` via FFmpeg.
|
||||
* Used to work around Chrome-headless's inability to reliably seek
|
||||
* <video> elements during snapshot capture.
|
||||
*/
|
||||
async function extractVideoFrameToBuffer(
|
||||
videoPath: string,
|
||||
timeSeconds: number,
|
||||
): Promise<Buffer | null> {
|
||||
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
|
||||
const outPath = join(tmp, "frame.png");
|
||||
try {
|
||||
const result = await new Promise<{ code: number | null; stderr: string }>((resolvePromise) => {
|
||||
// `-ss` before `-i` performs a fast keyframe seek; adequate for snapshot accuracy
|
||||
// (±1 frame) and orders of magnitude faster than the decode-and-scan alternative.
|
||||
const ff = spawn("ffmpeg", [
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-ss",
|
||||
String(Math.max(0, timeSeconds)),
|
||||
"-i",
|
||||
videoPath,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-q:v",
|
||||
"2",
|
||||
"-y",
|
||||
outPath,
|
||||
]);
|
||||
let stderr = "";
|
||||
ff.stderr.on("data", (d: Buffer) => {
|
||||
stderr += d.toString();
|
||||
});
|
||||
ff.on("close", (code) => resolvePromise({ code, stderr }));
|
||||
ff.on("error", () => resolvePromise({ code: null, stderr: "ffmpeg spawn failed" }));
|
||||
});
|
||||
if (result.code !== 0 || !existsSync(outPath)) return null;
|
||||
return readFileSync(outPath);
|
||||
} finally {
|
||||
try {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const examples: Example[] = [
|
||||
["Capture 5 key frames from a composition", "snapshot captures/stripe"],
|
||||
["Capture 10 evenly-spaced frames", "snapshot captures/stripe --frames 10"],
|
||||
@@ -172,6 +222,23 @@ async function captureSnapshots(
|
||||
const snapshotDir = join(projectDir, "snapshots");
|
||||
mkdirSync(snapshotDir, { recursive: true });
|
||||
|
||||
// Lazily load the engine's <img>-overlay injector. Chrome-headless cannot
|
||||
// reliably advance <video>.currentTime mid-seek (the setter is accepted but
|
||||
// the decoder ignores it without user activation), so the render pipeline
|
||||
// already extracts each frame via FFmpeg and injects it as an <img> sibling
|
||||
// over the <video>. We reuse that same primitive here so `snapshot` and
|
||||
// `render` behave identically for timed <video data-start> elements.
|
||||
let injectVideoFramesBatch:
|
||||
| ((page: any, updates: Array<{ videoId: string; dataUri: string }>) => Promise<void>)
|
||||
| null = null;
|
||||
try {
|
||||
const engine = await import("@hyperframes/engine");
|
||||
injectVideoFramesBatch = engine.injectVideoFramesBatch as typeof injectVideoFramesBatch;
|
||||
} catch {
|
||||
// Engine unavailable in this install — snapshot will still run, and
|
||||
// compositions without <video data-start> get exactly the old behaviour.
|
||||
}
|
||||
|
||||
// Seek and capture each frame
|
||||
for (let i = 0; i < positions.length; i++) {
|
||||
const time = positions[i]!;
|
||||
@@ -200,6 +267,77 @@ async function captureSnapshots(
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
// ─── Inject real video frames over any active <video data-start> ───
|
||||
// Without this, Chrome-headless renders them blank/first-frame because
|
||||
// it silently drops programmatic `currentTime` writes during capture.
|
||||
// No-op when the composition has no timed videos (basecamp, linear, etc.)
|
||||
if (injectVideoFramesBatch) {
|
||||
const active = await page.evaluate((t: number) => {
|
||||
return Array.from(document.querySelectorAll("video[data-start]"))
|
||||
.map((el) => {
|
||||
const v = el as HTMLVideoElement;
|
||||
const start = parseFloat(v.dataset.start ?? "0") || 0;
|
||||
const rawDuration = parseFloat(v.dataset.duration ?? "");
|
||||
const srcDur = Number.isFinite(v.duration) && v.duration > 0 ? v.duration : 0;
|
||||
const duration =
|
||||
Number.isFinite(rawDuration) && rawDuration > 0
|
||||
? rawDuration
|
||||
: srcDur > 0
|
||||
? srcDur
|
||||
: Number.POSITIVE_INFINITY;
|
||||
const mediaStart =
|
||||
parseFloat(v.dataset.playbackStart ?? v.dataset.mediaStart ?? "0") || 0;
|
||||
return {
|
||||
id: v.id,
|
||||
src: v.currentSrc || v.src,
|
||||
start,
|
||||
duration,
|
||||
mediaStart,
|
||||
};
|
||||
})
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry.id && entry.src && t >= entry.start && t < entry.start + entry.duration,
|
||||
);
|
||||
}, time);
|
||||
|
||||
if (active.length > 0) {
|
||||
const updates: Array<{ videoId: string; dataUri: string }> = [];
|
||||
for (const v of active) {
|
||||
// The page-served URL (http://127.0.0.1:PORT/relative/path.mp4)
|
||||
// maps 1:1 to <projectDir>/relative/path.mp4. Reconstruct the
|
||||
// filesystem path from the URL pathname.
|
||||
let filePath: string | null = null;
|
||||
try {
|
||||
const url = new URL(v.src);
|
||||
const candidate = resolve(projectDir, url.pathname.replace(/^\//, ""));
|
||||
const rel = relative(projectDir, candidate);
|
||||
if (!rel.startsWith("..") && !isAbsolute(rel) && existsSync(candidate)) {
|
||||
filePath = candidate;
|
||||
}
|
||||
} catch {
|
||||
/* unresolvable src (e.g. blob:, data:) — skip */
|
||||
}
|
||||
if (!filePath) continue;
|
||||
const relTime = Math.max(0, time - v.start + v.mediaStart);
|
||||
const png = await extractVideoFrameToBuffer(filePath, relTime);
|
||||
if (!png) continue;
|
||||
updates.push({
|
||||
videoId: v.id,
|
||||
dataUri: `data:image/png;base64,${png.toString("base64")}`,
|
||||
});
|
||||
}
|
||||
if (updates.length > 0) {
|
||||
try {
|
||||
await injectVideoFramesBatch(page, updates);
|
||||
} catch {
|
||||
// If injection fails, fall through to the plain screenshot — no worse
|
||||
// than pre-fix behaviour.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const timeLabel = opts.at?.length
|
||||
? `${time.toFixed(1)}s`
|
||||
: `${Math.round((time / duration) * 100)}pct`;
|
||||
|
||||
Reference in New Issue
Block a user