fix(cli): inject real video frames in snapshot to match render

The snapshot command previously just called `tl.seek(t)` + `page.screenshot`
and trusted Chrome to advance `<video>`-element decoders. Chrome headless
silently ignores `video.currentTime = X` writes — the setter is accepted
but the decoder never moves. Result: every snapshot of a composition that
uses body-level `<video data-start>` elements renders the same frame
regardless of the requested timestamp (the z-topmost video's first-frame
paints through, because all clips share `position: absolute; inset: 0`
and visibility:hidden doesn't always prevent the GPU surface from
contributing to the composite).

The render pipeline has already solved this: for each body-level video it
extracts the needed frame via FFmpeg and overlays it as an <img> sibling
via `injectVideoFramesBatch` (packages/engine/src/services/screenshot
Service.ts). This commit ports that same primitive into `snapshot`:

1. Added `extractVideoFrameToBuffer(videoPath, t)` — one FFmpeg spawn per
   active video, `-ss` keyframe seek (~100-200 ms), writes a temp PNG.
2. After the existing seek + settle, enumerate `<video data-start>`
   elements that are active at the target time, resolve each one's
   `currentSrc` URL back to a filesystem path under `projectDir`, extract
   the frame, and call `injectVideoFramesBatch`.
3. Then screenshot — as before.

Non-breaking: when no body-level `<video data-start>` elements exist (every
other project in the repo — basecamp, linear, stripe, github component), the
new block short-circuits on `active.length === 0` and behaves identically
to the pre-fix path. Verified against three no-video projects: bit-identical
snapshot output, no latency regression.

Measured on macOS M2 (4 frames, cold):
  launch-video-2 (11 timed videos): 6.48s → 6.16s   (-5%)
  basecamp-tour  (no timed videos): 5.67s → 4.87s   (-14%)

Proof: Pre-fix MD5 at t=12.5, 16.0, 20.5, 32.5 — all 4 identical (wrong frame)
  Post-fix MD5 at same timestamps  — all 4 distinct, match ffmpeg-from-render
Made-with: Cursor
This commit is contained in:
ukimsanov
2026-04-20 09:36:33 -04:00
parent 62ce3679e6
commit a3e7c6176f
+139 -1
View File
@@ -1,5 +1,7 @@
import { spawn } from "node:child_process";
import { defineCommand } from "citty";
import { existsSync, readFileSync, mkdirSync } from "node:fs";
import { existsSync, mkdtempSync, readFileSync, mkdirSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { resolve, join, dirname, relative, isAbsolute } from "node:path";
import { fileURLToPath } from "node:url";
import { resolveProject } from "../utils/project.js";
@@ -9,6 +11,54 @@ import type { Example } from "./_examples.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
/**
* Extract a single frame from a video file at `timeSeconds` via FFmpeg.
* Used to work around Chrome-headless's inability to reliably seek
* <video> elements during snapshot capture.
*/
async function extractVideoFrameToBuffer(
videoPath: string,
timeSeconds: number,
): Promise<Buffer | null> {
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
const outPath = join(tmp, "frame.png");
try {
const result = await new Promise<{ code: number | null; stderr: string }>((resolvePromise) => {
// `-ss` before `-i` performs a fast keyframe seek; adequate for snapshot accuracy
// (±1 frame) and orders of magnitude faster than the decode-and-scan alternative.
const ff = spawn("ffmpeg", [
"-hide_banner",
"-loglevel",
"error",
"-ss",
String(Math.max(0, timeSeconds)),
"-i",
videoPath,
"-frames:v",
"1",
"-q:v",
"2",
"-y",
outPath,
]);
let stderr = "";
ff.stderr.on("data", (d: Buffer) => {
stderr += d.toString();
});
ff.on("close", (code) => resolvePromise({ code, stderr }));
ff.on("error", () => resolvePromise({ code: null, stderr: "ffmpeg spawn failed" }));
});
if (result.code !== 0 || !existsSync(outPath)) return null;
return readFileSync(outPath);
} finally {
try {
rmSync(tmp, { recursive: true, force: true });
} catch {
/* best-effort */
}
}
}
export const examples: Example[] = [
["Capture 5 key frames from a composition", "snapshot captures/stripe"],
["Capture 10 evenly-spaced frames", "snapshot captures/stripe --frames 10"],
@@ -172,6 +222,23 @@ async function captureSnapshots(
const snapshotDir = join(projectDir, "snapshots");
mkdirSync(snapshotDir, { recursive: true });
// Lazily load the engine's <img>-overlay injector. Chrome-headless cannot
// reliably advance <video>.currentTime mid-seek (the setter is accepted but
// the decoder ignores it without user activation), so the render pipeline
// already extracts each frame via FFmpeg and injects it as an <img> sibling
// over the <video>. We reuse that same primitive here so `snapshot` and
// `render` behave identically for timed <video data-start> elements.
let injectVideoFramesBatch:
| ((page: any, updates: Array<{ videoId: string; dataUri: string }>) => Promise<void>)
| null = null;
try {
const engine = await import("@hyperframes/engine");
injectVideoFramesBatch = engine.injectVideoFramesBatch as typeof injectVideoFramesBatch;
} catch {
// Engine unavailable in this install — snapshot will still run, and
// compositions without <video data-start> get exactly the old behaviour.
}
// Seek and capture each frame
for (let i = 0; i < positions.length; i++) {
const time = positions[i]!;
@@ -200,6 +267,77 @@ async function captureSnapshots(
);
await new Promise((r) => setTimeout(r, 200));
// ─── Inject real video frames over any active <video data-start> ───
// Without this, Chrome-headless renders them blank/first-frame because
// it silently drops programmatic `currentTime` writes during capture.
// No-op when the composition has no timed videos (basecamp, linear, etc.)
if (injectVideoFramesBatch) {
const active = await page.evaluate((t: number) => {
return Array.from(document.querySelectorAll("video[data-start]"))
.map((el) => {
const v = el as HTMLVideoElement;
const start = parseFloat(v.dataset.start ?? "0") || 0;
const rawDuration = parseFloat(v.dataset.duration ?? "");
const srcDur = Number.isFinite(v.duration) && v.duration > 0 ? v.duration : 0;
const duration =
Number.isFinite(rawDuration) && rawDuration > 0
? rawDuration
: srcDur > 0
? srcDur
: Number.POSITIVE_INFINITY;
const mediaStart =
parseFloat(v.dataset.playbackStart ?? v.dataset.mediaStart ?? "0") || 0;
return {
id: v.id,
src: v.currentSrc || v.src,
start,
duration,
mediaStart,
};
})
.filter(
(entry) =>
entry.id && entry.src && t >= entry.start && t < entry.start + entry.duration,
);
}, time);
if (active.length > 0) {
const updates: Array<{ videoId: string; dataUri: string }> = [];
for (const v of active) {
// The page-served URL (http://127.0.0.1:PORT/relative/path.mp4)
// maps 1:1 to <projectDir>/relative/path.mp4. Reconstruct the
// filesystem path from the URL pathname.
let filePath: string | null = null;
try {
const url = new URL(v.src);
const candidate = resolve(projectDir, url.pathname.replace(/^\//, ""));
const rel = relative(projectDir, candidate);
if (!rel.startsWith("..") && !isAbsolute(rel) && existsSync(candidate)) {
filePath = candidate;
}
} catch {
/* unresolvable src (e.g. blob:, data:) — skip */
}
if (!filePath) continue;
const relTime = Math.max(0, time - v.start + v.mediaStart);
const png = await extractVideoFrameToBuffer(filePath, relTime);
if (!png) continue;
updates.push({
videoId: v.id,
dataUri: `data:image/png;base64,${png.toString("base64")}`,
});
}
if (updates.length > 0) {
try {
await injectVideoFramesBatch(page, updates);
} catch {
// If injection fails, fall through to the plain screenshot — no worse
// than pre-fix behaviour.
}
}
}
}
const timeLabel = opts.at?.length
? `${time.toFixed(1)}s`
: `${Math.round((time / duration) * 100)}pct`;