mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-11 06:30:03 +00:00
feat(capture): pipeline improvements — contact sheets, design styles, snapshot
Capture pipeline work that came out of the 11-round website-to-video
eval branch. The wins that actually moved quality were the artifacts
agents read (contact sheets, design-styles) and the snapshot tool
visual-verification fixes; the rest are smaller follow-ons.
**Contact sheets (`contactSheet.ts`, new)**
- Replaces the embedded one-image-per-asset listing with paginated
labeled grids (3-col screenshots / 4-col raster / 5-col SVG). Each
page contains 9–15 cells with filename labels baked in via SVG
text overlay (`escapeXml` covers `&<>"'`).
- `fit: "contain"` keeps every asset visible at its real aspect
ratio; the old `fit: "cover"` cropped to the first image's box.
- Returns `string[]` (page paths) — single-page captures get one
file, multi-page produce `contact-sheet-1.jpg`, `contact-sheet-2.jpg`,
etc.
- `createSvgContactSheet` scans both `assets/svgs/` (inline-extracted
SVGs) and `assets/` root (external SVGs from `<img src="*.svg">`)
and de-dupes by filename. Sites with all-external SVGs (huly.io)
now get coverage they previously didn't.
**Design styles extractor (`designStyleExtractor.ts`, new)**
- Walks the live DOM and reads computed styles to produce
`extracted/design-styles.json`: typography hierarchy (every text
role with exact font-size / weight / line-height / letter-spacing),
button variants (background / padding / radius / shadow), card /
container / nav styles, spacing scale with base unit, border-radius
scale, box-shadow values with usage counts.
- Primary data source for DESIGN.md authoring at Step 1. Replaces
the prior "guess from screenshots" workflow.
**Snapshot tool (`snapshot.ts`)**
- HyperShader pre-rendering used to swallow the entire snapshot
capture window (every frame after the first showed the loading
overlay or final-opacity-zero exit fades). Wait signal is now
`window.__hf.shaderTransitions[].ready` (set after both warm and
cold cache paths complete); local-time seek for sub-comps means
exit fades read at their own t=0..duration, not global time.
- Gemini vision per-frame analysis runs by default (`descriptions.md`
next to the contact sheet). `--describe "custom Q"` overrides the
prompt; `--describe false` opts out.
- 3-column contact sheet generation for snapshot frames so reviewers
see all beats at a glance.
**Screenshot capture (`screenshotCapture.ts`)**
- Replaces `querySelectorAll('*') + getComputedStyle` overlay scan
with a TreeWalker that early-exits on cheap rect checks before
reaching the expensive style read. Caps at 5000 elements per page.
- Cookie/consent dismissal selectors are scoped under cookie /
consent / gdpr ancestors so we don't click "Accept invitation" or
similar unrelated buttons.
**Agent prompt (`agentPromptGenerator.ts`)**
- Auto-discovers contact-sheet page count (matches base name plus
paginated `-NNN` variants only, with regex escaping on the base
name and numeric sort for 10+ pages).
- `inferColorRole`: classifies extracted hex colors as bg-dark /
bg-light / accent / surface / neutral via luminance + saturation,
so the agent prompt shows `#533AFD (accent)` instead of bare hex.
- `design-styles.json` row is gated on `existsSync` — the upstream
write is wrapped in try/catch and may skip on failure, so the
prompt only points to files actually on disk.
**Other CLI ergonomics**
- `cli.ts`: auto-load `.env` from CWD on startup so subcommands like
`snapshot` don't need explicit `export GEMINI_API_KEY=…`. Handles
`export FOO=bar`, quoted values, inline `# comments`.
- `commands/transcribe.ts`: default output dir is the input file's
directory, not CWD. Stops the "wrote transcript.json somewhere
unexpected" footgun.
- `assetDownloader.ts`: improved asset naming uses catalog context;
de-duplicates inline SVG filenames.
- `contentExtractor.ts`: captions SVGs via Gemini (code-as-text) and
integrates them into asset descriptions.
- `tokenExtractor.ts` + `types.ts`: SVG bounding box dimensions and
new DesignStyles schema added.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { defineCommand } from "citty";
|
||||
import { existsSync, mkdtempSync, readFileSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { existsSync, mkdtempSync, readFileSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve, join, relative, isAbsolute } from "node:path";
|
||||
import { resolveProject } from "../utils/project.js";
|
||||
@@ -139,25 +139,88 @@ async function captureSnapshots(
|
||||
})
|
||||
.catch(() => {});
|
||||
|
||||
// Wait for sub-compositions to be mounted by the runtime
|
||||
// (they're fetched and injected asynchronously via data-composition-src)
|
||||
// Wait for ALL sub-compositions to be mounted by the runtime.
|
||||
// The old check resolved when the first sub-timeline registered, causing
|
||||
// "last beat black" bugs: beat-5's sub-comp hadn't loaded yet when the
|
||||
// snapshot seeked into its time range. Now we count data-composition-src
|
||||
// host elements and wait until we have a matching number of sub-timelines.
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => {
|
||||
const tls = (window as any).__timelines;
|
||||
if (!tls) return false;
|
||||
const keys = Object.keys(tls);
|
||||
// Wait until at least one sub-composition timeline is registered
|
||||
// (not counting "main" or empty registrations)
|
||||
return keys.length >= 2 || keys.some((k) => k !== "main");
|
||||
const hosts = document.querySelectorAll("[data-composition-src]").length;
|
||||
if (hosts === 0) return Object.keys(tls).length >= 1;
|
||||
const subKeys = Object.keys(tls).filter((k) => k !== "main");
|
||||
return subKeys.length >= hosts;
|
||||
},
|
||||
{ timeout: timeoutMs },
|
||||
)
|
||||
.catch(() => {});
|
||||
|
||||
// Wait for shader transition pre-rendering to complete (if active).
|
||||
//
|
||||
// Two failure modes existed with the previous overlay-only check:
|
||||
// 1. Cold cache: HyperShader creates [data-hyper-shader-loading] but never
|
||||
// removes it from the DOM — it only sets display:none. Checking for
|
||||
// element *absence* never resolved, so the wait always timed out at 60s.
|
||||
// 2. Warm cache: HyperShader loads frames from IndexedDB without showing
|
||||
// the overlay at all. Checking for element absence resolved instantly
|
||||
// (no element) while hydration was still running in the background.
|
||||
//
|
||||
// Fix: use window.__hf.shaderTransitions[].ready as the primary signal
|
||||
// (set after both warm and cold cache paths complete), with the overlay
|
||||
// display:none as a fallback for older builds that lack the ready state.
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => {
|
||||
const win = window as unknown as {
|
||||
__hf?: { shaderTransitions?: Record<string, { ready?: boolean }> };
|
||||
};
|
||||
// Primary: HyperShader ready state — authoritative for both cache paths
|
||||
const shaderTransitions = win.__hf?.shaderTransitions;
|
||||
if (shaderTransitions !== undefined) {
|
||||
return Object.values(shaderTransitions).every((s) => s.ready === true);
|
||||
}
|
||||
// Fallback: overlay visibility (older builds without ready state).
|
||||
// Check display:none rather than element absence — element stays in
|
||||
// the DOM when hidden.
|
||||
const overlay = document.querySelector(
|
||||
"[data-hyper-shader-loading]",
|
||||
) as HTMLElement | null;
|
||||
if (!overlay) return true;
|
||||
return window.getComputedStyle(overlay).display === "none";
|
||||
},
|
||||
{ timeout: 90_000 },
|
||||
)
|
||||
.catch(() => {});
|
||||
|
||||
// Extra settle time for media, fonts, and animations to initialize
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
|
||||
// Font verification — report which fonts loaded vs fell back
|
||||
const fontReport = await page
|
||||
.evaluate(() => {
|
||||
const loaded: string[] = [];
|
||||
const failed: string[] = [];
|
||||
(document as any).fonts.forEach((f: any) => {
|
||||
const entry = `${f.family} (${f.weight} ${f.style})`;
|
||||
if (f.status === "loaded") loaded.push(entry);
|
||||
else failed.push(entry + ` [${f.status}]`);
|
||||
});
|
||||
return { loaded, failed };
|
||||
})
|
||||
.catch(() => ({ loaded: [] as string[], failed: [] as string[] }));
|
||||
|
||||
if (fontReport.loaded.length > 0 || fontReport.failed.length > 0) {
|
||||
console.log(
|
||||
`\n ${c.dim("Fonts loaded:")} ${fontReport.loaded.length > 0 ? fontReport.loaded.join(", ") : "none"}`,
|
||||
);
|
||||
if (fontReport.failed.length > 0) {
|
||||
console.log(` ${c.error("Fonts FAILED:")} ${fontReport.failed.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Get composition duration
|
||||
const duration = await page.evaluate(() => {
|
||||
const win = window as any;
|
||||
@@ -186,9 +249,20 @@ async function captureSnapshots(
|
||||
? [duration / 2]
|
||||
: Array.from({ length: numFrames }, (_, i) => (i / (numFrames - 1)) * duration);
|
||||
|
||||
// Create output directory
|
||||
// Create output directory and clear previous frames so old captures
|
||||
// don't mix with the current run in contact sheets.
|
||||
const snapshotDir = join(projectDir, "snapshots");
|
||||
mkdirSync(snapshotDir, { recursive: true });
|
||||
try {
|
||||
const { readdirSync, rmSync } = await import("node:fs");
|
||||
for (const file of readdirSync(snapshotDir)) {
|
||||
if (/\.(png|jpg|jpeg)$/i.test(file)) {
|
||||
rmSync(join(snapshotDir, file), { force: true });
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* best-effort clear — proceed even if cleanup fails */
|
||||
}
|
||||
|
||||
// Lazily load the engine's <img>-overlay injector. Chrome-headless cannot
|
||||
// reliably advance <video>.currentTime mid-seek (the setter is accepted but
|
||||
@@ -237,18 +311,30 @@ async function captureSnapshots(
|
||||
const time = positions[i]!;
|
||||
|
||||
await page.evaluate((t: number) => {
|
||||
const w = window as Window & {
|
||||
__player?: { seek?: (time: number) => void };
|
||||
__timelines?: Record<string, { pause?: (time?: number) => void }>;
|
||||
gsap?: { ticker?: { tick?: () => void } };
|
||||
};
|
||||
if (typeof w.__player?.seek === "function") {
|
||||
w.__player.seek(t);
|
||||
} else if (w.__timelines) {
|
||||
for (const tl of Object.values(w.__timelines)) {
|
||||
tl?.pause?.(t);
|
||||
const win = window as any;
|
||||
if (win.__player?.seek) {
|
||||
win.__player.seek(t);
|
||||
} else {
|
||||
const tls = win.__timelines;
|
||||
if (tls) {
|
||||
for (const key in tls) {
|
||||
if (tls[key]?.seek) {
|
||||
// Sub-composition timelines run in local time relative to
|
||||
// their data-start. Seeking them to global time causes beats
|
||||
// with exit animations to appear black (global t clamps past
|
||||
// the exit). Compute local time: global_t - data_start.
|
||||
const host = document.querySelector<HTMLElement>(
|
||||
`[data-composition-id="${key}"]`,
|
||||
);
|
||||
const dataStart = host
|
||||
? parseFloat(host.getAttribute("data-start") ?? "0") || 0
|
||||
: 0;
|
||||
const localTime = Math.max(0, t - dataStart);
|
||||
tls[key].pause();
|
||||
tls[key].seek(localTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
w.gsap?.ticker?.tick?.();
|
||||
}
|
||||
}, time);
|
||||
|
||||
@@ -356,9 +442,7 @@ async function captureSnapshots(
|
||||
}
|
||||
}
|
||||
|
||||
const timeLabel = opts.at?.length
|
||||
? `${time.toFixed(1)}s`
|
||||
: `${Math.round((time / duration) * 100)}pct`;
|
||||
const timeLabel = `${time.toFixed(1)}s`;
|
||||
const filename = `frame-${String(i).padStart(2, "0")}-at-${timeLabel}.png`;
|
||||
const framePath = join(snapshotDir, filename);
|
||||
|
||||
@@ -400,6 +484,11 @@ export default defineCommand({
|
||||
description: "Ms to wait for runtime to initialize (default: 5000)",
|
||||
default: "5000",
|
||||
},
|
||||
describe: {
|
||||
type: "string",
|
||||
description:
|
||||
"Gemini vision frame analysis. Runs by default when GEMINI_API_KEY is set. Pass a custom question (e.g. --describe 'Is the logo visible in every beat?') to override the default prompt, or --describe false to opt out.",
|
||||
},
|
||||
},
|
||||
async run({ args }) {
|
||||
const project = resolveProject(args.dir);
|
||||
@@ -411,6 +500,16 @@ export default defineCommand({
|
||||
.map((s) => parseFloat(s.trim()))
|
||||
.filter((n) => !isNaN(n))
|
||||
: undefined;
|
||||
// Gemini frame analysis runs by default (silently skipped if
|
||||
// GEMINI_API_KEY is not set). `--describe "custom question"` overrides
|
||||
// the default prompt with a targeted question. `--describe false` opts
|
||||
// out entirely.
|
||||
const describeArg =
|
||||
args.describe === undefined
|
||||
? "true"
|
||||
: String(args.describe) === "false"
|
||||
? null
|
||||
: String(args.describe);
|
||||
|
||||
const label = atTimestamps
|
||||
? `${atTimestamps.length} frames at [${atTimestamps.map((t) => t.toFixed(1) + "s").join(", ")}]`
|
||||
@@ -431,6 +530,121 @@ export default defineCommand({
|
||||
for (const p of paths) {
|
||||
console.log(` ${p}`);
|
||||
}
|
||||
|
||||
// Generate contact sheet for quick AI review
|
||||
try {
|
||||
const { createSnapshotContactSheet } = await import("../capture/contactSheet.js");
|
||||
const snapshotDir = join(project.dir, "snapshots");
|
||||
const sheets = await createSnapshotContactSheet(
|
||||
snapshotDir,
|
||||
join(snapshotDir, "contact-sheet.jpg"),
|
||||
);
|
||||
if (sheets.length > 0) {
|
||||
const label =
|
||||
sheets.length === 1 ? "contact-sheet.jpg" : `contact-sheet-1..${sheets.length}.jpg`;
|
||||
console.log(` ${c.dim(label)} (grid view for AI review)`);
|
||||
}
|
||||
} catch {
|
||||
/* non-critical */
|
||||
}
|
||||
|
||||
// Gemini vision descriptions. Runs by default — see describeArg
|
||||
// resolution above. `null` means the user explicitly opted out with
|
||||
// `--describe false`; missing GEMINI_API_KEY logs a skip and continues.
|
||||
if (describeArg !== null) {
|
||||
try {
|
||||
const geminiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
|
||||
if (!geminiKey) {
|
||||
console.log(` ${c.dim("--describe: GEMINI_API_KEY not set, skipping")}`);
|
||||
} else if (paths.length > 0) {
|
||||
console.log(` ${c.dim("Describing frames with Gemini vision...")}`);
|
||||
const { GoogleGenAI } = await import("@google/genai");
|
||||
const ai = new GoogleGenAI({ apiKey: geminiKey });
|
||||
const model = process.env.HYPERFRAMES_GEMINI_MODEL || "gemini-3.1-flash-lite-preview";
|
||||
const snapshotDir = join(project.dir, "snapshots");
|
||||
|
||||
const customQuestion =
|
||||
describeArg === "true"
|
||||
? "Describe this video composition frame in 1-2 sentences. Be specific and factual: what elements are visible, what text appears, is the frame blank/black/loading, what is the composition. Flag any obvious problems."
|
||||
: describeArg;
|
||||
|
||||
const descriptions: string[] = [
|
||||
`# Snapshot Frame Descriptions`,
|
||||
``,
|
||||
`**Question asked:** ${customQuestion}`,
|
||||
``,
|
||||
`Compare each description against your storyboard spec. A "black frame" or "loading screen" for a content beat is a bug.`,
|
||||
``,
|
||||
];
|
||||
|
||||
// Scale down PNGs before sending to stay under Gemini's 4 MB inline
|
||||
// limit. Full 1920×1080 PNGs are typically 3-6 MB. Use sharp if
|
||||
// available; otherwise skip files over the limit.
|
||||
type SharpFn = (buf: Buffer) => {
|
||||
resize: (w: number) => { jpeg: () => { toBuffer: () => Promise<Buffer> } };
|
||||
};
|
||||
let sharpFn: SharpFn | null = null;
|
||||
try {
|
||||
const s = await import("sharp");
|
||||
sharpFn = (s.default ?? s) as unknown as SharpFn;
|
||||
} catch {
|
||||
/* sharp not installed — fall back to size check */
|
||||
}
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
paths.map(async (p) => {
|
||||
const filename = p.replace("snapshots/", "");
|
||||
const filePath = join(snapshotDir, filename);
|
||||
if (!existsSync(filePath)) return { filename, desc: "file not found" };
|
||||
const raw = readFileSync(filePath);
|
||||
let imageData: Buffer;
|
||||
let mimeType = "image/png";
|
||||
if (sharpFn) {
|
||||
imageData = await sharpFn(raw).resize(960).jpeg().toBuffer();
|
||||
mimeType = "image/jpeg";
|
||||
} else {
|
||||
if (raw.length > 3_800_000)
|
||||
return {
|
||||
filename,
|
||||
desc: "file too large for Gemini — install sharp to enable auto-resize",
|
||||
};
|
||||
imageData = raw;
|
||||
}
|
||||
const base64 = imageData.toString("base64");
|
||||
const response = await ai.models.generateContent({
|
||||
model,
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ inlineData: { mimeType, data: base64 } }, { text: customQuestion }],
|
||||
},
|
||||
],
|
||||
config: { maxOutputTokens: 250 },
|
||||
});
|
||||
return { filename, desc: response.text?.trim() || "no description" };
|
||||
}),
|
||||
);
|
||||
|
||||
for (const result of results) {
|
||||
if (result.status === "fulfilled") {
|
||||
descriptions.push(`## ${result.value.filename}`, `${result.value.desc}`, ``);
|
||||
} else {
|
||||
// Log first failure so Gemini issues are visible rather than silent
|
||||
const errMsg =
|
||||
result.reason instanceof Error ? result.reason.message : String(result.reason);
|
||||
descriptions.push(`## (error)`, `Gemini call failed: ${errMsg.slice(0, 120)}`, ``);
|
||||
}
|
||||
}
|
||||
|
||||
const descPath = join(snapshotDir, "descriptions.md");
|
||||
writeFileSync(descPath, descriptions.join("\n"));
|
||||
console.log(` ${c.dim("descriptions.md")} (Gemini frame analysis)`);
|
||||
}
|
||||
} catch (descErr) {
|
||||
const msg = descErr instanceof Error ? descErr.message : String(descErr);
|
||||
console.log(` ${c.dim(`--describe failed: ${msg.slice(0, 80)}`)}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.error(`\n${c.error("✗")} Snapshot failed: ${msg}`);
|
||||
|
||||
Reference in New Issue
Block a user