mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-08-31 02:41:44 +00:00
feat(capture): pipeline improvements — contact sheets, design styles, snapshot
Capture pipeline work that came out of the 11-round website-to-video
eval branch. The wins that actually moved quality were the artifacts
agents read (contact sheets, design-styles) and the snapshot tool
visual-verification fixes; the rest are smaller follow-ons.
**Contact sheets (`contactSheet.ts`, new)**
- Replaces the embedded one-image-per-asset listing with paginated
labeled grids (3-col screenshots / 4-col raster / 5-col SVG). Each
page contains 9–15 cells with filename labels baked in via SVG
text overlay (`escapeXml` covers `&<>"'`).
- `fit: "contain"` keeps every asset visible at its real aspect
ratio; the old `fit: "cover"` cropped to the first image's box.
- Returns `string[]` (page paths) — single-page captures get one
file, multi-page produce `contact-sheet-1.jpg`, `contact-sheet-2.jpg`,
etc.
- `createSvgContactSheet` scans both `assets/svgs/` (inline-extracted
SVGs) and `assets/` root (external SVGs from `<img src="*.svg">`)
and de-dupes by filename. Sites with all-external SVGs (huly.io)
now get coverage they previously didn't.
**Design styles extractor (`designStyleExtractor.ts`, new)**
- Walks the live DOM and reads computed styles to produce
`extracted/design-styles.json`: typography hierarchy (every text
role with exact font-size / weight / line-height / letter-spacing),
button variants (background / padding / radius / shadow), card /
container / nav styles, spacing scale with base unit, border-radius
scale, box-shadow values with usage counts.
- Primary data source for DESIGN.md authoring at Step 1. Replaces
the prior "guess from screenshots" workflow.
**Snapshot tool (`snapshot.ts`)**
- HyperShader pre-rendering used to swallow the entire snapshot
capture window (every frame after the first showed the loading
overlay or final-opacity-zero exit fades). Wait signal is now
`window.__hf.shaderTransitions[].ready` (set after both warm and
cold cache paths complete); local-time seek for sub-comps means
exit fades read at their own t=0..duration, not global time.
- Gemini vision per-frame analysis runs by default (`descriptions.md`
next to the contact sheet). `--describe "custom Q"` overrides the
prompt; `--describe false` opts out.
- 3-column contact sheet generation for snapshot frames so reviewers
see all beats at a glance.
**Screenshot capture (`screenshotCapture.ts`)**
- Replaces `querySelectorAll('*') + getComputedStyle` overlay scan
with a TreeWalker that early-exits on cheap rect checks before
reaching the expensive style read. Caps at 5000 elements per page.
- Cookie/consent dismissal selectors are scoped under cookie /
consent / gdpr ancestors so we don't click "Accept invitation" or
similar unrelated buttons.
**Agent prompt (`agentPromptGenerator.ts`)**
- Auto-discovers contact-sheet page count (matches base name plus
paginated `-NNN` variants only, with regex escaping on the base
name and numeric sort for 10+ pages).
- `inferColorRole`: classifies extracted hex colors as bg-dark /
bg-light / accent / surface / neutral via luminance + saturation,
so the agent prompt shows `#533AFD (accent)` instead of bare hex.
- `design-styles.json` row is gated on `existsSync` — the upstream
write is wrapped in try/catch and may skip on failure, so the
prompt only points to files actually on disk.
**Other CLI ergonomics**
- `cli.ts`: auto-load `.env` from CWD on startup so subcommands like
`snapshot` don't need explicit `export GEMINI_API_KEY=…`. Handles
`export FOO=bar`, quoted values, inline `# comments`.
- `commands/transcribe.ts`: default output dir is the input file's
directory, not CWD. Stops the "wrote transcript.json somewhere
unexpected" footgun.
- `assetDownloader.ts`: improved asset naming uses catalog context;
de-duplicates inline SVG filenames.
- `contentExtractor.ts`: captions SVGs via Gemini (code-as-text) and
integrates them into asset descriptions.
- `tokenExtractor.ts` + `types.ts`: SVG bounding box dimensions and
new DesignStyles schema added.
This commit is contained in:
@@ -10,12 +10,35 @@
|
||||
* website-to-hyperframes skill — this file points agents there.
|
||||
*/
|
||||
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { writeFileSync, readdirSync, existsSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { DesignTokens } from "./types.js";
|
||||
import type { AnimationCatalog } from "./animationCataloger.js";
|
||||
import type { CatalogedAsset } from "./assetCataloger.js";
|
||||
|
||||
/**
|
||||
* Infer a human-readable role hint from a hex color based on luminance and saturation.
|
||||
* Not a substitute for DESIGN.md — just helps orient agents scanning the brand summary.
|
||||
*/
|
||||
function inferColorRole(hex: string): string {
|
||||
const r = parseInt(hex.slice(1, 3), 16) / 255;
|
||||
const g = parseInt(hex.slice(3, 5), 16) / 255;
|
||||
const b = parseInt(hex.slice(5, 7), 16) / 255;
|
||||
if (isNaN(r) || isNaN(g) || isNaN(b)) return "color";
|
||||
|
||||
const max = Math.max(r, g, b);
|
||||
const min = Math.min(r, g, b);
|
||||
const luminance = 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
||||
const saturation = max === 0 ? 0 : (max - min) / max;
|
||||
|
||||
if (luminance < 0.04) return "bg-dark";
|
||||
if (luminance > 0.9) return "bg-light";
|
||||
if (saturation > 0.4 && luminance > 0.05 && luminance < 0.7) return "accent";
|
||||
if (luminance < 0.2) return "surface-dark";
|
||||
if (luminance > 0.7) return "surface-light";
|
||||
return "neutral";
|
||||
}
|
||||
|
||||
export function generateAgentPrompt(
|
||||
outputDir: string,
|
||||
url: string,
|
||||
@@ -25,25 +48,28 @@ export function generateAgentPrompt(
|
||||
hasLottie?: boolean,
|
||||
hasShaders?: boolean,
|
||||
_catalogedAssets?: CatalogedAsset[], // reserved for future asset inventory
|
||||
detectedLibraries?: string[],
|
||||
_detectedLibraries?: string[],
|
||||
): void {
|
||||
const prompt = buildPrompt(url, tokens, hasScreenshot, hasLottie, hasShaders, detectedLibraries);
|
||||
const prompt = buildPrompt(outputDir, url, tokens, hasScreenshot, hasLottie, hasShaders);
|
||||
writeFileSync(join(outputDir, "AGENTS.md"), prompt, "utf-8");
|
||||
writeFileSync(join(outputDir, "CLAUDE.md"), prompt, "utf-8");
|
||||
writeFileSync(join(outputDir, ".cursorrules"), prompt, "utf-8");
|
||||
}
|
||||
|
||||
function buildPrompt(
|
||||
outputDir: string,
|
||||
url: string,
|
||||
tokens: DesignTokens,
|
||||
hasScreenshot: boolean,
|
||||
hasLottie?: boolean,
|
||||
hasShaders?: boolean,
|
||||
detectedLibraries?: string[],
|
||||
): string {
|
||||
const title = tokens.title || new URL(url).hostname.replace(/^www\./, "");
|
||||
|
||||
const colorSummary = tokens.colors.slice(0, 10).join(", ");
|
||||
const colorSummary = tokens.colors
|
||||
.slice(0, 10)
|
||||
.map((hex) => `${hex} (${inferColorRole(hex)})`)
|
||||
.join(", ");
|
||||
const fontSummary =
|
||||
tokens.fonts
|
||||
.map(
|
||||
@@ -58,18 +84,67 @@ function buildPrompt(
|
||||
.join(", ") || "none detected";
|
||||
|
||||
// Build the data inventory table rows
|
||||
// Helper: find all contact sheet pages for a given base name. Matches the
|
||||
// exact base file plus paginated variants only (e.g. `contact-sheet.jpg`,
|
||||
// `contact-sheet-2.jpg`, `contact-sheet-3.jpg`). The "-NNN" suffix is digits
|
||||
// only, so unrelated files that happen to share the prefix (notably the
|
||||
// `contact-sheet-svgs.jpg` SVG fallback sheet in assets/) don't get mixed in.
|
||||
function contactSheetRows(dir: string, baseFile: string, label: string): string[] {
|
||||
const fullDir = join(outputDir, dir);
|
||||
if (!existsSync(fullDir)) return [];
|
||||
const baseName = baseFile.replace(/\.jpg$/, "");
|
||||
// Escape regex metacharacters in baseName so future callers can pass
|
||||
// filenames containing `.`, `+`, `(`, etc. without the regex breaking.
|
||||
const escapedBase = baseName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const paginatedRe = new RegExp(`^${escapedBase}(?:-(\\d+))?\\.jpg$`);
|
||||
// Sort by the numeric page suffix so `contact-sheet-10.jpg` lands after
|
||||
// `contact-sheet-2.jpg`, not before (default string sort orders them
|
||||
// lexicographically and breaks at 10+ pages). Unpaginated `contact-sheet.jpg`
|
||||
// gets page 0 so it sorts first if it co-exists with paginated files.
|
||||
const all = readdirSync(fullDir)
|
||||
.filter((f) => paginatedRe.test(f))
|
||||
.map((f) => ({ name: f, page: parseInt(f.match(paginatedRe)?.[1] ?? "0", 10) }))
|
||||
.sort((a, b) => a.page - b.page)
|
||||
.map((entry) => entry.name);
|
||||
if (all.length === 0) return [];
|
||||
if (all.length === 1) {
|
||||
return [`| \`${dir}/${all[0]}\` | ${label} |`];
|
||||
}
|
||||
return all.map((f, i) => `| \`${dir}/${f}\` | ${label} — page ${i + 1} of ${all.length} |`);
|
||||
}
|
||||
|
||||
const tableRows: string[] = [];
|
||||
if (hasScreenshot) {
|
||||
const screenshotRows = contactSheetRows(
|
||||
"screenshots",
|
||||
"contact-sheet.jpg",
|
||||
"**View this first.** All scroll screenshots in labeled grid — see the entire page at a glance",
|
||||
);
|
||||
if (screenshotRows.length > 0) {
|
||||
tableRows.push(...screenshotRows);
|
||||
} else {
|
||||
tableRows.push(
|
||||
"| `screenshots/contact-sheet.jpg` | **View this first.** All scroll screenshots in one labeled grid. |",
|
||||
);
|
||||
}
|
||||
tableRows.push(
|
||||
"| `screenshots/scroll-*.png` | Viewport screenshots of the full page. Start with `scroll-000.png` (hero). |",
|
||||
"| `screenshots/scroll-*.png` | Individual viewport screenshots if you need detail on a specific section. |",
|
||||
);
|
||||
}
|
||||
tableRows.push(
|
||||
"| `extracted/asset-descriptions.md` | One-line description of every downloaded asset. **Read this first.** |",
|
||||
);
|
||||
tableRows.push(
|
||||
`| \`extracted/tokens.json\` | Design tokens: ${tokens.colors.length} colors, ${tokens.fonts.length} fonts, ${tokens.headings?.length ?? 0} headings, ${tokens.ctas?.length ?? 0} CTAs |`,
|
||||
);
|
||||
// design-styles.json is written from a try/catch in capture/index.ts and
|
||||
// gets skipped when the live-DOM style extraction fails. Only list it in the
|
||||
// agent prompt when it actually exists, so the agent isn't pointed at a 404.
|
||||
if (existsSync(join(outputDir, "extracted", "design-styles.json"))) {
|
||||
tableRows.push(
|
||||
"| `extracted/design-styles.json` | Computed styles from live DOM: typography hierarchy, button/card/nav styles, spacing scale, border-radius, box shadows. Primary data source for DESIGN.md. |",
|
||||
);
|
||||
}
|
||||
tableRows.push(
|
||||
"| `extracted/asset-descriptions.md` | One-line description of every downloaded asset. Read this for asset selection — only open individual files for safe-zone checking. |",
|
||||
);
|
||||
tableRows.push(
|
||||
"| `extracted/visible-text.txt` | Page text in DOM order, prefixed with HTML tag (`[h1]`, `[p]`, `[a]`). Use as context — rephrase freely. |",
|
||||
);
|
||||
@@ -81,20 +156,41 @@ function buildPrompt(
|
||||
if (hasShaders) {
|
||||
tableRows.push("| `extracted/shaders.json` | WebGL shader source (GLSL). |");
|
||||
}
|
||||
if (detectedLibraries && detectedLibraries.length > 0) {
|
||||
tableRows.push(
|
||||
`| \`extracted/detected-libraries.json\` | Libraries: ${detectedLibraries.join(", ")} |`,
|
||||
);
|
||||
|
||||
// Asset contact sheets — dynamically list all pages
|
||||
const assetSheetRows = contactSheetRows(
|
||||
"assets",
|
||||
"contact-sheet.jpg",
|
||||
"Downloaded images in labeled grid — view before opening individual files",
|
||||
);
|
||||
if (assetSheetRows.length > 0) {
|
||||
tableRows.push(...assetSheetRows);
|
||||
} else {
|
||||
tableRows.push("| `assets/contact-sheet.jpg` | All downloaded images in one labeled grid. |");
|
||||
}
|
||||
tableRows.push("| `assets/` | Downloaded images, SVGs, and font files. |");
|
||||
|
||||
// SVG contact sheets — check both assets/svgs/ and assets/ root fallback
|
||||
const svgSubdirRows = contactSheetRows(
|
||||
"assets/svgs",
|
||||
"contact-sheet.jpg",
|
||||
"SVGs rendered as thumbnails in labeled grid",
|
||||
);
|
||||
const svgRootRows = contactSheetRows(
|
||||
"assets",
|
||||
"contact-sheet-svgs.jpg",
|
||||
"SVGs rendered as thumbnails in labeled grid",
|
||||
);
|
||||
const svgRows = svgSubdirRows.length > 0 ? svgSubdirRows : svgRootRows;
|
||||
if (svgRows.length > 0) {
|
||||
tableRows.push(...svgRows);
|
||||
}
|
||||
|
||||
tableRows.push("| `assets/` | Individual downloaded images, SVGs, and font files. |");
|
||||
|
||||
// Brand summary — just the essentials
|
||||
const brandLines: string[] = [];
|
||||
brandLines.push(`- **Colors**: ${colorSummary || "see tokens.json"}`);
|
||||
brandLines.push(`- **Fonts**: ${fontSummary}`);
|
||||
if (detectedLibraries && detectedLibraries.length > 0) {
|
||||
brandLines.push(`- **Built with**: ${detectedLibraries.join(", ")}`);
|
||||
}
|
||||
|
||||
return `# ${title}
|
||||
|
||||
|
||||
@@ -24,11 +24,21 @@ export async function downloadAssets(
|
||||
|
||||
// 1. ALL inline SVGs — save as files (logos get priority naming)
|
||||
mkdirSync(join(outputDir, "assets", "svgs"), { recursive: true });
|
||||
const usedSvgNames = new Set<string>();
|
||||
for (let i = 0; i < tokens.svgs.length && i < 30; i++) {
|
||||
const svg = tokens.svgs[i]!;
|
||||
if (!svg.outerHTML || svg.outerHTML.length < 50) continue;
|
||||
const label = svg.label?.replace(/[^a-zA-Z0-9-_ ]/g, "").trim();
|
||||
const name = label ? slugify(label) + ".svg" : svg.isLogo ? `logo-${i}.svg` : `icon-${i}.svg`;
|
||||
let slug = label ? slugify(label) : svg.isLogo ? `logo-${i}` : `icon-${i}`;
|
||||
// Deduplicate — two SVGs with same aria-label get suffixed
|
||||
let finalSlug = slug;
|
||||
let suffix = 2;
|
||||
while (usedSvgNames.has(finalSlug)) {
|
||||
finalSlug = `${slug}-${suffix}`;
|
||||
suffix++;
|
||||
}
|
||||
usedSvgNames.add(finalSlug);
|
||||
const name = `${finalSlug}.svg`;
|
||||
const localPath = `assets/svgs/${name}`;
|
||||
try {
|
||||
writeFileSync(join(outputDir, localPath), svg.outerHTML, "utf-8");
|
||||
@@ -86,55 +96,49 @@ export async function downloadAssets(
|
||||
}
|
||||
}
|
||||
|
||||
// Download all images (no arbitrary cap) — Claude Code needs to see every asset to use them creatively.
|
||||
// The 10KB minimum size filter handles tracking pixels and tiny icons.
|
||||
// Download all images — use catalog context for human-readable filenames.
|
||||
// Pre-filter to deduplicate before downloading.
|
||||
const toDownload: { url: string; isPoster: boolean; normalized: string }[] = [];
|
||||
const toDownload: {
|
||||
url: string;
|
||||
isPoster: boolean;
|
||||
normalized: string;
|
||||
catalog?: CatalogedAsset;
|
||||
}[] = [];
|
||||
for (const { url, isPoster } of imageUrls) {
|
||||
const normalized = normalizeUrl(url);
|
||||
if (downloadedUrls.has(normalized)) continue;
|
||||
downloadedUrls.add(normalized); // Reserve to prevent duplicates in parallel batches
|
||||
toDownload.push({ url, isPoster, normalized });
|
||||
downloadedUrls.add(normalized);
|
||||
const catalog = catalogedAssets?.find((a) => normalizeUrl(a.url) === normalized);
|
||||
toDownload.push({ url, isPoster, normalized, catalog });
|
||||
}
|
||||
|
||||
// Download in parallel batches of 5
|
||||
const BATCH_SIZE = 5;
|
||||
let imgIdx = 0;
|
||||
const usedNames = new Set<string>();
|
||||
for (let i = 0; i < toDownload.length; i += BATCH_SIZE) {
|
||||
const batch = toDownload.slice(i, i + BATCH_SIZE);
|
||||
const results = await Promise.allSettled(
|
||||
batch.map(async ({ url, isPoster }) => {
|
||||
batch.map(async ({ url, isPoster, catalog }) => {
|
||||
const parsedUrl = new URL(url);
|
||||
const pathExt = extname(parsedUrl.pathname);
|
||||
const ext = pathExt && pathExt.length <= 5 ? pathExt : ".jpg";
|
||||
const buffer = await fetchBuffer(url);
|
||||
if (!buffer) return null;
|
||||
// SVGs are inherently small — don't apply the 10KB minimum to them
|
||||
const isSvg = ext === ".svg" || url.includes(".svg");
|
||||
const minSize = isSvg ? 200 : 10000;
|
||||
if (buffer.length < minSize) return null;
|
||||
return { url, isPoster, parsedUrl, ext, buffer };
|
||||
return { url, isPoster, parsedUrl, ext, buffer, catalog };
|
||||
}),
|
||||
);
|
||||
for (const result of results) {
|
||||
if (result.status !== "fulfilled" || !result.value) continue;
|
||||
const { url, isPoster, parsedUrl, ext, buffer } = result.value;
|
||||
const { url, isPoster, parsedUrl, ext, buffer, catalog } = result.value;
|
||||
try {
|
||||
const prefix = isPoster ? "poster" : "image";
|
||||
const rawName =
|
||||
parsedUrl.pathname
|
||||
.split("/")
|
||||
.pop()
|
||||
?.replace(/\.[^.]+$/, "") || "";
|
||||
const isMeaningful =
|
||||
rawName.length > 2 &&
|
||||
rawName.length < 50 &&
|
||||
!/^[a-f0-9]{8,}$/i.test(rawName) &&
|
||||
!/^\d+$/.test(rawName) &&
|
||||
!rawName.includes("_next") &&
|
||||
!rawName.includes("?");
|
||||
const slug = isMeaningful ? slugify(rawName) : `${prefix}-${imgIdx}`;
|
||||
// Generate human-readable name from catalog context
|
||||
const slug = deriveAssetName(parsedUrl, catalog, isPoster, imgIdx, usedNames);
|
||||
const name = `${slug}${ext}`;
|
||||
usedNames.add(slug);
|
||||
const localPath = `assets/${name}`;
|
||||
writeFileSync(join(outputDir, localPath), buffer);
|
||||
assets.push({ url, localPath, type: "image" });
|
||||
@@ -303,3 +307,77 @@ function slugify(text: string): string {
|
||||
.replace(/^-|-$/g, "")
|
||||
.slice(0, 40);
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a human-readable filename from catalog context.
|
||||
* Priority: alt text > nearest heading > meaningful URL path > fallback index.
|
||||
*/
|
||||
function deriveAssetName(
|
||||
parsedUrl: URL,
|
||||
catalog: CatalogedAsset | undefined,
|
||||
isPoster: boolean,
|
||||
idx: number,
|
||||
usedNames: Set<string>,
|
||||
): string {
|
||||
const candidates: string[] = [];
|
||||
|
||||
// 1. Alt text / description from catalog
|
||||
if (catalog?.description) {
|
||||
const desc = catalog.description.replace(/[^a-zA-Z0-9 -]/g, "").trim();
|
||||
if (desc.length > 3 && desc.length < 80) candidates.push(desc);
|
||||
}
|
||||
|
||||
// 2. Nearest heading context
|
||||
if (catalog?.nearestHeading) {
|
||||
const heading = catalog.nearestHeading.replace(/[^a-zA-Z0-9 -]/g, "").trim();
|
||||
if (heading.length > 3 && heading.length < 60) candidates.push(heading);
|
||||
}
|
||||
|
||||
// 3. Meaningful URL path segment
|
||||
const rawName =
|
||||
parsedUrl.pathname
|
||||
.split("/")
|
||||
.pop()
|
||||
?.replace(/\.[^.]+$/, "") || "";
|
||||
const isMeaningful =
|
||||
rawName.length > 2 &&
|
||||
rawName.length < 50 &&
|
||||
!/^[a-f0-9]{8,}$/i.test(rawName) &&
|
||||
!/^\d+$/.test(rawName) &&
|
||||
!rawName.includes("_next") &&
|
||||
!rawName.includes("?");
|
||||
if (isMeaningful) candidates.push(rawName);
|
||||
|
||||
// 4. Section classes as context
|
||||
if (catalog?.sectionClasses) {
|
||||
const classes = catalog.sectionClasses
|
||||
.split(/\s+/)
|
||||
.filter((c) => c.length > 3 && c.length < 30 && !/^(w-|h-|p-|m-|flex|grid|block)/.test(c))
|
||||
.slice(0, 2)
|
||||
.join("-");
|
||||
if (classes.length > 3) candidates.push(classes);
|
||||
}
|
||||
|
||||
// Pick the best candidate
|
||||
const prefix = isPoster ? "poster" : catalog?.aboveFold ? "hero" : "image";
|
||||
let slug = "";
|
||||
|
||||
for (const c of candidates) {
|
||||
slug = slugify(c);
|
||||
if (slug.length > 3 && !usedNames.has(slug)) break;
|
||||
}
|
||||
|
||||
if (!slug || slug.length <= 3 || usedNames.has(slug)) {
|
||||
slug = `${prefix}-${idx}`;
|
||||
}
|
||||
|
||||
// Deduplicate
|
||||
let final = slug;
|
||||
let suffix = 2;
|
||||
while (usedNames.has(final)) {
|
||||
final = `${slug}-${suffix}`;
|
||||
suffix++;
|
||||
}
|
||||
|
||||
return final;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
/**
|
||||
* Generate labeled contact sheet grids from images.
|
||||
*
|
||||
* Stitches images into a numbered grid with cell labels.
|
||||
* Saves 50-65% tokens vs. AI agents reading images individually.
|
||||
*/
|
||||
|
||||
import sharp from "sharp";
|
||||
import { readdirSync, readFileSync, writeFileSync, unlinkSync, existsSync } from "node:fs";
|
||||
import { join, extname, basename, dirname } from "node:path";
|
||||
|
||||
interface ContactSheetOptions {
|
||||
cols?: number;
|
||||
maxImages?: number;
|
||||
padding?: number;
|
||||
labelMode?: "index" | "filename" | "custom";
|
||||
labels?: string[];
|
||||
quality?: number;
|
||||
/** Target width per cell in pixels (default: 600) */
|
||||
cellWidth?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a contact sheet from a list of image paths.
|
||||
* Returns the output file path, or null if no images.
|
||||
*/
|
||||
export async function createContactSheet(
|
||||
imagePaths: string[],
|
||||
outputPath: string,
|
||||
opts: ContactSheetOptions = {},
|
||||
): Promise<string | null> {
|
||||
const {
|
||||
cols = 3,
|
||||
maxImages = 16,
|
||||
padding = 4,
|
||||
labelMode = "index",
|
||||
labels,
|
||||
quality = 88,
|
||||
cellWidth = 600,
|
||||
} = opts;
|
||||
|
||||
const files = imagePaths.slice(0, maxImages);
|
||||
if (files.length === 0) return null;
|
||||
|
||||
// Read first image to determine aspect ratio
|
||||
const firstMeta = await sharp(files[0]!).metadata();
|
||||
const srcW = firstMeta.width || 1920;
|
||||
const srcH = firstMeta.height || 1080;
|
||||
|
||||
// Scale to target cell width, maintain aspect ratio
|
||||
const scale = cellWidth / srcW;
|
||||
const cellW = cellWidth;
|
||||
const cellH = Math.round(srcH * scale);
|
||||
|
||||
const rows = Math.ceil(files.length / cols);
|
||||
const labelH = 26;
|
||||
const totalW = cols * cellW + (cols + 1) * padding;
|
||||
const totalH = rows * (cellH + labelH) + (rows + 1) * padding;
|
||||
|
||||
const overlays: sharp.OverlayOptions[] = [];
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const col = i % cols;
|
||||
const row = Math.floor(i / cols);
|
||||
const x = padding + col * (cellW + padding);
|
||||
const y = padding + row * (cellH + labelH + padding);
|
||||
|
||||
// Resize image to cell size — contain keeps full image visible (no cropping)
|
||||
const resized = await sharp(files[i]!)
|
||||
.resize(cellW, cellH, { fit: "contain", background: { r: 26, g: 26, b: 26 } })
|
||||
.toBuffer();
|
||||
|
||||
overlays.push({ input: resized, left: x, top: y + labelH });
|
||||
|
||||
// Label text
|
||||
let labelText = `${i + 1}`;
|
||||
if (labelMode === "filename") {
|
||||
labelText = `${i + 1}. ${basename(files[i]!).replace(extname(files[i]!), "")}`;
|
||||
} else if (labelMode === "custom" && labels?.[i]) {
|
||||
labelText = `${i + 1}. ${labels[i]}`;
|
||||
}
|
||||
|
||||
// Truncate label to fit cell
|
||||
if (labelText.length > 60) labelText = labelText.slice(0, 57) + "...";
|
||||
|
||||
const labelSvg = Buffer.from(
|
||||
`<svg width="${cellW}" height="${labelH}">` +
|
||||
`<rect width="${cellW}" height="${labelH}" fill="#1a1a1a"/>` +
|
||||
`<text x="8" y="18" font-family="Arial,Helvetica,sans-serif" font-size="13" font-weight="bold" fill="#ffffff">${escapeXml(labelText)}</text>` +
|
||||
`</svg>`,
|
||||
);
|
||||
|
||||
overlays.push({ input: labelSvg, left: x, top: y });
|
||||
}
|
||||
|
||||
await sharp({
|
||||
create: {
|
||||
width: totalW,
|
||||
height: totalH,
|
||||
channels: 3,
|
||||
background: { r: 26, g: 26, b: 26 },
|
||||
},
|
||||
})
|
||||
.composite(overlays)
|
||||
.jpeg({ quality })
|
||||
.toFile(outputPath);
|
||||
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
function escapeXml(s: string): string {
|
||||
return s
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
/**
|
||||
* Split imagePaths into pages of `pageSize`, write one contact sheet per page.
|
||||
* Output files: basePath → base-1.jpg, base-2.jpg, ...
|
||||
* Returns the list of written file paths (empty if no images).
|
||||
*/
|
||||
async function createContactSheetPages(
|
||||
imagePaths: string[],
|
||||
outputBasePath: string,
|
||||
opts: ContactSheetOptions & { pageSize?: number } = {},
|
||||
labelOffset = 0,
|
||||
customLabels?: string[],
|
||||
): Promise<string[]> {
|
||||
if (imagePaths.length === 0) return [];
|
||||
const { pageSize = imagePaths.length, ...sheetOpts } = opts;
|
||||
const ext = outputBasePath.match(/\.[^.]+$/)?.[0] ?? ".jpg";
|
||||
const base = outputBasePath.slice(0, -ext.length);
|
||||
|
||||
const pages = Math.ceil(imagePaths.length / pageSize);
|
||||
const results: string[] = [];
|
||||
|
||||
for (let p = 0; p < pages; p++) {
|
||||
const chunk = imagePaths.slice(p * pageSize, (p + 1) * pageSize);
|
||||
const chunkLabels = customLabels?.slice(p * pageSize, (p + 1) * pageSize);
|
||||
const outPath = pages === 1 ? outputBasePath : `${base}-${p + 1}${ext}`;
|
||||
|
||||
const labelsForChunk = chunkLabels
|
||||
? { labelMode: "custom" as const, labels: chunkLabels }
|
||||
: sheetOpts.labelMode === "filename"
|
||||
? { labelMode: "filename" as const }
|
||||
: { labelMode: "index" as const };
|
||||
|
||||
const written = await createContactSheet(chunk, outPath, {
|
||||
...sheetOpts,
|
||||
...labelsForChunk,
|
||||
maxImages: chunk.length,
|
||||
});
|
||||
if (written) results.push(written);
|
||||
void labelOffset; // used by callers that pre-compute labels
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Contact sheet for scroll screenshots. Paginated — all screenshots covered.
|
||||
* Labels: "1. 0% scroll", "2. 23% scroll", etc.
|
||||
* Returns array of written file paths.
|
||||
*/
|
||||
export async function createScrollContactSheet(
|
||||
screenshotsDir: string,
|
||||
outputPath: string,
|
||||
): Promise<string[]> {
|
||||
if (!existsSync(screenshotsDir)) return [];
|
||||
|
||||
const scrollFiles = readdirSync(screenshotsDir)
|
||||
.filter((f) => f.startsWith("scroll-") && f.endsWith(".png"))
|
||||
.sort();
|
||||
|
||||
if (scrollFiles.length === 0) return [];
|
||||
|
||||
const paths = scrollFiles.map((f) => join(screenshotsDir, f));
|
||||
const labels = scrollFiles.map((f) => {
|
||||
const m = f.match(/scroll-(\d+)\.png/);
|
||||
return m ? `${m[1]}% scroll` : f;
|
||||
});
|
||||
|
||||
// 3 cols max for readability; 9 per page (3×3) so cells stay large enough to read
|
||||
return createContactSheetPages(
|
||||
paths,
|
||||
outputPath,
|
||||
{ cols: 3, cellWidth: 600, pageSize: 9 },
|
||||
0,
|
||||
labels,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Contact sheet for snapshot frames. All frames covered across pages.
|
||||
* Labels: "1. 1.0s", "2. 3.0s", etc.
|
||||
* Returns array of written file paths.
|
||||
*/
|
||||
export async function createSnapshotContactSheet(
|
||||
snapshotsDir: string,
|
||||
outputPath: string,
|
||||
): Promise<string[]> {
|
||||
if (!existsSync(snapshotsDir)) return [];
|
||||
|
||||
const snapshotFiles = readdirSync(snapshotsDir)
|
||||
.filter((f) => f.startsWith("frame-") && f.endsWith(".png"))
|
||||
.sort();
|
||||
|
||||
if (snapshotFiles.length === 0) return [];
|
||||
|
||||
const paths = snapshotFiles.map((f) => join(snapshotsDir, f));
|
||||
const labels = snapshotFiles.map((f) => {
|
||||
const m = f.match(/at-([\d.]+)s/);
|
||||
return m ? `${m[1]}s` : f;
|
||||
});
|
||||
|
||||
// 3 cols, 9 per page (3×3)
|
||||
return createContactSheetPages(
|
||||
paths,
|
||||
outputPath,
|
||||
{ cols: 3, cellWidth: 600, pageSize: 9 },
|
||||
0,
|
||||
labels,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Contact sheet for captured assets. Paginated — all assets covered.
|
||||
* Labels: "1. filename"
|
||||
* Returns array of written file paths.
|
||||
*/
|
||||
export async function createAssetContactSheet(
|
||||
assetsDir: string,
|
||||
outputPath: string,
|
||||
): Promise<string[]> {
|
||||
if (!existsSync(assetsDir)) return [];
|
||||
|
||||
const imageExts = new Set([".png", ".jpg", ".jpeg", ".webp"]);
|
||||
const assetFiles = readdirSync(assetsDir)
|
||||
.filter((f) => imageExts.has(extname(f).toLowerCase()) && !f.includes("contact-sheet"))
|
||||
.sort();
|
||||
|
||||
if (assetFiles.length === 0) return [];
|
||||
|
||||
const paths = assetFiles.map((f) => join(assetsDir, f));
|
||||
|
||||
// 4 cols, 12 per page (4×3) — covers all assets across as many pages as needed
|
||||
return createContactSheetPages(paths, outputPath, {
|
||||
cols: 4,
|
||||
cellWidth: 480,
|
||||
labelMode: "filename",
|
||||
pageSize: 12,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Contact sheet for SVGs — renders each SVG to a thumbnail PNG, then grids them.
|
||||
* Sharp supports SVG input natively, so no browser needed.
|
||||
* Labels: "1. filename"
|
||||
*
|
||||
* Accepts one or two directories: the primary svgs/ subdir and optionally the
|
||||
* parent assets/ root (for external SVGs downloaded as <img src="*.svg">).
|
||||
* Files are deduplicated by basename so duplicates across dirs are collapsed.
|
||||
*/
|
||||
export async function createSvgContactSheet(
|
||||
svgsDir: string,
|
||||
outputPath: string,
|
||||
assetsRootDir?: string,
|
||||
): Promise<string[]> {
|
||||
const dirsToScan = [svgsDir, assetsRootDir].filter(
|
||||
(d): d is string => d !== undefined && existsSync(d),
|
||||
);
|
||||
if (dirsToScan.length === 0) return [];
|
||||
|
||||
const seen = new Set<string>();
|
||||
const svgPaths: string[] = [];
|
||||
|
||||
for (const dir of dirsToScan) {
|
||||
for (const f of readdirSync(dir)
|
||||
.filter((f) => f.endsWith(".svg"))
|
||||
.sort()) {
|
||||
if (!seen.has(f)) {
|
||||
seen.add(f);
|
||||
svgPaths.push(join(dir, f));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (svgPaths.length === 0) return [];
|
||||
|
||||
const svgFileNames = svgPaths.map((p) => p.split("/").pop()!);
|
||||
|
||||
// Render ALL SVGs to PNG thumbnails first, then paginate the sheets
|
||||
const thumbSize = 200;
|
||||
const tmpDir = dirname(outputPath);
|
||||
const tmpPaths: string[] = [];
|
||||
const labels: string[] = [];
|
||||
|
||||
for (let i = 0; i < svgPaths.length; i++) {
|
||||
const svgPath = svgPaths[i]!;
|
||||
const tmpPath = join(tmpDir, `.thumb-${i}.png`);
|
||||
try {
|
||||
const svgBuf = readFileSync(svgPath);
|
||||
const thumb = await sharp(svgBuf)
|
||||
.resize(thumbSize, thumbSize, {
|
||||
fit: "contain",
|
||||
background: { r: 245, g: 245, b: 245, alpha: 1 },
|
||||
})
|
||||
.flatten({ background: { r: 245, g: 245, b: 245 } })
|
||||
.png()
|
||||
.toBuffer();
|
||||
writeFileSync(tmpPath, thumb);
|
||||
tmpPaths.push(tmpPath);
|
||||
labels.push(svgFileNames[i]!.replace(".svg", ""));
|
||||
} catch {
|
||||
// SVG might be malformed — skip
|
||||
}
|
||||
}
|
||||
|
||||
if (tmpPaths.length === 0) return [];
|
||||
|
||||
// 5 cols, 15 per page (5×3) — all SVGs covered across pages
|
||||
let results: string[] = [];
|
||||
try {
|
||||
results = await createContactSheetPages(
|
||||
tmpPaths,
|
||||
outputPath,
|
||||
{
|
||||
cols: 5,
|
||||
cellWidth: thumbSize,
|
||||
pageSize: 15,
|
||||
},
|
||||
0,
|
||||
labels,
|
||||
);
|
||||
} finally {
|
||||
for (const tmp of tmpPaths) {
|
||||
try {
|
||||
unlinkSync(tmp);
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
@@ -9,7 +9,7 @@
|
||||
*/
|
||||
|
||||
import type { Page } from "puppeteer-core";
|
||||
import { readdirSync, statSync, readFileSync } from "node:fs";
|
||||
import { existsSync, readdirSync, statSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { CatalogedAsset } from "./assetCataloger.js";
|
||||
import type { DesignTokens } from "./types.js";
|
||||
@@ -231,6 +231,69 @@ export async function captionImagesWithGemini(
|
||||
);
|
||||
}
|
||||
progress("design", `${Object.keys(geminiCaptions).length} images captioned with Gemini`);
|
||||
|
||||
// Caption SVGs by sending source code as text (vision API rejects image/svg+xml).
|
||||
const svgFiles: Array<{ file: string; relPath: string }> = [];
|
||||
const assetsDir = join(outputDir, "assets");
|
||||
for (const f of readdirSync(assetsDir)) {
|
||||
if (/\.svg$/i.test(f)) svgFiles.push({ file: f, relPath: f });
|
||||
}
|
||||
const svgsSubdir = join(assetsDir, "svgs");
|
||||
if (existsSync(svgsSubdir)) {
|
||||
for (const f of readdirSync(svgsSubdir)) {
|
||||
if (/\.svg$/i.test(f)) svgFiles.push({ file: f, relPath: `svgs/${f}` });
|
||||
}
|
||||
}
|
||||
|
||||
if (svgFiles.length > 0) {
|
||||
progress("design", `Captioning ${svgFiles.length} SVGs via code analysis...`);
|
||||
const SVG_BATCH = 20;
|
||||
const MAX_SVG_CHARS = 10_000;
|
||||
for (let i = 0; i < svgFiles.length; i += SVG_BATCH) {
|
||||
const batch = svgFiles.slice(i, i + SVG_BATCH);
|
||||
const results = await Promise.allSettled(
|
||||
batch.map(async ({ relPath }) => {
|
||||
const filePath = join(assetsDir, relPath);
|
||||
let svgText = readFileSync(filePath, "utf-8");
|
||||
if (svgText.length > MAX_SVG_CHARS) {
|
||||
svgText = svgText.slice(0, MAX_SVG_CHARS) + "\n<!-- truncated -->";
|
||||
}
|
||||
const response = await ai.models.generateContent({
|
||||
model,
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{
|
||||
text:
|
||||
"This SVG code is from a website. Describe what it renders in ONE short sentence " +
|
||||
"for a video storyboard. Focus on: what shape/icon/illustration it is, its colors. " +
|
||||
"Be factual.\n\n" +
|
||||
svgText,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
config: { maxOutputTokens: 300 },
|
||||
});
|
||||
return { file: relPath, caption: response.text?.trim() || "" };
|
||||
}),
|
||||
);
|
||||
for (const result of results) {
|
||||
if (result.status === "fulfilled" && result.value.caption) {
|
||||
geminiCaptions[result.value.file] = result.value.caption;
|
||||
}
|
||||
}
|
||||
if (i + SVG_BATCH < svgFiles.length) {
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
progress(
|
||||
"design",
|
||||
`Captioned ${Math.min(i + SVG_BATCH, svgFiles.length)}/${svgFiles.length} SVGs...`,
|
||||
);
|
||||
}
|
||||
progress("design", `${Object.keys(geminiCaptions).length} total assets captioned`);
|
||||
}
|
||||
} catch (err) {
|
||||
warnings.push(`Gemini captioning failed: ${err}`);
|
||||
}
|
||||
@@ -295,6 +358,11 @@ export function generateAssetDescriptions(
|
||||
const svgsPath = join(assetsPath, "svgs");
|
||||
for (const file of readdirSync(svgsPath)) {
|
||||
if (!file.endsWith(".svg")) continue;
|
||||
const geminiCaption = geminiCaptions[`svgs/${file}`];
|
||||
if (geminiCaption) {
|
||||
svgLines.push(`svgs/${file} — ${geminiCaption}`);
|
||||
continue;
|
||||
}
|
||||
const svgMatch = tokens.svgs.find(
|
||||
(s) =>
|
||||
s.label &&
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
/**
|
||||
* Extract computed design styles from key DOM elements.
|
||||
*
|
||||
* Targets ~50 elements (headings, body text, buttons, cards, nav) and extracts
|
||||
* only design-relevant CSS properties. Output is a compact, pre-clustered
|
||||
* design system summary — not raw computed styles per element.
|
||||
*
|
||||
* All page.evaluate() calls use string expressions to avoid
|
||||
* tsx/esbuild __name injection (see esbuild issue #1031).
|
||||
*/
|
||||
|
||||
import type { Page } from "puppeteer-core";
|
||||
import type { DesignStyles } from "./types.js";
|
||||
|
||||
const EXTRACT_DESIGN_STYLES_SCRIPT = `(() => {
|
||||
var isVisible = (el) => {
|
||||
var s = getComputedStyle(el);
|
||||
return s.display !== "none" && s.visibility !== "hidden" && s.opacity !== "0" && el.getBoundingClientRect().height > 0;
|
||||
};
|
||||
|
||||
function rgbToHex(color) {
|
||||
if (!color) return "";
|
||||
if (color.startsWith('#')) return color.toUpperCase();
|
||||
var m = color.match(/rgba?\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*,\\s*(\\d+)/);
|
||||
if (!m) return color;
|
||||
return '#' + ((1<<24) + (parseInt(m[1])<<16) + (parseInt(m[2])<<8) + parseInt(m[3])).toString(16).slice(1).toUpperCase();
|
||||
}
|
||||
|
||||
function cleanFont(f) {
|
||||
return f.split(",")[0].replace(/['"]/g, "").trim();
|
||||
}
|
||||
|
||||
function getStyles(el) {
|
||||
var s = getComputedStyle(el);
|
||||
return {
|
||||
fontFamily: cleanFont(s.fontFamily),
|
||||
fontSize: s.fontSize,
|
||||
fontWeight: s.fontWeight,
|
||||
lineHeight: s.lineHeight,
|
||||
letterSpacing: s.letterSpacing,
|
||||
color: rgbToHex(s.color),
|
||||
background: rgbToHex(s.backgroundColor),
|
||||
padding: s.padding,
|
||||
borderRadius: s.borderRadius,
|
||||
border: s.border,
|
||||
boxShadow: s.boxShadow === "none" ? "none" : s.boxShadow,
|
||||
height: s.height
|
||||
};
|
||||
}
|
||||
|
||||
// ── 1. Typography hierarchy ──
|
||||
// Sample each text role and deduplicate by fontSize
|
||||
var typographyMap = {};
|
||||
var roleSelectors = [
|
||||
{ role: "display", sel: "h1", max: 3 },
|
||||
{ role: "heading-2", sel: "h2", max: 5 },
|
||||
{ role: "heading-3", sel: "h3", max: 5 },
|
||||
{ role: "heading-4", sel: "h4", max: 3 },
|
||||
{ role: "body", sel: "p", max: 10 },
|
||||
{ role: "body-small", sel: "figcaption, .caption, [class*='caption'], [class*='subtitle'], small", max: 5 },
|
||||
{ role: "label", sel: "label, [class*='label'], [class*='tag'], [class*='badge']", max: 5 },
|
||||
{ role: "link", sel: "a:not([class*='btn']):not([class*='button']):not([role='button'])", max: 5 },
|
||||
{ role: "code", sel: "code, pre, [class*='mono']", max: 3 }
|
||||
];
|
||||
|
||||
for (var ri = 0; ri < roleSelectors.length; ri++) {
|
||||
var spec = roleSelectors[ri];
|
||||
var els = Array.from(document.querySelectorAll(spec.sel)).slice(0, spec.max);
|
||||
for (var ei = 0; ei < els.length; ei++) {
|
||||
if (!isVisible(els[ei])) continue;
|
||||
var s = getStyles(els[ei]);
|
||||
var key = s.fontSize + "|" + s.fontWeight + "|" + s.fontFamily;
|
||||
if (!typographyMap[key]) {
|
||||
var text = (els[ei].textContent || "").trim().replace(/\\s+/g, " ").slice(0, 60);
|
||||
typographyMap[key] = {
|
||||
role: spec.role,
|
||||
fontFamily: s.fontFamily,
|
||||
fontSize: s.fontSize,
|
||||
fontWeight: s.fontWeight,
|
||||
lineHeight: s.lineHeight,
|
||||
letterSpacing: s.letterSpacing,
|
||||
color: s.color,
|
||||
sampleText: text
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by font size descending
|
||||
var typography = Object.values(typographyMap);
|
||||
typography.sort(function(a, b) {
|
||||
return parseFloat(b.fontSize) - parseFloat(a.fontSize);
|
||||
});
|
||||
|
||||
// Deduplicate roles — keep only the first (largest) for each role prefix
|
||||
var seenRoles = {};
|
||||
var uniqueTypo = [];
|
||||
for (var ti = 0; ti < typography.length; ti++) {
|
||||
var baseRole = typography[ti].role.replace(/-\\d+$/, "");
|
||||
if (!seenRoles[baseRole]) {
|
||||
seenRoles[baseRole] = true;
|
||||
typography[ti].role = baseRole;
|
||||
uniqueTypo.push(typography[ti]);
|
||||
} else if (baseRole === "heading") {
|
||||
// Keep multiple heading levels
|
||||
uniqueTypo.push(typography[ti]);
|
||||
}
|
||||
}
|
||||
|
||||
// ── 2. Buttons ──
|
||||
var buttonEls = Array.from(document.querySelectorAll(
|
||||
'button, a[class*="btn"], a[class*="button"], a[role="button"], ' +
|
||||
'[class*="btn-"], [class*="button-"], [class*="cta"]'
|
||||
)).filter(function(el) {
|
||||
return isVisible(el) && !el.closest('nav, [role="navigation"]');
|
||||
}).slice(0, 10);
|
||||
|
||||
var buttonMap = {};
|
||||
for (var bi = 0; bi < buttonEls.length; bi++) {
|
||||
var bs = getStyles(buttonEls[bi]);
|
||||
// Deduplicate by visual appearance
|
||||
var bKey = bs.background + "|" + bs.borderRadius + "|" + bs.border;
|
||||
if (!buttonMap[bKey]) {
|
||||
var btnText = (buttonEls[bi].textContent || "").trim().slice(0, 40);
|
||||
buttonMap[bKey] = {
|
||||
label: btnText || "button",
|
||||
background: bs.background,
|
||||
color: bs.color,
|
||||
padding: bs.padding,
|
||||
borderRadius: bs.borderRadius,
|
||||
border: bs.border,
|
||||
boxShadow: bs.boxShadow,
|
||||
fontSize: bs.fontSize,
|
||||
fontWeight: bs.fontWeight,
|
||||
height: bs.height
|
||||
};
|
||||
}
|
||||
}
|
||||
var buttons = Object.values(buttonMap).slice(0, 4);
|
||||
|
||||
// ── 3. Cards / containers ──
|
||||
var cardEls = Array.from(document.querySelectorAll(
|
||||
'[class*="card"], [class*="Card"], [class*="tile"], [class*="Tile"], ' +
|
||||
'[class*="panel"], [class*="Panel"], [class*="feature"], ' +
|
||||
'article, [class*="box"]:not(select):not(input)'
|
||||
)).filter(function(el) {
|
||||
var rect = el.getBoundingClientRect();
|
||||
return isVisible(el) && rect.width > 100 && rect.height > 80;
|
||||
}).slice(0, 10);
|
||||
|
||||
var cardMap = {};
|
||||
for (var ci = 0; ci < cardEls.length; ci++) {
|
||||
var cs = getStyles(cardEls[ci]);
|
||||
var cKey = cs.background + "|" + cs.borderRadius + "|" + cs.border;
|
||||
if (!cardMap[cKey]) {
|
||||
cardMap[cKey] = {
|
||||
label: "card",
|
||||
background: cs.background,
|
||||
color: cs.color,
|
||||
padding: cs.padding,
|
||||
borderRadius: cs.borderRadius,
|
||||
border: cs.border,
|
||||
boxShadow: cs.boxShadow,
|
||||
fontSize: cs.fontSize,
|
||||
fontWeight: cs.fontWeight,
|
||||
height: cs.height
|
||||
};
|
||||
}
|
||||
}
|
||||
var cards = Object.values(cardMap).slice(0, 4);
|
||||
|
||||
// ── 4. Navigation ──
|
||||
var navEl = document.querySelector('nav, header, [role="navigation"], [class*="navbar"], [class*="header"]');
|
||||
var nav = null;
|
||||
if (navEl && isVisible(navEl)) {
|
||||
var ns = getStyles(navEl);
|
||||
nav = {
|
||||
label: "navigation",
|
||||
background: ns.background,
|
||||
color: ns.color,
|
||||
padding: ns.padding,
|
||||
borderRadius: ns.borderRadius,
|
||||
border: ns.border,
|
||||
boxShadow: ns.boxShadow,
|
||||
fontSize: ns.fontSize,
|
||||
fontWeight: ns.fontWeight,
|
||||
height: ns.height
|
||||
};
|
||||
}
|
||||
|
||||
// ── 5. Spacing scale ──
|
||||
// Collect padding and margin values from visible elements
|
||||
var spacingCounts = {};
|
||||
var spacingSamples = Array.from(document.querySelectorAll(
|
||||
"section, div, article, main, aside, header, footer, nav, " +
|
||||
"button, a, p, h1, h2, h3, h4, li, ul, ol"
|
||||
)).slice(0, 200);
|
||||
|
||||
for (var si = 0; si < spacingSamples.length; si++) {
|
||||
if (!isVisible(spacingSamples[si])) continue;
|
||||
var ss = getComputedStyle(spacingSamples[si]);
|
||||
var props = [ss.paddingTop, ss.paddingRight, ss.paddingBottom, ss.paddingLeft,
|
||||
ss.marginTop, ss.marginRight, ss.marginBottom, ss.marginLeft,
|
||||
ss.gap, ss.rowGap, ss.columnGap];
|
||||
for (var pi = 0; pi < props.length; pi++) {
|
||||
var val = parseFloat(props[pi]);
|
||||
if (val > 0 && val <= 200) {
|
||||
var rounded = Math.round(val);
|
||||
spacingCounts[rounded] = (spacingCounts[rounded] || 0) + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find the most common spacing values (at least 3 occurrences)
|
||||
var spacingEntries = Object.entries(spacingCounts)
|
||||
.filter(function(e) { return e[1] >= 3; })
|
||||
.sort(function(a, b) { return b[1] - a[1]; });
|
||||
var observedSpacing = spacingEntries.map(function(e) { return parseInt(e[0]); }).sort(function(a,b) { return a - b; });
|
||||
|
||||
// Detect base unit — GCD of the top spacing values, clamped to 4 or 8
|
||||
var baseUnit = 8;
|
||||
if (observedSpacing.length >= 3) {
|
||||
var divisible4 = observedSpacing.filter(function(v) { return v % 4 === 0; }).length;
|
||||
var divisible8 = observedSpacing.filter(function(v) { return v % 8 === 0; }).length;
|
||||
baseUnit = (divisible4 > divisible8 * 1.5) ? 4 : 8;
|
||||
}
|
||||
|
||||
// ── 6. Border radius scale ──
|
||||
var radiusCounts = {};
|
||||
var radiusSamples = Array.from(document.querySelectorAll(
|
||||
"button, a, [class*='card'], [class*='btn'], input, select, textarea, " +
|
||||
"[class*='badge'], [class*='tag'], [class*='chip'], img, video"
|
||||
)).slice(0, 100);
|
||||
|
||||
for (var rsi = 0; rsi < radiusSamples.length; rsi++) {
|
||||
if (!isVisible(radiusSamples[rsi])) continue;
|
||||
var br = getComputedStyle(radiusSamples[rsi]).borderRadius;
|
||||
if (br && br !== "0px") {
|
||||
radiusCounts[br] = (radiusCounts[br] || 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
var radius = Object.entries(radiusCounts)
|
||||
.filter(function(e) { return e[1] >= 2; })
|
||||
.sort(function(a, b) { return parseFloat(a[0]) - parseFloat(b[0]); })
|
||||
.map(function(e) { return e[0]; });
|
||||
|
||||
// ── 7. Box shadows ──
|
||||
var shadowCounts = {};
|
||||
var shadowSamples = Array.from(document.querySelectorAll(
|
||||
"[class*='card'], [class*='Card'], button, [class*='btn'], " +
|
||||
"[class*='dropdown'], [class*='modal'], [class*='popover'], " +
|
||||
"nav, header, [class*='panel'], article"
|
||||
)).slice(0, 100);
|
||||
|
||||
for (var shi = 0; shi < shadowSamples.length; shi++) {
|
||||
if (!isVisible(shadowSamples[shi])) continue;
|
||||
var shVal = getComputedStyle(shadowSamples[shi]).boxShadow;
|
||||
if (shVal && shVal !== "none") {
|
||||
shadowCounts[shVal] = (shadowCounts[shVal] || 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
var shadows = Object.entries(shadowCounts)
|
||||
.sort(function(a, b) { return b[1] - a[1]; })
|
||||
.slice(0, 5)
|
||||
.map(function(e) { return { value: e[0], count: e[1] }; });
|
||||
|
||||
return {
|
||||
typography: uniqueTypo,
|
||||
spacing: { observed: observedSpacing.slice(0, 15), baseUnit: baseUnit },
|
||||
radius: radius,
|
||||
shadows: shadows,
|
||||
buttons: buttons,
|
||||
cards: cards,
|
||||
nav: nav
|
||||
};
|
||||
})()`;
|
||||
|
||||
export async function extractDesignStyles(page: Page): Promise<DesignStyles> {
|
||||
return page.evaluate(EXTRACT_DESIGN_STYLES_SCRIPT) as Promise<DesignStyles>;
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { join } from "node:path";
|
||||
import { extractHtml } from "./htmlExtractor.js";
|
||||
// captureScreenshots removed — full-page screenshot replaces per-section shots
|
||||
import { extractTokens } from "./tokenExtractor.js";
|
||||
import { extractDesignStyles } from "./designStyleExtractor.js";
|
||||
import { downloadAssets, downloadAndRewriteFonts } from "./assetDownloader.js";
|
||||
import { extractFontMetadata } from "./fontMetadataExtractor.js";
|
||||
// briefGenerator.ts, visual-style, capture-summary removed — DESIGN.md replaces them
|
||||
@@ -317,12 +318,36 @@ export async function captureWebsite(
|
||||
// Extract design tokens
|
||||
progress("tokens", "Extracting design tokens...");
|
||||
const tokens = await extractTokens(page1);
|
||||
// Save tokens.json without SVG outerHTML (kept in memory for asset downloader)
|
||||
const tokensForDisk = {
|
||||
...tokens,
|
||||
svgs: tokens.svgs.map(({ outerHTML: _, ...rest }) => rest),
|
||||
};
|
||||
writeFileSync(
|
||||
join(outputDir, "extracted", "tokens.json"),
|
||||
JSON.stringify(tokens, null, 2),
|
||||
JSON.stringify(tokensForDisk, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
// Extract computed design styles (typography, buttons, cards, spacing, shadows)
|
||||
progress("style", "Extracting design styles...");
|
||||
try {
|
||||
const designStyles = await extractDesignStyles(page1);
|
||||
writeFileSync(
|
||||
join(outputDir, "extracted", "design-styles.json"),
|
||||
JSON.stringify(designStyles, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
progress(
|
||||
"tokens",
|
||||
`${designStyles.typography.length} typography roles, ${designStyles.buttons.length} button styles, ${designStyles.shadows.length} shadow values extracted`,
|
||||
);
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? `${err.message}\n${err.stack}` : String(err);
|
||||
console.error(` ⚠ Design style extraction failed: ${errMsg}`);
|
||||
warnings.push(`Design style extraction failed: ${errMsg}`);
|
||||
}
|
||||
|
||||
// Collect animation catalog
|
||||
progress("animations", "Cataloging animations...");
|
||||
animationCatalog = await collectAnimationCatalog(page1, cdpAnims, cdp);
|
||||
@@ -486,23 +511,7 @@ export async function captureWebsite(
|
||||
writeFileSync(join(outputDir, "extracted", "visible-text.txt"), visibleTextContent, "utf-8");
|
||||
}
|
||||
|
||||
// Save cataloged assets as JSON for AI agent
|
||||
if (catalogedAssets.length > 0) {
|
||||
writeFileSync(
|
||||
join(outputDir, "extracted", "assets-catalog.json"),
|
||||
JSON.stringify(catalogedAssets, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
}
|
||||
|
||||
// Save detected libraries
|
||||
if (detectedLibraries.length > 0) {
|
||||
writeFileSync(
|
||||
join(outputDir, "extracted", "detected-libraries.json"),
|
||||
JSON.stringify(detectedLibraries, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
}
|
||||
// detected-libraries and assets-catalog removed — 0/8 agents read them in v6 testing
|
||||
|
||||
// AI-powered image captioning via Gemini (optional — enriches asset descriptions)
|
||||
const geminiCaptions = await captionImagesWithGemini(outputDir, progress, warnings);
|
||||
@@ -528,6 +537,52 @@ export async function captureWebsite(
|
||||
|
||||
progress("design", "DESIGN.md will be created by your AI agent");
|
||||
|
||||
// Generate contact sheets (saves AI agents 50-65% tokens vs reading images individually)
|
||||
// All functions return string[] — paginated so every image is covered
|
||||
try {
|
||||
const { createScrollContactSheet, createAssetContactSheet, createSvgContactSheet } =
|
||||
await import("./contactSheet.js");
|
||||
|
||||
const scrollSheets = await createScrollContactSheet(
|
||||
join(outputDir, "screenshots"),
|
||||
join(outputDir, "screenshots", "contact-sheet.jpg"),
|
||||
);
|
||||
if (scrollSheets.length > 0)
|
||||
progress(
|
||||
"design",
|
||||
`Screenshot contact sheet generated (${scrollSheets.length} page${scrollSheets.length > 1 ? "s" : ""})`,
|
||||
);
|
||||
|
||||
const assetsImgDir = join(outputDir, "assets");
|
||||
if (existsSync(assetsImgDir)) {
|
||||
const assetSheets = await createAssetContactSheet(
|
||||
assetsImgDir,
|
||||
join(outputDir, "assets", "contact-sheet.jpg"),
|
||||
);
|
||||
if (assetSheets.length > 0)
|
||||
progress(
|
||||
"design",
|
||||
`Asset contact sheet generated (${assetSheets.length} page${assetSheets.length > 1 ? "s" : ""})`,
|
||||
);
|
||||
}
|
||||
|
||||
// Scan assets/svgs/ (inline SVGs) AND assets/ root (external SVGs from <img src="*.svg">)
|
||||
// so sites like huly.io that only use external SVGs still get a grid
|
||||
const svgsDir = join(outputDir, "assets", "svgs");
|
||||
const assetsRootDir = join(outputDir, "assets");
|
||||
const svgOutputPath = existsSync(svgsDir)
|
||||
? join(outputDir, "assets", "svgs", "contact-sheet.jpg")
|
||||
: join(outputDir, "assets", "contact-sheet-svgs.jpg");
|
||||
const svgSheets = await createSvgContactSheet(svgsDir, svgOutputPath, assetsRootDir);
|
||||
if (svgSheets.length > 0)
|
||||
progress(
|
||||
"design",
|
||||
`SVG contact sheet generated (${svgSheets.length} page${svgSheets.length > 1 ? "s" : ""})`,
|
||||
);
|
||||
} catch {
|
||||
/* contact sheets are non-critical — agent can still read images individually */
|
||||
}
|
||||
|
||||
// Generate project scaffold (index.html, meta.json, CLAUDE.md)
|
||||
await generateProjectScaffold(
|
||||
outputDir,
|
||||
|
||||
@@ -29,6 +29,82 @@ export async function captureScrollScreenshots(page: Page, outputDir: string): P
|
||||
const filePaths: string[] = [];
|
||||
|
||||
try {
|
||||
// Dismiss marketing banners, cookie consents, and popups before scrolling.
|
||||
// These overlay content and contaminate screenshots with UI that doesn't
|
||||
// belong in video compositions (cookie popups, newsletter modals, etc.)
|
||||
await page
|
||||
.evaluate(() => {
|
||||
// Click common dismiss/accept buttons
|
||||
const selectors = [
|
||||
// Cookie consent
|
||||
'[id*="cookie"] button[class*="accept"]',
|
||||
'[id*="cookie"] button[class*="agree"]',
|
||||
'[id*="cookie"] button[class*="allow"]',
|
||||
'[class*="cookie"] button[class*="accept"]',
|
||||
'[class*="consent"] button',
|
||||
// Generic close buttons on overlays/modals
|
||||
'[class*="banner"] [class*="close"]',
|
||||
'[class*="banner"] [class*="dismiss"]',
|
||||
'[class*="popup"] [class*="close"]',
|
||||
'[class*="modal"] [class*="close"]',
|
||||
'[class*="overlay"] [class*="close"]',
|
||||
// Common GDPR patterns — scoped under a cookie/consent/gdpr ancestor
|
||||
// so we don't click "Accept invitation" / "Accept terms" / etc. on
|
||||
// unrelated buttons elsewhere on the page.
|
||||
'[id*="cookie" i] button[id*="accept" i]',
|
||||
'[id*="consent" i] button[id*="accept" i]',
|
||||
'[id*="gdpr" i] button[id*="accept" i]',
|
||||
'[class*="cookie" i] button[class*="accept-all" i]',
|
||||
'[class*="cookie" i] button[class*="acceptAll" i]',
|
||||
'[class*="consent" i] button[class*="accept-all" i]',
|
||||
// Notification prompts
|
||||
'button[class*="decline"]',
|
||||
'button[class*="not-now"]',
|
||||
'button[class*="no-thanks"]',
|
||||
];
|
||||
for (const sel of selectors) {
|
||||
try {
|
||||
const el = document.querySelector<HTMLElement>(sel);
|
||||
if (el) el.click();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
// Hide fixed/sticky overlays that aren't the main nav. Scanning every
|
||||
// element with querySelectorAll('*') + getComputedStyle is O(n) DOM
|
||||
// calls and can dominate evaluate() time on large pages. Narrow the
|
||||
// candidate set with a TreeWalker that early-exits on viewport-sized
|
||||
// rect checks (cheap) before reaching the expensive getComputedStyle.
|
||||
const SCAN_CAP = 5000;
|
||||
const minWidth = window.innerWidth * 0.3;
|
||||
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
|
||||
let visited = 0;
|
||||
let node = walker.nextNode();
|
||||
while (node && visited < SCAN_CAP) {
|
||||
visited++;
|
||||
const el = node as HTMLElement;
|
||||
const rect = el.getBoundingClientRect();
|
||||
// Cheap viewport-size filter first — eliminates the vast majority of
|
||||
// tiny / hidden / off-screen elements without touching getComputedStyle.
|
||||
if (rect.height > 80 && rect.width > minWidth) {
|
||||
const tag = el.tagName;
|
||||
if (tag !== "HEADER" && tag !== "NAV" && !el.closest("header") && !el.closest("nav")) {
|
||||
const style = window.getComputedStyle(el);
|
||||
if (
|
||||
(style.position === "fixed" || style.position === "sticky") &&
|
||||
style.zIndex !== "auto" &&
|
||||
parseInt(style.zIndex) > 100
|
||||
) {
|
||||
el.style.display = "none";
|
||||
}
|
||||
}
|
||||
}
|
||||
node = walker.nextNode();
|
||||
}
|
||||
})
|
||||
.catch(() => {});
|
||||
await new Promise((r) => setTimeout(r, 400));
|
||||
|
||||
const scrollHeight = (await page.evaluate(
|
||||
`Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)`,
|
||||
)) as number;
|
||||
@@ -74,6 +150,8 @@ export async function captureScrollScreenshots(page: Page, outputDir: string): P
|
||||
// Reset scroll
|
||||
await page.evaluate(`window.scrollTo(0, 0)`);
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
// full-page.png removed — 1/8 agents read it, contact sheet covers the same content
|
||||
} catch {
|
||||
/* scroll screenshots are non-critical */
|
||||
}
|
||||
|
||||
@@ -304,9 +304,12 @@ const EXTRACT_SCRIPT = `(() => {
|
||||
// Keep SVGs that have a label OR are at least 16px wide OR are inside a logo/brand context
|
||||
var inLogoContext = svg.closest('[class*="logo"], [class*="brand"], [class*="partner"], [class*="customer"], [class*="marquee"]') !== null;
|
||||
if (!label && !inLogoContext && (!w || parseInt(w) < 16)) return null;
|
||||
var rect = svg.getBoundingClientRect();
|
||||
return {
|
||||
label: label || undefined,
|
||||
viewBox: svg.getAttribute("viewBox") || undefined,
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
outerHTML: svg.outerHTML.slice(0, 10000),
|
||||
isLogo: (label && label.toLowerCase().indexOf("logo") !== -1) || svg.closest('[class*="logo"], [class*="brand"], [class*="home"], [class*="marquee"], [class*="partner"], [class*="customer"]') !== null
|
||||
};
|
||||
|
||||
@@ -102,10 +102,12 @@ export interface DesignTokens {
|
||||
}>;
|
||||
/** CTA button/link text */
|
||||
ctas: Array<{ text: string; href?: string }>;
|
||||
/** SVG elements with labels */
|
||||
/** SVG elements with labels (outerHTML kept in memory for asset downloader, stripped from saved JSON) */
|
||||
svgs: Array<{
|
||||
label?: string;
|
||||
viewBox?: string;
|
||||
width: number;
|
||||
height: number;
|
||||
outerHTML: string;
|
||||
isLogo: boolean;
|
||||
}>;
|
||||
@@ -121,6 +123,45 @@ export interface DesignTokens {
|
||||
}>;
|
||||
}
|
||||
|
||||
// ── Design Styles (computed from live DOM) ──────────────────────────────────
|
||||
|
||||
export interface TypographyRole {
|
||||
role: string;
|
||||
fontFamily: string;
|
||||
fontSize: string;
|
||||
fontWeight: string;
|
||||
lineHeight: string;
|
||||
letterSpacing: string;
|
||||
color: string;
|
||||
sampleText: string;
|
||||
}
|
||||
|
||||
export interface ComponentStyle {
|
||||
label: string;
|
||||
background: string;
|
||||
color: string;
|
||||
padding: string;
|
||||
borderRadius: string;
|
||||
border: string;
|
||||
boxShadow: string;
|
||||
fontSize: string;
|
||||
fontWeight: string;
|
||||
height: string;
|
||||
}
|
||||
|
||||
export interface DesignStyles {
|
||||
typography: TypographyRole[];
|
||||
spacing: {
|
||||
observed: number[];
|
||||
baseUnit: number;
|
||||
};
|
||||
radius: string[];
|
||||
shadows: Array<{ value: string; count: number }>;
|
||||
buttons: ComponentStyle[];
|
||||
cards: ComponentStyle[];
|
||||
nav: ComponentStyle | null;
|
||||
}
|
||||
|
||||
// ── Assets ──────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface DownloadedAsset {
|
||||
|
||||
@@ -18,6 +18,7 @@ import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { existsSync } from "node:fs";
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
(() => {
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const shader = join(here, "shaderTransitionWorker.js");
|
||||
@@ -47,6 +48,42 @@ if (rootVersionRequested) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── Load .env from CWD ─────────────────────────────────────────────────────
|
||||
// Agents run from the project directory where .env holds API keys (Gemini,
|
||||
// HeyGen, ElevenLabs). Load it automatically so they don't need `source .env`.
|
||||
try {
|
||||
const { readFileSync } = await import("node:fs");
|
||||
const { resolve } = await import("node:path");
|
||||
const envPath = resolve(process.cwd(), ".env");
|
||||
const envContent = readFileSync(envPath, "utf-8");
|
||||
for (const rawLine of envContent.split("\n")) {
|
||||
let line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
// Tolerate `export FOO=bar` (common in dotfile-style .env files).
|
||||
if (line.startsWith("export ")) line = line.slice(7).trim();
|
||||
const eqIdx = line.indexOf("=");
|
||||
if (eqIdx < 1) continue;
|
||||
const key = line.slice(0, eqIdx).trim();
|
||||
let val = line.slice(eqIdx + 1).trim();
|
||||
if (val.startsWith('"') || val.startsWith("'")) {
|
||||
// Quoted value: take until the matching closing quote; leave the rest.
|
||||
// Anything after a closing quote (including `# comment`) is dropped.
|
||||
const quote = val.charAt(0);
|
||||
const end = val.indexOf(quote, 1);
|
||||
if (end > 0) val = val.slice(1, end);
|
||||
else val = val.slice(1); // unterminated quote — best-effort, strip opener
|
||||
} else {
|
||||
// Unquoted value: strip inline `# comment` (requires whitespace before #
|
||||
// to avoid eating `pass#word` style values).
|
||||
const commentMatch = val.match(/\s+#/);
|
||||
if (commentMatch?.index !== undefined) val = val.slice(0, commentMatch.index).trim();
|
||||
}
|
||||
if (key && !(key in process.env)) process.env[key] = val;
|
||||
}
|
||||
} catch {
|
||||
/* .env not present — fine, env vars may be set another way */
|
||||
}
|
||||
|
||||
// ── Lazy imports ────────────────────────────────────────────────────────────
|
||||
// Telemetry, update checks, and heavy modules are imported only when needed.
|
||||
// For --help we skip telemetry entirely.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { defineCommand } from "citty";
|
||||
import { existsSync, mkdtempSync, readFileSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { existsSync, mkdtempSync, readFileSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve, join, relative, isAbsolute } from "node:path";
|
||||
import { resolveProject } from "../utils/project.js";
|
||||
@@ -139,25 +139,88 @@ async function captureSnapshots(
|
||||
})
|
||||
.catch(() => {});
|
||||
|
||||
// Wait for sub-compositions to be mounted by the runtime
|
||||
// (they're fetched and injected asynchronously via data-composition-src)
|
||||
// Wait for ALL sub-compositions to be mounted by the runtime.
|
||||
// The old check resolved when the first sub-timeline registered, causing
|
||||
// "last beat black" bugs: beat-5's sub-comp hadn't loaded yet when the
|
||||
// snapshot seeked into its time range. Now we count data-composition-src
|
||||
// host elements and wait until we have a matching number of sub-timelines.
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => {
|
||||
const tls = (window as any).__timelines;
|
||||
if (!tls) return false;
|
||||
const keys = Object.keys(tls);
|
||||
// Wait until at least one sub-composition timeline is registered
|
||||
// (not counting "main" or empty registrations)
|
||||
return keys.length >= 2 || keys.some((k) => k !== "main");
|
||||
const hosts = document.querySelectorAll("[data-composition-src]").length;
|
||||
if (hosts === 0) return Object.keys(tls).length >= 1;
|
||||
const subKeys = Object.keys(tls).filter((k) => k !== "main");
|
||||
return subKeys.length >= hosts;
|
||||
},
|
||||
{ timeout: timeoutMs },
|
||||
)
|
||||
.catch(() => {});
|
||||
|
||||
// Wait for shader transition pre-rendering to complete (if active).
|
||||
//
|
||||
// Two failure modes existed with the previous overlay-only check:
|
||||
// 1. Cold cache: HyperShader creates [data-hyper-shader-loading] but never
|
||||
// removes it from the DOM — it only sets display:none. Checking for
|
||||
// element *absence* never resolved, so the wait always timed out at 60s.
|
||||
// 2. Warm cache: HyperShader loads frames from IndexedDB without showing
|
||||
// the overlay at all. Checking for element absence resolved instantly
|
||||
// (no element) while hydration was still running in the background.
|
||||
//
|
||||
// Fix: use window.__hf.shaderTransitions[].ready as the primary signal
|
||||
// (set after both warm and cold cache paths complete), with the overlay
|
||||
// display:none as a fallback for older builds that lack the ready state.
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => {
|
||||
const win = window as unknown as {
|
||||
__hf?: { shaderTransitions?: Record<string, { ready?: boolean }> };
|
||||
};
|
||||
// Primary: HyperShader ready state — authoritative for both cache paths
|
||||
const shaderTransitions = win.__hf?.shaderTransitions;
|
||||
if (shaderTransitions !== undefined) {
|
||||
return Object.values(shaderTransitions).every((s) => s.ready === true);
|
||||
}
|
||||
// Fallback: overlay visibility (older builds without ready state).
|
||||
// Check display:none rather than element absence — element stays in
|
||||
// the DOM when hidden.
|
||||
const overlay = document.querySelector(
|
||||
"[data-hyper-shader-loading]",
|
||||
) as HTMLElement | null;
|
||||
if (!overlay) return true;
|
||||
return window.getComputedStyle(overlay).display === "none";
|
||||
},
|
||||
{ timeout: 90_000 },
|
||||
)
|
||||
.catch(() => {});
|
||||
|
||||
// Extra settle time for media, fonts, and animations to initialize
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
|
||||
// Font verification — report which fonts loaded vs fell back
|
||||
const fontReport = await page
|
||||
.evaluate(() => {
|
||||
const loaded: string[] = [];
|
||||
const failed: string[] = [];
|
||||
(document as any).fonts.forEach((f: any) => {
|
||||
const entry = `${f.family} (${f.weight} ${f.style})`;
|
||||
if (f.status === "loaded") loaded.push(entry);
|
||||
else failed.push(entry + ` [${f.status}]`);
|
||||
});
|
||||
return { loaded, failed };
|
||||
})
|
||||
.catch(() => ({ loaded: [] as string[], failed: [] as string[] }));
|
||||
|
||||
if (fontReport.loaded.length > 0 || fontReport.failed.length > 0) {
|
||||
console.log(
|
||||
`\n ${c.dim("Fonts loaded:")} ${fontReport.loaded.length > 0 ? fontReport.loaded.join(", ") : "none"}`,
|
||||
);
|
||||
if (fontReport.failed.length > 0) {
|
||||
console.log(` ${c.error("Fonts FAILED:")} ${fontReport.failed.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Get composition duration
|
||||
const duration = await page.evaluate(() => {
|
||||
const win = window as any;
|
||||
@@ -186,9 +249,20 @@ async function captureSnapshots(
|
||||
? [duration / 2]
|
||||
: Array.from({ length: numFrames }, (_, i) => (i / (numFrames - 1)) * duration);
|
||||
|
||||
// Create output directory
|
||||
// Create output directory and clear previous frames so old captures
|
||||
// don't mix with the current run in contact sheets.
|
||||
const snapshotDir = join(projectDir, "snapshots");
|
||||
mkdirSync(snapshotDir, { recursive: true });
|
||||
try {
|
||||
const { readdirSync, rmSync } = await import("node:fs");
|
||||
for (const file of readdirSync(snapshotDir)) {
|
||||
if (/\.(png|jpg|jpeg)$/i.test(file)) {
|
||||
rmSync(join(snapshotDir, file), { force: true });
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* best-effort clear — proceed even if cleanup fails */
|
||||
}
|
||||
|
||||
// Lazily load the engine's <img>-overlay injector. Chrome-headless cannot
|
||||
// reliably advance <video>.currentTime mid-seek (the setter is accepted but
|
||||
@@ -237,18 +311,30 @@ async function captureSnapshots(
|
||||
const time = positions[i]!;
|
||||
|
||||
await page.evaluate((t: number) => {
|
||||
const w = window as Window & {
|
||||
__player?: { seek?: (time: number) => void };
|
||||
__timelines?: Record<string, { pause?: (time?: number) => void }>;
|
||||
gsap?: { ticker?: { tick?: () => void } };
|
||||
};
|
||||
if (typeof w.__player?.seek === "function") {
|
||||
w.__player.seek(t);
|
||||
} else if (w.__timelines) {
|
||||
for (const tl of Object.values(w.__timelines)) {
|
||||
tl?.pause?.(t);
|
||||
const win = window as any;
|
||||
if (win.__player?.seek) {
|
||||
win.__player.seek(t);
|
||||
} else {
|
||||
const tls = win.__timelines;
|
||||
if (tls) {
|
||||
for (const key in tls) {
|
||||
if (tls[key]?.seek) {
|
||||
// Sub-composition timelines run in local time relative to
|
||||
// their data-start. Seeking them to global time causes beats
|
||||
// with exit animations to appear black (global t clamps past
|
||||
// the exit). Compute local time: global_t - data_start.
|
||||
const host = document.querySelector<HTMLElement>(
|
||||
`[data-composition-id="${key}"]`,
|
||||
);
|
||||
const dataStart = host
|
||||
? parseFloat(host.getAttribute("data-start") ?? "0") || 0
|
||||
: 0;
|
||||
const localTime = Math.max(0, t - dataStart);
|
||||
tls[key].pause();
|
||||
tls[key].seek(localTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
w.gsap?.ticker?.tick?.();
|
||||
}
|
||||
}, time);
|
||||
|
||||
@@ -356,9 +442,7 @@ async function captureSnapshots(
|
||||
}
|
||||
}
|
||||
|
||||
const timeLabel = opts.at?.length
|
||||
? `${time.toFixed(1)}s`
|
||||
: `${Math.round((time / duration) * 100)}pct`;
|
||||
const timeLabel = `${time.toFixed(1)}s`;
|
||||
const filename = `frame-${String(i).padStart(2, "0")}-at-${timeLabel}.png`;
|
||||
const framePath = join(snapshotDir, filename);
|
||||
|
||||
@@ -400,6 +484,11 @@ export default defineCommand({
|
||||
description: "Ms to wait for runtime to initialize (default: 5000)",
|
||||
default: "5000",
|
||||
},
|
||||
describe: {
|
||||
type: "string",
|
||||
description:
|
||||
"Gemini vision frame analysis. Runs by default when GEMINI_API_KEY is set. Pass a custom question (e.g. --describe 'Is the logo visible in every beat?') to override the default prompt, or --describe false to opt out.",
|
||||
},
|
||||
},
|
||||
async run({ args }) {
|
||||
const project = resolveProject(args.dir);
|
||||
@@ -411,6 +500,16 @@ export default defineCommand({
|
||||
.map((s) => parseFloat(s.trim()))
|
||||
.filter((n) => !isNaN(n))
|
||||
: undefined;
|
||||
// Gemini frame analysis runs by default (silently skipped if
|
||||
// GEMINI_API_KEY is not set). `--describe "custom question"` overrides
|
||||
// the default prompt with a targeted question. `--describe false` opts
|
||||
// out entirely.
|
||||
const describeArg =
|
||||
args.describe === undefined
|
||||
? "true"
|
||||
: String(args.describe) === "false"
|
||||
? null
|
||||
: String(args.describe);
|
||||
|
||||
const label = atTimestamps
|
||||
? `${atTimestamps.length} frames at [${atTimestamps.map((t) => t.toFixed(1) + "s").join(", ")}]`
|
||||
@@ -431,6 +530,121 @@ export default defineCommand({
|
||||
for (const p of paths) {
|
||||
console.log(` ${p}`);
|
||||
}
|
||||
|
||||
// Generate contact sheet for quick AI review
|
||||
try {
|
||||
const { createSnapshotContactSheet } = await import("../capture/contactSheet.js");
|
||||
const snapshotDir = join(project.dir, "snapshots");
|
||||
const sheets = await createSnapshotContactSheet(
|
||||
snapshotDir,
|
||||
join(snapshotDir, "contact-sheet.jpg"),
|
||||
);
|
||||
if (sheets.length > 0) {
|
||||
const label =
|
||||
sheets.length === 1 ? "contact-sheet.jpg" : `contact-sheet-1..${sheets.length}.jpg`;
|
||||
console.log(` ${c.dim(label)} (grid view for AI review)`);
|
||||
}
|
||||
} catch {
|
||||
/* non-critical */
|
||||
}
|
||||
|
||||
// Gemini vision descriptions. Runs by default — see describeArg
|
||||
// resolution above. `null` means the user explicitly opted out with
|
||||
// `--describe false`; missing GEMINI_API_KEY logs a skip and continues.
|
||||
if (describeArg !== null) {
|
||||
try {
|
||||
const geminiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
|
||||
if (!geminiKey) {
|
||||
console.log(` ${c.dim("--describe: GEMINI_API_KEY not set, skipping")}`);
|
||||
} else if (paths.length > 0) {
|
||||
console.log(` ${c.dim("Describing frames with Gemini vision...")}`);
|
||||
const { GoogleGenAI } = await import("@google/genai");
|
||||
const ai = new GoogleGenAI({ apiKey: geminiKey });
|
||||
const model = process.env.HYPERFRAMES_GEMINI_MODEL || "gemini-3.1-flash-lite-preview";
|
||||
const snapshotDir = join(project.dir, "snapshots");
|
||||
|
||||
const customQuestion =
|
||||
describeArg === "true"
|
||||
? "Describe this video composition frame in 1-2 sentences. Be specific and factual: what elements are visible, what text appears, is the frame blank/black/loading, what is the composition. Flag any obvious problems."
|
||||
: describeArg;
|
||||
|
||||
const descriptions: string[] = [
|
||||
`# Snapshot Frame Descriptions`,
|
||||
``,
|
||||
`**Question asked:** ${customQuestion}`,
|
||||
``,
|
||||
`Compare each description against your storyboard spec. A "black frame" or "loading screen" for a content beat is a bug.`,
|
||||
``,
|
||||
];
|
||||
|
||||
// Scale down PNGs before sending to stay under Gemini's 4 MB inline
|
||||
// limit. Full 1920×1080 PNGs are typically 3-6 MB. Use sharp if
|
||||
// available; otherwise skip files over the limit.
|
||||
type SharpFn = (buf: Buffer) => {
|
||||
resize: (w: number) => { jpeg: () => { toBuffer: () => Promise<Buffer> } };
|
||||
};
|
||||
let sharpFn: SharpFn | null = null;
|
||||
try {
|
||||
const s = await import("sharp");
|
||||
sharpFn = (s.default ?? s) as unknown as SharpFn;
|
||||
} catch {
|
||||
/* sharp not installed — fall back to size check */
|
||||
}
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
paths.map(async (p) => {
|
||||
const filename = p.replace("snapshots/", "");
|
||||
const filePath = join(snapshotDir, filename);
|
||||
if (!existsSync(filePath)) return { filename, desc: "file not found" };
|
||||
const raw = readFileSync(filePath);
|
||||
let imageData: Buffer;
|
||||
let mimeType = "image/png";
|
||||
if (sharpFn) {
|
||||
imageData = await sharpFn(raw).resize(960).jpeg().toBuffer();
|
||||
mimeType = "image/jpeg";
|
||||
} else {
|
||||
if (raw.length > 3_800_000)
|
||||
return {
|
||||
filename,
|
||||
desc: "file too large for Gemini — install sharp to enable auto-resize",
|
||||
};
|
||||
imageData = raw;
|
||||
}
|
||||
const base64 = imageData.toString("base64");
|
||||
const response = await ai.models.generateContent({
|
||||
model,
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ inlineData: { mimeType, data: base64 } }, { text: customQuestion }],
|
||||
},
|
||||
],
|
||||
config: { maxOutputTokens: 250 },
|
||||
});
|
||||
return { filename, desc: response.text?.trim() || "no description" };
|
||||
}),
|
||||
);
|
||||
|
||||
for (const result of results) {
|
||||
if (result.status === "fulfilled") {
|
||||
descriptions.push(`## ${result.value.filename}`, `${result.value.desc}`, ``);
|
||||
} else {
|
||||
// Log first failure so Gemini issues are visible rather than silent
|
||||
const errMsg =
|
||||
result.reason instanceof Error ? result.reason.message : String(result.reason);
|
||||
descriptions.push(`## (error)`, `Gemini call failed: ${errMsg.slice(0, 120)}`, ``);
|
||||
}
|
||||
}
|
||||
|
||||
const descPath = join(snapshotDir, "descriptions.md");
|
||||
writeFileSync(descPath, descriptions.join("\n"));
|
||||
console.log(` ${c.dim("descriptions.md")} (Gemini frame analysis)`);
|
||||
}
|
||||
} catch (descErr) {
|
||||
const msg = descErr instanceof Error ? descErr.message : String(descErr);
|
||||
console.log(` ${c.dim(`--describe failed: ${msg.slice(0, 80)}`)}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.error(`\n${c.error("✗")} Snapshot failed: ${msg}`);
|
||||
|
||||
@@ -10,7 +10,7 @@ export const examples: Example[] = [
|
||||
["Import an existing SRT file", "hyperframes transcribe subtitles.srt"],
|
||||
["Import an OpenAI Whisper JSON response", "hyperframes transcribe response.json"],
|
||||
];
|
||||
import { resolve, join, extname } from "node:path";
|
||||
import { resolve, join, extname, dirname } from "node:path";
|
||||
import * as clack from "@clack/prompts";
|
||||
import { c } from "../ui/colors.js";
|
||||
import { DEFAULT_MODEL } from "../whisper/manager.js";
|
||||
@@ -56,7 +56,10 @@ export default defineCommand({
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const dir = resolve(args.dir ?? ".");
|
||||
// Default to the directory containing the input file so transcript.json
|
||||
// lands next to narration.wav regardless of where the command is run from.
|
||||
// Explicit --dir overrides this (e.g. for import mode targeting a project dir).
|
||||
const dir = resolve(args.dir ?? dirname(inputPath));
|
||||
const ext = extname(inputPath).toLowerCase();
|
||||
|
||||
// ── Import mode: convert existing transcript ──────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user