mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-05 17:30:50 +00:00
Capture pipeline work that came out of the 11-round website-to-video
eval branch. The wins that actually moved quality were the artifacts
agents read (contact sheets, design-styles) and the snapshot tool
visual-verification fixes; the rest are smaller follow-ons.
**Contact sheets (`contactSheet.ts`, new)**
- Replaces the embedded one-image-per-asset listing with paginated
labeled grids (3-col screenshots / 4-col raster / 5-col SVG). Each
page contains 9–15 cells with filename labels baked in via SVG
text overlay (`escapeXml` covers `&<>"'`).
- `fit: "contain"` keeps every asset visible at its real aspect
ratio; the old `fit: "cover"` cropped to the first image's box.
- Returns `string[]` (page paths) — single-page captures get one
file, multi-page produce `contact-sheet-1.jpg`, `contact-sheet-2.jpg`,
etc.
- `createSvgContactSheet` scans both `assets/svgs/` (inline-extracted
SVGs) and `assets/` root (external SVGs from `<img src="*.svg">`)
and de-dupes by filename. Sites with all-external SVGs (huly.io)
now get coverage they previously didn't.
**Design styles extractor (`designStyleExtractor.ts`, new)**
- Walks the live DOM and reads computed styles to produce
`extracted/design-styles.json`: typography hierarchy (every text
role with exact font-size / weight / line-height / letter-spacing),
button variants (background / padding / radius / shadow), card /
container / nav styles, spacing scale with base unit, border-radius
scale, box-shadow values with usage counts.
- Primary data source for DESIGN.md authoring at Step 1. Replaces
the prior "guess from screenshots" workflow.
**Snapshot tool (`snapshot.ts`)**
- HyperShader pre-rendering used to swallow the entire snapshot
capture window (every frame after the first showed the loading
overlay or final-opacity-zero exit fades). Wait signal is now
`window.__hf.shaderTransitions[].ready` (set after both warm and
cold cache paths complete); local-time seek for sub-comps means
exit fades read at their own t=0..duration, not global time.
- Gemini vision per-frame analysis runs by default (`descriptions.md`
next to the contact sheet). `--describe "custom Q"` overrides the
prompt; `--describe false` opts out.
- 3-column contact sheet generation for snapshot frames so reviewers
see all beats at a glance.
**Screenshot capture (`screenshotCapture.ts`)**
- Replaces `querySelectorAll('*') + getComputedStyle` overlay scan
with a TreeWalker that early-exits on cheap rect checks before
reaching the expensive style read. Caps at 5000 elements per page.
- Cookie/consent dismissal selectors are scoped under cookie /
consent / gdpr ancestors so we don't click "Accept invitation" or
similar unrelated buttons.
**Agent prompt (`agentPromptGenerator.ts`)**
- Auto-discovers contact-sheet page count (matches base name plus
paginated `-NNN` variants only, with regex escaping on the base
name and numeric sort for 10+ pages).
- `inferColorRole`: classifies extracted hex colors as bg-dark /
bg-light / accent / surface / neutral via luminance + saturation,
so the agent prompt shows `#533AFD (accent)` instead of bare hex.
- `design-styles.json` row is gated on `existsSync` — the upstream
write is wrapped in try/catch and may skip on failure, so the
prompt only points to files actually on disk.
**Other CLI ergonomics**
- `cli.ts`: auto-load `.env` from CWD on startup so subcommands like
`snapshot` don't need explicit `export GEMINI_API_KEY=…`. Handles
`export FOO=bar`, quoted values, inline `# comments`.
- `commands/transcribe.ts`: default output dir is the input file's
directory, not CWD. Stops the "wrote transcript.json somewhere
unexpected" footgun.
- `assetDownloader.ts`: improved asset naming uses catalog context;
de-duplicates inline SVG filenames.
- `contentExtractor.ts`: captions SVGs via Gemini (code-as-text) and
integrates them into asset descriptions.
- `tokenExtractor.ts` + `types.ts`: SVG bounding box dimensions and
new DesignStyles schema added.
161 lines
6.5 KiB
TypeScript
161 lines
6.5 KiB
TypeScript
/**
|
|
* Screenshot capture for the website capture pipeline.
|
|
*
|
|
* All page.evaluate() calls use string expressions to avoid
|
|
* tsx/esbuild __name injection (see esbuild issue #1031).
|
|
*/
|
|
|
|
import type { Page } from "puppeteer-core";
|
|
import { writeFileSync, mkdirSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
|
|
/**
|
|
* Capture viewport screenshots covering the entire page height.
|
|
*
|
|
* Scrolls down the page in viewport-sized steps (with slight overlap),
|
|
* taking a 1920x1080 screenshot at each position. The number of screenshots
|
|
* depends on the page height — short pages get fewer, long pages get more.
|
|
* Capped at 20 to avoid excessive output on extremely long pages.
|
|
*
|
|
* Unlike the old section-tiling approach, this does NOT disable sticky/fixed
|
|
* elements — screenshots show the page in its natural browsing state with
|
|
* scroll-triggered animations fired.
|
|
*/
|
|
export async function captureScrollScreenshots(page: Page, outputDir: string): Promise<string[]> {
|
|
const screenshotsDir = join(outputDir, "screenshots");
|
|
mkdirSync(screenshotsDir, { recursive: true });
|
|
|
|
const MAX_SCREENSHOTS = 20;
|
|
const filePaths: string[] = [];
|
|
|
|
try {
|
|
// Dismiss marketing banners, cookie consents, and popups before scrolling.
|
|
// These overlay content and contaminate screenshots with UI that doesn't
|
|
// belong in video compositions (cookie popups, newsletter modals, etc.)
|
|
await page
|
|
.evaluate(() => {
|
|
// Click common dismiss/accept buttons
|
|
const selectors = [
|
|
// Cookie consent
|
|
'[id*="cookie"] button[class*="accept"]',
|
|
'[id*="cookie"] button[class*="agree"]',
|
|
'[id*="cookie"] button[class*="allow"]',
|
|
'[class*="cookie"] button[class*="accept"]',
|
|
'[class*="consent"] button',
|
|
// Generic close buttons on overlays/modals
|
|
'[class*="banner"] [class*="close"]',
|
|
'[class*="banner"] [class*="dismiss"]',
|
|
'[class*="popup"] [class*="close"]',
|
|
'[class*="modal"] [class*="close"]',
|
|
'[class*="overlay"] [class*="close"]',
|
|
// Common GDPR patterns — scoped under a cookie/consent/gdpr ancestor
|
|
// so we don't click "Accept invitation" / "Accept terms" / etc. on
|
|
// unrelated buttons elsewhere on the page.
|
|
'[id*="cookie" i] button[id*="accept" i]',
|
|
'[id*="consent" i] button[id*="accept" i]',
|
|
'[id*="gdpr" i] button[id*="accept" i]',
|
|
'[class*="cookie" i] button[class*="accept-all" i]',
|
|
'[class*="cookie" i] button[class*="acceptAll" i]',
|
|
'[class*="consent" i] button[class*="accept-all" i]',
|
|
// Notification prompts
|
|
'button[class*="decline"]',
|
|
'button[class*="not-now"]',
|
|
'button[class*="no-thanks"]',
|
|
];
|
|
for (const sel of selectors) {
|
|
try {
|
|
const el = document.querySelector<HTMLElement>(sel);
|
|
if (el) el.click();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
// Hide fixed/sticky overlays that aren't the main nav. Scanning every
|
|
// element with querySelectorAll('*') + getComputedStyle is O(n) DOM
|
|
// calls and can dominate evaluate() time on large pages. Narrow the
|
|
// candidate set with a TreeWalker that early-exits on viewport-sized
|
|
// rect checks (cheap) before reaching the expensive getComputedStyle.
|
|
const SCAN_CAP = 5000;
|
|
const minWidth = window.innerWidth * 0.3;
|
|
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
|
|
let visited = 0;
|
|
let node = walker.nextNode();
|
|
while (node && visited < SCAN_CAP) {
|
|
visited++;
|
|
const el = node as HTMLElement;
|
|
const rect = el.getBoundingClientRect();
|
|
// Cheap viewport-size filter first — eliminates the vast majority of
|
|
// tiny / hidden / off-screen elements without touching getComputedStyle.
|
|
if (rect.height > 80 && rect.width > minWidth) {
|
|
const tag = el.tagName;
|
|
if (tag !== "HEADER" && tag !== "NAV" && !el.closest("header") && !el.closest("nav")) {
|
|
const style = window.getComputedStyle(el);
|
|
if (
|
|
(style.position === "fixed" || style.position === "sticky") &&
|
|
style.zIndex !== "auto" &&
|
|
parseInt(style.zIndex) > 100
|
|
) {
|
|
el.style.display = "none";
|
|
}
|
|
}
|
|
}
|
|
node = walker.nextNode();
|
|
}
|
|
})
|
|
.catch(() => {});
|
|
await new Promise((r) => setTimeout(r, 400));
|
|
|
|
const scrollHeight = (await page.evaluate(
|
|
`Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)`,
|
|
)) as number;
|
|
const viewportHeight = (await page.evaluate(`window.innerHeight`)) as number;
|
|
|
|
// Calculate scroll positions: step by 70% of viewport (30% overlap between shots)
|
|
const step = Math.floor(viewportHeight * 0.7);
|
|
const positions: number[] = [0];
|
|
for (let y = step; y < scrollHeight - viewportHeight; y += step) {
|
|
positions.push(y);
|
|
}
|
|
// Always include the bottom of the page
|
|
const lastPos = Math.max(0, scrollHeight - viewportHeight);
|
|
if (positions[positions.length - 1] !== lastPos) {
|
|
positions.push(lastPos);
|
|
}
|
|
|
|
// Downsample if too many positions
|
|
let finalPositions = positions;
|
|
if (positions.length > MAX_SCREENSHOTS) {
|
|
finalPositions = [positions[0]!];
|
|
const stride = (positions.length - 1) / (MAX_SCREENSHOTS - 1);
|
|
for (let i = 1; i < MAX_SCREENSHOTS - 1; i++) {
|
|
finalPositions.push(positions[Math.round(i * stride)]!);
|
|
}
|
|
finalPositions.push(positions[positions.length - 1]!);
|
|
}
|
|
|
|
for (let i = 0; i < finalPositions.length; i++) {
|
|
await page.evaluate(`window.scrollTo(0, ${finalPositions[i]})`);
|
|
await new Promise((r) => setTimeout(r, 400));
|
|
|
|
const pct = Math.round(
|
|
(finalPositions[i]! / Math.max(1, scrollHeight - viewportHeight)) * 100,
|
|
);
|
|
const filename = `scroll-${String(Math.min(pct, 100)).padStart(3, "0")}.png`;
|
|
const filePath = join(screenshotsDir, filename);
|
|
const buffer = await page.screenshot({ type: "png" });
|
|
writeFileSync(filePath, buffer);
|
|
filePaths.push(`screenshots/${filename}`);
|
|
}
|
|
|
|
// Reset scroll
|
|
await page.evaluate(`window.scrollTo(0, 0)`);
|
|
await new Promise((r) => setTimeout(r, 200));
|
|
|
|
// full-page.png removed — 1/8 agents read it, contact sheet covers the same content
|
|
} catch {
|
|
/* scroll screenshots are non-critical */
|
|
}
|
|
|
|
return filePaths;
|
|
}
|