mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 16:08:56 +00:00
* fix(cli): keep OpenRouter as a vision provider for capture Reverts #7188. * fix(cli): shorten the restored thinking budget comment
447 lines
18 KiB
TypeScript
447 lines
18 KiB
TypeScript
import { existsSync } from "node:fs";
|
||
import { ensureCaptureDirSync, writeCaptureFileSync } from "./captureFile.js";
|
||
import { CaptureDirRefusedError } from "./captureErrors.js";
|
||
import { join } from "node:path";
|
||
import {
|
||
downloadAssets,
|
||
downloadAndRewriteFonts,
|
||
mergeDrops,
|
||
noDrops,
|
||
totalDrops,
|
||
} from "./assetDownloader.js";
|
||
import { extractFontMetadata } from "./fontMetadataExtractor.js";
|
||
import { normalizeErrorMessage } from "../utils/errorMessage.js";
|
||
import { diag } from "../ui/diagnostics.js";
|
||
import { serializeTokensForCapture } from "./partialCapture.js";
|
||
import type { PartialCaptureState } from "./partialCapture.js";
|
||
import type {
|
||
CapturePhase,
|
||
CapturePhaseProgress,
|
||
CaptureResult,
|
||
DesignTokens,
|
||
ExtractedHtml,
|
||
} from "./types.js";
|
||
import type { CatalogedAsset } from "./assetCataloger.js";
|
||
import type { IconCandidate } from "./faviconRanker.js";
|
||
import { createCaptureDownloadBudget } from "./readBoundedResponse.js";
|
||
import {
|
||
captionImagesWithGemini,
|
||
generateAssetDescriptions,
|
||
resolveVisionPhaseCompletion,
|
||
} from "./contentExtractor.js";
|
||
import type { VisionCaptionOutcome } from "./contentExtractor.js";
|
||
|
||
function hasVisionCredentials(skipVision: boolean): boolean {
|
||
return Boolean(
|
||
!skipVision &&
|
||
(process.env.OPENROUTER_API_KEY ||
|
||
process.env.GEMINI_API_KEY ||
|
||
process.env.GOOGLE_API_KEY ||
|
||
(process.env.HYPERFRAMES_VERTEX_PROJECT_ID &&
|
||
process.env.HYPERFRAMES_VERTEX_SERVICE_ACCOUNT)),
|
||
);
|
||
}
|
||
|
||
function writeAssetDescriptionsFile(outputDir: string, lines: string[], header: string): void {
|
||
writeCaptureFileSync(
|
||
join(outputDir, "extracted", "asset-descriptions.md"),
|
||
header + lines.map((line) => `- ${line}`).join("\n") + "\n",
|
||
"utf-8",
|
||
);
|
||
}
|
||
|
||
export interface PostExtractionInput {
|
||
state: PartialCaptureState;
|
||
outputDir: string;
|
||
warnings: string[];
|
||
progress: (stage: string, detail?: string) => void;
|
||
remainingMs: () => number;
|
||
phase: (
|
||
name: CapturePhase,
|
||
status: CapturePhaseProgress["status"],
|
||
reason?: CapturePhaseProgress["reason"],
|
||
) => void;
|
||
animationCatalog: CaptureResult["animationCatalog"];
|
||
catalogedAssets: CatalogedAsset[];
|
||
visibleTextContent: string;
|
||
faviconLinks: IconCandidate[];
|
||
tokens: DesignTokens;
|
||
extracted: ExtractedHtml;
|
||
skipAssets: boolean;
|
||
skipVision: boolean;
|
||
downloadByteBudget: ReturnType<typeof createCaptureDownloadBudget>;
|
||
assets: CaptureResult["assets"];
|
||
dropped: CaptureResult["dropped"];
|
||
fontDrops: CaptureResult["dropped"];
|
||
canWrite: () => boolean;
|
||
}
|
||
|
||
export interface PostExtractionResult {
|
||
assets: CaptureResult["assets"];
|
||
dropped: CaptureResult["dropped"];
|
||
fontDrops: CaptureResult["dropped"];
|
||
extracted: ExtractedHtml;
|
||
tokens: DesignTokens;
|
||
animationCatalog: CaptureResult["animationCatalog"];
|
||
}
|
||
|
||
export async function runPostExtraction(input: PostExtractionInput): Promise<PostExtractionResult> {
|
||
let {
|
||
state,
|
||
outputDir,
|
||
warnings,
|
||
progress,
|
||
remainingMs,
|
||
phase,
|
||
animationCatalog,
|
||
catalogedAssets,
|
||
visibleTextContent,
|
||
faviconLinks,
|
||
tokens,
|
||
extracted,
|
||
skipAssets,
|
||
skipVision,
|
||
downloadByteBudget,
|
||
assets,
|
||
dropped,
|
||
fontDrops,
|
||
canWrite,
|
||
} = input;
|
||
const runFontExtraction = async (): Promise<void> => {
|
||
if (!canWrite()) return;
|
||
// Download fonts and preserve per-font budget exhaustion in the capture tally.
|
||
phase("fonts", "started");
|
||
const fontPass = await downloadAndRewriteFonts(extracted.headHtml, outputDir, {
|
||
remainingMs,
|
||
byteBudget: downloadByteBudget,
|
||
});
|
||
fontDrops = fontPass.drops;
|
||
extracted.headHtml = fontPass.css;
|
||
phase(
|
||
"fonts",
|
||
remainingMs() > 0 ? "completed" : "degraded",
|
||
remainingMs() > 0 ? undefined : "budget-exhausted",
|
||
);
|
||
|
||
// Identify each downloaded font by reading its OpenType name table.
|
||
// Modern frameworks hash font filenames; this manifest tells the
|
||
// downstream pipeline (DESIGN.md authoring, beat sub-agents) which file
|
||
// belongs to which family without guessing from filename patterns.
|
||
try {
|
||
const fontsManifest = extractFontMetadata(
|
||
join(outputDir, "assets", "fonts"),
|
||
join(outputDir, "extracted", "fonts-manifest.json"),
|
||
);
|
||
if (fontsManifest.families.length > 0) {
|
||
const summary = fontsManifest.families
|
||
.map((f) => `${f.family}${f.variable ? " (variable)" : ""} × ${f.fileCount}`)
|
||
.join(", ");
|
||
// stderr (via diag): `capture --json` writes its envelope to stdout, so
|
||
// these progress/advisory lines must not land there.
|
||
diag.notice(`Font metadata extracted: ${summary}`);
|
||
if (fontsManifest.unidentified.length > 0) {
|
||
diag.warn(
|
||
` ${fontsManifest.unidentified.length} font file(s) could not be identified — DESIGN.md should flag these explicitly.`,
|
||
);
|
||
}
|
||
}
|
||
} catch (err) {
|
||
diag.warn("Font metadata extraction failed (non-fatal):", normalizeErrorMessage(err));
|
||
}
|
||
};
|
||
await runFontExtraction();
|
||
|
||
const runAnimationCatalog = async (): Promise<void> => {
|
||
// Save animation catalog — lean version for the agent (not 745 raw CSS declarations)
|
||
if (animationCatalog) {
|
||
// Extract just what's useful: counts, named animations, a few representative keyframed entries
|
||
const uniqueAnimNames = new Set<string>();
|
||
for (const d of animationCatalog.cssDeclarations || []) {
|
||
if (d.animation?.name) uniqueAnimNames.add(d.animation.name);
|
||
}
|
||
|
||
// Keep up to 10 Web Animations that have actual keyframe data (most useful for recreation)
|
||
const representativeAnims = (animationCatalog.webAnimations || [])
|
||
.filter((a: { keyframes?: unknown[] }) => a.keyframes && a.keyframes.length > 0)
|
||
.slice(0, 10);
|
||
|
||
const leanCatalog = {
|
||
summary: animationCatalog.summary,
|
||
namedAnimations: Array.from(uniqueAnimNames),
|
||
scrollTriggeredElements: (animationCatalog.scrollTargets || []).length,
|
||
representativeAnimations: representativeAnims,
|
||
};
|
||
|
||
if (canWrite()) {
|
||
writeCaptureFileSync(
|
||
join(outputDir, "extracted", "animations.json"),
|
||
JSON.stringify(leanCatalog, null, 2),
|
||
"utf-8",
|
||
);
|
||
}
|
||
}
|
||
};
|
||
await runAnimationCatalog();
|
||
|
||
const runAssetDownload = async (): Promise<void> => {
|
||
// Download assets — single pass using the catalog for best image quality
|
||
let assetDrops = noDrops();
|
||
const runAssetFetch = async (): Promise<void> => {
|
||
if (!skipAssets && canWrite()) {
|
||
// Called even with the budget already gone, for the reason the font pass is: the loop that
|
||
// skips an asset is the only thing that can say how many it skipped.
|
||
phase("assets", "started");
|
||
progress("assets", "Downloading assets...");
|
||
const assetPass = await downloadAssets(tokens, outputDir, catalogedAssets, faviconLinks, {
|
||
remainingMs,
|
||
byteBudget: downloadByteBudget,
|
||
});
|
||
assets = assetPass.assets;
|
||
assetDrops = assetPass.drops;
|
||
state.assets = assets;
|
||
// Which icons the site declared, what each one is, and why one became favicon.<ext>.
|
||
// The brand-kit consumer needs the shape to decide which tile an icon belongs in; the
|
||
// reason is what stops a substituted headline from being silent again.
|
||
if (canWrite()) {
|
||
writeCaptureFileSync(
|
||
join(outputDir, "extracted", "icons-manifest.json"),
|
||
JSON.stringify(assetPass.icons, null, 2),
|
||
"utf-8",
|
||
);
|
||
}
|
||
phase(
|
||
"assets",
|
||
remainingMs() > 0 ? "completed" : "degraded",
|
||
remainingMs() > 0 ? undefined : "budget-exhausted",
|
||
);
|
||
} else {
|
||
phase("assets", "degraded", "disabled");
|
||
}
|
||
};
|
||
await runAssetFetch();
|
||
|
||
const recordAssetDrops = (): void => {
|
||
// One capture-wide tally, summed from the two passes that own the drops. The warning is
|
||
// DERIVED from it rather than written alongside it, so the prose and the number cannot
|
||
// disagree the way two separately-authored budget strings could.
|
||
dropped = mergeDrops(fontDrops, assetDrops);
|
||
state.dropped = dropped;
|
||
const droppedTotal = totalDrops(dropped);
|
||
if (droppedTotal > 0) {
|
||
const breakdown = Object.entries(dropped)
|
||
.filter(([, n]) => typeof n === "number" && n > 0)
|
||
.map(([reason, n]) => `${n} ${reason}`)
|
||
.join(", ");
|
||
warnings.push(
|
||
`${droppedTotal} referenced asset(s) are not in this capture (${breakdown}). ` +
|
||
"A thin capture with no drops is a thin page; this one was truncated.",
|
||
);
|
||
}
|
||
};
|
||
recordAssetDrops();
|
||
|
||
const rewriteTokenAssets = (): void => {
|
||
// Join in-section media URLs → downloaded local paths, then re-write
|
||
// tokens.json. Downstream page recreation MUST reference local files:
|
||
// remote URLs fail at render time (hotlink/CORS 403, no egress in
|
||
// Docker/Lambda, frame-timing blanks for not-yet-loaded images).
|
||
const base = (u: string): string => u.split(/[#?]/)[0] ?? u;
|
||
const localByUrl = new Map<string, string>();
|
||
const collectLocalAssetPaths = (): void => {
|
||
for (const a of assets) {
|
||
if (!a.url || !a.localPath) continue;
|
||
localByUrl.set(a.url, a.localPath);
|
||
localByUrl.set(base(a.url), a.localPath);
|
||
}
|
||
};
|
||
collectLocalAssetPaths();
|
||
const applyLocalAssetPaths = (): void => {
|
||
for (const sec of tokens.sections || []) {
|
||
const local: string[] = [];
|
||
for (const u of sec.assetUrls || []) {
|
||
const hit = localByUrl.get(u) || localByUrl.get(base(u));
|
||
if (hit && !local.includes(hit)) local.push(hit);
|
||
}
|
||
if (local.length) sec.assets = local;
|
||
}
|
||
};
|
||
applyLocalAssetPaths();
|
||
if (assets.length && Array.isArray(tokens.sections)) {
|
||
if (canWrite()) {
|
||
writeCaptureFileSync(
|
||
join(outputDir, "extracted", "tokens.json"),
|
||
serializeTokensForCapture(tokens),
|
||
"utf-8",
|
||
);
|
||
}
|
||
}
|
||
};
|
||
await rewriteTokenAssets();
|
||
};
|
||
await runAssetDownload();
|
||
|
||
// Persist a self-contained page recreation under extracted/, with images already inlined.
|
||
try {
|
||
const pageHtml = `<!doctype html>\n<html ${extracted.htmlAttrs || ""}>\n<head>\n${extracted.headHtml}\n</head>\n<body>\n${extracted.bodyHtml}\n</body>\n</html>\n`;
|
||
state.pageHtml = pageHtml;
|
||
if (canWrite())
|
||
writeCaptureFileSync(join(outputDir, "extracted", "page.html"), pageHtml, "utf-8");
|
||
} catch (err) {
|
||
warnings.push(`page.html write failed: ${err}`);
|
||
}
|
||
// Save visible text content for AI agent to use
|
||
if (visibleTextContent) {
|
||
if (canWrite()) {
|
||
writeCaptureFileSync(
|
||
join(outputDir, "extracted", "visible-text.txt"),
|
||
visibleTextContent,
|
||
"utf-8",
|
||
);
|
||
}
|
||
}
|
||
|
||
// detected-libraries and assets-catalog removed — 0/8 agents read them in v6 testing
|
||
|
||
// AI-powered image captioning via Gemini (optional — enriches asset descriptions)
|
||
let geminiCaptions: Record<string, string> = {};
|
||
const runVision = async (): Promise<void> => {
|
||
if (skipVision) {
|
||
phase("vision", "degraded", "disabled");
|
||
} else if (remainingMs() <= 0 || !canWrite()) {
|
||
warnings.push(
|
||
"Capture budget exhausted before vision captioning; catalog descriptions were preserved.",
|
||
);
|
||
phase("vision", "degraded", "budget-exhausted");
|
||
} else {
|
||
phase("vision", "started");
|
||
let visionOutcome: VisionCaptionOutcome = {
|
||
timedOutRequests: 0,
|
||
failedRequests: 0,
|
||
budgetExhausted: false,
|
||
};
|
||
geminiCaptions = await captionImagesWithGemini(outputDir, progress, warnings, {
|
||
remainingMs,
|
||
onOutcome: (outcome) => {
|
||
visionOutcome = outcome;
|
||
},
|
||
});
|
||
const completion = resolveVisionPhaseCompletion(visionOutcome, remainingMs());
|
||
phase(
|
||
"vision",
|
||
completion.status,
|
||
completion.status === "degraded" ? completion.reason : undefined,
|
||
);
|
||
}
|
||
};
|
||
await runVision();
|
||
|
||
// Generate asset descriptions for the AI agent
|
||
const runAssetDescriptions = (): void => {
|
||
progress("design", "Generating asset descriptions...");
|
||
try {
|
||
const lines = generateAssetDescriptions(outputDir, tokens, catalogedAssets, geminiCaptions);
|
||
if (lines.length === 0 || !canWrite()) return;
|
||
const hasVisionKey = hasVisionCredentials(skipVision);
|
||
const header = hasVisionKey
|
||
? "# Asset Descriptions\n\nOne line per file. Read this instead of opening every image individually.\n\nTo find a specific brand or icon, **grep this file for the brand name in the description text** (e.g. `grep -i 'autodesk' asset-descriptions.md`). The Gemini Vision captions identify what's actually in each file — that's the agent's selector.\n\nThe `logo-<hash>.svg` filename prefix is a cheap structural hint (DOM said this SVG was inside a `<header>`, home-link `<a>`, or had an aria-label matching the page brand). It is NOT a content claim — many `logo-*` files are nav icons or decorative shapes. Trust the captions, not the filename prefix.\n\n"
|
||
: "# Asset Descriptions\n\n⚠️ No vision credentials — descriptions below are catalog-derived (alt text, headings, section context, filename) instead of Vision-generated. To get richer Vision descriptions on the next capture, set GEMINI_API_KEY (or GOOGLE_API_KEY), or HYPERFRAMES_VERTEX_PROJECT_ID plus HYPERFRAMES_VERTEX_SERVICE_ACCOUNT for Vertex service-account auth, and re-run.\n\nThe `logo-<hash>.svg` filename prefix is a structural hint (DOM said this SVG was inside a `<header>`, home-link `<a>`, or had an aria-label matching the page brand). To pick the actual brand logo without Vision, open the `logo-*` candidates in a previewer or rasterize them with `sharp` before referencing — composing a fake logo ships off-brand in the final video.\n\n";
|
||
writeAssetDescriptionsFile(outputDir, lines, header);
|
||
progress(
|
||
"design",
|
||
`${lines.length} asset descriptions written${hasVisionKey ? "" : " (no vision provider — catalog-fallback mode)"}`,
|
||
);
|
||
} catch {
|
||
/* non-critical */
|
||
}
|
||
};
|
||
runAssetDescriptions();
|
||
|
||
progress("design", "DESIGN.md will be created by your AI agent");
|
||
|
||
// Generate contact sheets (saves AI agents 50-65% tokens vs reading images individually)
|
||
// All functions return string[] — paginated so every image is covered
|
||
const runContactSheets = async (): Promise<void> => {
|
||
if (remainingMs() > 0 && canWrite()) {
|
||
phase("contact-sheets", "started");
|
||
try {
|
||
const { createScrollContactSheet, createAssetContactSheet, createSvgContactSheet } =
|
||
await import("./contactSheet.js");
|
||
|
||
const contactSheetBudget = { remainingMs };
|
||
|
||
const runScrollContactSheet = async (): Promise<void> => {
|
||
const scrollSheets = await createScrollContactSheet(
|
||
join(outputDir, "screenshots"),
|
||
join(outputDir, "screenshots", "contact-sheet.jpg"),
|
||
contactSheetBudget,
|
||
);
|
||
if (scrollSheets.length > 0)
|
||
progress(
|
||
"design",
|
||
`Screenshot contact sheet generated (${scrollSheets.length} page${scrollSheets.length > 1 ? "s" : ""})`,
|
||
);
|
||
};
|
||
await runScrollContactSheet();
|
||
|
||
const runAssetContactSheet = async (): Promise<void> => {
|
||
const assetsImgDir = join(outputDir, "assets");
|
||
if (existsSync(assetsImgDir)) {
|
||
const assetSheets = await createAssetContactSheet(
|
||
assetsImgDir,
|
||
join(outputDir, "assets", "contact-sheet.jpg"),
|
||
contactSheetBudget,
|
||
);
|
||
if (assetSheets.length > 0)
|
||
progress(
|
||
"design",
|
||
`Asset contact sheet generated (${assetSheets.length} page${assetSheets.length > 1 ? "s" : ""})`,
|
||
);
|
||
}
|
||
};
|
||
await runAssetContactSheet();
|
||
|
||
const runSvgContactSheet = async (): Promise<void> => {
|
||
// Scan assets/svgs/ (inline SVGs) AND assets/ root (external SVGs from <img src="*.svg">)
|
||
// so sites like huly.io that only use external SVGs still get a grid
|
||
const svgsDir = join(outputDir, "assets", "svgs");
|
||
const assetsRootDir = join(outputDir, "assets");
|
||
const hasSvgsDir = existsSync(svgsDir);
|
||
if (hasSvgsDir) ensureCaptureDirSync(outputDir, svgsDir);
|
||
const svgOutputPath = hasSvgsDir
|
||
? join(outputDir, "assets", "svgs", "contact-sheet.jpg")
|
||
: join(outputDir, "assets", "contact-sheet-svgs.jpg");
|
||
const svgSheets = await createSvgContactSheet(
|
||
svgsDir,
|
||
svgOutputPath,
|
||
assetsRootDir,
|
||
contactSheetBudget,
|
||
);
|
||
if (svgSheets.length > 0)
|
||
progress(
|
||
"design",
|
||
`SVG contact sheet generated (${svgSheets.length} page${svgSheets.length > 1 ? "s" : ""})`,
|
||
);
|
||
};
|
||
await runSvgContactSheet();
|
||
} catch (err) {
|
||
if (err instanceof CaptureDirRefusedError)
|
||
warnings.push(`SVG contact sheet skipped: ${err.message}`);
|
||
/* otherwise non-critical — agent can still read images individually */
|
||
}
|
||
phase(
|
||
"contact-sheets",
|
||
remainingMs() > 0 ? "completed" : "degraded",
|
||
remainingMs() > 0 ? undefined : "budget-exhausted",
|
||
);
|
||
} else {
|
||
warnings.push(
|
||
"Capture budget exhausted before contact sheets; source images were preserved.",
|
||
);
|
||
phase("contact-sheets", "degraded", "budget-exhausted");
|
||
}
|
||
};
|
||
await runContactSheets();
|
||
|
||
return { assets, dropped, fontDrops, extracted, tokens, animationCatalog };
|
||
}
|