Files
hyperframes/packages/cli/src/capture/coreExtractionPhase.ts
T
Miguel Ángel 5b5633fbbc fix(cli): website capture ends with what it has instead of running until it is killed (#6662)
* fix(cli): website capture stops inlining slow images and stylesheets and says what it skipped

* fix(cli): a capture ends at four minutes with its partial bundle unless told otherwise

* fix(cli): a capture's deadline starts once its browser is up, and the kept-image count is exact

* fix(cli): an image whose fetch the deadline cut off always counts as kept
2026-10-28 13:56:48 +00:00

465 lines
17 KiB
TypeScript

import type { ScreenshotInterruption } from "./screenshotCapture.js";
import type { Browser, Page } from "puppeteer-core";
import type { LottieDiscovery } from "./lottieDiscovery.js";
import type { DiscoveredLottie } from "./mediaCapture.js";
import type { CatalogedAsset } from "./assetCataloger.js";
import { serializeTokensForCapture } from "./partialCapture.js";
import type { PartialCaptureState } from "./partialCapture.js";
import type { CaptureResult, DesignTokens, ExtractedHtml } from "./types.js";
import type { IconCandidate } from "./faviconRanker.js";
import { startCdpAnimationCapture } from "./animationCataloger.js";
import { createCaptureDownloadBudget } from "./readBoundedResponse.js";
import { detectLibraries } from "./contentExtractor.js";
import { writeCaptureFileSync } from "./captureFile.js";
import { CaptureDirRefusedError } from "./captureErrors.js";
import { join } from "node:path";
import { extractHtml } from "./htmlExtractor.js";
import { extractTokens } from "./tokenExtractor.js";
import { extractDesignStyles } from "./designStyleExtractor.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { collectAnimationCatalog } from "./animationCataloger.js";
import {
saveLottieAnimations,
renderLottiePreviews,
captureVideoManifest,
} from "./mediaCapture.js";
import { extractVisibleText } from "./contentExtractor.js";
import { isDegradableEvaluateTimeoutError } from "./captureTimeout.js";
import { waitForPageReady } from "./navigateForCapture.js";
import { lazyScrollForCapture } from "./lazyScrollForCapture.js";
import { filterExtractedScripts } from "./filterExtractedScripts.js";
export interface CoreExtractionInput {
page1: Page;
chromeBrowser: Browser;
cdp: Awaited<ReturnType<typeof startCdpAnimationCapture>>["cdp"];
cdpAnims: Awaited<ReturnType<typeof startCdpAnimationCapture>>["animations"];
state: PartialCaptureState;
outputDir: string;
warnings: string[];
progress: (stage: string, detail?: string) => void;
remainingMs: () => number;
maxScreenshots: number;
pageContentCheck: {
textLength: number;
title: string;
hasChallengeElement: boolean;
bodyChildCount: number;
};
contentCheckTimedOut: boolean;
discoveredLotties: DiscoveredLottie[];
lottieDiscovery: LottieDiscovery;
discoveredVideoUrls: Set<string>;
animationCatalog: CaptureResult["animationCatalog"];
capturedShaders: Array<{ type: string; source: string }> | undefined;
catalogedAssets: CatalogedAsset[];
detectedLibraries: Awaited<ReturnType<typeof detectLibraries>>;
visibleTextContent: string;
faviconLinks: IconCandidate[];
tokens: DesignTokens;
extracted: ExtractedHtml;
screenshots: string[];
downloadByteBudget: ReturnType<typeof createCaptureDownloadBudget>;
canWrite: () => boolean;
}
export interface CoreExtractionResult {
animationCatalog: CaptureResult["animationCatalog"];
capturedShaders: Array<{ type: string; source: string }> | undefined;
catalogedAssets: CatalogedAsset[];
detectedLibraries: Awaited<ReturnType<typeof detectLibraries>>;
visibleTextContent: string;
faviconLinks: IconCandidate[];
tokens: DesignTokens;
extracted: ExtractedHtml;
screenshots: string[];
screenshotOutcome:
| { kind: "complete" }
| {
kind: "partial" | "failed";
reason: ScreenshotInterruption["reason"] | "deadline";
};
}
export async function runCoreExtraction(input: CoreExtractionInput): Promise<CoreExtractionResult> {
let {
page1,
chromeBrowser,
cdp,
cdpAnims,
state,
outputDir,
warnings,
progress,
remainingMs,
pageContentCheck,
contentCheckTimedOut,
discoveredLotties,
lottieDiscovery,
discoveredVideoUrls,
animationCatalog,
capturedShaders,
catalogedAssets,
detectedLibraries,
visibleTextContent,
faviconLinks,
tokens,
extracted,
screenshots,
downloadByteBudget,
canWrite,
} = input;
const { maxScreenshots } = input;
let exhaustedPhase: string | undefined = remainingMs() <= 0 ? "navigation checks" : undefined;
const recordBudget = (phase: string): void => {
if (exhaustedPhase === undefined && remainingMs() <= 0) exhaustedPhase = phase;
};
const animationWarning = (): string =>
remainingMs() > 0
? "animation catalog evaluate timed out; continuing without animation catalog"
: "--capture-budget spent before the animation catalog finished; continuing without animation catalog";
let screenshotOutcome: CoreExtractionResult["screenshotOutcome"] = { kind: "complete" };
const runLazyAndLottie = async (): Promise<void> => {
if (!contentCheckTimedOut && pageContentCheck.textLength < 100) {
const reason =
"Page has very little text content (" +
pageContentCheck.textLength +
" chars) — may be blocked or a client-rendered SPA that needs more time";
warnings.push(reason);
progress("warn", reason);
}
const lazyLoadBudgetMs = Math.min(15_000, remainingMs());
const lazyScroll = await lazyScrollForCapture(page1, lazyLoadBudgetMs, {
onWarning: (message) => {
warnings.push(message);
progress("warn", message);
},
});
if (lazyScroll.timedOut && !lazyScroll.degraded) {
const message = `lazy-scroll stopped after ${lazyScroll.steps} steps (budget ${lazyLoadBudgetMs}ms)`;
warnings.push(message);
progress("warn", message);
}
await new Promise((r) => setTimeout(r, 300));
};
await runLazyAndLottie();
recordBudget("lazy scrolling");
const runAnimationDiscovery = async (): Promise<void> => {
const scanDomLotties = async (): Promise<void> => {
// Save discovered Lottie animations
// Also scan DOM for Lottie web components not caught by network interception
try {
const domLotties = await page1.evaluate(`(() => {
var urls = [];
document.querySelectorAll('dotlottie-wc, lottie-player, dotlottie-player').forEach(function(el) {
var src = el.getAttribute('src');
if (src) urls.push(src);
});
// Also check lottie-web registered animations
if (window.lottie && window.lottie.getRegisteredAnimations) {
window.lottie.getRegisteredAnimations().forEach(function(anim) {
if (anim.path) urls.push(anim.path);
});
}
return urls;
})()`);
if (Array.isArray(domLotties)) {
for (const lottieUrl of domLotties) {
if (
typeof lottieUrl === "string" &&
!discoveredLotties.some((l: { url: string }) => l.url === lottieUrl)
) {
discoveredLotties.push({ url: lottieUrl });
}
}
}
} catch {
/* DOM scan failed — non-critical */
}
};
await scanDomLotties();
const collectNetworkLotties = async (): Promise<void> => {
for (const found of await lottieDiscovery.run(downloadByteBudget, remainingMs)) {
const existing = discoveredLotties.findIndex(
(item: { url: string }) => item.url === found.url,
);
if (existing < 0) discoveredLotties.push(found);
else discoveredLotties[existing] = found;
}
};
await collectNetworkLotties();
const saveDiscoveredLotties = async (): Promise<void> => {
if (discoveredLotties.length > 0 && remainingMs() > 0) {
const lottieDir = join(outputDir, "assets", "lottie");
if (!canWrite()) return;
const lottieBudget = { remainingMs, byteBudget: downloadByteBudget };
const savedCount = await saveLottieAnimations(
discoveredLotties,
lottieDir,
outputDir,
lottieBudget,
);
// Generate manifest + preview thumbnails so the agent can SEE what each animation is
if (savedCount > 0 && remainingMs() > 0 && canWrite()) {
await renderLottiePreviews(chromeBrowser, lottieDir, outputDir, lottieBudget);
progress("lottie", `${savedCount} Lottie animation(s) saved`);
}
}
};
await saveDiscoveredLotties();
};
await runAnimationDiscovery();
recordBudget("animation discovery");
// Save captured WebGL shaders (useful context for shader transitions + library detection)
const runShaderCapture = async (): Promise<void> => {
try {
const shaders = await page1.evaluate(`window.__capturedShaders || []`);
if (Array.isArray(shaders) && shaders.length > 0) {
const seen = new Set<string>();
const unique = (shaders as Array<{ type: string; source: string }>).filter((s) => {
if (seen.has(s.source)) return false;
seen.add(s.source);
return true;
});
capturedShaders = unique;
if (canWrite()) {
writeCaptureFileSync(
join(outputDir, "extracted", "shaders.json"),
JSON.stringify(unique, null, 2),
"utf-8",
);
}
progress("shaders", `${unique.length} WebGL shader(s) captured`);
}
} catch {
/* shader extraction failed — non-critical */
}
};
await runShaderCapture();
recordBudget("shader capture");
const runTokenExtraction = async (): Promise<void> => {
// Extract DOM data before extractHtml mutates image URLs and removes scripts.
// Extract design tokens
progress("tokens", "Extracting design tokens...");
tokens = await extractTokens(page1);
state.tokens = tokens;
// Save tokens.json without SVG outerHTML (kept in memory for asset downloader)
if (canWrite()) {
writeCaptureFileSync(
join(outputDir, "extracted", "tokens.json"),
serializeTokensForCapture(tokens),
"utf-8",
);
}
// Extract computed design styles (typography, buttons, cards, spacing, shadows)
progress("style", "Extracting design styles...");
try {
const designStyles = await extractDesignStyles(page1);
state.designStyles = designStyles;
if (canWrite()) {
writeCaptureFileSync(
join(outputDir, "extracted", "design-styles.json"),
JSON.stringify(designStyles, null, 2),
"utf-8",
);
}
progress(
"tokens",
`${designStyles.typography.length} typography roles, ${designStyles.buttons.length} button styles, ${designStyles.shadows.length} shadow values extracted`,
);
} catch (err) {
const errMsg =
err instanceof Error ? `${err.message}\n${err.stack}` : normalizeErrorMessage(err);
console.error(` ⚠ Design style extraction failed: ${errMsg}`);
warnings.push(`Design style extraction failed: ${errMsg}`);
}
};
await runTokenExtraction();
recordBudget("design token extraction");
const runAnimationCapture = async (): Promise<void> => {
progress("animations", "Cataloging animations...");
try {
const animationOutcome = await collectAnimationCatalog(page1, cdpAnims, cdp, {
scrollBudgetMs: Math.min(8_000, remainingMs()),
evaluateBudgetMs: Math.min(15_000, remainingMs()),
});
animationCatalog = animationOutcome.catalog;
if (animationOutcome.timedOut) {
const message = animationWarning();
warnings.push(message);
progress("warn", message);
}
} catch (err) {
if (!isDegradableEvaluateTimeoutError(err)) {
throw err;
}
const message = animationWarning();
warnings.push(message);
progress("warn", message);
try {
await cdp.send("Animation.disable");
} catch {
/* ignore */
}
}
recordBudget("animation catalog");
};
await runAnimationCapture();
const runScreenshotCapture = async (): Promise<void> => {
progress("screenshots", "Capturing scroll screenshots...");
const { captureScrollScreenshots } = await import("./screenshotCapture.js");
if (!canWrite()) {
screenshotOutcome = {
kind: "failed",
reason: "deadline",
};
return;
}
screenshots = [];
state.screenshots = screenshots;
const capture = await captureScrollScreenshots(page1, outputDir, {
remainingMs,
maxScreenshots,
files: screenshots,
});
const failed = capture.failures.map(
({ file, scrollY, message }) =>
`could not capture ${file}${scrollY === null ? "" : ` (scrolled to ${scrollY}px)`}: ${message}`,
);
if (capture.interruption || failed.length > 0 || screenshots.length === 0) {
recordBudget("screenshots");
const { reason, message: detail }: ScreenshotInterruption = capture.interruption ?? {
reason: "internal-error",
message: failed.length > 0 ? "" : "screenshot capture produced no files",
};
const cause = [
reason === "budget-exhausted"
? `--capture-budget exhausted during ${exhaustedPhase}`
: detail,
...failed,
]
.filter(Boolean)
.join("; ");
const message = `${screenshots.length}/${maxScreenshots} requested screenshot files captured: ${cause}`;
screenshotOutcome = {
kind: screenshots.length === 0 ? "failed" : "partial",
reason,
};
warnings.push(message);
progress("warn", message);
}
progress("screenshots", `${screenshots.length} scroll screenshots captured`);
};
await runScreenshotCapture();
const runHtmlExtraction = async (): Promise<void> => {
await waitForPageReady(page1, {
timeoutMs: Math.min(15_000, remainingMs()),
networkIdle: "after-load",
});
// Catalog all assets (must run before extractHtml which converts img src to data URLs)
progress("design", "Cataloging assets...");
try {
const { catalogAssets } = await import("./assetCataloger.js");
catalogedAssets = await catalogAssets(page1);
progress("design", `${catalogedAssets.length} assets cataloged`);
if (catalogedAssets.length === 0) {
warnings.push(
"Asset catalog is empty — no images will be downloaded. The page may use non-standard image loading.",
);
}
} catch (err) {
warnings.push(`Asset cataloging failed (no images will be downloaded): ${err}`);
}
// ── MUTATION phase: extractHtml modifies the live DOM (converts images to data URLs) ──
progress("extract", "Extracting HTML & CSS...");
extracted = await extractHtml(page1, {
settleTime: 1000,
onWarning: (message) => {
warnings.push(message);
progress("warn", message);
},
});
state.extracted = extracted;
// Strip framework scripts from the extracted body — keep visual library scripts
// IMPORTANT: Use non-greedy matching within individual script tags only
extracted.bodyHtml = extracted.bodyHtml
// Remove React hydration markers
.replace(/\s*data-reactroot="[^"]*"/g, "")
.replace(/\s*data-reactroot/g, "");
const filteredScripts = filterExtractedScripts(extracted.bodyHtml, extracted.headHtml);
extracted.bodyHtml = filteredScripts.bodyHtml;
extracted.headHtml = filteredScripts.headHtml;
// Generate video manifest — screenshot each <video> element + extract surrounding context
// so Claude Code can SEE what each video shows and WHERE it was used on the page.
try {
const videoBudgetMs = remainingMs();
if (videoBudgetMs > 0 && canWrite()) {
await captureVideoManifest(page1, outputDir, progress, {
networkVideoUrls: discoveredVideoUrls, // Layer 1 (live Set, read after sampling)
sampleMs: Math.min(12000, videoBudgetMs), // Layer 2: poll DOM within the shared budget
downloadBudgetMs: videoBudgetMs,
remainingMs,
});
}
} catch (err) {
if (err instanceof CaptureDirRefusedError)
warnings.push(`Video manifest skipped: ${err.message}`);
/* otherwise non-blocking — video manifest is best-effort */
}
// Detect JS libraries via globals, DOM fingerprints, script URLs, and shaders
detectedLibraries = await detectLibraries(page1, capturedShaders);
// Extract all visible text in DOM order
visibleTextContent = await extractVisibleText(page1);
// Extract favicon links before closing page (removed from tokens to reduce noise)
// `sizes` and `type` are the only evidence of icon quality: page.html on disk does not
// keep the <link> tags, and the bytes are only fetched for the candidate that wins, so
// dropping these attributes here makes the choice unrecoverable downstream.
faviconLinks = (await page1.evaluate(`(() => {
var iconEls = Array.from(document.querySelectorAll('link[rel*="icon"], link[rel="apple-touch-icon"]'));
return iconEls.map(function(l) {
return {
rel: l.rel,
href: l.href,
sizes: l.getAttribute('sizes'),
type: l.getAttribute('type'),
};
});
})()`)) as IconCandidate[];
await page1.close();
};
await runHtmlExtraction();
return {
screenshotOutcome,
animationCatalog,
capturedShaders,
catalogedAssets,
detectedLibraries,
visibleTextContent,
faviconLinks,
tokens,
extracted,
screenshots,
};
}