/**
* Extract full-page HTML from a website using Puppeteer CDP.
*
* All page.evaluate() calls use string expressions to avoid
* tsx/esbuild __name injection (see esbuild issue #1031).
*/
import type { Page } from "puppeteer-core";
import type { ExtractedHtml } from "./types.js";
import { isPrivateUrl } from "./assetDownloader.js";
const DEFAULT_SETTLE_TIME = 3000;
// Pre-existing capture pipeline size — surfaced by a one-line escape fix, not new logic.
// fallow-ignore-next-line complexity
export async function extractHtml(
page: Page,
opts: { settleTime?: number } = {},
): Promise {
const settleTime = opts.settleTime ?? DEFAULT_SETTLE_TIME;
// Lazy-load scroll removed — index.ts already scrolls before calling extractHtml.
// Images are loaded by the time we get here.
// Settle wait kept as buffer before DOM extraction.
await new Promise((r) => setTimeout(r, settleTime));
// Step 2: Inline external stylesheets
// Fetch CSS from Node.js (bypasses CORS) then inject into page
const stylesheetUrls = (await page.evaluate(`(() => {
return Array.from(document.querySelectorAll('link[rel="stylesheet"][href]')).map(function(l) { return l.href; });
})()`)) as string[];
for (const href of stylesheetUrls) {
try {
if (isPrivateUrl(href)) continue;
const res = await fetch(href, {
signal: AbortSignal.timeout(10000),
headers: { "User-Agent": "Mozilla/5.0" },
});
if (!res.ok) continue;
let css = await res.text();
// Fix relative url() references
// fallow-ignore-next-line complexity
css = css.replace(/url\(\s*['"]?([^'")\s]+)['"]?\s*\)/g, (match: string, url: string) => {
if (url.startsWith("data:") || url.startsWith("http") || url.startsWith("//")) return match;
try {
return `url('${new URL(url, href).href}')`;
} catch {
return match;
}
});
// Add the CSS as a