/**
* Extract full-page HTML from a website using Puppeteer CDP.
*
* All page.evaluate() calls use string expressions to avoid
* tsx/esbuild __name injection (see esbuild issue #1031).
*/
import type { Page } from "puppeteer-core";
import type { ExtractedHtml } from "./types.js";
import { safeFetch } from "./assetDownloader.js";
import { CAPTURE_USER_AGENT } from "./userAgent.js";
const DEFAULT_SETTLE_TIME = 3000;
// Best-effort inlining: a page with hundreds of slow images or sheets must not hold the capture.
const DEFAULT_INLINE_BUDGET_MS = 20_000;
// Pre-existing capture pipeline size — surfaced by a one-line escape fix, not new logic.
// fallow-ignore-next-line complexity
export async function extractHtml(
page: Pick,
opts: {
settleTime?: number;
inlineBudgetMs?: number;
onWarning?: (message: string) => void;
} = {},
): Promise {
const settleTime = opts.settleTime ?? DEFAULT_SETTLE_TIME;
const inlineBudgetMs = opts.inlineBudgetMs ?? DEFAULT_INLINE_BUDGET_MS;
// Lazy-load scroll removed — index.ts already scrolls before calling extractHtml.
// Images are loaded by the time we get here.
// Settle wait kept as buffer before DOM extraction.
await new Promise((r) => setTimeout(r, settleTime));
// Step 2: Inline external stylesheets
// Fetch CSS from Node.js (bypasses CORS) then inject into page
const stylesheetUrls = (await page.evaluate(`(() => {
return Array.from(document.querySelectorAll('link[rel="stylesheet"][href]')).map(function(l) { return l.href; });
})()`)) as string[];
const sheetsDeadline = Date.now() + inlineBudgetMs;
for (const [index, href] of stylesheetUrls.entries()) {
if (Date.now() >= sheetsDeadline) {
opts.onWarning?.(
`${stylesheetUrls.length - index} stylesheet(s) stayed linked: inlining stopped after ${inlineBudgetMs}ms`,
);
break;
}
try {
const res = await safeFetch(href, {
signal: AbortSignal.timeout(10000),
headers: { "User-Agent": CAPTURE_USER_AGENT },
});
if (!res?.ok) {
await res?.body?.cancel().catch(() => {});
continue;
}
let css = await res.text();
// Fix relative url() references
// fallow-ignore-next-line complexity
css = css.replace(/url\(\s*['"]?([^'")\s]+)['"]?\s*\)/g, (match: string, url: string) => {
if (url.startsWith("data:") || url.startsWith("http") || url.startsWith("//")) return match;
try {
return `url('${new URL(url, href).href}')`;
} catch {
return match;
}
});
await page.evaluate(
(targetHref: string, content: string) => {
const links = document.querySelectorAll('link[rel="stylesheet"]');
for (const link of links) {
if (link instanceof HTMLLinkElement && link.href === targetHref) {
const style = document.createElement("style");
style.textContent = content;
style.media = link.media;
link.replaceWith(style);
break;
}
}
},
href,
css,
);
} catch {
/* network error — skip */
}
}
// Step 3: Make URLs absolute and fix HTML entity encoding in src attributes
await page.evaluate(`(() => {
document.querySelectorAll("img[src]").forEach(function(el) {
try {
// getAttribute returns the raw HTML attribute (with &)
// .src returns the resolved URL (with &) — use .src for the correct value
var resolved = el.src;
if (resolved) el.setAttribute("src", resolved);
} catch(e) {}
});
// Fix srcset attributes too (Next.js image optimization)
document.querySelectorAll("img[srcset]").forEach(function(el) {
try {
var srcset = el.getAttribute("srcset") || "";
// Decode & entities in srcset
srcset = srcset.replace(/&/g, "&");
el.setAttribute("srcset", srcset);
} catch(e) {}
});
document.querySelectorAll('[style*="url("]').forEach(function(el) {
el.style.cssText = el.style.cssText.replace(/url\\(['"]?([^'"\\)\\s]+)['"]?\\)/g, function(_, url) {
try { return "url('" + new URL(url, location.href).href + "')"; } catch(e) { return "url('" + url + "')"; }
});
});
// A page's own