/** * Extract full-page HTML from a website using Puppeteer CDP. * * All page.evaluate() calls use string expressions to avoid * tsx/esbuild __name injection (see esbuild issue #1031). */ import type { Page } from "puppeteer-core"; import type { ExtractedHtml } from "./types.js"; import { safeFetch } from "./assetDownloader.js"; import { CAPTURE_USER_AGENT } from "./userAgent.js"; const DEFAULT_SETTLE_TIME = 3000; // Best-effort inlining: a page with hundreds of slow images or sheets must not hold the capture. const DEFAULT_INLINE_BUDGET_MS = 20_000; // Pre-existing capture pipeline size — surfaced by a one-line escape fix, not new logic. // fallow-ignore-next-line complexity export async function extractHtml( page: Pick, opts: { settleTime?: number; inlineBudgetMs?: number; onWarning?: (message: string) => void; } = {}, ): Promise { const settleTime = opts.settleTime ?? DEFAULT_SETTLE_TIME; const inlineBudgetMs = opts.inlineBudgetMs ?? DEFAULT_INLINE_BUDGET_MS; // Lazy-load scroll removed — index.ts already scrolls before calling extractHtml. // Images are loaded by the time we get here. // Settle wait kept as buffer before DOM extraction. await new Promise((r) => setTimeout(r, settleTime)); // Step 2: Inline external stylesheets // Fetch CSS from Node.js (bypasses CORS) then inject into page const stylesheetUrls = (await page.evaluate(`(() => { return Array.from(document.querySelectorAll('link[rel="stylesheet"][href]')).map(function(l) { return l.href; }); })()`)) as string[]; const sheetsDeadline = Date.now() + inlineBudgetMs; for (const [index, href] of stylesheetUrls.entries()) { if (Date.now() >= sheetsDeadline) { opts.onWarning?.( `${stylesheetUrls.length - index} stylesheet(s) stayed linked: inlining stopped after ${inlineBudgetMs}ms`, ); break; } try { const res = await safeFetch(href, { signal: AbortSignal.timeout(10000), headers: { "User-Agent": CAPTURE_USER_AGENT }, }); if (!res?.ok) { await res?.body?.cancel().catch(() => {}); continue; } let css = await res.text(); // Fix relative url() references // fallow-ignore-next-line complexity css = css.replace(/url\(\s*['"]?([^'")\s]+)['"]?\s*\)/g, (match: string, url: string) => { if (url.startsWith("data:") || url.startsWith("http") || url.startsWith("//")) return match; try { return `url('${new URL(url, href).href}')`; } catch { return match; } }); await page.evaluate( (targetHref: string, content: string) => { const links = document.querySelectorAll('link[rel="stylesheet"]'); for (const link of links) { if (link instanceof HTMLLinkElement && link.href === targetHref) { const style = document.createElement("style"); style.textContent = content; style.media = link.media; link.replaceWith(style); break; } } }, href, css, ); } catch { /* network error — skip */ } } // Step 3: Make URLs absolute and fix HTML entity encoding in src attributes await page.evaluate(`(() => { document.querySelectorAll("img[src]").forEach(function(el) { try { // getAttribute returns the raw HTML attribute (with &) // .src returns the resolved URL (with &) — use .src for the correct value var resolved = el.src; if (resolved) el.setAttribute("src", resolved); } catch(e) {} }); // Fix srcset attributes too (Next.js image optimization) document.querySelectorAll("img[srcset]").forEach(function(el) { try { var srcset = el.getAttribute("srcset") || ""; // Decode & entities in srcset srcset = srcset.replace(/&/g, "&"); el.setAttribute("srcset", srcset); } catch(e) {} }); document.querySelectorAll('[style*="url("]').forEach(function(el) { el.style.cssText = el.style.cssText.replace(/url\\(['"]?([^'"\\)\\s]+)['"]?\\)/g, function(_, url) { try { return "url('" + new URL(url, location.href).href + "')"; } catch(e) { return "url('" + url + "')"; } }); }); // A page's own