diff --git a/packages/cli/src/capture/captureWatchdog.test.ts b/packages/cli/src/capture/captureWatchdog.test.ts index 1e41e5267e..6f16f753da 100644 --- a/packages/cli/src/capture/captureWatchdog.test.ts +++ b/packages/cli/src/capture/captureWatchdog.test.ts @@ -27,9 +27,21 @@ describe("createCaptureWatchdog", () => { watchdog.dispose(); }); + it("starts its clock when the capture's browser is up, so a first-run Chrome download is not counted", async () => { + vi.useFakeTimers(); + const watchdog = createCaptureWatchdog(1000); + await vi.advanceTimersByTimeAsync(5000); + expect(watchdog.expired()).toBe(false); + watchdog.registerBrowser({ close: vi.fn().mockResolvedValue(undefined) }); + await vi.advanceTimersByTimeAsync(1000); + expect(watchdog.expired()).toBe(true); + watchdog.dispose(); + }); + it("observes a losing attempt rejection instead of leaving it unhandled", async () => { vi.useFakeTimers(); const watchdog = createCaptureWatchdog(1); + watchdog.registerBrowser({ close: vi.fn().mockResolvedValue(undefined) }); const work = new Promise((_resolve, reject) => { setTimeout(() => reject(new Error("browser closed")), 10); }); diff --git a/packages/cli/src/capture/captureWatchdog.ts b/packages/cli/src/capture/captureWatchdog.ts index 8e9986e763..3f5afec9a5 100644 --- a/packages/cli/src/capture/captureWatchdog.ts +++ b/packages/cli/src/capture/captureWatchdog.ts @@ -29,36 +29,39 @@ export async function runWithWatchdog( ]); } -export function createCaptureWatchdog(deadlineMs: number | undefined): CaptureWatchdog { +const DEADLINE_UNDER_AGENT_SHELL_LIMITS_MS = 4 * 60_000; + +export function createCaptureWatchdog( + deadlineMs: number = DEADLINE_UNDER_AGENT_SHELL_LIMITS_MS, +): CaptureWatchdog { let browser: CaptureWatchdogBrowser | undefined; let didExpire = false; - let timer: ReturnType | undefined; let resolveDeadline: () => void = () => {}; const promise = new Promise((resolve) => { resolveDeadline = resolve; }); - if (deadlineMs !== undefined) { - timer = setTimeout(() => { - didExpire = true; - resolveDeadline(); - void browser?.close().catch(() => undefined); - }, deadlineMs); - } + let deadlineFromFirstBrowser: ReturnType | undefined; + const expire = () => { + didExpire = true; + resolveDeadline(); + void browser?.close().catch(() => undefined); + }; return { expired: () => didExpire, promise, registerBrowser: (nextBrowser) => { browser = nextBrowser; + deadlineFromFirstBrowser ??= setTimeout(expire, deadlineMs); if (didExpire) void browser.close().catch(() => undefined); }, unregisterBrowser: (closedBrowser) => { if (browser === closedBrowser) browser = undefined; }, dispose: () => { - if (timer !== undefined) clearTimeout(timer); + clearTimeout(deadlineFromFirstBrowser); }, }; } diff --git a/packages/cli/src/capture/coreExtractionPhase.ts b/packages/cli/src/capture/coreExtractionPhase.ts index b1864148fa..c8232e596d 100644 --- a/packages/cli/src/capture/coreExtractionPhase.ts +++ b/packages/cli/src/capture/coreExtractionPhase.ts @@ -385,7 +385,13 @@ export async function runCoreExtraction(input: CoreExtractionInput): Promise { + warnings.push(message); + progress("warn", message); + }, + }); state.extracted = extracted; // Strip framework scripts from the extracted body — keep visual library scripts diff --git a/packages/cli/src/capture/htmlExtractor.test.ts b/packages/cli/src/capture/htmlExtractor.test.ts index c5ff84f803..e1e33d689e 100644 --- a/packages/cli/src/capture/htmlExtractor.test.ts +++ b/packages/cli/src/capture/htmlExtractor.test.ts @@ -209,6 +209,57 @@ describe("extractHtml", () => { }); }); +it("stops inlining images when a CDN never answers and says how many kept their URLs", async () => { + const img = (n: number) => ({ + src: `https://cdn.example/${n}.png`, + naturalWidth: 100, + naturalHeight: 100, + setAttribute: () => {}, + removeAttribute: () => {}, + }); + const sameOrigin = { ...img(4), src: "https://brand.example/4.png" }; + const tiny = { ...img(5), naturalWidth: 4 }; + const imgs = [img(1), sameOrigin, img(2), tiny, img(3)]; + // A fetch that only ends when its abort signal fires, like a CDN that holds the socket open. + const hungFetch = (_url: string, init: { signal?: AbortSignal }) => + new Promise((_resolve, reject) => { + init.signal?.addEventListener("abort", () => reject(init.signal?.reason)); + }); + const context = { + Date, + URL, + AbortSignal, + fetch: hungFetch, + location: { origin: "https://brand.example" }, + document: { + querySelectorAll: (selector: string) => (selector === "img" ? imgs : []), + createElement: () => ({ + getContext: () => ({ + drawImage: () => { + throw new Error("tainted canvas"); + }, + }), + }), + }, + }; + const page = { + evaluate: async (script: string) => { + if (script.includes("naturalWidth")) return runInNewContext(script, context); + if (script.includes('link[rel="stylesheet"][href]')) return []; + return { headHtml: "", bodyHtml: "", cssomRules: "", htmlAttrs: "", fullPageHeight: 0 }; + }, + url: () => "https://brand.example/", + } as unknown as Page; + const warnings: string[] = []; + await extractHtml(page, { + settleTime: 0, + inlineBudgetMs: 100, + onWarning: (message) => warnings.push(message), + }); + // The one cut off mid-fetch and the two after it; never the same-origin or tiny ones it would not inline. + expect(warnings).toEqual(["3 image(s) kept their original URLs: inlining stopped after 100ms"]); +}, 10_000); + it("fetches Node stylesheets through validated redirects and refuses metadata redirects", async () => { const window = new Window({ url: "https://brand.example/" }); window.document.head.innerHTML = diff --git a/packages/cli/src/capture/htmlExtractor.ts b/packages/cli/src/capture/htmlExtractor.ts index 2f9fbecc4c..70b53087c7 100644 --- a/packages/cli/src/capture/htmlExtractor.ts +++ b/packages/cli/src/capture/htmlExtractor.ts @@ -11,14 +11,21 @@ import { safeFetch } from "./assetDownloader.js"; import { CAPTURE_USER_AGENT } from "./userAgent.js"; const DEFAULT_SETTLE_TIME = 3000; +// Best-effort inlining: a page with hundreds of slow images or sheets must not hold the capture. +const DEFAULT_INLINE_BUDGET_MS = 20_000; // Pre-existing capture pipeline size — surfaced by a one-line escape fix, not new logic. // fallow-ignore-next-line complexity export async function extractHtml( page: Pick, - opts: { settleTime?: number } = {}, + opts: { + settleTime?: number; + inlineBudgetMs?: number; + onWarning?: (message: string) => void; + } = {}, ): Promise { const settleTime = opts.settleTime ?? DEFAULT_SETTLE_TIME; + const inlineBudgetMs = opts.inlineBudgetMs ?? DEFAULT_INLINE_BUDGET_MS; // Lazy-load scroll removed — index.ts already scrolls before calling extractHtml. // Images are loaded by the time we get here. @@ -31,7 +38,14 @@ export async function extractHtml( return Array.from(document.querySelectorAll('link[rel="stylesheet"][href]')).map(function(l) { return l.href; }); })()`)) as string[]; - for (const href of stylesheetUrls) { + const sheetsDeadline = Date.now() + inlineBudgetMs; + for (const [index, href] of stylesheetUrls.entries()) { + if (Date.now() >= sheetsDeadline) { + opts.onWarning?.( + `${stylesheetUrls.length - index} stylesheet(s) stayed linked: inlining stopped after ${inlineBudgetMs}ms`, + ); + break; + } try { const res = await safeFetch(href, { signal: AbortSignal.timeout(10000), @@ -111,8 +125,10 @@ export async function extractHtml( // Step 3b: Convert cross-origin images to data URLs // Some CDNs (Contentful, etc.) block direct access but images are already // loaded in the browser. We convert loaded images to data URLs via canvas. - await page.evaluate(`(async () => { + const unconverted = (await page.evaluate(`(async () => { var imgs = Array.from(document.querySelectorAll("img")); + var deadline = Date.now() + ${inlineBudgetMs}; + var kept = 0; for (var i = 0; i < imgs.length; i++) { var img = imgs[i]; try { @@ -121,6 +137,10 @@ export async function extractHtml( // Only convert cross-origin images (same-origin ones will load fine) var imgUrl = new URL(img.src); if (imgUrl.origin === location.origin) continue; + if (Date.now() >= deadline) { + kept++; + continue; + } // Try to draw to canvas — will fail if CORS blocks it var canvas = document.createElement("canvas"); canvas.width = img.naturalWidth; @@ -134,8 +154,9 @@ export async function extractHtml( } } catch(e) { // Canvas CORS failed — try fetch + blob as fallback + var stop = AbortSignal.timeout(Math.max(1, deadline - Date.now())); try { - var resp = await fetch(img.src, { mode: "cors" }); + var resp = await fetch(img.src, { mode: "cors", signal: stop }); if (resp.ok) { var blob = await resp.blob(); var reader = new FileReader(); @@ -150,10 +171,17 @@ export async function extractHtml( } } catch(e2) { // Both methods failed — image stays as original URL + if (stop.aborted) kept++; } } } - })()`); + return kept; + })()`)) as number; + if (unconverted > 0) { + opts.onWarning?.( + `${unconverted} image(s) kept their original URLs: inlining stopped after ${inlineBudgetMs}ms`, + ); + } // Step 4: Extract everything const result = (await page.evaluate(`(() => { diff --git a/packages/cli/src/capture/index.watchdog.test.ts b/packages/cli/src/capture/index.watchdog.test.ts index 156b3135e3..0619aa79ce 100644 --- a/packages/cli/src/capture/index.watchdog.test.ts +++ b/packages/cli/src/capture/index.watchdog.test.ts @@ -1,9 +1,13 @@ +import { mkdtempSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createPartialCaptureState } from "./partialCapture.js"; import { CAPTURE_PHASE_SCHEMA } from "./types.js"; import type { CaptureOptions, CaptureResult } from "./types.js"; import { captureBrowserArgs } from "./browserLaunchArgs.js"; import { NavigationDeadlineError } from "./captureErrors.js"; +import type { CaptureWatchdog } from "./captureWatchdog.js"; const { captureWebsiteAttemptMock } = vi.hoisted(() => ({ captureWebsiteAttemptMock: vi.fn(), @@ -92,4 +96,26 @@ describe("capture navigation retry launch", () => { await expect(captureWebsite(options)).rejects.toThrow("capture navigation timed out"); expect(captureWebsiteAttemptMock).toHaveBeenCalledOnce(); }); + it("a capture whose network step never answers ends at four minutes with its partial bundle", async () => { + vi.useFakeTimers(); + try { + const outputDir = mkdtempSync(join(tmpdir(), "hf-capture-deadline-")); + captureWebsiteAttemptMock.mockImplementationOnce( + (_opts, _progress, _webgl, watchdog: CaptureWatchdog) => { + watchdog.registerBrowser({ close: async () => {} }); + return new Promise(() => {}); + }, + ); + const { captureWebsite } = await import("./index.js"); + let settled: CaptureResult | undefined; + void captureWebsite({ url: "https://example.com", outputDir }).then((result) => { + settled = result; + }); + await vi.advanceTimersByTimeAsync(4 * 60_000); + expect(settled?.lastPhase).toMatchObject({ status: "degraded", reason: "deadline" }); + expect(readdirSync(outputDir).length).toBeGreaterThan(0); + } finally { + vi.useRealTimers(); + } + }); });