diff --git a/packages/cli/src/capture/animationCataloger.degrade.test.ts b/packages/cli/src/capture/animationCataloger.degrade.test.ts new file mode 100644 index 000000000..2e5aa69ff --- /dev/null +++ b/packages/cli/src/capture/animationCataloger.degrade.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it, vi } from "vitest"; +import { collectAnimationCatalog } from "./animationCataloger.js"; + +describe("collectAnimationCatalog degradation", () => { + it("surfaces evaluate timeouts via timedOut instead of silent empty success", async () => { + const page = { + evaluate: vi.fn(() => new Promise(() => undefined)), + }; + const cdp = { + send: vi.fn(async () => undefined), + }; + + const outcome = await collectAnimationCatalog(page as never, [], cdp as never, { + scrollBudgetMs: 20, + evaluateBudgetMs: 20, + }); + + expect(outcome.timedOut).toBe(true); + expect(outcome.catalog.summary.webAnimations).toBe(0); + expect(cdp.send).toHaveBeenCalledWith("Animation.disable"); + }); +}); diff --git a/packages/cli/src/capture/animationCataloger.ts b/packages/cli/src/capture/animationCataloger.ts index acb46212c..c064bf6f4 100644 --- a/packages/cli/src/capture/animationCataloger.ts +++ b/packages/cli/src/capture/animationCataloger.ts @@ -16,6 +16,8 @@ */ import type { Page, CDPSession } from "puppeteer-core"; +import { isDegradableEvaluateTimeoutError, withRemainingBudget } from "./captureTimeout.js"; +import { lazyScrollForCapture } from "./lazyScrollForCapture.js"; export interface AnimationCatalog { /** Active animations via document.getAnimations() — includes keyframes */ @@ -127,32 +129,36 @@ export async function startCdpAnimationCapture( return { cdp, animations }; } -/** - * Collect the full animation catalog after page has loaded and settled. - * Should be called after scrolling through the page to trigger all animations. - */ +const EMPTY_ANIMATION_SCAN = { + webAnimations: [] as WebAnimationEntry[], + cssDeclarations: [] as CssAnimationEntry[], + scrollTargets: [] as ScrollTarget[], + canvasCount: 0, +}; + +export interface CollectAnimationCatalogOutcome { + catalog: AnimationCatalog; + timedOut: boolean; +} + export async function collectAnimationCatalog( page: Page, cdpAnimations: CdpAnimationEntry[], cdp: CDPSession, -): Promise { - // Scroll through page to trigger scroll-based animations - await page.evaluate(`(async () => { - var height = document.body.scrollHeight; - for (var y = 0; y < height; y += window.innerHeight * 0.5) { - window.scrollTo(0, y); - await new Promise(function(r) { setTimeout(r, 400); }); - } - window.scrollTo(0, 0); - await new Promise(function(r) { setTimeout(r, 1000); }); - })()`); + opts: { scrollBudgetMs?: number; evaluateBudgetMs?: number } = {}, +): Promise { + const scrollBudgetMs = opts.scrollBudgetMs ?? 8_000; + const evaluateBudgetMs = opts.evaluateBudgetMs ?? 15_000; + const scroll = await lazyScrollForCapture(page, scrollBudgetMs); + let timedOut = scroll.degraded || scroll.timedOut; - // Collect from Web Animations API + computed styles + IO targets - const result = (await page.evaluate(`(() => { + let result = EMPTY_ANIMATION_SCAN; + try { + result = (await withRemainingBudget( + page.evaluate(`(() => { var webAnimations = []; var cssDeclarations = []; - // 1. Web Animations API try { var anims = document.getAnimations(); webAnimations = anims.map(function(anim) { @@ -180,7 +186,6 @@ export async function collectAnimationCatalog( }); } catch(e) {} - // 2. CSS animation/transition scan var allEls = document.querySelectorAll('*'); for (var i = 0; i < allEls.length && i < 5000; i++) { var el = allEls[i]; @@ -202,31 +207,44 @@ export async function collectAnimationCatalog( } catch(e) {} } - // 3. IO targets (collected by monkey-patch) var scrollTargets = (window.__hf_io_targets || []).map(function(t) { return { selector: t.selector, rect: t.rect }; }); - // 4. Canvas summary var canvasCount = document.querySelectorAll('canvas').length; return { webAnimations: webAnimations, cssDeclarations: cssDeclarations, scrollTargets: scrollTargets, canvasCount: canvasCount }; - })()`)) as any; + })()`), + evaluateBudgetMs, + "animation-catalog", + )) as typeof EMPTY_ANIMATION_SCAN; + } catch (err) { + if (!isDegradableEvaluateTimeoutError(err)) { + throw err; + } + timedOut = true; + } - // Stop CDP listener - await cdp.send("Animation.disable"); + try { + await cdp.send("Animation.disable"); + } catch { + /* session may already be closed */ + } return { - webAnimations: result.webAnimations, - cssDeclarations: result.cssDeclarations, - scrollTargets: result.scrollTargets, - cdpAnimations, - summary: { - webAnimations: result.webAnimations.length, - cssDeclarations: result.cssDeclarations.length, - scrollTargets: result.scrollTargets.length, - cdpAnimations: cdpAnimations.length, - canvases: result.canvasCount, + catalog: { + webAnimations: result.webAnimations, + cssDeclarations: result.cssDeclarations, + scrollTargets: result.scrollTargets, + cdpAnimations, + summary: { + webAnimations: result.webAnimations.length, + cssDeclarations: result.cssDeclarations.length, + scrollTargets: result.scrollTargets.length, + cdpAnimations: cdpAnimations.length, + canvases: result.canvasCount, + }, }, + timedOut, }; } diff --git a/packages/cli/src/capture/captureTimeout.test.ts b/packages/cli/src/capture/captureTimeout.test.ts new file mode 100644 index 000000000..b6ca53bb3 --- /dev/null +++ b/packages/cli/src/capture/captureTimeout.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import { TimeoutError } from "puppeteer-core"; +import { + PUPPETEER_DEFAULT_PROTOCOL_TIMEOUT_MS, + StageBudgetTimeoutError, + captureProtocolTimeoutMs, + formatCaptureFailureReason, + isNavigationTimeoutError, + isProtocolEvaluateTimeoutError, + withRemainingBudget, +} from "./captureTimeout.js"; + +describe("captureProtocolTimeoutMs", () => { + it("uses the larger of nav timeout and post-nav budget", () => { + expect(captureProtocolTimeoutMs(120_000, 90_000)).toBe(120_000); + expect(captureProtocolTimeoutMs(30_000, 120_000)).toBe(120_000); + }); + + it("floors at 60s", () => { + expect(captureProtocolTimeoutMs(5_000, 5_000)).toBe(60_000); + }); + + it("uses puppeteer default when inputs are non-finite", () => { + expect(captureProtocolTimeoutMs(Number.POSITIVE_INFINITY, 120_000)).toBe( + PUPPETEER_DEFAULT_PROTOCOL_TIMEOUT_MS, + ); + }); +}); + +describe("isNavigationTimeoutError", () => { + it("matches Puppeteer TimeoutError navigation timeouts", () => { + expect( + isNavigationTimeoutError(new TimeoutError("Navigation timeout of 30000 ms exceeded")), + ).toBe(true); + }); + + it("does not treat plain Error as navigation timeout", () => { + expect(isNavigationTimeoutError(new Error("Navigation timeout of 30000 ms exceeded"))).toBe( + false, + ); + }); + + it("still classifies message strings for BLOCKED.md formatting", () => { + expect(isNavigationTimeoutError("Navigation timeout of 30000 ms exceeded")).toBe(true); + }); +}); + +describe("isProtocolEvaluateTimeoutError", () => { + it("matches Puppeteer TimeoutError evaluate timeouts by instanceof", () => { + expect( + isProtocolEvaluateTimeoutError( + new TimeoutError( + "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting in launch/connect calls for a higher timeout if needed.", + ), + ), + ).toBe(true); + }); + + it("treats non-navigation TimeoutError as evaluate/protocol timeout even if wording drifts", () => { + expect( + isProtocolEvaluateTimeoutError(new TimeoutError("Timed out after waiting 180000ms")), + ).toBe(true); + }); + + it("ignores navigation TimeoutErrors", () => { + expect( + isProtocolEvaluateTimeoutError(new TimeoutError("Navigation timeout of 30000 ms exceeded")), + ).toBe(false); + }); +}); + +describe("withRemainingBudget", () => { + it("returns before a never-resolving promise when the budget elapses", async () => { + const never = new Promise(() => undefined); + const started = Date.now(); + await expect(withRemainingBudget(never, 20, "never-resolve")).rejects.toBeInstanceOf( + StageBudgetTimeoutError, + ); + expect(Date.now() - started).toBeLessThan(200); + }); + + it("resolves when work finishes inside the budget", async () => { + await expect(withRemainingBudget(Promise.resolve("ok"), 100, "fast")).resolves.toBe("ok"); + }); + + it("fails immediately when no budget remains", async () => { + await expect(withRemainingBudget(Promise.resolve("ok"), 0, "empty")).rejects.toBeInstanceOf( + StageBudgetTimeoutError, + ); + }); +}); + +describe("formatCaptureFailureReason", () => { + it("describes evaluate/protocol timeouts as extraction failures", () => { + const reason = formatCaptureFailureReason( + "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting in launch/connect calls for a higher timeout if needed.", + ); + expect(reason).toMatch(/extraction timed out/i); + expect(reason).not.toMatch(/navigation timed out/i); + }); + + it("keeps navigation timeout wording for nav failures", () => { + expect(formatCaptureFailureReason("Navigation timeout of 30000 ms exceeded")).toMatch( + /navigation timed out/i, + ); + }); +}); diff --git a/packages/cli/src/capture/captureTimeout.ts b/packages/cli/src/capture/captureTimeout.ts new file mode 100644 index 000000000..e171a59b7 --- /dev/null +++ b/packages/cli/src/capture/captureTimeout.ts @@ -0,0 +1,97 @@ +import { TimeoutError } from "puppeteer-core"; + +export const PUPPETEER_DEFAULT_PROTOCOL_TIMEOUT_MS = 180_000; + +export class StageBudgetTimeoutError extends Error { + readonly budgetMs: number; + + constructor(label: string, budgetMs: number) { + super(`stage budget exceeded while ${label} (${budgetMs}ms)`); + this.name = "StageBudgetTimeoutError"; + this.budgetMs = budgetMs; + } +} + +export function captureProtocolTimeoutMs(navTimeoutMs: number, postNavBudgetMs: number): number { + if (!Number.isFinite(navTimeoutMs) || !Number.isFinite(postNavBudgetMs)) { + return PUPPETEER_DEFAULT_PROTOCOL_TIMEOUT_MS; + } + return Math.max(60_000, Math.max(0, navTimeoutMs), Math.max(0, postNavBudgetMs)); +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +function hasNavigationTimeoutMessage(msg: string): boolean { + return /navigation timeout/i.test(msg); +} + +function hasProtocolEvaluateTimeoutMessage(msg: string): boolean { + return /Runtime\.evaluate timed out|protocolTimeout|protocol timeout/i.test(msg); +} + +export function isNavigationTimeoutError(err: unknown): boolean { + if (err instanceof TimeoutError) { + return hasNavigationTimeoutMessage(err.message); + } + // String path used by BLOCKED.md formatting. + return typeof err === "string" && hasNavigationTimeoutMessage(err); +} + +function isStageBudgetTimeoutError(err: unknown): boolean { + return err instanceof StageBudgetTimeoutError; +} + +export function isProtocolEvaluateTimeoutError(err: unknown): boolean { + if (err instanceof TimeoutError) { + return !hasNavigationTimeoutMessage(err.message); + } + return hasProtocolEvaluateTimeoutMessage(errorMessage(err)); +} + +export function isDegradableEvaluateTimeoutError(err: unknown): boolean { + return isStageBudgetTimeoutError(err) || isProtocolEvaluateTimeoutError(err); +} + +export async function withRemainingBudget( + work: Promise, + remainingMs: number, + label: string, +): Promise { + if (!(remainingMs > 0)) { + throw new StageBudgetTimeoutError(label, Math.max(0, remainingMs)); + } + + let timer: ReturnType | undefined; + try { + return await Promise.race([ + work, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + reject(new StageBudgetTimeoutError(label, remainingMs)); + }, remainingMs); + }), + ]); + } finally { + if (timer !== undefined) { + clearTimeout(timer); + } + } +} + +export function formatCaptureFailureReason(errMsg: string): string { + if (hasProtocolEvaluateTimeoutMessage(errMsg) || /stage budget exceeded/i.test(errMsg)) { + return ( + `Page extraction timed out while running in-page script (${errMsg}). ` + + "The page likely opened, but a later capture step hung." + ); + } + if (hasNavigationTimeoutMessage(errMsg)) { + return "Page navigation timed out — the site may be blocking headless browsers or requires authentication."; + } + if (/timeout|timed out/i.test(errMsg)) { + return `Capture timed out: ${errMsg}`; + } + return `Capture failed: ${errMsg}`; +} diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index 5d9dfc92d..cdcbfd1e9 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -43,6 +43,12 @@ import type { VisionCaptionOutcome } from "./contentExtractor.js"; import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js"; import { detectBlockedPage } from "./pageBlockDetection.js"; import { navigateForCapture } from "./navigateForCapture.js"; +import { + captureProtocolTimeoutMs, + isDegradableEvaluateTimeoutError, + withRemainingBudget, +} from "./captureTimeout.js"; +import { lazyScrollForCapture } from "./lazyScrollForCapture.js"; import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js"; export type { CaptureOptions, CaptureResult } from "./types.js"; @@ -123,6 +129,7 @@ export async function captureWebsite( const chromeBrowser = await puppeteer.default.launch({ headless: true, executablePath: browser.executablePath, + protocolTimeout: captureProtocolTimeoutMs(timeout, budgetMs), args: [ "--no-sandbox", "--disable-dev-shm-usage", @@ -253,27 +260,51 @@ export async function captureWebsite( postNavigationDeadline = Date.now() + budgetMs; await new Promise((r) => setTimeout(r, settleTime)); - // Check if the page loaded real content or an anti-bot challenge - // Combine structural evidence with the main response status/title. Low text - // alone stays non-fatal so image-led sites are not rejected. - const pageContentCheck = (await page1.evaluate(`(() => { - var text = (document.body.innerText || "").trim(); - var title = document.title || ""; - // Structural: Cloudflare Turnstile widget or challenge iframe - var hasCfTurnstile = !!document.querySelector('.cf-turnstile, [data-sitekey], iframe[src*="challenges.cloudflare.com"], #challenge-running, #challenge-form'); - // Structural: page is almost empty (challenge pages have minimal DOM) - var bodyChildCount = document.body.children.length; - return { textLength: text.length, title: title, hasChallengeElement: hasCfTurnstile, bodyChildCount: bodyChildCount }; - })()`)) as { + let pageContentCheck: { textLength: number; title: string; hasChallengeElement: boolean; bodyChildCount: number; + } = { + textLength: 0, + title: "", + hasChallengeElement: false, + bodyChildCount: Number.POSITIVE_INFINITY, }; + let contentCheckTimedOut = false; + try { + pageContentCheck = (await withRemainingBudget( + page1.evaluate(`(() => { + var text = (document.body && document.body.innerText || "").trim(); + var title = document.title || ""; + var hasCfTurnstile = !!document.querySelector('.cf-turnstile, [data-sitekey], iframe[src*="challenges.cloudflare.com"], #challenge-running, #challenge-form'); + var bodyChildCount = document.body ? document.body.children.length : 0; + return { textLength: text.length, title: title, hasChallengeElement: hasCfTurnstile, bodyChildCount: bodyChildCount }; + })()`), + Math.min(5_000, remainingMs()), + "content-check", + )) as typeof pageContentCheck; + } catch (err) { + if (!isDegradableEvaluateTimeoutError(err)) { + throw err; + } + contentCheckTimedOut = true; + const message = + "post-navigation content check timed out; continuing with HTTP-status blocked-page detection only"; + warnings.push(message); + progress("warn", message); + } const blockedReason = detectBlockedPage({ httpStatus: navigationResponse?.status() ?? null, - ...pageContentCheck, + ...(contentCheckTimedOut + ? { + title: "", + textLength: 0, + bodyChildCount: 0, + hasChallengeElement: false, + } + : pageContentCheck), }); if (blockedReason) { phase("navigation", "degraded", "blocked"); @@ -283,7 +314,7 @@ export async function captureWebsite( phase("navigation", "completed"); phase("core-extraction", "started"); - if (pageContentCheck.textLength < 100) { + if (!contentCheckTimedOut && pageContentCheck.textLength < 100) { const reason = "Page has very little text content (" + pageContentCheck.textLength + @@ -292,42 +323,18 @@ export async function captureWebsite( progress("warn", reason); } - // Scroll through page to trigger lazy-loaded images and Lottie animations - // Framer and other modern sites use IntersectionObserver — images only load - // when scrolled into view. We scroll the full page, then wait for all images - // to finish loading before proceeding. const lazyLoadBudgetMs = Math.min(15_000, remainingMs()); - if (lazyLoadBudgetMs > 0) { - await page1.evaluate(`(async () => { - var lazyLoadDeadline = Date.now() + ${lazyLoadBudgetMs}; - var h = document.body.scrollHeight; - for (var y = 0; y < h; y += window.innerHeight * 0.7) { - if (Date.now() >= lazyLoadDeadline) break; - window.scrollTo(0, y); - await new Promise(function(r) { setTimeout(r, 400); }); - } - // Scroll to very bottom to catch footer lazy-loads - if (Date.now() < lazyLoadDeadline) { - window.scrollTo(0, document.body.scrollHeight); - await new Promise(function(r) { setTimeout(r, Math.min(800, Math.max(0, lazyLoadDeadline - Date.now()))); }); - } - // Wait for all images to finish loading - var imgs = Array.from(document.querySelectorAll('img')); - var pending = imgs.filter(function(img) { return !img.complete; }); - var imageWaitMs = Math.min(5000, Math.max(0, lazyLoadDeadline - Date.now())); - if (pending.length > 0 && imageWaitMs > 0) { - await Promise.race([ - Promise.all(pending.map(function(img) { - return new Promise(function(r) { img.onload = r; img.onerror = r; }); - })), - new Promise(function(r) { setTimeout(r, imageWaitMs); }) - ]); - } - window.scrollTo(0, 0); - })()`); + const lazyScroll = await lazyScrollForCapture(page1, lazyLoadBudgetMs, { + onWarning: (message) => { + warnings.push(message); + progress("warn", message); + }, + }); + if (lazyScroll.timedOut && !lazyScroll.degraded) { + const message = `lazy-scroll stopped after ${lazyScroll.steps} steps (budget ${lazyLoadBudgetMs}ms)`; + warnings.push(message); + progress("warn", message); } - - await page1.evaluate(`window.scrollTo(0, 0)`); await new Promise((r) => setTimeout(r, 300)); // Save discovered Lottie animations @@ -434,15 +441,47 @@ export async function captureWebsite( warnings.push(`Design style extraction failed: ${errMsg}`); } - // Collect animation catalog progress("animations", "Cataloging animations..."); - animationCatalog = await collectAnimationCatalog(page1, cdpAnims, cdp); + try { + const animationOutcome = await collectAnimationCatalog(page1, cdpAnims, cdp, { + scrollBudgetMs: Math.min(8_000, remainingMs()), + evaluateBudgetMs: Math.min(15_000, remainingMs()), + }); + animationCatalog = animationOutcome.catalog; + if (animationOutcome.timedOut) { + const message = + "animation catalog evaluate timed out; continuing without animation catalog"; + warnings.push(message); + progress("warn", message); + } + } catch (err) { + if (!isDegradableEvaluateTimeoutError(err)) { + throw err; + } + const message = "animation catalog evaluate timed out; continuing without animation catalog"; + warnings.push(message); + progress("warn", message); + try { + await cdp.send("Animation.disable"); + } catch { + /* ignore */ + } + } - // Capture scroll-position viewport screenshots progress("screenshots", "Capturing scroll screenshots..."); const { captureScrollScreenshots } = await import("./screenshotCapture.js"); - const screenshots = await captureScrollScreenshots(page1, outputDir, { remainingMs }); - progress("screenshots", `${screenshots.length} scroll screenshots captured`); + let screenshots: string[] = []; + try { + screenshots = await captureScrollScreenshots(page1, outputDir, { remainingMs }); + progress("screenshots", `${screenshots.length} scroll screenshots captured`); + } catch (err) { + if (!isDegradableEvaluateTimeoutError(err)) { + throw err; + } + const message = "scroll screenshots timed out; continuing without screenshots"; + warnings.push(message); + progress("warn", message); + } // Catalog all assets (must run before extractHtml which converts img src to data URLs) progress("design", "Cataloging assets..."); diff --git a/packages/cli/src/capture/lazyScrollForCapture.test.ts b/packages/cli/src/capture/lazyScrollForCapture.test.ts new file mode 100644 index 000000000..7814a9dc6 --- /dev/null +++ b/packages/cli/src/capture/lazyScrollForCapture.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from "vitest"; +import { lazyScrollForCapture } from "./lazyScrollForCapture.js"; + +describe("lazyScrollForCapture", () => { + it("no-ops when budget is empty", async () => { + const evaluate = vi.fn(); + const result = await lazyScrollForCapture({ evaluate }, 0); + expect(result).toEqual({ steps: 0, timedOut: false, degraded: false }); + expect(evaluate).not.toHaveBeenCalled(); + }); + + it("scrolls in node-driven steps until the bottom", async () => { + const heights = [{ atBottom: false }, { atBottom: false }, { atBottom: true }]; + const evaluate = vi.fn(async (expr: string) => { + if (expr.includes("atBottom")) { + return heights.shift() ?? { atBottom: true }; + } + return 0; + }); + const sleep = vi.fn(async () => undefined); + + const result = await lazyScrollForCapture({ evaluate }, 15_000, { stepDelayMs: 10, sleep }); + + expect(result.steps).toBe(3); + expect(result.timedOut).toBe(false); + expect(result.degraded).toBe(false); + }); + + it("returns within the stage budget when evaluate never resolves", async () => { + const evaluate = vi.fn(() => new Promise(() => undefined)); + const started = Date.now(); + const warnings: string[] = []; + + const result = await lazyScrollForCapture({ evaluate }, 25, { + onWarning: (message) => warnings.push(message), + sleep: async () => undefined, + }); + + expect(result.degraded).toBe(true); + expect(result.timedOut).toBe(true); + expect(Date.now() - started).toBeLessThan(250); + expect(warnings[0]).toMatch(/lazy-scroll evaluate timed out/i); + }); + + it("does not issue a recovery evaluate after the budget is exhausted", async () => { + let now = 1_000_000; + const dateSpy = vi.spyOn(Date, "now").mockImplementation(() => now); + const evaluate = vi.fn(() => { + now += 100; + return new Promise(() => undefined); + }); + + try { + const result = await lazyScrollForCapture({ evaluate }, 30, { sleep: async () => undefined }); + expect(result.degraded).toBe(true); + expect(evaluate).toHaveBeenCalledTimes(1); + } finally { + dateSpy.mockRestore(); + } + }); +}); diff --git a/packages/cli/src/capture/lazyScrollForCapture.ts b/packages/cli/src/capture/lazyScrollForCapture.ts new file mode 100644 index 000000000..cc520a5e3 --- /dev/null +++ b/packages/cli/src/capture/lazyScrollForCapture.ts @@ -0,0 +1,141 @@ +import { isDegradableEvaluateTimeoutError, withRemainingBudget } from "./captureTimeout.js"; + +const LAZY_SCROLL_STEP_DELAY_MS = 400; +const LAZY_SCROLL_MAX_IMAGE_WAIT_MS = 5_000; +const LAZY_SCROLL_BOTTOM_SETTLE_MS = 800; +const LAZY_SCROLL_IMAGE_POLL_MS = 250; + +export interface LazyScrollPage { + evaluate(pageFunction: string): Promise; +} + +export interface LazyScrollResult { + steps: number; + timedOut: boolean; + degraded: boolean; +} + +async function evaluateWithinBudget( + page: LazyScrollPage, + expression: string, + deadline: number, + label: string, +): Promise { + return withRemainingBudget(page.evaluate(expression), deadline - Date.now(), label); +} + +async function settleAfterScroll( + page: LazyScrollPage, + deadline: number, + sleep: (ms: number) => Promise, +): Promise { + const bottomSettleMs = Math.min(LAZY_SCROLL_BOTTOM_SETTLE_MS, Math.max(0, deadline - Date.now())); + if (bottomSettleMs > 0) { + await evaluateWithinBudget( + page, + `window.scrollTo(0, document.body.scrollHeight)`, + deadline, + "lazy-scroll-bottom", + ); + await sleep(bottomSettleMs); + } + + const imageDeadline = Math.min(deadline, Date.now() + LAZY_SCROLL_MAX_IMAGE_WAIT_MS); + while (Date.now() < imageDeadline) { + const pending = (await evaluateWithinBudget( + page, + `Array.from(document.images).filter(function(img) { return !img.complete; }).length`, + imageDeadline, + "lazy-scroll-image-pending", + )) as number; + if (!(pending > 0)) { + break; + } + const waitMs = Math.min(LAZY_SCROLL_IMAGE_POLL_MS, imageDeadline - Date.now()); + if (waitMs <= 0) { + break; + } + await sleep(waitMs); + } + + if (deadline - Date.now() > 0) { + await evaluateWithinBudget(page, `window.scrollTo(0, 0)`, deadline, "lazy-scroll-reset"); + } +} + +// fallow-ignore-next-line complexity +export async function lazyScrollForCapture( + page: LazyScrollPage, + budgetMs: number, + opts: { + stepDelayMs?: number; + onWarning?: (message: string) => void; + sleep?: (ms: number) => Promise; + } = {}, +): Promise { + if (budgetMs <= 0) { + return { steps: 0, timedOut: false, degraded: false }; + } + + const stepDelayMs = opts.stepDelayMs ?? LAZY_SCROLL_STEP_DELAY_MS; + const sleep = opts.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const deadline = Date.now() + budgetMs; + let steps = 0; + let timedOut = false; + let degraded = false; + + try { + while (Date.now() < deadline) { + const state = (await evaluateWithinBudget( + page, + `(() => { + var y = window.scrollY || window.pageYOffset || 0; + var view = window.innerHeight || 0; + var height = document.body ? document.body.scrollHeight : 0; + var next = Math.min(y + view * 0.7, Math.max(0, height - view)); + var atBottom = height <= view + 2 || next <= y + 1; + window.scrollTo(0, atBottom ? height : next); + return { atBottom: atBottom }; + })()`, + deadline, + "lazy-scroll-step", + )) as { atBottom: boolean }; + + steps += 1; + if (state.atBottom) { + break; + } + + const remaining = deadline - Date.now(); + if (remaining <= 0) { + timedOut = true; + break; + } + await sleep(Math.min(stepDelayMs, remaining)); + } + + if (Date.now() >= deadline) { + timedOut = true; + } + + if (deadline - Date.now() > 0) { + await settleAfterScroll(page, deadline, sleep); + } + } catch (err) { + if (!isDegradableEvaluateTimeoutError(err)) { + throw err; + } + degraded = true; + timedOut = true; + opts.onWarning?.("lazy-scroll evaluate timed out; continuing with current page state"); + if (deadline - Date.now() > 0) { + try { + await evaluateWithinBudget(page, `window.scrollTo(0, 0)`, deadline, "lazy-scroll-recovery"); + } catch { + /* page may be wedged; do not spend another unbounded protocol wait */ + } + } + } + + return { steps, timedOut, degraded }; +} diff --git a/packages/cli/src/capture/navigateForCapture.test.ts b/packages/cli/src/capture/navigateForCapture.test.ts index 5fdec11aa..98c9517a9 100644 --- a/packages/cli/src/capture/navigateForCapture.test.ts +++ b/packages/cli/src/capture/navigateForCapture.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { TimeoutError } from "puppeteer-core"; import { isNavigationTimeoutError, navigateForCapture, @@ -17,10 +18,10 @@ describe("networkIdleAttemptTimeoutMs", () => { }); describe("isNavigationTimeoutError", () => { - it("matches Puppeteer navigation timeouts", () => { - expect(isNavigationTimeoutError(new Error("Navigation timeout of 30000 ms exceeded"))).toBe( - true, - ); + it("matches Puppeteer TimeoutError navigation timeouts", () => { + expect( + isNavigationTimeoutError(new TimeoutError("Navigation timeout of 30000 ms exceeded")), + ).toBe(true); }); it("ignores unrelated failures", () => { @@ -51,7 +52,7 @@ describe("navigateForCapture", () => { const response = { status: () => 200 }; const goto = vi .fn() - .mockRejectedValueOnce(new Error("Navigation timeout of 30000 ms exceeded")) + .mockRejectedValueOnce(new TimeoutError("Navigation timeout of 30000 ms exceeded")) .mockResolvedValueOnce(response); const result = await navigateForCapture({ goto }, "https://www.yahoo.com/", 120_000); @@ -81,7 +82,7 @@ describe("navigateForCapture", () => { const response = { status: () => 200 }; const goto = vi .fn() - .mockRejectedValueOnce(new Error("Navigation timeout of 30000 ms exceeded")) + .mockRejectedValueOnce(new TimeoutError("Navigation timeout of 30000 ms exceeded")) .mockResolvedValueOnce(response); await navigateForCapture({ goto }, "https://www.yahoo.com/", totalTimeoutMs); @@ -106,7 +107,7 @@ describe("navigateForCapture", () => { }); it("does not fall back when the caller timeout already equals the idle attempt", async () => { - const err = new Error("Navigation timeout of 10000 ms exceeded"); + const err = new TimeoutError("Navigation timeout of 10000 ms exceeded"); const goto = vi.fn().mockRejectedValue(err); await expect(navigateForCapture({ goto }, "https://example.com", 10_000)).rejects.toBe(err); diff --git a/packages/cli/src/capture/navigateForCapture.ts b/packages/cli/src/capture/navigateForCapture.ts index 38eed6df0..92b659759 100644 --- a/packages/cli/src/capture/navigateForCapture.ts +++ b/packages/cli/src/capture/navigateForCapture.ts @@ -1,3 +1,7 @@ +import { isNavigationTimeoutError } from "./captureTimeout.js"; + +export { isNavigationTimeoutError }; + export const NETWORK_IDLE_ATTEMPT_MS = 30_000; export type CaptureGotoWaitUntil = "networkidle2" | "domcontentloaded"; @@ -22,11 +26,6 @@ export function networkIdleAttemptTimeoutMs(totalTimeoutMs: number): number { return Math.min(NETWORK_IDLE_ATTEMPT_MS, Math.max(0, totalTimeoutMs)); } -export function isNavigationTimeoutError(err: unknown): boolean { - const msg = err instanceof Error ? err.message : String(err); - return /navigation timeout/i.test(msg); -} - export async function navigateForCapture( page: CaptureGotoPage, url: string, diff --git a/packages/cli/src/capture/screenshotCapture.degrade.test.ts b/packages/cli/src/capture/screenshotCapture.degrade.test.ts new file mode 100644 index 000000000..3fafa235e --- /dev/null +++ b/packages/cli/src/capture/screenshotCapture.degrade.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it, vi } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { TimeoutError } from "puppeteer-core"; +import type { Page } from "puppeteer-core"; +import { captureScrollScreenshots } from "./screenshotCapture.js"; + +describe("captureScrollScreenshots degradation", () => { + it("rethrows protocol evaluate timeouts for the caller warning path", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-scroll-degrade-")); + const page = { + evaluate: vi.fn(async () => { + throw new TimeoutError( + "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting in launch/connect calls for a higher timeout if needed.", + ); + }), + screenshot: vi.fn(), + } as unknown as Page; + + await expect( + captureScrollScreenshots(page, dir, { remainingMs: () => 5_000 }), + ).rejects.toBeInstanceOf(TimeoutError); + }); +}); diff --git a/packages/cli/src/capture/screenshotCapture.ts b/packages/cli/src/capture/screenshotCapture.ts index 159ac3cd2..53a547dd8 100644 --- a/packages/cli/src/capture/screenshotCapture.ts +++ b/packages/cli/src/capture/screenshotCapture.ts @@ -6,6 +6,7 @@ */ import type { Page } from "puppeteer-core"; +import { isDegradableEvaluateTimeoutError } from "./captureTimeout.js"; import { writeFileSync, mkdirSync } from "node:fs"; import { join } from "node:path"; @@ -197,7 +198,11 @@ export async function captureScrollScreenshots( node = walker.nextNode(); } }) - .catch(() => {}); + .catch((err: unknown) => { + if (isDegradableEvaluateTimeoutError(err)) { + throw err; + } + }); await new Promise((r) => setTimeout(r, 400)); const scrollHeight = (await page.evaluate( @@ -257,7 +262,10 @@ export async function captureScrollScreenshots( const plate = await captureFullPagePlate(page, screenshotsDir, budget); if (plate) filePaths.push(plate); } - } catch { + } catch (err) { + if (isDegradableEvaluateTimeoutError(err)) { + throw err; + } /* scroll screenshots are non-critical */ } diff --git a/packages/cli/src/commands/capture.ts b/packages/cli/src/commands/capture.ts index f9b4c11ad..7d4d3a2ed 100644 --- a/packages/cli/src/commands/capture.ts +++ b/packages/cli/src/commands/capture.ts @@ -250,14 +250,11 @@ export default defineCommand({ } } catch (err) { const errMsg = normalizeErrorMessage(err); - // Write BLOCKED.md so the user/agent knows the capture failed try { const { mkdirSync, writeFileSync } = await import("node:fs"); + const { formatCaptureFailureReason } = await import("../capture/captureTimeout.js"); mkdirSync(outputDir, { recursive: true }); - const isTimeout = /timeout|timed out/i.test(errMsg); - const reason = isTimeout - ? "Page navigation timed out — the site may be blocking headless browsers or requires authentication." - : `Capture failed: ${errMsg}`; + const reason = formatCaptureFailureReason(errMsg); writeFileSync( `${outputDir}/BLOCKED.md`, `# Capture Failed\n\n${reason}\n\nURL: ${url}\n\n## What to try\n\n- Re-run with a longer timeout: \`--timeout 60000\`\n- The site may block headless browsers (anti-bot protection)\n- Try capturing a different page on the same domain\n`,