diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index b9955f121..fa5627de8 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -41,6 +41,7 @@ import { } from "./contentExtractor.js"; import type { VisionCaptionOutcome } from "./contentExtractor.js"; import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js"; +import { detectBlockedPage } from "./pageBlockDetection.js"; import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js"; export type { CaptureOptions, CaptureResult } from "./types.js"; @@ -240,15 +241,13 @@ export async function captureWebsite( // Use networkidle2 (allows 2 ongoing connections) instead of networkidle0 — // modern SPAs often have persistent WebSocket/analytics connections that // prevent networkidle0 from ever resolving. - await page1.goto(url, { waitUntil: "networkidle2", timeout }); + const navigationResponse = await page1.goto(url, { waitUntil: "networkidle2", timeout }); postNavigationDeadline = Date.now() + budgetMs; await new Promise((r) => setTimeout(r, settleTime)); - phase("navigation", "completed"); - phase("core-extraction", "started"); // Check if the page loaded real content or an anti-bot challenge - // Use structural detection (DOM elements + cookies), not text regex matching — - // text matching causes false positives on sites that mention "blocked" or "verify" in copy + // Combine structural evidence with the main response status/title. Low text + // alone stays non-fatal so image-led sites are not rejected. const pageContentCheck = (await page1.evaluate(`(() => { var text = (document.body.innerText || "").trim(); var title = document.title || ""; @@ -256,19 +255,28 @@ export async function captureWebsite( var hasCfTurnstile = !!document.querySelector('.cf-turnstile, [data-sitekey], iframe[src*="challenges.cloudflare.com"], #challenge-running, #challenge-form'); // Structural: page is almost empty (challenge pages have minimal DOM) var bodyChildCount = document.body.children.length; - var isMinimalDom = bodyChildCount <= 5 && text.length < 500; - // Title-based: only check title on near-empty pages - var hasChallengeTitle = isMinimalDom && /just a moment|attention required|access denied/i.test(title); - var isChallenged = hasCfTurnstile || hasChallengeTitle; - return { textLength: text.length, title: title, isChallenged: isChallenged, bodyChildCount: bodyChildCount }; - })()`)) as { textLength: number; title: string; isChallenged: boolean; bodyChildCount: number }; + return { textLength: text.length, title: title, hasChallengeElement: hasCfTurnstile, bodyChildCount: bodyChildCount }; + })()`)) as { + textLength: number; + title: string; + hasChallengeElement: boolean; + bodyChildCount: number; + }; - if (pageContentCheck.isChallenged || pageContentCheck.textLength < 100) { - const reason = pageContentCheck.isChallenged - ? "Anti-bot protection detected (Cloudflare challenge or similar)" - : "Page has very little text content (" + - pageContentCheck.textLength + - " chars) — may be blocked or a client-rendered SPA that needs more time"; + const blockedReason = detectBlockedPage({ + httpStatus: navigationResponse?.status() ?? null, + ...pageContentCheck, + }); + if (blockedReason) throw new Error(blockedReason); + + phase("navigation", "completed"); + phase("core-extraction", "started"); + + if (pageContentCheck.textLength < 100) { + const reason = + "Page has very little text content (" + + pageContentCheck.textLength + + " chars) — may be blocked or a client-rendered SPA that needs more time"; warnings.push(reason); progress("warn", reason); } diff --git a/packages/cli/src/capture/pageBlockDetection.test.ts b/packages/cli/src/capture/pageBlockDetection.test.ts new file mode 100644 index 000000000..b180e55fb --- /dev/null +++ b/packages/cli/src/capture/pageBlockDetection.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { detectBlockedPage } from "./pageBlockDetection.js"; + +describe("detectBlockedPage", () => { + it.each([ + { + label: "local Fuji response", + evidence: { + httpStatus: 403, + title: "403 - Forbidden", + textLength: 50, + bodyChildCount: 1, + hasChallengeElement: false, + }, + }, + { + label: "EC2-style soft-block response", + evidence: { + httpStatus: 200, + title: "Access denied", + textLength: 18, + bodyChildCount: 2, + hasChallengeElement: false, + }, + }, + ])("rejects a minimal protection page: $label", ({ evidence }) => { + expect(detectBlockedPage(evidence)).toMatch(/capture blocked/i); + }); + + it("rejects a structural challenge on a minimal page", () => { + expect( + detectBlockedPage({ + httpStatus: 200, + title: "Please verify you are human", + textLength: 300, + bodyChildCount: 3, + hasChallengeElement: true, + }), + ).toMatch(/capture blocked/i); + }); + + it.each([ + { + label: "an image-led page with little text", + evidence: { + httpStatus: 200, + title: "Photography portfolio", + textLength: 18, + bodyChildCount: 4, + hasChallengeElement: false, + }, + }, + { + label: "a real page discussing access denial", + evidence: { + httpStatus: 200, + title: "Why websites say Access Denied", + textLength: 2_000, + bodyChildCount: 20, + hasChallengeElement: false, + }, + }, + { + label: "a full page with an embedded CAPTCHA", + evidence: { + httpStatus: 200, + title: "Contact us", + textLength: 2_000, + bodyChildCount: 20, + hasChallengeElement: true, + }, + }, + ])("allows $label", ({ evidence }) => { + expect(detectBlockedPage(evidence)).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/capture/pageBlockDetection.ts b/packages/cli/src/capture/pageBlockDetection.ts new file mode 100644 index 000000000..a103c081b --- /dev/null +++ b/packages/cli/src/capture/pageBlockDetection.ts @@ -0,0 +1,34 @@ +export interface PageLoadEvidence { + httpStatus: number | null; + title: string; + textLength: number; + bodyChildCount: number; + hasChallengeElement: boolean; +} + +/** + * Returns an actionable failure only for strong protection-page signals. + * Low text by itself is intentionally not fatal because image-led sites and + * client-rendered applications can legitimately have very little body copy. + */ +export function detectBlockedPage(evidence: PageLoadEvidence): string | undefined { + const isMinimalDom = evidence.bodyChildCount <= 5 && evidence.textLength < 500; + const hasBlockedStatus = + evidence.httpStatus === 401 || evidence.httpStatus === 403 || evidence.httpStatus === 429; + const hasBlockedTitle = + /^(?:error\s*)?(?:401|403|429)(?:\s*[-:—]\s*|\s+)|forbidden|access denied|attention required|just a moment/i.test( + evidence.title, + ); + + if (!isMinimalDom || (!evidence.hasChallengeElement && !hasBlockedStatus && !hasBlockedTitle)) { + return undefined; + } + + const statusDetail = + evidence.httpStatus === null ? "no HTTP status" : `HTTP ${evidence.httpStatus}`; + return ( + `Website capture blocked: the loaded page matched an access-protection response ` + + `(${statusDetail}, title ${JSON.stringify(evidence.title)}, ${evidence.textLength} text chars). ` + + "The site may reject automated or data-center traffic; retry from an allowed network or provide source assets directly." + ); +}