fix(cli): reject blocked website captures

packages/cli/src/capture/index.ts

@@ -41,6 +41,7 @@ import {
 } from "./contentExtractor.js";
 import type { VisionCaptionOutcome } from "./contentExtractor.js";
 import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js";
+import { detectBlockedPage } from "./pageBlockDetection.js";
 import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js";
 
 export type { CaptureOptions, CaptureResult } from "./types.js";
@@ -240,15 +241,13 @@ export async function captureWebsite(
     // Use networkidle2 (allows 2 ongoing connections) instead of networkidle0 —
     // modern SPAs often have persistent WebSocket/analytics connections that
     // prevent networkidle0 from ever resolving.
-    await page1.goto(url, { waitUntil: "networkidle2", timeout });
+    const navigationResponse = await page1.goto(url, { waitUntil: "networkidle2", timeout });
     postNavigationDeadline = Date.now() + budgetMs;
     await new Promise((r) => setTimeout(r, settleTime));
-    phase("navigation", "completed");
-    phase("core-extraction", "started");
 
     // Check if the page loaded real content or an anti-bot challenge
-    // Use structural detection (DOM elements + cookies), not text regex matching —
+    // Combine structural evidence with the main response status/title. Low text
-    // text matching causes false positives on sites that mention "blocked" or "verify" in copy
+    // alone stays non-fatal so image-led sites are not rejected.
     const pageContentCheck = (await page1.evaluate(`(() => {
       var text = (document.body.innerText || "").trim();
       var title = document.title || "";
@@ -256,17 +255,26 @@ export async function captureWebsite(
       var hasCfTurnstile = !!document.querySelector('.cf-turnstile, [data-sitekey], iframe[src*="challenges.cloudflare.com"], #challenge-running, #challenge-form');
       // Structural: page is almost empty (challenge pages have minimal DOM)
       var bodyChildCount = document.body.children.length;
-      var isMinimalDom = bodyChildCount <= 5 && text.length < 500;
+      return { textLength: text.length, title: title, hasChallengeElement: hasCfTurnstile, bodyChildCount: bodyChildCount };
-      // Title-based: only check title on near-empty pages
+    })()`)) as {
-      var hasChallengeTitle = isMinimalDom && /just a moment|attention required|access denied/i.test(title);
+      textLength: number;
-      var isChallenged = hasCfTurnstile || hasChallengeTitle;
+      title: string;
-      return { textLength: text.length, title: title, isChallenged: isChallenged, bodyChildCount: bodyChildCount };
+      hasChallengeElement: boolean;
-    })()`)) as { textLength: number; title: string; isChallenged: boolean; bodyChildCount: number };
+      bodyChildCount: number;
+    };
 
-    if (pageContentCheck.isChallenged || pageContentCheck.textLength < 100) {
+    const blockedReason = detectBlockedPage({
-      const reason = pageContentCheck.isChallenged
+      httpStatus: navigationResponse?.status() ?? null,
-        ? "Anti-bot protection detected (Cloudflare challenge or similar)"
+      ...pageContentCheck,
-        : "Page has very little text content (" +
+    });
+    if (blockedReason) throw new Error(blockedReason);
+
+    phase("navigation", "completed");
+    phase("core-extraction", "started");
+
+    if (pageContentCheck.textLength < 100) {
+      const reason =
+        "Page has very little text content (" +
         pageContentCheck.textLength +
         " chars) — may be blocked or a client-rendered SPA that needs more time";
       warnings.push(reason);

packages/cli/src/capture/pageBlockDetection.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, it } from "vitest";
+import { detectBlockedPage } from "./pageBlockDetection.js";
+
+describe("detectBlockedPage", () => {
+  it.each([
+    {
+      label: "local Fuji response",
+      evidence: {
+        httpStatus: 403,
+        title: "403 - Forbidden",
+        textLength: 50,
+        bodyChildCount: 1,
+        hasChallengeElement: false,
+      },
+    },
+    {
+      label: "EC2-style soft-block response",
+      evidence: {
+        httpStatus: 200,
+        title: "Access denied",
+        textLength: 18,
+        bodyChildCount: 2,
+        hasChallengeElement: false,
+      },
+    },
+  ])("rejects a minimal protection page: $label", ({ evidence }) => {
+    expect(detectBlockedPage(evidence)).toMatch(/capture blocked/i);
+  });
+
+  it("rejects a structural challenge on a minimal page", () => {
+    expect(
+      detectBlockedPage({
+        httpStatus: 200,
+        title: "Please verify you are human",
+        textLength: 300,
+        bodyChildCount: 3,
+        hasChallengeElement: true,
+      }),
+    ).toMatch(/capture blocked/i);
+  });
+
+  it.each([
+    {
+      label: "an image-led page with little text",
+      evidence: {
+        httpStatus: 200,
+        title: "Photography portfolio",
+        textLength: 18,
+        bodyChildCount: 4,
+        hasChallengeElement: false,
+      },
+    },
+    {
+      label: "a real page discussing access denial",
+      evidence: {
+        httpStatus: 200,
+        title: "Why websites say Access Denied",
+        textLength: 2_000,
+        bodyChildCount: 20,
+        hasChallengeElement: false,
+      },
+    },
+    {
+      label: "a full page with an embedded CAPTCHA",
+      evidence: {
+        httpStatus: 200,
+        title: "Contact us",
+        textLength: 2_000,
+        bodyChildCount: 20,
+        hasChallengeElement: true,
+      },
+    },
+  ])("allows $label", ({ evidence }) => {
+    expect(detectBlockedPage(evidence)).toBeUndefined();
+  });
+});

packages/cli/src/capture/pageBlockDetection.ts

@@ -0,0 +1,34 @@
+export interface PageLoadEvidence {
+  httpStatus: number | null;
+  title: string;
+  textLength: number;
+  bodyChildCount: number;
+  hasChallengeElement: boolean;
+}
+
+/**
+ * Returns an actionable failure only for strong protection-page signals.
+ * Low text by itself is intentionally not fatal because image-led sites and
+ * client-rendered applications can legitimately have very little body copy.
+ */
+export function detectBlockedPage(evidence: PageLoadEvidence): string | undefined {
+  const isMinimalDom = evidence.bodyChildCount <= 5 && evidence.textLength < 500;
+  const hasBlockedStatus =
+    evidence.httpStatus === 401 || evidence.httpStatus === 403 || evidence.httpStatus === 429;
+  const hasBlockedTitle =
+    /^(?:error\s*)?(?:401|403|429)(?:\s*[-:—]\s*|\s+)|forbidden|access denied|attention required|just a moment/i.test(
+      evidence.title,
+    );
+
+  if (!isMinimalDom || (!evidence.hasChallengeElement && !hasBlockedStatus && !hasBlockedTitle)) {
+    return undefined;
+  }
+
+  const statusDetail =
+    evidence.httpStatus === null ? "no HTTP status" : `HTTP ${evidence.httpStatus}`;
+  return (
+    `Website capture blocked: the loaded page matched an access-protection response ` +
+    `(${statusDetail}, title ${JSON.stringify(evidence.title)}, ${evidence.textLength} text chars). ` +
+    "The site may reject automated or data-center traffic; retry from an allowed network or provide source assets directly."
+  );
+}