fix(capture): fall back from networkidle2 to domcontentloaded on nav timeout (#3224)

* fix(cli): fall back from networkidle2 to domcontentloaded on capture nav timeout

Sites like yahoo.com never reach network idle, so website capture hung for
the full 120s navigation budget. Prefer a short networkidle2 attempt, then
continue with domcontentloaded and the existing settle/scroll path.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(capture): use remaining timeout budget for domcontentloaded fallback

Keep total navigation time within the caller --timeout instead of
re-applying a full 30s floor after networkidle2.

Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Cursor <cursoragent@cursor.com>

packages/cli/src/capture/index.ts

@@ -42,6 +42,7 @@ import {
 import type { VisionCaptionOutcome } from "./contentExtractor.js";
 import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js";
 import { detectBlockedPage } from "./pageBlockDetection.js";
+import { navigateForCapture } from "./navigateForCapture.js";
 import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js";
 
 export type { CaptureOptions, CaptureResult } from "./types.js";
@@ -238,10 +239,17 @@ export async function captureWebsite(
       }
     });
 
-    // Use networkidle2 (allows 2 ongoing connections) instead of networkidle0 —
-    // modern SPAs often have persistent WebSocket/analytics connections that
-    // prevent networkidle0 from ever resolving.
-    const navigationResponse = await page1.goto(url, { waitUntil: "networkidle2", timeout });
+    const navigation = await navigateForCapture(page1, url, timeout);
+    const navigationResponse = navigation.response;
+    if (navigation.fellBackFromNetworkIdle) {
+      warnings.push(
+        `networkidle2 timed out after ${navigation.networkIdleTimeoutMs}ms; continued with domcontentloaded`,
+      );
+      progress(
+        "warn",
+        `networkidle2 timed out after ${navigation.networkIdleTimeoutMs}ms; continuing with domcontentloaded`,
+      );
+    }
     postNavigationDeadline = Date.now() + budgetMs;
     await new Promise((r) => setTimeout(r, settleTime));
 

packages/cli/src/capture/navigateForCapture.test.ts

@@ -0,0 +1,118 @@
+import { describe, expect, it, vi } from "vitest";
+import {
+  isNavigationTimeoutError,
+  navigateForCapture,
+  networkIdleAttemptTimeoutMs,
+  NETWORK_IDLE_ATTEMPT_MS,
+} from "./navigateForCapture.js";
+
+describe("networkIdleAttemptTimeoutMs", () => {
+  it("caps at the network-idle attempt budget", () => {
+    expect(networkIdleAttemptTimeoutMs(120_000)).toBe(NETWORK_IDLE_ATTEMPT_MS);
+  });
+
+  it("honors a caller timeout below the attempt budget", () => {
+    expect(networkIdleAttemptTimeoutMs(10_000)).toBe(10_000);
+  });
+});
+
+describe("isNavigationTimeoutError", () => {
+  it("matches Puppeteer navigation timeouts", () => {
+    expect(isNavigationTimeoutError(new Error("Navigation timeout of 30000 ms exceeded"))).toBe(
+      true,
+    );
+  });
+
+  it("ignores unrelated failures", () => {
+    expect(isNavigationTimeoutError(new Error("net::ERR_NAME_NOT_RESOLVED"))).toBe(false);
+  });
+});
+
+describe("navigateForCapture", () => {
+  it("returns after networkidle2 when the page settles", async () => {
+    const response = { status: () => 200 };
+    const goto = vi.fn().mockResolvedValue(response);
+    const result = await navigateForCapture({ goto }, "https://example.com", 120_000);
+
+    expect(result).toEqual({
+      response,
+      waitUntil: "networkidle2",
+      networkIdleTimeoutMs: NETWORK_IDLE_ATTEMPT_MS,
+      fellBackFromNetworkIdle: false,
+    });
+    expect(goto).toHaveBeenCalledTimes(1);
+    expect(goto).toHaveBeenCalledWith("https://example.com", {
+      waitUntil: "networkidle2",
+      timeout: NETWORK_IDLE_ATTEMPT_MS,
+    });
+  });
+
+  it("falls back to domcontentloaded after a networkidle2 navigation timeout", async () => {
+    const response = { status: () => 200 };
+    const goto = vi
+      .fn()
+      .mockRejectedValueOnce(new Error("Navigation timeout of 30000 ms exceeded"))
+      .mockResolvedValueOnce(response);
+
+    const result = await navigateForCapture({ goto }, "https://www.yahoo.com/", 120_000);
+
+    expect(result).toEqual({
+      response,
+      waitUntil: "domcontentloaded",
+      networkIdleTimeoutMs: NETWORK_IDLE_ATTEMPT_MS,
+      fellBackFromNetworkIdle: true,
+    });
+    expect(goto).toHaveBeenNthCalledWith(1, "https://www.yahoo.com/", {
+      waitUntil: "networkidle2",
+      timeout: NETWORK_IDLE_ATTEMPT_MS,
+    });
+    expect(goto).toHaveBeenNthCalledWith(2, "https://www.yahoo.com/", {
+      waitUntil: "domcontentloaded",
+      timeout: 90_000,
+    });
+  });
+
+  it.each([
+    { totalTimeoutMs: 31_000, fallbackTimeoutMs: 1_000 },
+    { totalTimeoutMs: 45_000, fallbackTimeoutMs: 15_000 },
+  ])(
+    "uses only the remaining budget for fallback when totalTimeoutMs=$totalTimeoutMs",
+    async ({ totalTimeoutMs, fallbackTimeoutMs }) => {
+      const response = { status: () => 200 };
+      const goto = vi
+        .fn()
+        .mockRejectedValueOnce(new Error("Navigation timeout of 30000 ms exceeded"))
+        .mockResolvedValueOnce(response);
+
+      await navigateForCapture({ goto }, "https://www.yahoo.com/", totalTimeoutMs);
+
+      expect(goto).toHaveBeenNthCalledWith(1, "https://www.yahoo.com/", {
+        waitUntil: "networkidle2",
+        timeout: NETWORK_IDLE_ATTEMPT_MS,
+      });
+      expect(goto).toHaveBeenNthCalledWith(2, "https://www.yahoo.com/", {
+        waitUntil: "domcontentloaded",
+        timeout: fallbackTimeoutMs,
+      });
+    },
+  );
+
+  it("does not fall back for non-timeout navigation errors", async () => {
+    const err = new Error("net::ERR_CONNECTION_REFUSED");
+    const goto = vi.fn().mockRejectedValue(err);
+
+    await expect(navigateForCapture({ goto }, "https://example.com", 120_000)).rejects.toBe(err);
+    expect(goto).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not fall back when the caller timeout already equals the idle attempt", async () => {
+    const err = new Error("Navigation timeout of 10000 ms exceeded");
+    const goto = vi.fn().mockRejectedValue(err);
+
+    await expect(navigateForCapture({ goto }, "https://example.com", 10_000)).rejects.toBe(err);
+    expect(goto).toHaveBeenCalledWith("https://example.com", {
+      waitUntil: "networkidle2",
+      timeout: 10_000,
+    });
+  });
+});

packages/cli/src/capture/navigateForCapture.ts

@@ -0,0 +1,64 @@
+export const NETWORK_IDLE_ATTEMPT_MS = 30_000;
+
+export type CaptureGotoWaitUntil = "networkidle2" | "domcontentloaded";
+
+export interface CaptureGotoOptions {
+  waitUntil: CaptureGotoWaitUntil;
+  timeout: number;
+}
+
+export interface CaptureGotoPage<TResponse = unknown> {
+  goto(url: string, options: CaptureGotoOptions): Promise<TResponse>;
+}
+
+export interface NavigateForCaptureResult<TResponse = unknown> {
+  response: TResponse;
+  waitUntil: CaptureGotoWaitUntil;
+  networkIdleTimeoutMs: number;
+  fellBackFromNetworkIdle: boolean;
+}
+
+export function networkIdleAttemptTimeoutMs(totalTimeoutMs: number): number {
+  return Math.min(NETWORK_IDLE_ATTEMPT_MS, Math.max(0, totalTimeoutMs));
+}
+
+export function isNavigationTimeoutError(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /navigation timeout/i.test(msg);
+}
+
+export async function navigateForCapture<TResponse>(
+  page: CaptureGotoPage<TResponse>,
+  url: string,
+  totalTimeoutMs: number,
+): Promise<NavigateForCaptureResult<TResponse>> {
+  const networkIdleTimeoutMs = networkIdleAttemptTimeoutMs(totalTimeoutMs);
+  try {
+    const response = await page.goto(url, {
+      waitUntil: "networkidle2",
+      timeout: networkIdleTimeoutMs,
+    });
+    return {
+      response,
+      waitUntil: "networkidle2",
+      networkIdleTimeoutMs,
+      fellBackFromNetworkIdle: false,
+    };
+  } catch (err) {
+    if (!isNavigationTimeoutError(err) || networkIdleTimeoutMs >= totalTimeoutMs) {
+      throw err;
+    }
+  }
+
+  const fallbackTimeoutMs = totalTimeoutMs - networkIdleTimeoutMs;
+  const response = await page.goto(url, {
+    waitUntil: "domcontentloaded",
+    timeout: fallbackTimeoutMs,
+  });
+  return {
+    response,
+    waitUntil: "domcontentloaded",
+    networkIdleTimeoutMs,
+    fellBackFromNetworkIdle: true,
+  };
+}