From 7860d19433e45507c9544f6e6b542e7e572e47c9 Mon Sep 17 00:00:00 2001 From: Xuanru Li <157947275+xuanruli@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:31:59 -0700 Subject: [PATCH] fix(capture): fall back from networkidle2 to domcontentloaded on nav timeout (#3224) * fix(cli): fall back from networkidle2 to domcontentloaded on capture nav timeout Sites like yahoo.com never reach network idle, so website capture hung for the full 120s navigation budget. Prefer a short networkidle2 attempt, then continue with domcontentloaded and the existing settle/scroll path. Co-authored-by: Cursor * fix(capture): use remaining timeout budget for domcontentloaded fallback Keep total navigation time within the caller --timeout instead of re-applying a full 30s floor after networkidle2. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- packages/cli/src/capture/index.ts | 16 ++- .../src/capture/navigateForCapture.test.ts | 118 ++++++++++++++++++ .../cli/src/capture/navigateForCapture.ts | 64 ++++++++++ 3 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 packages/cli/src/capture/navigateForCapture.test.ts create mode 100644 packages/cli/src/capture/navigateForCapture.ts diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index b57be6e4a..5d9dfc92d 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -42,6 +42,7 @@ import { import type { VisionCaptionOutcome } from "./contentExtractor.js"; import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js"; import { detectBlockedPage } from "./pageBlockDetection.js"; +import { navigateForCapture } from "./navigateForCapture.js"; import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js"; export type { CaptureOptions, CaptureResult } from "./types.js"; @@ -238,10 +239,17 @@ export async function captureWebsite( } }); - // Use networkidle2 (allows 2 ongoing connections) instead of networkidle0 — - // modern SPAs often have persistent WebSocket/analytics connections that - // prevent networkidle0 from ever resolving. - const navigationResponse = await page1.goto(url, { waitUntil: "networkidle2", timeout }); + const navigation = await navigateForCapture(page1, url, timeout); + const navigationResponse = navigation.response; + if (navigation.fellBackFromNetworkIdle) { + warnings.push( + `networkidle2 timed out after ${navigation.networkIdleTimeoutMs}ms; continued with domcontentloaded`, + ); + progress( + "warn", + `networkidle2 timed out after ${navigation.networkIdleTimeoutMs}ms; continuing with domcontentloaded`, + ); + } postNavigationDeadline = Date.now() + budgetMs; await new Promise((r) => setTimeout(r, settleTime)); diff --git a/packages/cli/src/capture/navigateForCapture.test.ts b/packages/cli/src/capture/navigateForCapture.test.ts new file mode 100644 index 000000000..5fdec11aa --- /dev/null +++ b/packages/cli/src/capture/navigateForCapture.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it, vi } from "vitest"; +import { + isNavigationTimeoutError, + navigateForCapture, + networkIdleAttemptTimeoutMs, + NETWORK_IDLE_ATTEMPT_MS, +} from "./navigateForCapture.js"; + +describe("networkIdleAttemptTimeoutMs", () => { + it("caps at the network-idle attempt budget", () => { + expect(networkIdleAttemptTimeoutMs(120_000)).toBe(NETWORK_IDLE_ATTEMPT_MS); + }); + + it("honors a caller timeout below the attempt budget", () => { + expect(networkIdleAttemptTimeoutMs(10_000)).toBe(10_000); + }); +}); + +describe("isNavigationTimeoutError", () => { + it("matches Puppeteer navigation timeouts", () => { + expect(isNavigationTimeoutError(new Error("Navigation timeout of 30000 ms exceeded"))).toBe( + true, + ); + }); + + it("ignores unrelated failures", () => { + expect(isNavigationTimeoutError(new Error("net::ERR_NAME_NOT_RESOLVED"))).toBe(false); + }); +}); + +describe("navigateForCapture", () => { + it("returns after networkidle2 when the page settles", async () => { + const response = { status: () => 200 }; + const goto = vi.fn().mockResolvedValue(response); + const result = await navigateForCapture({ goto }, "https://example.com", 120_000); + + expect(result).toEqual({ + response, + waitUntil: "networkidle2", + networkIdleTimeoutMs: NETWORK_IDLE_ATTEMPT_MS, + fellBackFromNetworkIdle: false, + }); + expect(goto).toHaveBeenCalledTimes(1); + expect(goto).toHaveBeenCalledWith("https://example.com", { + waitUntil: "networkidle2", + timeout: NETWORK_IDLE_ATTEMPT_MS, + }); + }); + + it("falls back to domcontentloaded after a networkidle2 navigation timeout", async () => { + const response = { status: () => 200 }; + const goto = vi + .fn() + .mockRejectedValueOnce(new Error("Navigation timeout of 30000 ms exceeded")) + .mockResolvedValueOnce(response); + + const result = await navigateForCapture({ goto }, "https://www.yahoo.com/", 120_000); + + expect(result).toEqual({ + response, + waitUntil: "domcontentloaded", + networkIdleTimeoutMs: NETWORK_IDLE_ATTEMPT_MS, + fellBackFromNetworkIdle: true, + }); + expect(goto).toHaveBeenNthCalledWith(1, "https://www.yahoo.com/", { + waitUntil: "networkidle2", + timeout: NETWORK_IDLE_ATTEMPT_MS, + }); + expect(goto).toHaveBeenNthCalledWith(2, "https://www.yahoo.com/", { + waitUntil: "domcontentloaded", + timeout: 90_000, + }); + }); + + it.each([ + { totalTimeoutMs: 31_000, fallbackTimeoutMs: 1_000 }, + { totalTimeoutMs: 45_000, fallbackTimeoutMs: 15_000 }, + ])( + "uses only the remaining budget for fallback when totalTimeoutMs=$totalTimeoutMs", + async ({ totalTimeoutMs, fallbackTimeoutMs }) => { + const response = { status: () => 200 }; + const goto = vi + .fn() + .mockRejectedValueOnce(new Error("Navigation timeout of 30000 ms exceeded")) + .mockResolvedValueOnce(response); + + await navigateForCapture({ goto }, "https://www.yahoo.com/", totalTimeoutMs); + + expect(goto).toHaveBeenNthCalledWith(1, "https://www.yahoo.com/", { + waitUntil: "networkidle2", + timeout: NETWORK_IDLE_ATTEMPT_MS, + }); + expect(goto).toHaveBeenNthCalledWith(2, "https://www.yahoo.com/", { + waitUntil: "domcontentloaded", + timeout: fallbackTimeoutMs, + }); + }, + ); + + it("does not fall back for non-timeout navigation errors", async () => { + const err = new Error("net::ERR_CONNECTION_REFUSED"); + const goto = vi.fn().mockRejectedValue(err); + + await expect(navigateForCapture({ goto }, "https://example.com", 120_000)).rejects.toBe(err); + expect(goto).toHaveBeenCalledTimes(1); + }); + + it("does not fall back when the caller timeout already equals the idle attempt", async () => { + const err = new Error("Navigation timeout of 10000 ms exceeded"); + const goto = vi.fn().mockRejectedValue(err); + + await expect(navigateForCapture({ goto }, "https://example.com", 10_000)).rejects.toBe(err); + expect(goto).toHaveBeenCalledWith("https://example.com", { + waitUntil: "networkidle2", + timeout: 10_000, + }); + }); +}); diff --git a/packages/cli/src/capture/navigateForCapture.ts b/packages/cli/src/capture/navigateForCapture.ts new file mode 100644 index 000000000..38eed6df0 --- /dev/null +++ b/packages/cli/src/capture/navigateForCapture.ts @@ -0,0 +1,64 @@ +export const NETWORK_IDLE_ATTEMPT_MS = 30_000; + +export type CaptureGotoWaitUntil = "networkidle2" | "domcontentloaded"; + +export interface CaptureGotoOptions { + waitUntil: CaptureGotoWaitUntil; + timeout: number; +} + +export interface CaptureGotoPage { + goto(url: string, options: CaptureGotoOptions): Promise; +} + +export interface NavigateForCaptureResult { + response: TResponse; + waitUntil: CaptureGotoWaitUntil; + networkIdleTimeoutMs: number; + fellBackFromNetworkIdle: boolean; +} + +export function networkIdleAttemptTimeoutMs(totalTimeoutMs: number): number { + return Math.min(NETWORK_IDLE_ATTEMPT_MS, Math.max(0, totalTimeoutMs)); +} + +export function isNavigationTimeoutError(err: unknown): boolean { + const msg = err instanceof Error ? err.message : String(err); + return /navigation timeout/i.test(msg); +} + +export async function navigateForCapture( + page: CaptureGotoPage, + url: string, + totalTimeoutMs: number, +): Promise> { + const networkIdleTimeoutMs = networkIdleAttemptTimeoutMs(totalTimeoutMs); + try { + const response = await page.goto(url, { + waitUntil: "networkidle2", + timeout: networkIdleTimeoutMs, + }); + return { + response, + waitUntil: "networkidle2", + networkIdleTimeoutMs, + fellBackFromNetworkIdle: false, + }; + } catch (err) { + if (!isNavigationTimeoutError(err) || networkIdleTimeoutMs >= totalTimeoutMs) { + throw err; + } + } + + const fallbackTimeoutMs = totalTimeoutMs - networkIdleTimeoutMs; + const response = await page.goto(url, { + waitUntil: "domcontentloaded", + timeout: fallbackTimeoutMs, + }); + return { + response, + waitUntil: "domcontentloaded", + networkIdleTimeoutMs, + fellBackFromNetworkIdle: true, + }; +}