import type { CDPSession, HTTPResponse, Page } from "puppeteer-core"; import { isDegradableEvaluateTimeoutError, isNavigationTimeoutError, withRemainingBudget, } from "./captureTimeout.js"; export { isNavigationTimeoutError }; export const NETWORK_IDLE_ATTEMPT_MS = 30_000; export type CaptureGotoWaitUntil = "networkidle2" | "domcontentloaded"; export interface CaptureGotoOptions { waitUntil: CaptureGotoWaitUntil; timeout: number; } export interface CaptureGotoPage { goto(url: string, options: CaptureGotoOptions): Promise; } export interface NavigateForCaptureResult { response: TResponse; waitUntil: CaptureGotoWaitUntil; networkIdleTimeoutMs: number; fellBackFromNetworkIdle: boolean; } export function networkIdleAttemptTimeoutMs(totalTimeoutMs: number): number { return Math.min(NETWORK_IDLE_ATTEMPT_MS, Math.max(0, totalTimeoutMs)); } export async function navigateForCapture( page: CaptureGotoPage, url: string, totalTimeoutMs: number, ): Promise> { const networkIdleTimeoutMs = networkIdleAttemptTimeoutMs(totalTimeoutMs); try { const response = await page.goto(url, { waitUntil: "networkidle2", timeout: networkIdleTimeoutMs, }); return { response, waitUntil: "networkidle2", networkIdleTimeoutMs, fellBackFromNetworkIdle: false, }; } catch (err) { if (!isNavigationTimeoutError(err) || networkIdleTimeoutMs >= totalTimeoutMs) { throw err; } } const fallbackTimeoutMs = totalTimeoutMs - networkIdleTimeoutMs; const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: fallbackTimeoutMs, }); return { response, waitUntil: "domcontentloaded", networkIdleTimeoutMs, fellBackFromNetworkIdle: true, }; } export interface PageReadyPage { waitForNetworkIdle?(options: { idleTime: number; timeout: number; concurrency?: number; }): Promise; evaluate(fn: () => unknown): Promise; } export interface WaitForPageReadyOptions { timeoutMs?: number; networkIdle?: "always" | "after-load"; } export async function waitForPageReady( page: PageReadyPage, options: WaitForPageReadyOptions = {}, ): Promise { const { timeoutMs = 15_000, networkIdle = "always" } = options; const deadline = Date.now() + timeoutMs; const remaining = () => Math.max(0, deadline - Date.now()); const idle = async () => { try { await page.waitForNetworkIdle?.({ idleTime: 500, timeout: remaining(), concurrency: 2, }); } catch {} }; if (networkIdle === "always" && remaining() > 0) await idle(); let waitedOutLoad = false; while (remaining() > 0) { let timer: ReturnType | undefined; const loading = await Promise.race([ page .evaluate(() => document.readyState === "loading") .catch((err: unknown) => { if (isDegradableEvaluateTimeoutError(err)) throw err; return true; }), new Promise((resolve) => { timer = setTimeout(() => resolve(true), Math.max(1, remaining())); }), ]).finally(() => { if (timer !== undefined) clearTimeout(timer); }); if (loading !== true) break; waitedOutLoad = true; if (remaining() > 0) { await new Promise((resolve) => setTimeout(resolve, Math.min(500, remaining()))); } } if (waitedOutLoad && remaining() > 0) await idle(); } const PUPPETEER_LAUNCH_DEFAULT_TIMEOUT_MS = 30_000; const PAGE_LOAD_BLOCKING_TYPES = new Set(["Document", "Script", "Stylesheet"]); const NAMED_BLOCKING_REQUESTS = 3; const SENT_OR_ANSWERED_WITHOUT_NETWORK = [ "Network.requestWillBeSentExtraInfo", "Network.responseReceived", ] as const; export async function gotoWhenSent( page: Page, url: string, timeoutMs: number, coldChromeFirstRequestTimeoutMs = PUPPETEER_LAUNCH_DEFAULT_TIMEOUT_MS, ): Promise { const { TimeoutError } = await import("puppeteer-core"); const cdp = await page.createCDPSession(); try { await cdp.send("Network.enable"); const sent = documentRequestSent(cdp); const unfinished = unfinishedPageLoadRequests(cdp); const navigation = page.goto(url, { waitUntil: "domcontentloaded", timeout: 0 }); const navigationAwaitedBelow = navigation.catch(() => undefined); await withRemainingBudget( Promise.race([sent, navigationAwaitedBelow]), coldChromeFirstRequestTimeoutMs, "sending the page request", () => new TimeoutError( `Navigation timeout: Chrome did not send the page request within ${coldChromeFirstRequestTimeoutMs} ms`, ), ); return await withRemainingBudget( navigation, timeoutMs, "loading the page", () => new TimeoutError(pageLoadTimeoutMessage(timeoutMs, url, unfinished())), ); } finally { await cdp.detach().catch(() => undefined); } } export function documentRequestSent(cdp: CDPSession): Promise { const documents = new Set(); const sent = new Set(); return new Promise((resolve) => { const settle = (requestId: string) => { if (documents.has(requestId) && sent.has(requestId)) resolve(); }; cdp.on("Network.requestWillBeSent", ({ requestId, type }) => { if (type === "Document") documents.add(requestId); settle(requestId); }); for (const event of SENT_OR_ANSWERED_WITHOUT_NETWORK) { cdp.on(event, ({ requestId }) => { sent.add(requestId); settle(requestId); }); } }); } export function unfinishedPageLoadRequests(cdp: CDPSession): () => string[] { const unfinished = new Map(); cdp.on("Network.requestWillBeSent", ({ requestId, request, type }) => { if (type && PAGE_LOAD_BLOCKING_TYPES.has(type)) unfinished.set(requestId, request.url); }); for (const event of ["Network.loadingFinished", "Network.loadingFailed"] as const) { cdp.on(event, ({ requestId }) => unfinished.delete(requestId)); } return () => [...unfinished.values()]; } function urlWithoutSecrets(raw: string): string { try { const url = new URL(raw); if (url.protocol !== "http:" && url.protocol !== "https:") return `${url.protocol}…`; return `${url.origin}${url.pathname}${url.search ? "?…" : ""}`; } catch { return "(unreadable URL)"; } } function originOf(raw: string): string | null { try { return new URL(raw).origin; } catch { return null; } } function pageLoadTimeoutMessage(timeoutMs: number, pageUrl: string, waitingOn: string[]): string { const exceeded = `Navigation timeout of ${timeoutMs} ms exceeded`; if (waitingOn.length === 0) return exceeded; const named = waitingOn.slice(0, NAMED_BLOCKING_REQUESTS).map(urlWithoutSecrets).join(", "); const unnamed = waitingOn.length - NAMED_BLOCKING_REQUESTS; const more = unnamed > 0 ? ` and ${unnamed} more` : ""; const pageOrigin = new URL(pageUrl).origin; const fromAnotherServer = waitingOn.some((request) => originOf(request) !== pageOrigin); const advice = fromAnotherServer ? " A