mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 23:47:33 +00:00
* perf(cli): commands run without esbuild, puppeteer-core or sharp until they need them A host that bundles the CLI can now leave these packages out: each command loads them only where it uses them, and sharp installs once on first use when it is missing. The build check fails on any static import of them again. * fix(cli): use a host's sharp from NODE_PATH and let hosts turn first-use installs off * fix(cli): capture fails when sharp is missing, and one install serves every caller A missing sharp now stops the asset download up front instead of counting every image as unavailable. Concurrent first uses share one install, a failed install is not retried in the same run, and preview thumbnails report why. Hosts can turn off installs for a list of packages, e.g. HYPERFRAMES_NO_OPTIONAL_INSTALL=sharp. * fix(cli): snapshot --describe still works without sharp, resizing only when it is installed * style(cli): format the sharp loader and its tests * build(cli): sharp is an optional dependency, so a host can ship the CLI without it * build(studio-server): keep sharp out of the bundle now that it is optional
233 lines
7.3 KiB
TypeScript
233 lines
7.3 KiB
TypeScript
import type { CDPSession, HTTPResponse, Page } from "puppeteer-core";
|
|
import {
|
|
isDegradableEvaluateTimeoutError,
|
|
isNavigationTimeoutError,
|
|
withRemainingBudget,
|
|
} from "./captureTimeout.js";
|
|
|
|
export { isNavigationTimeoutError };
|
|
|
|
export const NETWORK_IDLE_ATTEMPT_MS = 30_000;
|
|
|
|
export type CaptureGotoWaitUntil = "networkidle2" | "domcontentloaded";
|
|
|
|
export interface CaptureGotoOptions {
|
|
waitUntil: CaptureGotoWaitUntil;
|
|
timeout: number;
|
|
}
|
|
|
|
export interface CaptureGotoPage<TResponse = unknown> {
|
|
goto(url: string, options: CaptureGotoOptions): Promise<TResponse>;
|
|
}
|
|
|
|
export interface NavigateForCaptureResult<TResponse = unknown> {
|
|
response: TResponse;
|
|
waitUntil: CaptureGotoWaitUntil;
|
|
networkIdleTimeoutMs: number;
|
|
fellBackFromNetworkIdle: boolean;
|
|
}
|
|
|
|
export function networkIdleAttemptTimeoutMs(totalTimeoutMs: number): number {
|
|
return Math.min(NETWORK_IDLE_ATTEMPT_MS, Math.max(0, totalTimeoutMs));
|
|
}
|
|
|
|
export async function navigateForCapture<TResponse>(
|
|
page: CaptureGotoPage<TResponse>,
|
|
url: string,
|
|
totalTimeoutMs: number,
|
|
): Promise<NavigateForCaptureResult<TResponse>> {
|
|
const networkIdleTimeoutMs = networkIdleAttemptTimeoutMs(totalTimeoutMs);
|
|
try {
|
|
const response = await page.goto(url, {
|
|
waitUntil: "networkidle2",
|
|
timeout: networkIdleTimeoutMs,
|
|
});
|
|
return {
|
|
response,
|
|
waitUntil: "networkidle2",
|
|
networkIdleTimeoutMs,
|
|
fellBackFromNetworkIdle: false,
|
|
};
|
|
} catch (err) {
|
|
if (!isNavigationTimeoutError(err) || networkIdleTimeoutMs >= totalTimeoutMs) {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
const fallbackTimeoutMs = totalTimeoutMs - networkIdleTimeoutMs;
|
|
const response = await page.goto(url, {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: fallbackTimeoutMs,
|
|
});
|
|
return {
|
|
response,
|
|
waitUntil: "domcontentloaded",
|
|
networkIdleTimeoutMs,
|
|
fellBackFromNetworkIdle: true,
|
|
};
|
|
}
|
|
|
|
export interface PageReadyPage {
|
|
waitForNetworkIdle?(options: {
|
|
idleTime: number;
|
|
timeout: number;
|
|
concurrency?: number;
|
|
}): Promise<unknown>;
|
|
evaluate(fn: () => unknown): Promise<unknown>;
|
|
}
|
|
|
|
export interface WaitForPageReadyOptions {
|
|
timeoutMs?: number;
|
|
networkIdle?: "always" | "after-load";
|
|
}
|
|
|
|
export async function waitForPageReady(
|
|
page: PageReadyPage,
|
|
options: WaitForPageReadyOptions = {},
|
|
): Promise<void> {
|
|
const { timeoutMs = 15_000, networkIdle = "always" } = options;
|
|
const deadline = Date.now() + timeoutMs;
|
|
const remaining = () => Math.max(0, deadline - Date.now());
|
|
const idle = async () => {
|
|
try {
|
|
await page.waitForNetworkIdle?.({
|
|
idleTime: 500,
|
|
timeout: remaining(),
|
|
concurrency: 2,
|
|
});
|
|
} catch {}
|
|
};
|
|
|
|
if (networkIdle === "always" && remaining() > 0) await idle();
|
|
|
|
let waitedOutLoad = false;
|
|
while (remaining() > 0) {
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
const loading = await Promise.race([
|
|
page
|
|
.evaluate(() => document.readyState === "loading")
|
|
.catch((err: unknown) => {
|
|
if (isDegradableEvaluateTimeoutError(err)) throw err;
|
|
return true;
|
|
}),
|
|
new Promise<true>((resolve) => {
|
|
timer = setTimeout(() => resolve(true), Math.max(1, remaining()));
|
|
}),
|
|
]).finally(() => {
|
|
if (timer !== undefined) clearTimeout(timer);
|
|
});
|
|
if (loading !== true) break;
|
|
waitedOutLoad = true;
|
|
if (remaining() > 0) {
|
|
await new Promise((resolve) => setTimeout(resolve, Math.min(500, remaining())));
|
|
}
|
|
}
|
|
|
|
if (waitedOutLoad && remaining() > 0) await idle();
|
|
}
|
|
|
|
const PUPPETEER_LAUNCH_DEFAULT_TIMEOUT_MS = 30_000;
|
|
const PAGE_LOAD_BLOCKING_TYPES = new Set(["Document", "Script", "Stylesheet"]);
|
|
const NAMED_BLOCKING_REQUESTS = 3;
|
|
const SENT_OR_ANSWERED_WITHOUT_NETWORK = [
|
|
"Network.requestWillBeSentExtraInfo",
|
|
"Network.responseReceived",
|
|
] as const;
|
|
|
|
export async function gotoWhenSent(
|
|
page: Page,
|
|
url: string,
|
|
timeoutMs: number,
|
|
coldChromeFirstRequestTimeoutMs = PUPPETEER_LAUNCH_DEFAULT_TIMEOUT_MS,
|
|
): Promise<HTTPResponse | null> {
|
|
const { TimeoutError } = await import("puppeteer-core");
|
|
const cdp = await page.createCDPSession();
|
|
try {
|
|
await cdp.send("Network.enable");
|
|
const sent = documentRequestSent(cdp);
|
|
const unfinished = unfinishedPageLoadRequests(cdp);
|
|
const navigation = page.goto(url, { waitUntil: "domcontentloaded", timeout: 0 });
|
|
const navigationAwaitedBelow = navigation.catch(() => undefined);
|
|
await withRemainingBudget(
|
|
Promise.race([sent, navigationAwaitedBelow]),
|
|
coldChromeFirstRequestTimeoutMs,
|
|
"sending the page request",
|
|
() =>
|
|
new TimeoutError(
|
|
`Navigation timeout: Chrome did not send the page request within ${coldChromeFirstRequestTimeoutMs} ms`,
|
|
),
|
|
);
|
|
return await withRemainingBudget(
|
|
navigation,
|
|
timeoutMs,
|
|
"loading the page",
|
|
() => new TimeoutError(pageLoadTimeoutMessage(timeoutMs, url, unfinished())),
|
|
);
|
|
} finally {
|
|
await cdp.detach().catch(() => undefined);
|
|
}
|
|
}
|
|
|
|
export function documentRequestSent(cdp: CDPSession): Promise<void> {
|
|
const documents = new Set<string>();
|
|
const sent = new Set<string>();
|
|
return new Promise((resolve) => {
|
|
const settle = (requestId: string) => {
|
|
if (documents.has(requestId) && sent.has(requestId)) resolve();
|
|
};
|
|
cdp.on("Network.requestWillBeSent", ({ requestId, type }) => {
|
|
if (type === "Document") documents.add(requestId);
|
|
settle(requestId);
|
|
});
|
|
for (const event of SENT_OR_ANSWERED_WITHOUT_NETWORK) {
|
|
cdp.on(event, ({ requestId }) => {
|
|
sent.add(requestId);
|
|
settle(requestId);
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
export function unfinishedPageLoadRequests(cdp: CDPSession): () => string[] {
|
|
const unfinished = new Map<string, string>();
|
|
cdp.on("Network.requestWillBeSent", ({ requestId, request, type }) => {
|
|
if (type && PAGE_LOAD_BLOCKING_TYPES.has(type)) unfinished.set(requestId, request.url);
|
|
});
|
|
for (const event of ["Network.loadingFinished", "Network.loadingFailed"] as const) {
|
|
cdp.on(event, ({ requestId }) => unfinished.delete(requestId));
|
|
}
|
|
return () => [...unfinished.values()];
|
|
}
|
|
|
|
function urlWithoutSecrets(raw: string): string {
|
|
try {
|
|
const url = new URL(raw);
|
|
if (url.protocol !== "http:" && url.protocol !== "https:") return `${url.protocol}…`;
|
|
return `${url.origin}${url.pathname}${url.search ? "?…" : ""}`;
|
|
} catch {
|
|
return "(unreadable URL)";
|
|
}
|
|
}
|
|
|
|
function originOf(raw: string): string | null {
|
|
try {
|
|
return new URL(raw).origin;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function pageLoadTimeoutMessage(timeoutMs: number, pageUrl: string, waitingOn: string[]): string {
|
|
const exceeded = `Navigation timeout of ${timeoutMs} ms exceeded`;
|
|
if (waitingOn.length === 0) return exceeded;
|
|
const named = waitingOn.slice(0, NAMED_BLOCKING_REQUESTS).map(urlWithoutSecrets).join(", ");
|
|
const unnamed = waitingOn.length - NAMED_BLOCKING_REQUESTS;
|
|
const more = unnamed > 0 ? ` and ${unnamed} more` : "";
|
|
const pageOrigin = new URL(pageUrl).origin;
|
|
const fromAnotherServer = waitingOn.some((request) => originOf(request) !== pageOrigin);
|
|
const advice = fromAnotherServer
|
|
? " A <script> or stylesheet from another server holds the page until it arrives; serve it from the project instead."
|
|
: "";
|
|
return `${exceeded} while the page was still loading ${named}${more}.${advice}`;
|
|
}
|