fix(cli): pin website capture connections to public peers (#5668)

* fix(cli): pin website capture connections to public peers

* style(cli): keep capture policy explanations within comment limits

* test(cli): separate case-sensitive dotenv assertions

* fix(cli): honor capture deadlines and bootstrap isolation

* chore: trim redundant capture dotenv comments

* fix(capture): join signal cleanup and guard leading options

* docs(capture): trim redundant cleanup comment
This commit is contained in:
Miguel Ángel
2026-10-16 02:37:14 +00:00
committed by GitHub
parent 490ecafbc4
commit 8be76fc86c
29 changed files with 1869 additions and 135 deletions
+19
View File
@@ -207,6 +207,25 @@ npx hyperframes capture https://example.com --json
Fonts: sohne-var
```
Website capture accepts public HTTP(S) destinations without URL credentials.
Browser traffic goes through a per-capture proxy, and asset downloads use the
same checked-address policy. Each connection uses a validated numeric IPv4
address; redirects and later requests are checked again. Private, loopback,
link-local and special-use addresses are refused. IPv6-only sites are currently
unsupported. Plain WebSocket upgrades are refused; slow or refused resources
may be absent from the capture.
Capture uses a temporary browser profile with non-proxied WebRTC UDP disabled.
The profile and proxy are removed when the browser closes. The Chromium sandbox
is retained on macOS and Windows; Linux retains the container compatibility
exception.
Ancestor `.env` files can supply the supported vision API keys, models, Vertex
credentials/location and vision timeout. They cannot change the browser path,
executable discovery, TLS, proxy or process runtime settings. Set
`HYPERFRAMES_NO_CAPTURE_DOTENV=1` in the calling environment to skip these files.
An inherited `HYPERFRAMES_BROWSER_PATH` still selects a browser.
`Dropped` is how many assets the page referenced that are **not** in the
folder, and why. It is printed only when it is non-zero, and `--json` always
carries it as a `dropped` object. Without it, a capture of a spare page and a
+12 -1
View File
@@ -13,6 +13,7 @@ export class BrowserCleanupError extends AggregateError {}
const browsers = new Set<Browser>();
const closing = new WeakMap<Browser, Promise<void>>();
const pending = new Map<Promise<Browser>, AbortController>();
const sessionCleanups = new Set<() => Promise<void>>();
const CLOSE_GRACE_MS = 1_000;
const EXIT_POLL_MS = 50;
let listening = false;
@@ -102,7 +103,17 @@ export async function launchManagedBrowser(
async function closeManagedBrowsers(): Promise<void> {
for (const abort of pending.values()) abort.abort();
await Promise.allSettled(pending.keys());
await Promise.all([...browsers].map(closeManagedBrowser));
const results = await Promise.allSettled([...browsers].map(closeManagedBrowser));
results.push(...(await Promise.allSettled([...sessionCleanups].map((cleanup) => cleanup()))));
const errors = results.flatMap((result) => (result.status === "rejected" ? [result.reason] : []));
if (errors.length) throw new AggregateError(errors, "Browser session shutdown failed.");
}
export function registerBrowserSessionCleanup(cleanup: () => Promise<void>): () => void {
if (shutdown) throw new Error("The CLI is stopping; no browser session can start.");
ownSignals();
sessionCleanups.add(cleanup);
return () => sessionCleanups.delete(cleanup);
}
/** Manual teardown and signal teardown share the same bounded close, exactly once. */
@@ -1,4 +1,8 @@
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import { publicRequest } from "./publicNetwork.js";
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
afterEach(() => vi.mocked(publicRequest).mockReset());
import { annotateGifAssetMetadata, type CatalogedAsset } from "./assetCataloger.js";
function u16(value: number): number[] {
@@ -102,3 +106,40 @@ describe("annotateGifAssetMetadata", () => {
expect(annotated[1]?.notes).toBe("hero");
});
});
it("uses safe redirects for GIF metadata and never follows a private redirect", async () => {
const assets: CatalogedAsset[] = [
{ url: "https://cdn.example/animated.gif", type: "Image", contexts: ["img[src]"] },
{ url: "https://cdn.example/private.gif", type: "Image", contexts: ["img[src]"] },
];
const canceled = vi.fn();
vi.mocked(publicRequest).mockImplementation(async (url) => {
if (String(url).endsWith("animated.gif"))
return new Response(null, {
status: 302,
headers: { location: "https://images.example/final.gif" },
});
if (String(url).endsWith("private.gif"))
return new Response(new ReadableStream({ cancel: canceled }), {
status: 302,
headers: { location: "http://127.0.0.1/private.gif" },
});
if (String(url) === "https://images.example/final.gif")
return new Response(new Uint8Array(gif([...frame(5), ...frame(15)], 0)));
throw new Error(`Unexpected request: ${url}`);
});
const annotated = await annotateGifAssetMetadata(assets);
expect(annotated[0]?.notes).toBe("animated GIF: 2 frames, 0.200s, loops forever");
expect(annotated[1]?.notes).toBeUndefined();
expect(
vi
.mocked(publicRequest)
.mock.calls.map(([url]) => String(url))
.sort(),
).toEqual([
"https://cdn.example/animated.gif",
"https://cdn.example/private.gif",
"https://images.example/final.gif",
]);
expect(canceled).toHaveBeenCalledOnce();
});
+8 -5
View File
@@ -11,6 +11,8 @@
import type { Page } from "puppeteer-core";
import { parseAnimatedGifMetadata } from "@hyperframes/core";
import { safeFetch } from "./assetDownloader.js";
import { readBoundedResponse } from "./readBoundedResponse.js";
export interface CatalogedAsset {
url: string;
@@ -276,11 +278,12 @@ function appendNote(existing: string | undefined, note: string): string {
async function readAssetBytes(url: string): Promise<Uint8Array | null> {
try {
const response = await fetch(url, { signal: AbortSignal.timeout(15_000) });
if (!response.ok) return null;
const contentLength = response.headers.get("content-length");
if (contentLength && Number.parseInt(contentLength, 10) > 25 * 1024 * 1024) return null;
return new Uint8Array(await response.arrayBuffer());
const response = await safeFetch(url, { signal: AbortSignal.timeout(15_000) });
if (!response?.ok) {
await response?.body?.cancel().catch(() => {});
return null;
}
return await readBoundedResponse(response, 25 * 1024 * 1024);
} catch {
return null;
}
@@ -22,6 +22,14 @@ import type { DesignTokens } from "./types.js";
import type { IconCandidate } from "./faviconRanker.js";
import { CAPTURE_USER_AGENT } from "./userAgent.js";
import sharp from "sharp";
import { publicRequest } from "./publicNetwork.js";
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
// Exercise production URL/redirect checks while replacing only the socket transport.
function mockRequest(implementation: (url: string, init?: RequestInit) => Promise<Response>) {
vi.mocked(publicRequest).mockImplementation((url, init) => implementation(String(url), init));
}
describe("isPrivateUrl — SSRF denylist (security: F-003)", () => {
it("blocks loopback, private, and metadata IPv4", () => {
@@ -79,7 +87,7 @@ describe("isPrivateUrl — SSRF denylist (security: F-003)", () => {
});
describe("safeFetch — re-validates the denylist on every redirect hop (security: F-002)", () => {
afterEach(() => vi.unstubAllGlobals());
afterEach(() => vi.mocked(publicRequest).mockReset());
it("blocks a public URL that redirects to a private/metadata host", async () => {
const fetchMock = vi.fn(async (input: string, _init?: RequestInit) => {
@@ -92,7 +100,7 @@ describe("safeFetch — re-validates the denylist on every redirect hop (securit
// The metadata host must NEVER be fetched.
throw new Error(`safeFetch followed a redirect to a private host: ${input}`);
});
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
const res = await safeFetch("https://public.example/logo.png");
expect(res).toBeNull();
@@ -107,7 +115,7 @@ describe("safeFetch — re-validates the denylist on every redirect hop (securit
return new Response(null, { status: 301, headers: { location: "https://b.example/y" } });
return new Response("ok", { status: 200 });
});
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
const res = await safeFetch("https://a.example/x");
expect(res?.status).toBe(200);
@@ -130,7 +138,7 @@ describe("safeFetch — re-validates the denylist on every redirect hop (securit
headers: { location: target },
}),
);
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
expect(isPrivateUrl(target), target).toBe(true);
expect(await safeFetch(target), target).toBeNull();
expect(fetchMock).not.toHaveBeenCalled();
@@ -139,9 +147,25 @@ describe("safeFetch — re-validates the denylist on every redirect hop (securit
}
});
it("cancels redirect bodies before validating the next hop", async () => {
const canceled = vi.fn();
mockRequest(
vi.fn(
async () =>
new Response(new ReadableStream({ cancel: canceled }), {
status: 302,
headers: { location: "http://169.254.169.254/metadata" },
}),
),
);
expect(await safeFetch("https://public.example/asset")).toBeNull();
expect(canceled).toHaveBeenCalledOnce();
expect(publicRequest).toHaveBeenCalledTimes(1);
});
it("returns null when the initial URL is private", async () => {
const fetchMock = vi.fn(async () => new Response("ok"));
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
const res = await safeFetch("http://169.254.169.254/");
expect(res).toBeNull();
expect(fetchMock).not.toHaveBeenCalled();
@@ -185,12 +209,12 @@ describe("toStandaloneSvg — scraped inline SVGs must survive as .svg files", (
});
describe("downloadAndRewriteFonts — attempt caps", () => {
afterEach(() => vi.unstubAllGlobals());
afterEach(() => vi.mocked(publicRequest).mockReset());
async function expectFailedFontAttempts(css: string, expectedAttempts: number): Promise<void> {
const dir = mkdtempSync(join(tmpdir(), "hf-font-attempts-"));
const fetchMock = vi.fn(async () => new Response("failed", { status: 503 }));
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
try {
await downloadAndRewriteFonts(css, dir);
@@ -222,7 +246,7 @@ describe("downloadAndRewriteFonts — attempt caps", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-font-budget-"));
const css = "@font-face { font-family: Budget; src: url(https://fonts.example/budget.woff2); }";
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
try {
await downloadAndRewriteFonts(css, dir, { remainingMs: () => 0 });
@@ -269,7 +293,7 @@ const OPENAI_ICONS: IconCandidate[] = [
];
describe("drop counts — why a referenced asset is not in the capture", () => {
afterEach(() => vi.unstubAllGlobals());
afterEach(() => vi.mocked(publicRequest).mockReset());
/** `n` @font-face rules, each naming a DIFFERENT family, so only the global cap can bite. */
function fontCss(n: number): string {
@@ -285,7 +309,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
// ever have said "some".
await withTempDir(async (dir) => {
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
const { drops } = await downloadAndRewriteFonts(fontCss(4), dir, { remainingMs: () => 0 });
expect(fetchMock).not.toHaveBeenCalled();
expect(drops["budget-exhausted"]).toBe(4);
@@ -297,10 +321,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
it("separates the faces the global cap refused from the ones that failed", async () => {
// 35 declared, cap 30: 30 are attempted and every attempt 503s, 5 are never reached.
await withTempDir(async (dir) => {
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response("no", { status: 503 })),
);
mockRequest(vi.fn(async () => new Response("no", { status: 503 })));
const { drops } = await downloadAndRewriteFonts(fontCss(35), dir);
expect(drops["cap-reached"]).toBe(5);
expect(drops.unavailable).toBe(30);
@@ -311,10 +332,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
it("counts the faces the per-family cap refused", async () => {
// 10 rules, all one family, per-family cap 6: 6 attempted, 4 refused.
await withTempDir(async (dir) => {
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response("no", { status: 503 })),
);
mockRequest(vi.fn(async () => new Response("no", { status: 503 })));
const css = Array.from(
{ length: 10 },
(_, i) =>
@@ -330,8 +348,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
// The whole point of the tally: this page declared one face and we have it. A reader can now
// tell this apart from a page that declared thirty and got truncated to one.
await withTempDir(async (dir) => {
vi.stubGlobal(
"fetch",
mockRequest(
vi.fn(
async () =>
new Response(
@@ -362,8 +379,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
it("counts an image dropped for being under the raster floor", async () => {
// A valid small PNG is under the 10 KB floor; preserve the recorded drop reason.
await withTempDir(async (dir) => {
vi.stubGlobal(
"fetch",
mockRequest(
vi.fn(
async () =>
new Response(
@@ -388,7 +404,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
it("counts every catalogued image the budget never let it reach", async () => {
await withTempDir(async (dir) => {
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
const catalog = Array.from({ length: 7 }, (_, i) => ({
type: "Image",
url: `https://cdn.example/img-${i}.png`,
@@ -451,7 +467,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
});
describe("asset fetches present the same identity as the page navigation", () => {
afterEach(() => vi.unstubAllGlobals());
afterEach(() => vi.mocked(publicRequest).mockReset());
/**
* Measured against the real origin: `GET /favicon.svg` answers `403 text/html` to
@@ -477,7 +493,7 @@ describe("asset fetches present the same identity as the page navigation", () =>
// With a bot UA the ranker's winner 403s and the capture silently falls through to the
// next candidate, so the file on disk is chosen by the CDN rather than by the ranker.
await withTempDir(async (dir) => {
vi.stubGlobal("fetch", serveLikeAnAntiBotEdge());
mockRequest(serveLikeAnAntiBotEdge());
const { icons } = await downloadAssets(tokensWithNoSvgs(), dir, [], OPENAI_ICONS);
expect(icons.headline?.file).toBe("assets/favicon.svg");
expect(icons.icons[0]?.url).toBe("https://openai.example/favicon.svg");
@@ -487,7 +503,7 @@ describe("asset fetches present the same identity as the page navigation", () =>
it("sends one User-Agent for every asset request", async () => {
await withTempDir(async (dir) => {
const fetchMock = serveLikeAnAntiBotEdge();
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
await downloadAssets(tokensWithNoSvgs(), dir, [], OPENAI_ICONS);
const agents = fetchMock.mock.calls.map((call) =>
new Headers(call[1]?.headers).get("user-agent"),
@@ -515,7 +531,7 @@ async function testIco(): Promise<Buffer> {
}
describe("declared icons — keep them all, headline the bare mark", () => {
afterEach(() => vi.unstubAllGlobals());
afterEach(() => vi.mocked(publicRequest).mockReset());
/** A mark knocked out of a full-bleed rounded rect: the shape openai.com's favicon.svg has. */
const BADGE_SVG = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 180 180">
@@ -557,8 +573,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
it("writes one file per declared icon, named for its rel and sizes", async () => {
await withTempDir(async (dir) => {
vi.stubGlobal(
"fetch",
mockRequest(
serve({
"favicon.svg": BADGE_SVG,
"favicon.ico": await testIco(),
@@ -580,8 +595,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
// opaque tile. openai.com declares no bare mark at all, so the headline falls back to the
// ranker and the manifest has to say so rather than implying a preference was satisfied.
await withTempDir(async (dir) => {
vi.stubGlobal(
"fetch",
mockRequest(
serve({
"favicon.svg": BADGE_SVG,
"favicon.ico": await testIco(),
@@ -604,10 +618,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
// PNG does, and shape is what the brand band needs. Under the old rule this lands
// assets/favicon.svg.
await withTempDir(async (dir) => {
vi.stubGlobal(
"fetch",
serve({ "favicon.svg": BADGE_SVG, "apple-icon.png": await solidPng(0) }),
);
mockRequest(serve({ "favicon.svg": BADGE_SVG, "apple-icon.png": await solidPng(0) }));
const { icons } = await downloadAssets(tokensWithNoSvgs(), dir, [], [
{ rel: "icon", href: "https://x.test/favicon.svg", sizes: null, type: "image/svg+xml" },
{
@@ -629,7 +640,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
it("headlines the SVG when it is the bare mark", async () => {
await withTempDir(async (dir) => {
vi.stubGlobal("fetch", serve({ "favicon.svg": BARE_SVG }));
mockRequest(serve({ "favicon.svg": BARE_SVG }));
const { icons } = await downloadAssets(tokensWithNoSvgs(), dir, [], [
{ rel: "icon", href: "https://x.test/favicon.svg", sizes: null, type: "image/svg+xml" },
] as IconCandidate[]);
@@ -639,7 +650,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
it("keeps assets/favicon.<ext> so a stem match still finds the icon", async () => {
await withTempDir(async (dir) => {
vi.stubGlobal("fetch", serve({ "favicon.svg": BARE_SVG }));
mockRequest(serve({ "favicon.svg": BARE_SVG }));
const { assets } = await downloadAssets(tokensWithNoSvgs(), dir, [], [
{ rel: "icon", href: "https://x.test/favicon.svg", sizes: null, type: "image/svg+xml" },
] as IconCandidate[]);
@@ -658,8 +669,7 @@ describe("capture download security boundaries", () => {
new URL("../../../../docs/public/catalog/assets/8963f64fa28dc4ae.woff2", import.meta.url),
);
expect(first.equals(second)).toBe(false);
vi.stubGlobal(
"fetch",
mockRequest(
vi.fn(
async (url: string) =>
new Response(new Uint8Array(url.endsWith(".ttf") ? first : second)),
@@ -681,7 +691,7 @@ describe("capture download security boundaries", () => {
new URL("../../../../docs/public/catalog/assets/a634cb9e7783af7e.woff2", import.meta.url),
);
const fetchMock = vi.fn(async () => new Response(new Uint8Array(bytes)));
vi.stubGlobal("fetch", fetchMock);
mockRequest(fetchMock);
const byteBudget = { remainingBytes: bytes.length };
await downloadAndRewriteFonts(
"@font-face{font-family:A;src:url(https://fonts.example/site.woff2)}",
@@ -697,16 +707,13 @@ describe("capture download security boundaries", () => {
});
});
afterEach(() => vi.unstubAllGlobals());
afterEach(() => vi.mocked(publicRequest).mockReset());
it("canonicalizes an icon ADS suffix and its promoted copy while preserving SVG bytes", async () => {
await withTempDir(async (dir) => {
const body =
'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><rect width="24" height="24"/></svg>';
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response(body)),
);
mockRequest(vi.fn(async () => new Response(body)));
const { icons } = await downloadAssets(
tokensWithNoSvgs(),
dir,
@@ -728,10 +735,7 @@ describe("capture download security boundaries", () => {
it("does not publish arbitrary bytes as an OG image", async () => {
await withTempDir(async (dir) => {
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response(new Uint8Array(6000).fill(65))),
);
mockRequest(vi.fn(async () => new Response(new Uint8Array(6000).fill(65))));
const tokens = tokensWithNoSvgs();
tokens.ogImage = "https://public.example/og.jpg::$DATA";
const { assets, drops } = await downloadAssets(tokens, dir);
@@ -748,10 +752,7 @@ describe("capture download security boundaries", () => {
const bytes = readFileSync(
new URL("../../../../docs/public/catalog/assets/a634cb9e7783af7e.woff2", import.meta.url),
);
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response(new Uint8Array(bytes))),
);
mockRequest(vi.fn(async () => new Response(new Uint8Array(bytes))));
const source = `@font-face{font-family:Demo;src:url(https://public.example/${name})}`;
const result = await downloadAndRewriteFonts(source, dir);
const names = readdirSync(join(dir, "assets/fonts"));
@@ -765,10 +766,7 @@ describe("capture download security boundaries", () => {
it("leaves invalid font URLs unchanged and records their rejection", async () => {
await withTempDir(async (dir) => {
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response(new Uint8Array(2048))),
);
mockRequest(vi.fn(async () => new Response(new Uint8Array(2048))));
const source = "@font-face{font-family:Demo;src:url(https://public.example/font.woff2)}";
const result = await downloadAndRewriteFonts(source, dir);
expect(result.css).toBe(source);
+3 -1
View File
@@ -6,6 +6,7 @@
*/
import { isBlockedNetworkHost } from "@hyperframes/engine";
import { publicRequest } from "./publicNetwork.js";
import { ensureCaptureDirSync, writeCaptureFileSync } from "./captureFile.js";
import { join, extname } from "node:path";
import { createHash } from "node:crypto";
@@ -670,10 +671,11 @@ export async function safeFetch(url: string, init?: RequestInit): Promise<Respon
let current = url;
for (let hop = 0; hop <= MAX_FETCH_REDIRECTS; hop++) {
if (isPrivateUrl(current)) return null;
const res = await fetch(current, { ...init, redirect: "manual" });
const res = await publicRequest(current, { ...init, redirect: "manual" });
if (res.status >= 300 && res.status < 400) {
const loc = res.headers.get("location");
if (!loc) return res;
await res.body?.cancel().catch(() => {});
try {
current = new URL(loc, current).toString();
} catch {
@@ -0,0 +1,12 @@
import { describe, expect, it } from "vitest";
import { captureBrowserArgs } from "./browserLaunchArgs.js";
describe("capture browser sandbox", () => {
it.each(["darwin", "win32"] as const)("keeps Chromium's sandbox on %s", (platform) => {
expect(captureBrowserArgs(false, 800, 600, platform)).not.toContain("--no-sandbox");
});
it("retains the Linux container workaround", () => {
expect(captureBrowserArgs(false, 800, 600, "linux")).toContain("--no-sandbox");
});
});
@@ -2,9 +2,11 @@ export function captureBrowserArgs(
disableWebgl: boolean,
viewportWidth: number,
viewportHeight: number,
platform: NodeJS.Platform = process.platform,
): string[] {
const needsLinuxSandboxException = platform === "linux";
return [
"--no-sandbox",
...(needsLinuxSandboxException ? ["--no-sandbox"] : []),
"--disable-dev-shm-usage",
...(disableWebgl
? ["--disable-gpu"]
@@ -45,3 +45,21 @@ describe("captureWebsiteAttempt output folders", () => {
},
);
});
describe("captureWebsiteAttempt URL admission", () => {
it.each([
"file:///etc/passwd",
"data:text/html,hello",
"ftp://example.com/file",
"https://user:secret@example.com/",
])("rejects %s before creating capture folders or launching a browser", async (url) => {
const root = mkdtempSync(join(tmpdir(), "hf-capture-url-"));
onTestFinished(() => rmSync(root, { recursive: true, force: true }));
const opts = { url, outputDir: root };
await expect(
captureWebsiteAttempt(opts, undefined, false, watchdog, createPartialCaptureState(opts)),
).rejects.toThrow("HTTP or HTTPS URL without credentials");
expect(readdirSync(root)).toEqual([]);
expect(ensureBrowserMock).not.toHaveBeenCalled();
});
});
+18 -9
View File
@@ -1,4 +1,4 @@
import { launchManagedBrowser } from "../browser/launch.js";
import { launchCaptureBrowser } from "./launchCaptureBrowser.js";
import { LottieDiscovery } from "./lottieDiscovery.js";
import { createCaptureDownloadBudget } from "./readBoundedResponse.js";
/**
@@ -27,7 +27,7 @@ import type {
ExtractedHtml,
} from "./types.js";
import type { CaptureWatchdog } from "./captureWatchdog.js";
import { captureBrowserArgs } from "./browserLaunchArgs.js";
import type { PartialCaptureState } from "./partialCapture.js";
import { runNavigationChecks } from "./navigationPhase.js";
import { runCoreExtraction } from "./coreExtractionPhase.js";
@@ -56,6 +56,14 @@ export async function captureWebsiteAttempt(
onPhase,
} = opts;
const initialUrl = new URL(url);
if (
!["http:", "https:"].includes(initialUrl.protocol) ||
initialUrl.username ||
initialUrl.password
) {
throw new Error("Capture requires a public HTTP or HTTPS URL without credentials");
}
const downloadByteBudget = createCaptureDownloadBudget();
const warnings: string[] = [...state.warnings];
state.warnings = warnings;
@@ -111,14 +119,15 @@ export async function captureWebsiteAttempt(
progress("browser", "Launching headless Chrome...");
const { ensureBrowser } = await import("../browser/manager.js");
const browser = await ensureBrowser();
const puppeteer = await import("puppeteer-core");
const chromeBrowser = await launchManagedBrowser(puppeteer.default, {
headless: true,
const captureBrowser = await launchCaptureBrowser({
executablePath: browser.executablePath,
protocolTimeout: captureProtocolTimeoutMs(timeout, budgetMs),
args: captureBrowserArgs(disableWebgl, viewportWidth, viewportHeight),
disableWebgl,
width: viewportWidth,
height: viewportHeight,
});
watchdog.registerBrowser(chromeBrowser);
const chromeBrowser = captureBrowser.browser;
watchdog.registerBrowser(captureBrowser);
let animationCatalog: CaptureResult["animationCatalog"];
let tokens: DesignTokens = state.tokens;
@@ -339,7 +348,7 @@ export async function captureWebsiteAttempt(
lastPhase,
};
} finally {
watchdog.unregisterBrowser(chromeBrowser);
await chromeBrowser.close();
watchdog.unregisterBrowser(captureBrowser);
await captureBrowser.close();
}
}
@@ -0,0 +1,169 @@
import { createServer, type IncomingMessage, type RequestOptions } from "node:http";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { existsSync } from "node:fs";
import { createSocket } from "node:dgram";
import { createServer as createTcpServer } from "node:net";
const fixture = vi.hoisted(() => ({
port: 0,
privateHits: 0,
publicHits: 0,
rebound: false,
dialed: [] as string[],
}));
vi.mock("node:dns/promises", () => ({
lookup: vi.fn(async (host: string) => {
if (host === "capture-fixture.example.com") {
return [{ address: fixture.rebound ? "127.0.0.1" : "93.184.216.34", family: 4 }];
}
throw new Error("Fixture has no such DNS name");
}),
}));
vi.mock("node:http", async (importOriginal) => {
const original = await importOriginal<typeof import("node:http")>();
return {
...original,
request(options: RequestOptions, listener?: (response: IncomingMessage) => void) {
if (options.hostname === "93.184.216.34") {
fixture.dialed.push(options.hostname);
// Simulate the checked public peer with a local server; assert the actual dial input.
expect(Number(options.port)).toBe(fixture.port);
options = { ...options, hostname: "127.0.0.1" };
}
return original.request(options, listener);
},
};
});
import { launchCaptureBrowser } from "./launchCaptureBrowser.js";
const chrome = process.env.HYPERFRAMES_TEST_BROWSER_PATH;
describe.skipIf(!chrome || !existsSync(chrome))("real browser public transport", () => {
const server = createServer((req, res) => {
if (req.url?.startsWith("/private")) fixture.privateHits++;
else fixture.publicHits++;
if (req.url === "/redirect") {
res.writeHead(302, { Location: `http://127.0.0.1:${fixture.port}/private` });
res.end();
} else if (req.url === "/page") {
res.setHeader("Content-Type", "text/html");
res.end(
`<html><body>fixture<img src="http://127.0.0.1:${fixture.port}/private-image"><iframe src="http://127.0.0.1:${fixture.port}/private-frame"></iframe></body></html>`,
);
} else res.end("controlled fixture");
});
let capture: Awaited<ReturnType<typeof launchCaptureBrowser>>;
beforeAll(async () => {
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
fixture.port = (server.address() as { port: number }).port;
capture = await launchCaptureBrowser({
executablePath: chrome!,
protocolTimeout: 15_000,
disableWebgl: true,
width: 800,
height: 600,
});
}, 20_000);
afterAll(async () => {
await capture?.close();
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
});
it("renders the public peer but never requests a private redirect or subresource", async () => {
const page = await capture.browser.newPage();
try {
const base = `http://capture-fixture.example.com:${fixture.port}`;
await page.goto(base + "/page", { waitUntil: "networkidle0", timeout: 10_000 });
expect(await page.$eval("body", (body) => body.textContent)).toContain("fixture");
await page
.goto(base + "/redirect", { waitUntil: "domcontentloaded", timeout: 5_000 })
.catch(() => {});
expect(fixture.privateHits).toBe(0);
expect(fixture.publicHits).toBeGreaterThan(0);
expect(fixture.dialed.length).toBeGreaterThan(0);
} finally {
await page.close();
}
}, 20_000);
it("does not send direct WebRTC UDP to a private STUN peer", async () => {
const udp = createSocket("udp4");
let hits = 0;
udp.on("message", () => {
hits++;
});
await new Promise<void>((resolve) => udp.bind(0, "127.0.0.1", resolve));
const port = udp.address().port;
const page = await capture.browser.newPage();
try {
await page.evaluate(async (port) => {
const peer = new RTCPeerConnection({ iceServers: [{ urls: `stun:127.0.0.1:${port}` }] });
try {
peer.createDataChannel("probe");
await peer.setLocalDescription(await peer.createOffer());
await new Promise((resolve) => setTimeout(resolve, 2000));
} finally {
peer.close();
}
}, port);
expect(hits).toBe(0);
} finally {
await page.close();
await new Promise<void>((resolve) => udp.close(() => resolve()));
}
}, 10_000);
it("does not connect directly to a private WebRTC TURN TCP peer", async () => {
let hits = 0;
const tcp = createTcpServer((socket) => {
hits++;
socket.destroy();
});
await new Promise<void>((resolve) => tcp.listen(0, "127.0.0.1", resolve));
const address = tcp.address();
if (!address || typeof address === "string") throw new Error("Missing fixture address");
const page = await capture.browser.newPage();
try {
await page.evaluate(async (port) => {
const peer = new RTCPeerConnection({
iceServers: [
{
urls: `turn:127.0.0.1:${port}?transport=tcp`,
username: "fixture",
credential: "fixture",
},
],
});
try {
peer.createDataChannel("probe");
await peer.setLocalDescription(await peer.createOffer());
await new Promise((resolve) => setTimeout(resolve, 2000));
} finally {
peer.close();
}
}, address.port);
expect(hits).toBe(0);
} finally {
await page.close();
await new Promise<void>((resolve) => tcp.close(() => resolve()));
}
}, 10_000);
it("rechecks DNS on a later request and rejects a private rebinding answer", async () => {
const page = await capture.browser.newPage();
try {
const base = `http://capture-fixture.example.com:${fixture.port}`;
await page.goto(base + "/first", { waitUntil: "domcontentloaded" });
const reachedBefore = fixture.publicHits;
fixture.rebound = true;
await page
.goto(base + "/second", { waitUntil: "domcontentloaded", timeout: 5_000 })
.catch(() => {});
expect(fixture.publicHits).toBe(reachedBefore);
expect(fixture.privateHits).toBe(0);
} finally {
fixture.rebound = false;
await page.close();
}
}, 15_000);
});
@@ -0,0 +1,91 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { createServer, request } from "node:http";
import { connect } from "node:net";
import { startCaptureProxy } from "./captureProxy.js";
const { resolveAddress } = vi.hoisted(() => ({ resolveAddress: vi.fn() }));
vi.mock("./publicNetwork.js", () => ({ resolvePublicAddress: resolveAddress }));
describe("capture proxy", () => {
const proxies: Awaited<ReturnType<typeof startCaptureProxy>>[] = [];
afterEach(async () => {
await Promise.all(proxies.splice(0).map((proxy) => proxy.close()));
vi.restoreAllMocks();
resolveAddress.mockReset();
});
async function proxyGet(url: string): Promise<number> {
const proxy = await startCaptureProxy();
proxies.push(proxy);
return new Promise((resolve, reject) => {
const req = request(proxy.url, { path: url }, (res) => {
res.resume();
res.on("end", () => resolve(res.statusCode!));
});
req.on("error", reject);
req.end();
});
}
it("refuses a private target before opening an upstream connection", async () => {
resolveAddress.mockRejectedValue(new Error("Private destination"));
expect(await proxyGet("http://127.0.0.1/private")).toBe(403);
expect(resolveAddress).toHaveBeenCalledWith("127.0.0.1");
});
it("refuses URL credentials and non-http requests", async () => {
expect(await proxyGet("http://user:secret@example.com/")).toBe(403);
expect(await proxyGet("file:///etc/passwd")).toBe(403);
expect(resolveAddress).not.toHaveBeenCalled();
});
it("does not dial after teardown while DNS is pending", async () => {
let hits = 0;
const target = createServer((_req, res) => {
hits++;
res.end("fixture");
});
await new Promise<void>((resolve) => target.listen(0, "127.0.0.1", resolve));
const address = target.address();
if (!address || typeof address === "string") throw new Error("Missing fixture address");
let release: (address: string) => void = () => {};
let entered: () => void = () => {};
const resolving = new Promise<void>((resolve) => {
entered = resolve;
});
resolveAddress.mockImplementation(() => {
entered();
return new Promise<string>((resolve) => {
release = resolve;
});
});
const proxy = await startCaptureProxy();
proxies.push(proxy);
const finished = new Promise<void>((resolve) => {
const req = request(proxy.url, { path: `http://fixture.example.com:${address.port}/late` });
req.on("error", () => resolve());
req.end();
});
try {
await resolving;
await proxy.close();
release("127.0.0.1");
await finished;
await new Promise((resolve) => setTimeout(resolve, 30));
expect(hits).toBe(0);
} finally {
target.closeAllConnections();
await new Promise<void>((resolve) => target.close(() => resolve()));
}
});
it("closes idle CONNECT clients during teardown", async () => {
const proxy = await startCaptureProxy();
proxies.push(proxy);
const socket = connect(Number(new URL(proxy.url).port), "127.0.0.1");
await new Promise<void>((resolve) => socket.once("connect", resolve));
const closed = new Promise<void>((resolve) => socket.once("close", () => resolve()));
await proxy.close();
await closed;
});
});
+161
View File
@@ -0,0 +1,161 @@
import { createServer, request, type IncomingMessage, type ServerResponse } from "node:http";
import { connect, type Socket } from "node:net";
import type { Duplex } from "node:stream";
import { resolvePublicAddress } from "./publicNetwork.js";
const CONNECTION_TIMEOUT_MS = 15_000;
const HOP_HEADERS = new Set([
"connection",
"proxy-connection",
"proxy-authorization",
"proxy-authenticate",
"keep-alive",
"te",
"trailer",
"transfer-encoding",
"upgrade",
]);
function forwardHeaders(message: IncomingMessage): IncomingMessage["headers"] {
const headers = { ...message.headers };
const connection = String(headers.connection ?? "").split(",");
for (const name of [...HOP_HEADERS, ...connection]) delete headers[name.trim().toLowerCase()];
return headers;
}
function destination(raw: string, tunnel: boolean): URL {
const url = new URL(tunnel ? `https://${raw}` : raw);
if (
url.username ||
url.password ||
url.hash ||
(tunnel
? url.protocol !== "https:" || url.pathname !== "/" || !!url.search
: url.protocol !== "http:")
) {
throw new Error("Unsupported capture destination");
}
return url;
}
function refuse(response: ServerResponse): void {
if (!response.headersSent) response.writeHead(403, { Connection: "close" });
response.end();
}
async function forwardHttp(
incoming: IncomingMessage,
response: ServerResponse,
sockets: Set<Socket>,
): Promise<void> {
try {
const url = destination(incoming.url ?? "", false);
const address = await resolvePublicAddress(url.hostname);
if (incoming.destroyed || response.destroyed) return;
const headers = forwardHeaders(incoming);
headers.host = url.host;
const upstream = request(
{
hostname: address,
family: 4,
port: url.port || 80,
method: incoming.method,
path: url.pathname + url.search,
headers,
agent: false,
signal: AbortSignal.timeout(30_000),
},
(received) => {
response.writeHead(received.statusCode ?? 502, forwardHeaders(received));
received.on("error", () => response.destroy());
received.pipe(response);
},
);
upstream.on("socket", (socket) => trackSocket(socket, sockets));
upstream.setTimeout(CONNECTION_TIMEOUT_MS, () =>
upstream.destroy(new Error("Capture request timed out")),
);
upstream.on("error", () => {
if (!response.headersSent) response.writeHead(502, { Connection: "close" });
response.end();
});
response.on("close", () => upstream.destroy());
incoming.on("error", () => upstream.destroy());
incoming.pipe(upstream);
} catch {
refuse(response);
}
}
function trackSocket(socket: Socket, sockets: Set<Socket>): void {
sockets.add(socket);
socket.once("close", () => sockets.delete(socket));
// Teardown can destroy either half of a tunnel while its peer is still writing.
socket.on("error", () => socket.destroy());
}
async function forwardTunnel(
incoming: IncomingMessage,
client: Duplex,
head: Buffer,
sockets: Set<Socket>,
): Promise<void> {
try {
const url = destination(incoming.url ?? "", true);
const address = await resolvePublicAddress(url.hostname);
if (client.destroyed) return;
const upstream = connect({ host: address, family: 4, port: Number(url.port || 443) });
trackSocket(upstream, sockets);
const connectDeadline = setTimeout(() => upstream.destroy(), CONNECTION_TIMEOUT_MS);
connectDeadline.unref();
upstream.once("close", () => clearTimeout(connectDeadline));
upstream.setTimeout(CONNECTION_TIMEOUT_MS, () => upstream.destroy());
upstream.once("connect", () => {
clearTimeout(connectDeadline);
if (client.destroyed) return upstream.destroy();
client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
if (head.length) upstream.write(head);
upstream.pipe(client);
client.pipe(upstream);
});
upstream.once("close", () => client.destroy());
client.once("close", () => upstream.destroy());
} catch {
client.end("HTTP/1.1 403 Forbidden\r\nConnection: close\r\nContent-Length: 0\r\n\r\n");
}
}
/** Every browser target uses this proxy; only a checked numeric IPv4 address is dialed. */
export async function startCaptureProxy(): Promise<{ url: string; close(): Promise<void> }> {
const sockets = new Set<Socket>();
const server = createServer(
(incoming, response) => void forwardHttp(incoming, response, sockets),
);
server.on("connection", (socket) => trackSocket(socket, sockets));
server.on(
"connect",
(incoming, socket, head) => void forwardTunnel(incoming, socket, head, sockets),
);
server.on("upgrade", (_incoming, socket) => socket.destroy());
server.maxConnections = 128;
server.headersTimeout = CONNECTION_TIMEOUT_MS;
server.requestTimeout = 30_000;
server.keepAliveTimeout = 1_000;
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", resolve);
});
const address = server.address();
if (!address || typeof address === "string") throw new Error("Capture proxy did not bind");
let closing: Promise<void> | undefined;
return {
url: `http://127.0.0.1:${address.port}`,
close() {
closing ??= new Promise<void>((resolve) => {
server.close(() => resolve());
for (const socket of sockets) socket.destroy();
});
return closing;
},
};
}
+56 -12
View File
@@ -7,26 +7,36 @@ import { Window } from "happy-dom";
import type { Page } from "puppeteer-core";
import { downloadAndRewriteFonts } from "./assetDownloader.js";
import { extractHtml } from "./htmlExtractor.js";
import { publicRequest } from "./publicNetwork.js";
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
const FONT = readFileSync(
new URL("../../../../docs/public/catalog/assets/a634cb9e7783af7e.woff2", import.meta.url),
);
function fakePage(window: Window): Page {
const evaluate = async (script: unknown) => {
if (typeof script !== "string") throw new Error("Expected a page expression");
return runInNewContext(script, {
const evaluate = async (script: unknown, ...args: unknown[]) => {
const expression = typeof script === "function" ? `(${script.toString()})(...args)` : script;
if (typeof expression !== "string") throw new Error("Expected a page expression");
return runInNewContext(expression, {
args,
window,
document: window.document,
location: window.location,
URL,
});
};
return { evaluate, url: () => window.location.href } as unknown as Page;
const addStyleTag = async ({ content }: { content: string }) => {
const style = window.document.createElement("style");
style.textContent = content;
window.document.head.append(style);
};
return { evaluate, addStyleTag, url: () => window.location.href } as unknown as Page;
}
afterEach(() => {
vi.unstubAllGlobals();
vi.mocked(publicRequest).mockReset();
});
describe("extractHtml", () => {
@@ -41,13 +51,10 @@ describe("extractHtml", () => {
expect(extracted.headHtml).toContain("url(#glow)");
const fetched: string[] = [];
vi.stubGlobal(
"fetch",
vi.fn(async (url: string) => {
fetched.push(url);
return new Response(new Uint8Array(FONT));
}),
);
vi.mocked(publicRequest).mockImplementation(async (url) => {
fetched.push(String(url));
return new Response(new Uint8Array(FONT));
});
const fonts = await downloadAndRewriteFonts(extracted.headHtml, dir);
expect(fetched).toEqual(["https://brand.example/fonts/Display-Medium.woff2"]);
expect(fonts.css).toContain("assets/fonts/Display-Medium.woff2");
@@ -58,3 +65,40 @@ describe("extractHtml", () => {
}
});
});
it("fetches Node stylesheets through validated redirects and refuses metadata redirects", async () => {
const window = new Window({ url: "https://brand.example/" });
window.document.head.innerHTML =
'<link rel="stylesheet" href="https://cdn.example/start.css">' +
'<link rel="stylesheet" href="https://cdn.example/private.css">';
const canceled = vi.fn();
vi.mocked(publicRequest).mockImplementation(async (url) => {
if (String(url).endsWith("start.css"))
return new Response(null, {
status: 302,
headers: { location: "https://styles.example/final.css" },
});
if (String(url).endsWith("private.css"))
return new Response(new ReadableStream({ cancel: canceled }), {
status: 302,
headers: { location: "http://169.254.169.254/metadata" },
});
if (String(url) === "https://styles.example/final.css")
return new Response("body{color:purple}");
throw new Error(`Unexpected request: ${url}`);
});
try {
const html = await extractHtml(fakePage(window), { settleTime: 0 });
expect(html.headHtml).toContain("body{color:purple}");
expect(window.document.querySelector('link[href$="start.css"]')).toBeNull();
expect(window.document.querySelector('link[href$="private.css"]')).not.toBeNull();
expect(vi.mocked(publicRequest).mock.calls.map(([url]) => String(url))).toEqual([
"https://cdn.example/start.css",
"https://styles.example/final.css",
"https://cdn.example/private.css",
]);
expect(canceled).toHaveBeenCalledOnce();
} finally {
await window.happyDOM.close();
}
});
+6 -4
View File
@@ -7,7 +7,7 @@
import type { Page } from "puppeteer-core";
import type { ExtractedHtml } from "./types.js";
import { isPrivateUrl } from "./assetDownloader.js";
import { safeFetch } from "./assetDownloader.js";
import { CAPTURE_USER_AGENT } from "./userAgent.js";
const DEFAULT_SETTLE_TIME = 3000;
@@ -33,12 +33,14 @@ export async function extractHtml(
for (const href of stylesheetUrls) {
try {
if (isPrivateUrl(href)) continue;
const res = await fetch(href, {
const res = await safeFetch(href, {
signal: AbortSignal.timeout(10000),
headers: { "User-Agent": CAPTURE_USER_AGENT },
});
if (!res.ok) continue;
if (!res?.ok) {
await res?.body?.cancel().catch(() => {});
continue;
}
let css = await res.text();
// Fix relative url() references
// fallow-ignore-next-line complexity
@@ -0,0 +1,67 @@
import { EventEmitter } from "node:events";
import { existsSync } from "node:fs";
import { rm } from "node:fs/promises";
import { afterEach, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({ launch: vi.fn(), proxyClose: vi.fn(), exit: vi.fn() }));
vi.mock("puppeteer-core", () => ({ default: { launch: mocks.launch } }));
vi.mock("./captureProxy.js", () => ({
startCaptureProxy: async () => ({ url: "http://127.0.0.1:4567", close: mocks.proxyClose }),
}));
vi.mock("../utils/commandResult.js", () => ({ requestCliExit: mocks.exit }));
const originalSignals = new Map(
(["SIGINT", "SIGTERM"] as const).map((signal) => [signal, process.listeners(signal)]),
);
let profile: string | undefined;
afterEach(async () => {
for (const [signal, original] of originalSignals) {
for (const listener of process.listeners(signal)) {
if (!original.includes(listener)) process.removeListener(signal, listener);
}
}
if (profile) await rm(profile, { recursive: true, force: true });
vi.resetModules();
vi.resetAllMocks();
});
it.each(["SIGINT", "SIGTERM"] as const)(
"joins capture profile cleanup before root exit on %s without session.close",
async (signal) => {
const child = Object.assign(new EventEmitter(), {
pid: process.pid,
exitCode: null as number | null,
signalCode: null,
});
const browser = Object.assign(new EventEmitter(), {
process: () => child,
close: vi.fn(async () => {
child.exitCode = 0;
child.emit("exit", 0);
browser.emit("disconnected");
}),
});
mocks.launch.mockResolvedValue(browser);
mocks.proxyClose.mockResolvedValue(undefined);
const { launchCaptureBrowser } = await import("./launchCaptureBrowser.js");
const session = await launchCaptureBrowser({
executablePath: "/chrome",
protocolTimeout: 1000,
disableWebgl: false,
width: 800,
height: 600,
});
profile = mocks.launch.mock.calls[0]?.[0].userDataDir;
expect(existsSync(profile!)).toBe(true);
const exit = new Promise<{ code: number; profileExists: boolean }>((resolve) => {
mocks.exit.mockImplementation((code: number) => {
resolve({ code, profileExists: existsSync(profile!) });
});
});
process.emit(signal);
expect(await exit).toEqual({ code: signal === "SIGINT" ? 130 : 143, profileExists: false });
await session.close();
await session.close();
expect(browser.close).toHaveBeenCalledOnce();
},
);
@@ -0,0 +1,73 @@
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
proxyClose: vi.fn(),
browserClose: vi.fn(),
once: vi.fn(),
launch: vi.fn(),
}));
vi.mock("./captureProxy.js", () => ({
startCaptureProxy: async () => ({ url: "http://127.0.0.1:4567", close: mocks.proxyClose }),
}));
vi.mock("../browser/launch.js", () => ({
launchManagedBrowser: mocks.launch,
closeManagedBrowser: mocks.browserClose,
registerBrowserSessionCleanup: () => () => {},
}));
vi.mock("puppeteer-core", () => ({ default: {} }));
import { launchCaptureBrowser } from "./launchCaptureBrowser.js";
const options = {
executablePath: "/chrome",
protocolTimeout: 1000,
disableWebgl: false,
width: 800,
height: 600,
};
beforeEach(() => {
vi.resetAllMocks();
mocks.proxyClose.mockResolvedValue(undefined);
mocks.browserClose.mockResolvedValue(undefined);
mocks.launch.mockResolvedValue({ close: mocks.browserClose, once: mocks.once });
});
describe("capture browser transport lifecycle", () => {
it("forces proxy for loopback and disables direct UDP routes", async () => {
const session = await launchCaptureBrowser(options);
expect(mocks.launch.mock.calls[0]?.[1].args).toEqual(
expect.arrayContaining([
"--proxy-server=http://127.0.0.1:4567",
"--proxy-bypass-list=<-loopback>",
"--disable-quic",
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
]),
);
const profile = mocks.launch.mock.calls[0]?.[1].userDataDir;
expect(JSON.parse(readFileSync(join(profile, "Default", "Preferences"), "utf8"))).toEqual({
webrtc: { ip_handling_policy: "disable_non_proxied_udp" },
});
await session.close();
expect(existsSync(profile)).toBe(false);
expect(mocks.proxyClose.mock.invocationCallOrder[0] ?? 0).toBeLessThan(
mocks.browserClose.mock.invocationCallOrder[0] ?? 0,
);
});
it("closes proxy when launch fails", async () => {
mocks.launch.mockRejectedValue(new Error("launch failed"));
await expect(launchCaptureBrowser(options)).rejects.toThrow("launch failed");
expect(mocks.proxyClose).toHaveBeenCalledOnce();
});
it("closes proxy before a failing browser shutdown", async () => {
const session = await launchCaptureBrowser(options);
mocks.browserClose.mockRejectedValue(new Error("close failed"));
await expect(session.close()).rejects.toThrow("close failed");
expect(mocks.proxyClose).toHaveBeenCalledOnce();
});
it("closes proxy on unexpected browser disconnect", async () => {
const session = await launchCaptureBrowser(options);
const [event, callback] = mocks.once.mock.calls[0] ?? [];
expect(event).toBe("disconnected");
callback();
expect(mocks.proxyClose).toHaveBeenCalledOnce();
await session.close();
});
});
@@ -0,0 +1,76 @@
import type { Browser } from "puppeteer-core";
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
launchManagedBrowser,
closeManagedBrowser,
registerBrowserSessionCleanup,
} from "../browser/launch.js";
import { captureBrowserArgs } from "./browserLaunchArgs.js";
import { startCaptureProxy } from "./captureProxy.js";
/** The launch-wide proxy covers pages, popups, workers and both extraction passes. */
export async function launchCaptureBrowser(options: {
executablePath: string;
protocolTimeout: number;
disableWebgl: boolean;
width: number;
height: number;
}): Promise<{ browser: Browser; close(): Promise<void> }> {
const proxy = await startCaptureProxy();
let profile: string | undefined;
let browser: Browser | undefined;
let closing: Promise<void> | undefined;
let unregister: (() => void) | undefined;
const close = () => {
closing ??= (async () => {
try {
await proxy.close();
if (browser) await closeManagedBrowser(browser);
} finally {
try {
if (profile) await rm(profile, { recursive: true, force: true });
} finally {
unregister?.();
}
}
})();
return closing;
};
try {
const profilePath = await mkdtemp(join(tmpdir(), "hf-capture-profile-"));
profile = profilePath;
await mkdir(join(profile, "Default"));
await writeFile(
join(profile, "Default", "Preferences"),
JSON.stringify({
webrtc: { ip_handling_policy: "disable_non_proxied_udp" },
}),
);
unregister = registerBrowserSessionCleanup(close);
const puppeteer = await import("puppeteer-core");
browser = await launchManagedBrowser(puppeteer.default, {
headless: true,
userDataDir: profile,
executablePath: options.executablePath,
protocolTimeout: options.protocolTimeout,
// Linux containers may lack the user namespaces Chromium's sandbox requires.
args: [
...captureBrowserArgs(options.disableWebgl, options.width, options.height),
`--proxy-server=${proxy.url}`,
"--proxy-bypass-list=<-loopback>",
"--disable-quic",
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
],
});
browser.once("disconnected", () => void proxy.close());
return {
browser,
close,
};
} catch (error) {
await close();
throw error;
}
}
@@ -1,3 +1,5 @@
import { publicRequest } from "./publicNetwork.js";
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
import { afterEach, describe, expect, it, vi } from "vitest";
import { discoverLottieResponse, LottieDiscovery } from "./lottieDiscovery.js";
@@ -14,7 +16,7 @@ describe("bounded Lottie discovery", () => {
finish = resolve;
}),
);
vi.stubGlobal("fetch", fetchMock);
vi.mocked(publicRequest).mockImplementation(fetchMock);
const budget = { remainingBytes: 1000 };
let settled = false;
const pending = discovery
@@ -44,7 +46,7 @@ describe("bounded Lottie discovery", () => {
headers: () => ({ "content-type": "application/json" }),
});
const fetchMock = vi.fn(async (_url: string | URL | Request) => new Response("{}"));
vi.stubGlobal("fetch", fetchMock);
vi.mocked(publicRequest).mockImplementation(fetchMock);
const budget = { remainingBytes: 1000 };
expect(await discovery.run(budget, () => 10000)).toEqual([]);
expect(fetchMock).toHaveBeenCalledTimes(32);
@@ -60,14 +62,14 @@ describe("bounded Lottie discovery", () => {
});
discovery.collect({ url: () => "https://public.example/anim.lottie", headers: () => ({}) });
const fetchMock = vi.fn(async (_url: string | URL | Request) => new Response("{}"));
vi.stubGlobal("fetch", fetchMock);
vi.mocked(publicRequest).mockImplementation(fetchMock);
await discovery.run({ remainingBytes: 1000 }, () => 10000);
const urls = fetchMock.mock.calls.map((call) => call[0]);
expect(urls).toContain("https://public.example/real.json");
expect(urls).not.toContain("https://public.example/api/0");
expect(urls).toHaveLength(31);
});
afterEach(() => vi.unstubAllGlobals());
afterEach(() => vi.mocked(publicRequest).mockReset());
it.each([undefined, "1"])(
"does not trust intercepted Content-Length %s or materialize Puppeteer bodies",
async (length) => {
@@ -83,8 +85,7 @@ describe("bounded Lottie discovery", () => {
buffer,
};
const cancel = vi.fn();
vi.stubGlobal(
"fetch",
vi.mocked(publicRequest).mockImplementation(
vi.fn(
async () =>
new Response(
@@ -106,10 +107,7 @@ describe("bounded Lottie discovery", () => {
it("retains valid discovered JSON with its existing byte accounting", async () => {
const data = { v: "5.12.2", w: 100, h: 100, layers: [], fr: 30, ip: 0, op: 30 };
const body = JSON.stringify(data);
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response(body)),
);
vi.mocked(publicRequest).mockImplementation(vi.fn(async () => new Response(body)));
const budget = { remainingBytes: body.length };
const found = await discoverLottieResponse(
{ url: () => "https://public.example/animation.json?version=1", headers: () => ({}) },
+10 -14
View File
@@ -1,3 +1,5 @@
import { publicRequest } from "./publicNetwork.js";
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
import {
existsSync,
lstatSync,
@@ -31,7 +33,7 @@ function tempDir(): string {
afterEach(() => {
vi.useRealTimers();
vi.unstubAllGlobals();
vi.mocked(publicRequest).mockReset();
for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true });
tempDirs.length = 0;
});
@@ -44,7 +46,7 @@ describe("Lottie capture budget", () => {
remainingMs = 0;
return new Response(JSON.stringify({ w: 100, h: 100, layers: [] }), { status: 200 });
});
vi.stubGlobal("fetch", fetchMock);
vi.mocked(publicRequest).mockImplementation(fetchMock);
const saved = await saveLottieAnimations(
[{ url: "https://one.example/a.json" }, { url: "https://two.example/b.json" }],
@@ -173,7 +175,7 @@ describe("video capture live budget", () => {
headers: { "content-type": "video/mp4" },
}),
);
vi.stubGlobal("fetch", fetchMock);
vi.mocked(publicRequest).mockImplementation(fetchMock);
const capture = captureVideoManifest(page, dir, () => {}, {
sampleMs: 10_000,
@@ -205,10 +207,7 @@ describe("remainingVideoDownloadTimeoutMs", () => {
describe("Lottie capture rejects unsafe persistence", () => {
it("does not write corrupt archives as raw Lottie files", async () => {
const dir = tempDir();
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response("not a zip")),
);
vi.mocked(publicRequest).mockImplementation(vi.fn(async () => new Response("not a zip")));
expect(
await saveLottieAnimations([{ url: "https://public.example/bad.lottie" }], dir, dir),
).toBe(0);
@@ -216,8 +215,7 @@ describe("Lottie capture rejects unsafe persistence", () => {
});
it("does not publish truthy non-array layers", async () => {
const dir = tempDir();
vi.stubGlobal(
"fetch",
vi.mocked(publicRequest).mockImplementation(
vi.fn(async () => new Response('{"w":100,"h":100,"layers":true}')),
);
expect(await saveLottieAnimations([{ url: "https://public.example/bad.json" }], dir, dir)).toBe(
@@ -292,8 +290,7 @@ describe("media previews replace pre-planted links", () => {
typeof expression === "function" ? { x: 0, y: 0, width: 640, height: 360 } : [descriptor],
);
const page = { evaluate, screenshot: vi.fn(async () => PNG) } as unknown as Page;
vi.stubGlobal(
"fetch",
vi.mocked(publicRequest).mockImplementation(
vi.fn(async () => new Response(null, { status: 404 })),
);
@@ -327,8 +324,7 @@ describe("media folders refuse a planted directory link (#4304)", () => {
const { dir, outside } = plantedCapture(link);
const screenshot = vi.fn();
const page = { evaluate: vi.fn(async () => [descriptor]), screenshot } as unknown as Page;
vi.stubGlobal(
"fetch",
vi.mocked(publicRequest).mockImplementation(
vi.fn(async () => new Response(null, { status: 404 })),
);
@@ -364,7 +360,7 @@ describe("media folders refuse a planted directory link (#4304)", () => {
async () => {
const { dir, outside } = plantedCapture("assets/lottie");
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
vi.mocked(publicRequest).mockImplementation(fetchMock);
await expect(
saveLottieAnimations(
@@ -0,0 +1,328 @@
import { execFileSync } from "node:child_process";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import {
createServer,
type IncomingMessage,
type RequestOptions,
type ServerResponse,
} from "node:http";
import { createServer as createHttpsServer, type RequestOptions as HttpsOptions } from "node:https";
import { createServer as createTcpServer, type Socket } from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { TLSSocket } from "node:tls";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
const transport = vi.hoisted(() => ({
lookup: vi.fn(),
dials: [] as HttpsOptions[],
certificate: "",
trustCertificate: true,
}));
vi.mock("node:dns/promises", () => ({ lookup: transport.lookup }));
// Only the already-checked numeric peer is mapped to an isolated loopback server.
// The real Node HTTP/TLS transport, headers and certificate checks remain active.
vi.mock("node:http", async (importOriginal) => {
const original = await importOriginal<typeof import("node:http")>();
return {
...original,
request(options: RequestOptions, callback: (response: IncomingMessage) => void) {
if (options.hostname !== "93.184.216.34") throw new Error("Unexpected fixture peer");
transport.dials.push(options);
return original.request({ ...options, hostname: "127.0.0.1" }, callback);
},
};
});
vi.mock("node:https", async (importOriginal) => {
const original = await importOriginal<typeof import("node:https")>();
return {
...original,
request(options: HttpsOptions, callback: (response: IncomingMessage) => void) {
if (options.hostname !== "93.184.216.34") throw new Error("Unexpected fixture peer");
transport.dials.push(options);
return original.request(
{
...options,
hostname: "127.0.0.1",
ca: transport.trustCertificate ? transport.certificate : undefined,
},
callback,
);
},
};
});
import { publicRequest } from "./publicNetwork.js";
type RequestReceipt = {
host: string | undefined;
path: string | undefined;
servername: string | undefined;
};
const receipts: RequestReceipt[] = [];
const sockets = new Set<Socket>();
const streams = new Set<ServerResponse>();
let fixtureDirectory: string;
let httpPort: number;
let httpsPort: number;
let malformedPort: number;
function handleRequest(request: IncomingMessage, response: ServerResponse) {
const servername = (request.socket as TLSSocket).servername;
receipts.push({
host: request.headers.host,
path: request.url,
servername: typeof servername === "string" ? servername : undefined,
});
if (request.url === "/slow-transfer") {
streams.add(response);
response.writeHead(200, { "content-type": "video/mp4", "content-length": 35 * 1024 * 1024 });
response.flushHeaders();
let remaining = 35;
const interval = setInterval(() => {
response.write(Buffer.alloc(1024 * 1024));
if (--remaining === 0) response.end();
}, 1000);
response.once("close", () => {
clearInterval(interval);
streams.delete(response);
});
return;
}
if (request.url === "/stream") {
streams.add(response);
response.writeHead(200, { "content-type": "application/octet-stream" });
response.write("first chunk");
const interval = setInterval(() => response.write("next chunk"), 10);
response.once("close", () => {
clearInterval(interval);
streams.delete(response);
});
return;
}
response.end("fixture response");
}
const httpServer = createServer(handleRequest);
const httpsServer = createHttpsServer({}, handleRequest);
const malformedServer = createTcpServer((socket) => socket.end("not a TLS handshake\r\n"));
const servers = [httpServer, httpsServer, malformedServer];
async function listen(server: (typeof servers)[number]): Promise<number> {
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string") throw new Error("Missing fixture port");
return address.port;
}
async function waitForStreamClose() {
const deadline = Date.now() + 2000;
while (streams.size && Date.now() < deadline) {
await new Promise((resolve) => setTimeout(resolve, 10));
}
expect(streams.size).toBe(0);
}
describe("publicRequest controlled socket integration", () => {
beforeAll(async () => {
fixtureDirectory = await mkdtemp(join(tmpdir(), "capture-public-network-"));
const key = join(fixtureDirectory, "key.pem");
const certificate = join(fixtureDirectory, "certificate.pem");
execFileSync(
"openssl",
[
"req",
"-x509",
"-newkey",
"rsa:2048",
"-nodes",
"-keyout",
key,
"-out",
certificate,
"-days",
"1",
"-subj",
"/CN=assets.capture-fixture.com",
"-addext",
"subjectAltName=DNS:assets.capture-fixture.com",
],
{ stdio: "pipe" },
);
transport.certificate = await readFile(certificate, "utf8");
httpsServer.setSecureContext({ key: await readFile(key), cert: transport.certificate });
httpsServer.on("tlsClientError", () => {});
for (const server of servers) {
server.on("connection", (socket) => {
sockets.add(socket);
socket.once("close", () => sockets.delete(socket));
});
}
[httpPort, httpsPort, malformedPort] = await Promise.all([
listen(httpServer),
listen(httpsServer),
listen(malformedServer),
]);
});
beforeEach(() => {
receipts.length = 0;
transport.dials.length = 0;
transport.trustCertificate = true;
transport.lookup.mockReset();
transport.lookup.mockResolvedValue([{ address: "127.0.0.1", family: 4 }]);
transport.lookup.mockResolvedValueOnce([{ address: "93.184.216.34", family: 4 }]);
});
afterAll(async () => {
for (const socket of sockets) socket.destroy();
await Promise.all(
servers.map(
(server) =>
new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
}),
),
);
await rm(fixtureDirectory, { recursive: true, force: true });
});
it("pins the checked address without a second DNS lookup and preserves HTTP Host/path", async () => {
const response = await publicRequest(`http://assets.capture-fixture.com:${httpPort}/asset?q=1`);
expect(await response.text()).toBe("fixture response");
expect(transport.lookup).toHaveBeenCalledTimes(1);
expect(transport.lookup).toHaveBeenCalledWith("assets.capture-fixture.com", {
all: true,
family: 4,
verbatim: true,
});
expect(transport.dials).toHaveLength(1);
expect(transport.dials[0]?.hostname).toBe("93.184.216.34");
expect(receipts).toEqual([
{
host: `assets.capture-fixture.com:${httpPort}`,
path: "/asset?q=1",
servername: undefined,
},
]);
});
it("preserves HTTPS Host and SNI while verifying the trusted fixture certificate", async () => {
const response = await publicRequest(`https://assets.capture-fixture.com:${httpsPort}/secure`);
expect(await response.text()).toBe("fixture response");
expect(transport.dials[0]).toMatchObject({
hostname: "93.184.216.34",
servername: "assets.capture-fixture.com",
rejectUnauthorized: true,
});
expect(receipts).toEqual([
{
host: `assets.capture-fixture.com:${httpsPort}`,
path: "/secure",
servername: "assets.capture-fixture.com",
},
]);
});
it("rejects an untrusted certificate before any HTTP request reaches the target", async () => {
transport.trustCertificate = false;
await expect(
publicRequest(`https://assets.capture-fixture.com:${httpsPort}/secure`),
).rejects.toThrow(/self.signed|certificate/i);
expect(transport.dials[0]?.rejectUnauthorized).toBe(true);
expect(receipts).toHaveLength(0);
});
it("rejects a trusted certificate with the wrong hostname", async () => {
await expect(
publicRequest(`https://other.capture-fixture.com:${httpsPort}/secure`),
).rejects.toThrow(/hostname|altnames|certificate/i);
expect(transport.dials[0]).toMatchObject({
servername: "other.capture-fixture.com",
rejectUnauthorized: true,
});
expect(receipts).toHaveLength(0);
});
it("rejects a malformed TLS handshake", async () => {
await expect(
publicRequest(`https://assets.capture-fixture.com:${malformedPort}/secure`),
).rejects.toThrow(/ssl|tls|version|record/i);
expect(receipts).toHaveLength(0);
});
it("rejects private DNS answers before dialing or reaching a target", async () => {
transport.lookup.mockReset().mockResolvedValue([{ address: "127.0.0.1", family: 4 }]);
await expect(
publicRequest(`http://assets.capture-fixture.com:${httpPort}/asset`),
).rejects.toThrow(/non-public/);
expect(transport.dials).toHaveLength(0);
expect(receipts).toHaveLength(0);
});
it("cancels a real streaming response without uncaught teardown errors", async () => {
const response = await publicRequest(`http://assets.capture-fixture.com:${httpPort}/stream`);
const reader = response.body?.getReader();
if (!reader) throw new Error("Missing fixture stream");
expect((await reader.read()).value?.byteLength).toBeGreaterThan(0);
await reader.cancel();
await waitForStreamClose();
expect(receipts).toHaveLength(1);
});
it("finishes a 35-second HTTPS body within the caller's 120-second deadline", async () => {
const response = await publicRequest(
`https://assets.capture-fixture.com:${httpsPort}/slow-transfer`,
{ signal: AbortSignal.timeout(120_000) },
);
expect((await response.arrayBuffer()).byteLength).toBe(35 * 1024 * 1024);
await waitForStreamClose();
}, 60_000);
it.each([true, false])(
"uses the caller deadline when supplied, otherwise the default (%s)",
async (explicitDeadline) => {
const deadline = new AbortController();
const caller = new AbortController();
const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(deadline.signal);
let reader: ReadableStreamDefaultReader<Uint8Array> | undefined;
try {
const response = await publicRequest(
`https://assets.capture-fixture.com:${httpsPort}/stream`,
explicitDeadline ? { signal: caller.signal } : {},
);
reader = response.body!.getReader();
await reader.read();
// Advance only the transport's default deadline; real TLS/body I/O stays active.
deadline.abort(new Error("default deadline reached"));
if (explicitDeadline) {
expect((await reader.read()).value?.byteLength).toBeGreaterThan(0);
caller.abort();
await expect(reader.read()).rejects.toThrow();
} else {
await expect(reader.read()).rejects.toThrow();
}
} finally {
timeout.mockRestore();
await reader?.cancel().catch(() => {});
await waitForStreamClose();
}
},
);
it("propagates caller abort after HTTPS response headers and closes the socket", async () => {
const controller = new AbortController();
const response = await publicRequest(`https://assets.capture-fixture.com:${httpsPort}/stream`, {
signal: controller.signal,
});
const reader = response.body?.getReader();
if (!reader) throw new Error("Missing fixture stream");
await reader.read();
controller.abort();
await expect(reader.read()).rejects.toThrow();
await waitForStreamClose();
});
});
@@ -0,0 +1,177 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { EventEmitter } from "node:events";
import { PassThrough } from "node:stream";
const mocks = vi.hoisted(() => ({ lookup: vi.fn(), request: vi.fn() }));
vi.mock("node:dns/promises", () => ({ lookup: mocks.lookup }));
vi.mock("node:http", () => ({ request: mocks.request }));
vi.mock("node:https", () => ({ request: mocks.request }));
import { isPublicIPv4, publicRequest, resolvePublicAddress } from "./publicNetwork.js";
beforeEach(() => vi.resetAllMocks());
describe("public address policy", () => {
it.each([
"127.0.0.1",
"10.0.0.1",
"172.16.0.1",
"192.168.0.1",
"169.254.169.254",
"100.64.0.1",
"0.1.2.3",
"192.0.0.9",
"192.0.2.1",
"198.18.0.1",
"198.51.100.1",
"203.0.113.1",
"224.0.0.1",
"240.0.0.1",
"::1",
"::ffff:8.8.8.8",
"8.8.8.999",
])("rejects %s", (ip) => expect(isPublicIPv4(ip)).toBe(false));
it("accepts public literals without DNS", async () => {
await expect(resolvePublicAddress("8.8.8.8")).resolves.toBe("8.8.8.8");
expect(mocks.lookup).not.toHaveBeenCalled();
});
it.each([
"localhost",
"localhost.",
"foo.localhost",
"router.local",
"metadata.google.internal",
"[::1]",
])("blocks alias/literal %s before DNS", async (host) => {
await expect(resolvePublicAddress(host)).rejects.toThrow();
expect(mocks.lookup).not.toHaveBeenCalled();
});
it("rejects mixed DNS before opening a socket", async () => {
mocks.lookup.mockResolvedValue([
{ address: "8.8.8.8", family: 4 },
{ address: "127.0.0.1", family: 4 },
]);
await expect(publicRequest("http://example.com/")).rejects.toThrow();
expect(mocks.request).not.toHaveBeenCalled();
});
it.each([{ answers: [] }, { answers: [{ address: "8.8.8.8", family: 6 }] }])(
"rejects empty or malformed DNS",
async ({ answers }) => {
mocks.lookup.mockResolvedValue(answers);
await expect(resolvePublicAddress("example.com")).rejects.toThrow();
},
);
it("propagates DNS errors", async () => {
mocks.lookup.mockRejectedValue(new Error("DNS failed"));
await expect(resolvePublicAddress("example.com")).rejects.toThrow("DNS failed");
});
});
function fakeRequest() {
const req = Object.assign(new EventEmitter(), {
end: vi.fn(),
destroy: vi.fn(),
setTimeout: vi.fn(),
});
mocks.request.mockImplementation((_options, callback) => {
req.end.mockImplementation(() => {
const incoming = Object.assign(new PassThrough(), {
statusCode: 302,
statusMessage: "Found",
headers: { location: "http://127.0.0.1/" },
});
callback(incoming);
incoming.end("redirect");
});
return req;
});
return req;
}
describe("pinned request", () => {
it("dials the checked numeric IP only, preserving TLS identity and manual redirects", async () => {
mocks.lookup
.mockResolvedValueOnce([{ address: "8.8.8.8", family: 4 }])
.mockResolvedValue([{ address: "127.0.0.1", family: 4 }]);
fakeRequest();
const response = await publicRequest("https://example.com:8443/path?q=1", {
headers: { host: "evil.test", "accept-encoding": "gzip" },
});
expect(mocks.lookup).toHaveBeenCalledTimes(1);
expect(mocks.request).toHaveBeenCalledTimes(1);
expect(mocks.request.mock.calls[0]?.[0]).toMatchObject({
hostname: "8.8.8.8",
servername: "example.com",
rejectUnauthorized: true,
port: "8443",
path: "/path?q=1",
agent: false,
headers: { host: "example.com:8443", "accept-encoding": "identity" },
});
expect(response.status).toBe(302);
expect(await response.text()).toBe("redirect");
});
it.each(["file:///tmp/a", "http://user:secret@example.com"])(
"rejects unsafe URL %s",
async (url) => {
await expect(publicRequest(url)).rejects.toThrow();
expect(mocks.request).not.toHaveBeenCalled();
},
);
it("does not dial after abort during DNS", async () => {
const controller = new AbortController();
mocks.lookup.mockImplementation(async () => {
controller.abort();
return [{ address: "8.8.8.8", family: 4 }];
});
await expect(
publicRequest("https://example.com", { signal: controller.signal }),
).rejects.toThrow();
expect(mocks.request).not.toHaveBeenCalled();
});
});
describe("transport lifecycle", () => {
it("bounds stalled DNS without dialing", async () => {
vi.useFakeTimers();
try {
mocks.lookup.mockImplementation(() => new Promise(() => {}));
const result = expect(publicRequest("https://example.com")).rejects.toThrow("DNS timed out");
await vi.advanceTimersByTimeAsync(10_000);
await result;
expect(mocks.request).not.toHaveBeenCalled();
} finally {
vi.useRealTimers();
}
});
it("cancels the upstream stream and socket when the body is cancelled", async () => {
mocks.lookup.mockResolvedValue([{ address: "8.8.8.8", family: 4 }]);
const incoming = Object.assign(new PassThrough(), { statusCode: 200, headers: {} });
const req = Object.assign(new EventEmitter(), { end: vi.fn(), destroy: vi.fn() });
mocks.request.mockImplementation((_options, callback) => {
req.end.mockImplementation(() => callback(incoming));
return req;
});
const response = await publicRequest("https://example.com");
await response.body?.cancel();
expect(incoming.destroyed).toBe(true);
expect(req.destroy).toHaveBeenCalled();
// Socket teardown may race already queued terminal/data notifications.
expect(() => {
incoming.emit("aborted");
incoming.emit("error", new Error("late socket error"));
incoming.emit("data", Buffer.from("late bytes"));
incoming.emit("end");
}).not.toThrow();
});
it("fails unsupported compressed responses instead of returning corrupted asset bytes", async () => {
mocks.lookup.mockResolvedValue([{ address: "8.8.8.8", family: 4 }]);
const incoming = Object.assign(new PassThrough(), {
statusCode: 200,
headers: { "content-encoding": "gzip" },
});
const req = Object.assign(new EventEmitter(), { end: vi.fn(), destroy: vi.fn() });
mocks.request.mockImplementation((_options, callback) => {
req.end.mockImplementation(() => callback(incoming));
return req;
});
await expect(publicRequest("https://example.com")).rejects.toThrow("content encoding");
expect(incoming.destroyed).toBe(true);
});
});
+195
View File
@@ -0,0 +1,195 @@
import { lookup } from "node:dns/promises";
import { isIP } from "node:net";
import { request as httpRequest, type IncomingMessage, type ClientRequest } from "node:http";
import { request as httpsRequest } from "node:https";
function ipv4Number(address: string): number {
return address.split(".").reduce((value, octet) => value * 256 + Number(octet), 0);
}
// Conservatively exclude special-use networks, including all of 192.0/16.
const NON_PUBLIC_IPV4: ReadonlyArray<readonly [string, number]> = [
["0.0.0.0", 8],
["10.0.0.0", 8],
["100.64.0.0", 10],
["127.0.0.0", 8],
["169.254.0.0", 16],
["172.16.0.0", 12],
["192.0.0.0", 16],
["192.88.99.0", 24],
["192.168.0.0", 16],
["198.18.0.0", 15],
["198.51.100.0", 24],
["203.0.113.0", 24],
["224.0.0.0", 3],
];
const NON_PUBLIC_RANGES = NON_PUBLIC_IPV4.map(([network, prefix]) => ({
start: ipv4Number(network),
end: ipv4Number(network) + 2 ** (32 - prefix),
}));
/** Conservative IPv4-only policy: special-use addresses fail closed. */
export function isPublicIPv4(address: string): boolean {
if (isIP(address) !== 4) return false;
const value = ipv4Number(address);
return !NON_PUBLIC_RANGES.some(({ start, end }) => value >= start && value < end);
}
/** Validate every A answer, then return a numeric peer that callers must dial directly. */
export async function resolvePublicAddress(hostname: string): Promise<string> {
const host = hostname.toLowerCase().replace(/\.$/, "");
if (isIP(host) === 4) {
if (!isPublicIPv4(host)) throw new Error("Capture requires a public IPv4 address");
return host;
}
if (
!host.includes(".") ||
!/^[a-z0-9.-]+$/.test(host) ||
/(?:^|\.)(?:localhost|local|internal|home|lan|test|invalid|example)$/.test(host) ||
host.split(".").some((label) => !label || label.startsWith("-") || label.endsWith("-"))
) {
throw new Error("Capture requires a public hostname");
}
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const answers = await Promise.race([
lookup(host, { all: true, family: 4, verbatim: true }),
new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new Error("Capture DNS timed out")), 10_000);
timer.unref();
}),
]);
if (
!answers[0] ||
answers.some((answer) => answer.family !== 4 || !isPublicIPv4(answer.address))
) {
throw new Error("Capture DNS returned a non-public address");
}
const first = answers[0];
if (!first) throw new Error("Capture DNS returned no address");
return first.address;
} finally {
clearTimeout(timer);
}
}
/** Pinned asset transport; redirects are returned for caller validation. */
export async function publicRequest(url: string | URL, init: RequestInit = {}): Promise<Response> {
const target = new URL(url);
if (!["http:", "https:"].includes(target.protocol) || target.username || target.password) {
throw new Error("Capture requires an HTTP(S) URL without credentials");
}
const method = (init.method ?? "GET").toUpperCase();
if (!["GET", "HEAD"].includes(method) || init.body != null) {
throw new Error("Capture asset requests support only GET and HEAD without a body");
}
// Video callers budget the entire body transfer; a default must not shorten it.
const signal = init.signal ?? AbortSignal.timeout(30_000);
signal.throwIfAborted();
const address = await new Promise<string>((resolve, reject) => {
const onAbort = () => reject(signal.reason);
signal.addEventListener("abort", onAbort, { once: true });
resolvePublicAddress(target.hostname)
.then(resolve, reject)
.finally(() => signal.removeEventListener("abort", onAbort));
});
signal.throwIfAborted();
const headers = new Headers(init.headers);
for (const name of [
"connection",
"proxy-authorization",
"proxy-connection",
"transfer-encoding",
"upgrade",
"content-length",
])
headers.delete(name);
headers.set("host", target.host);
headers.set("accept-encoding", "identity");
return new Promise<Response>((resolve, reject) => {
const request = target.protocol === "https:" ? httpsRequest : httpRequest;
const req = request(
{
protocol: target.protocol,
hostname: address,
port: target.port || undefined,
path: `${target.pathname}${target.search}`,
method,
headers: Object.fromEntries(headers),
agent: false,
servername: target.hostname,
rejectUnauthorized: true,
signal,
},
(incoming) => {
try {
resolve(assetResponse(incoming, req, method));
} catch (error) {
incoming.destroy();
reject(error);
}
},
);
req.once("error", reject);
req.end();
});
}
function responseHeaders(incoming: IncomingMessage): Headers {
const headers = new Headers();
for (const [name, value] of Object.entries(incoming.headers)) {
if (value === undefined) continue;
for (const item of Array.isArray(value) ? value : [value]) headers.append(name, item);
}
return headers;
}
function assetResponse(incoming: IncomingMessage, req: ClientRequest, method: string): Response {
// Even discarded/null bodies can emit errors during socket teardown.
incoming.on("error", () => {});
const headers = responseHeaders(incoming);
const encoding = headers.get("content-encoding")?.toLowerCase();
if (encoding && encoding !== "identity") {
throw new Error("Capture server ignored identity content encoding");
}
const status = incoming.statusCode ?? 502;
if (method === "HEAD" || [204, 205, 304].includes(status)) {
incoming.resume();
return new Response(null, { status, headers });
}
return new Response(responseBody(incoming, req), { status, headers });
}
function responseBody(incoming: IncomingMessage, req: ClientRequest): ReadableStream<Uint8Array> {
let finished = false;
return new ReadableStream<Uint8Array>({
start(controller) {
incoming.on("data", (chunk: Buffer) => {
if (finished) return;
controller.enqueue(chunk);
incoming.pause();
});
incoming.once("end", () => {
if (finished) return;
finished = true;
controller.close();
});
const fail = (error: Error) => {
if (finished) return;
finished = true;
controller.error(error);
};
incoming.once("error", fail);
incoming.once("aborted", () => fail(new Error("Capture response aborted")));
incoming.pause();
},
pull() {
incoming.resume();
},
cancel() {
finished = true;
incoming.destroy();
req.destroy();
},
});
}
+100 -3
View File
@@ -125,24 +125,121 @@ describe("generateProjectScaffold metadata", () => {
});
describe("loadEnvFile", () => {
const keys = ["hyperframes_feedback_email", "HyperFrames_Feedback_Email", "HF_SCAFFOLD_TEST_KEY"];
const keys = ["hyperframes_feedback_email", "HyperFrames_Feedback_Email", "GEMINI_API_KEY"];
let dir: string;
beforeEach(() => {
dir = fs.mkdtempSync(join(tmpdir(), "hf-scaffold-env-"));
vi.stubEnv("GEMINI_API_KEY", undefined);
});
afterEach(() => {
for (const key of keys) delete process.env[key];
vi.unstubAllEnvs();
for (const key of keys.slice(0, 2)) delete process.env[key];
fs.rmSync(dir, { recursive: true, force: true });
});
it("never takes the feedback email from a project file, in any letter case", () => {
fs.writeFileSync(
join(dir, ".env"),
'hyperframes_feedback_email=a@example.com\nHyperFrames_Feedback_Email=b@example.com\nHF_SCAFFOLD_TEST_KEY="kept"\n',
'hyperframes_feedback_email=a@example.com\nHyperFrames_Feedback_Email=b@example.com\nGEMINI_API_KEY="kept"\n',
);
loadEnvFile(join(dir, "out"));
expect(keys.map((key) => process.env[key])).toEqual([undefined, undefined, "kept"]);
});
it("never accepts browser executables from an output ancestor", () => {
const browserKeys = [
"HYPERFRAMES_BROWSER_PATH",
"PRODUCER_HEADLESS_SHELL_PATH",
"hyperframes_browser_path",
];
const before = browserKeys.map((key) => process.env[key]);
try {
for (const key of browserKeys) delete process.env[key];
fs.writeFileSync(
join(dir, ".env"),
browserKeys.map((key) => `${key}=/untrusted/browser`).join("\n"),
);
loadEnvFile(join(dir, "out"));
expect(browserKeys.map((key) => process.env[key])).toEqual([undefined, undefined, undefined]);
} finally {
browserKeys.forEach((key, index) => {
const value = before[index];
if (value === undefined) delete process.env[key];
else process.env[key] = value;
});
}
});
it.each([
"PATH",
"Path",
"LOCALAPPDATA",
"PROGRAMFILES",
"PROGRAMFILES(X86)",
"HOME",
"LD_PRELOAD",
"ld_preload",
"LD_LIBRARY_PATH",
"DYLD_INSERT_LIBRARIES",
"NODE_OPTIONS",
"Node_Options",
"HYPERFRAMES_BROWSER_PATH",
"PRODUCER_HEADLESS_SHELL_PATH",
"localappdata",
"UNRECOGNIZED_FUTURE_SETTING",
])("does not import launcher or unknown setting %s", (key) => {
vi.stubEnv(key, undefined);
fs.writeFileSync(join(dir, ".env"), `${key}=/untrusted/value\nGEMINI_API_KEY=allowed\n`);
loadEnvFile(join(dir, "out"));
expect(process.env[key]).toBeUndefined();
expect(process.env.GEMINI_API_KEY).toBe("allowed");
});
it("does not import a supported provider setting with the wrong letter case", () => {
fs.writeFileSync(join(dir, ".env"), "gemini_api_key=/untrusted/value\n");
loadEnvFile(join(dir, "out"));
expect(process.env.gemini_api_key).toBeUndefined();
expect(process.env.GEMINI_API_KEY).toBeUndefined();
});
it("loads supported vision provider settings without replacing inherited credentials", () => {
const settings = {
OPENROUTER_API_KEY: "router-key",
GOOGLE_API_KEY: "google-key",
HYPERFRAMES_VERTEX_PROJECT_ID: "example-project",
HYPERFRAMES_VERTEX_SERVICE_ACCOUNT: '{"client_email":"test@example.com"}',
HYPERFRAMES_VERTEX_LOCATION: "us-central1",
HYPERFRAMES_VERTEX_MODEL: "vertex-model",
HYPERFRAMES_OPENROUTER_MODEL: "router-model",
HYPERFRAMES_GEMINI_MODEL: "gemini-model",
HYPERFRAMES_VISION_TIMEOUT_MS: "2000",
};
for (const key of Object.keys(settings)) vi.stubEnv(key, undefined);
vi.stubEnv("GEMINI_API_KEY", "inherited-key");
fs.writeFileSync(
join(dir, ".env"),
[
"GEMINI_API_KEY=project-key",
...Object.entries(settings).map(([key, value]) => `${key}=${value}`),
].join("\n"),
);
loadEnvFile(join(dir, "out"));
expect(process.env.GEMINI_API_KEY).toBe("inherited-key");
for (const [key, value] of Object.entries(settings)) expect(process.env[key]).toBe(value);
});
it("can skip loading the output's dotenv file entirely", () => {
const previous = process.env.HYPERFRAMES_NO_CAPTURE_DOTENV;
try {
process.env.HYPERFRAMES_NO_CAPTURE_DOTENV = "1";
fs.writeFileSync(join(dir, ".env"), "GEMINI_API_KEY=untrusted\n");
loadEnvFile(join(dir, "out"));
expect(process.env.GEMINI_API_KEY).toBeUndefined();
} finally {
if (previous === undefined) delete process.env.HYPERFRAMES_NO_CAPTURE_DOTENV;
else process.env.HYPERFRAMES_NO_CAPTURE_DOTENV = previous;
}
});
});
+5 -4
View File
@@ -7,22 +7,23 @@
import { existsSync, readFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { applyDotEnv } from "../utils/dotEnv.js";
import { applyDotEnv, CAPTURE_ENV_KEYS } from "../utils/dotEnv.js";
import { writeNewFileSync } from "../utils/writeNewFile.js";
import type { CatalogedAsset } from "./assetCataloger.js";
import type { CaptureResult, DesignTokens } from "./types.js";
/**
* Load .env file by walking up from startDir (up to 5 levels).
* Sets process.env keys that are not already set. Best-effort — never throws.
* Load supported vision settings from .env by walking up from startDir (up to 5 levels).
* Keeps inherited settings. Best-effort — never throws.
*/
export function loadEnvFile(startDir: string): void {
if (process.env.HYPERFRAMES_NO_CAPTURE_DOTENV === "1") return;
try {
let dir = resolve(startDir);
for (let i = 0; i < 5; i++) {
const envPath = resolve(dir, ".env");
try {
applyDotEnv(readFileSync(envPath, "utf-8"), process.env);
applyDotEnv(readFileSync(envPath, "utf-8"), process.env, CAPTURE_ENV_KEYS);
break;
} catch {
dir = resolve(dir, "..");
+88
View File
@@ -0,0 +1,88 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, it, vi } from "vitest";
import { mockTelemetry } from "./cliDispatchTestUtils.js";
const originalArgv = [...process.argv];
const originalExitCode = process.exitCode;
const dirs: string[] = [];
const eventNames = ["exit", "uncaughtException", "unhandledRejection"] as const;
const listeners = eventNames.map((event) => [event, process.listeners(event)] as const);
const streams = [process.stdout, process.stderr].map(
(stream) => [stream, stream.listeners("error")] as const,
);
afterEach(() => {
process.argv = [...originalArgv];
process.exitCode = originalExitCode;
vi.restoreAllMocks();
vi.unstubAllEnvs();
for (const path of [
"./commands/capture.js",
"./commands/init.js",
"./telemetry/index.js",
"./telemetry/events.js",
])
vi.doUnmock(path);
vi.resetModules();
for (const [event, original] of listeners) {
for (const listener of process.listeners(event)) {
if (!original.includes(listener)) process.removeListener(event, listener);
}
}
for (const [stream, original] of streams) {
for (const listener of stream.listeners("error")) {
if (!original.includes(listener)) stream.removeListener("error", listener);
}
}
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
it.each(
(
[
["capture", false, undefined, "fixture-key", undefined],
["capture", true, undefined, undefined, undefined],
["init", false, process.execPath, "fixture-key", undefined],
["init", true, process.execPath, "fixture-key", undefined],
["capture", false, process.execPath, "fixture-key", process.execPath],
] as const
).flatMap((test) =>
[[], ["--json"], ["--json", "--verbose"]].map((prefix) => [...test, prefix] as const),
),
)(
"applies CWD dotenv policy at real CLI startup: %s, opt-out=%s",
async (command, skip, browser, key, inheritedBrowser, prefix) => {
const dir = mkdtempSync(join(tmpdir(), "hf-cli-dotenv-"));
dirs.push(dir);
writeFileSync(
join(dir, ".env"),
`HYPERFRAMES_BROWSER_PATH=${process.execPath}\nNODE_OPTIONS=--no-warnings\nGEMINI_API_KEY=fixture-key\n`,
);
vi.spyOn(process, "cwd").mockReturnValue(dir);
vi.stubEnv("HYPERFRAMES_BROWSER_PATH", inheritedBrowser);
vi.stubEnv("NODE_OPTIONS", undefined);
vi.stubEnv("GEMINI_API_KEY", undefined);
vi.stubEnv("HYPERFRAMES_NO_CAPTURE_DOTENV", skip ? "1" : undefined);
let observed: unknown;
const definition = {
default: {
meta: { name: command },
args: {},
run: () => {
observed = [
process.env.HYPERFRAMES_BROWSER_PATH,
process.env.NODE_OPTIONS,
process.env.GEMINI_API_KEY,
];
},
},
};
vi.doMock("./commands/capture.js", () => definition);
vi.doMock("./commands/init.js", () => definition);
mockTelemetry();
process.argv = ["node", "cli.ts", ...prefix, command];
await import("./cli.js");
expect(observed).toEqual([browser, command === "capture" ? undefined : "--no-warnings", key]);
},
);
+20 -11
View File
@@ -60,17 +60,26 @@ if (rootVersionRequested) {
process.exit(0);
}
// ── Load .env from CWD ─────────────────────────────────────────────────────
// Agents run from the project directory where .env holds API keys (Gemini,
// HeyGen, ElevenLabs). Load it automatically so they don't need `source .env`.
const { applyDotEnv } = await import("./utils/dotEnv.js");
try {
const { readFileSync } = await import("node:fs");
const { resolve } = await import("node:path");
const envPath = resolve(process.cwd(), ".env");
applyDotEnv(readFileSync(envPath, "utf-8"), process.env);
} catch {
/* .env not present — fine, env vars may be set another way */
// Capture treats CWD as project data too: its early loader must use the same
// allowlist/opt-out as output ancestors, before browser selection can observe it.
const { applyDotEnv, CAPTURE_ENV_KEYS } = await import("./utils/dotEnv.js");
// The root has no value-taking options: Citty skips flags up to the first
// positional command, and stops subcommand selection at the -- separator.
const separator = argv.indexOf("--");
const commandArgs = separator < 0 ? argv : argv.slice(0, separator);
const captureEnv = commandArgs.find((arg) => !arg.startsWith("-")) === "capture";
if (!captureEnv || process.env.HYPERFRAMES_NO_CAPTURE_DOTENV !== "1") {
try {
const { readFileSync } = await import("node:fs");
const envPath = resolve(process.cwd(), ".env");
applyDotEnv(
readFileSync(envPath, "utf-8"),
process.env,
captureEnv ? CAPTURE_ENV_KEYS : undefined,
);
} catch {
/* .env not present — fine, env vars may be set another way */
}
}
// ── Lazy imports ────────────────────────────────────────────────────────────
+22
View File
@@ -22,4 +22,26 @@ describe("applyDotEnv", () => {
);
expect(env).toEqual({ OTHER: "1" });
});
it("copies only exact allowed names when a caller limits the imported settings", () => {
const env: NodeJS.ProcessEnv = { GEMINI_API_KEY: "inherited" };
applyDotEnv(
"GEMINI_API_KEY=project\nGOOGLE_API_KEY=allowed\ngoogle_api_key=other\nPATH=/untrusted\nLD_PRELOAD=payload",
env,
["GEMINI_API_KEY", "GOOGLE_API_KEY"],
);
expect(env).toEqual({ GEMINI_API_KEY: "inherited", GOOGLE_API_KEY: "allowed" });
});
it("an allowlist cannot opt the feedback email back in", () => {
const env: NodeJS.ProcessEnv = {};
applyDotEnv("HyperFrames_Feedback_Email=private", env, ["HyperFrames_Feedback_Email"]);
expect(env).toEqual({});
});
it("an empty allowlist imports nothing", () => {
const env: NodeJS.ProcessEnv = {};
applyDotEnv("GEMINI_API_KEY=project", env, []);
expect(env).toEqual({});
});
});
+27 -2
View File
@@ -1,5 +1,18 @@
import { FEEDBACK_EMAIL_ENV } from "./feedbackSource.js";
export const CAPTURE_ENV_KEYS = [
"OPENROUTER_API_KEY",
"GEMINI_API_KEY",
"GOOGLE_API_KEY",
"HYPERFRAMES_VERTEX_PROJECT_ID",
"HYPERFRAMES_VERTEX_SERVICE_ACCOUNT",
"HYPERFRAMES_VERTEX_LOCATION",
"HYPERFRAMES_VERTEX_MODEL",
"HYPERFRAMES_OPENROUTER_MODEL",
"HYPERFRAMES_GEMINI_MODEL",
"HYPERFRAMES_VISION_TIMEOUT_MS",
];
// A quoted value ends at its closing quote; unquoted, `#` starts a comment only after whitespace, so `pass#word` stays.
function valueOf(raw: string): string {
const quote = raw.charAt(0);
@@ -22,10 +35,22 @@ function entryOf(rawLine: string): [string, string] | null {
// Never over a key already set, and never the feedback email in any letter case (Windows env names are case-blind):
// only the launching app may attach it, and an agent can write a project `.env`.
export function applyDotEnv(content: string, env: NodeJS.ProcessEnv): void {
// An optional exact-name allowlist narrows callers that load less-trusted project settings.
export function applyDotEnv(
content: string,
env: NodeJS.ProcessEnv,
allowedKeys?: readonly string[],
): void {
const allowed = allowedKeys === undefined ? undefined : new Set(allowedKeys);
for (const entry of content.split("\n").map(entryOf)) {
if (!entry) continue;
const [key, value] = entry;
if (key && key.toUpperCase() !== FEEDBACK_EMAIL_ENV && !(key in env)) env[key] = value;
if (
key &&
key.toUpperCase() !== FEEDBACK_EMAIL_ENV &&
(allowed === undefined || allowed.has(key)) &&
!(key in env)
)
env[key] = value;
}
}