mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 17:27:09 +00:00
fix(cli): pin website capture connections to public peers (#5668)
* fix(cli): pin website capture connections to public peers * style(cli): keep capture policy explanations within comment limits * test(cli): separate case-sensitive dotenv assertions * fix(cli): honor capture deadlines and bootstrap isolation * chore: trim redundant capture dotenv comments * fix(capture): join signal cleanup and guard leading options * docs(capture): trim redundant cleanup comment
This commit is contained in:
@@ -207,6 +207,25 @@ npx hyperframes capture https://example.com --json
|
||||
Fonts: sohne-var
|
||||
```
|
||||
|
||||
Website capture accepts public HTTP(S) destinations without URL credentials.
|
||||
Browser traffic goes through a per-capture proxy, and asset downloads use the
|
||||
same checked-address policy. Each connection uses a validated numeric IPv4
|
||||
address; redirects and later requests are checked again. Private, loopback,
|
||||
link-local and special-use addresses are refused. IPv6-only sites are currently
|
||||
unsupported. Plain WebSocket upgrades are refused; slow or refused resources
|
||||
may be absent from the capture.
|
||||
|
||||
Capture uses a temporary browser profile with non-proxied WebRTC UDP disabled.
|
||||
The profile and proxy are removed when the browser closes. The Chromium sandbox
|
||||
is retained on macOS and Windows; Linux retains the container compatibility
|
||||
exception.
|
||||
|
||||
Ancestor `.env` files can supply the supported vision API keys, models, Vertex
|
||||
credentials/location and vision timeout. They cannot change the browser path,
|
||||
executable discovery, TLS, proxy or process runtime settings. Set
|
||||
`HYPERFRAMES_NO_CAPTURE_DOTENV=1` in the calling environment to skip these files.
|
||||
An inherited `HYPERFRAMES_BROWSER_PATH` still selects a browser.
|
||||
|
||||
`Dropped` is how many assets the page referenced that are **not** in the
|
||||
folder, and why. It is printed only when it is non-zero, and `--json` always
|
||||
carries it as a `dropped` object. Without it, a capture of a spare page and a
|
||||
|
||||
@@ -13,6 +13,7 @@ export class BrowserCleanupError extends AggregateError {}
|
||||
const browsers = new Set<Browser>();
|
||||
const closing = new WeakMap<Browser, Promise<void>>();
|
||||
const pending = new Map<Promise<Browser>, AbortController>();
|
||||
const sessionCleanups = new Set<() => Promise<void>>();
|
||||
const CLOSE_GRACE_MS = 1_000;
|
||||
const EXIT_POLL_MS = 50;
|
||||
let listening = false;
|
||||
@@ -102,7 +103,17 @@ export async function launchManagedBrowser(
|
||||
async function closeManagedBrowsers(): Promise<void> {
|
||||
for (const abort of pending.values()) abort.abort();
|
||||
await Promise.allSettled(pending.keys());
|
||||
await Promise.all([...browsers].map(closeManagedBrowser));
|
||||
const results = await Promise.allSettled([...browsers].map(closeManagedBrowser));
|
||||
results.push(...(await Promise.allSettled([...sessionCleanups].map((cleanup) => cleanup()))));
|
||||
const errors = results.flatMap((result) => (result.status === "rejected" ? [result.reason] : []));
|
||||
if (errors.length) throw new AggregateError(errors, "Browser session shutdown failed.");
|
||||
}
|
||||
|
||||
export function registerBrowserSessionCleanup(cleanup: () => Promise<void>): () => void {
|
||||
if (shutdown) throw new Error("The CLI is stopping; no browser session can start.");
|
||||
ownSignals();
|
||||
sessionCleanups.add(cleanup);
|
||||
return () => sessionCleanups.delete(cleanup);
|
||||
}
|
||||
|
||||
/** Manual teardown and signal teardown share the same bounded close, exactly once. */
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { publicRequest } from "./publicNetwork.js";
|
||||
|
||||
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
|
||||
afterEach(() => vi.mocked(publicRequest).mockReset());
|
||||
import { annotateGifAssetMetadata, type CatalogedAsset } from "./assetCataloger.js";
|
||||
|
||||
function u16(value: number): number[] {
|
||||
@@ -102,3 +106,40 @@ describe("annotateGifAssetMetadata", () => {
|
||||
expect(annotated[1]?.notes).toBe("hero");
|
||||
});
|
||||
});
|
||||
|
||||
it("uses safe redirects for GIF metadata and never follows a private redirect", async () => {
|
||||
const assets: CatalogedAsset[] = [
|
||||
{ url: "https://cdn.example/animated.gif", type: "Image", contexts: ["img[src]"] },
|
||||
{ url: "https://cdn.example/private.gif", type: "Image", contexts: ["img[src]"] },
|
||||
];
|
||||
const canceled = vi.fn();
|
||||
vi.mocked(publicRequest).mockImplementation(async (url) => {
|
||||
if (String(url).endsWith("animated.gif"))
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: "https://images.example/final.gif" },
|
||||
});
|
||||
if (String(url).endsWith("private.gif"))
|
||||
return new Response(new ReadableStream({ cancel: canceled }), {
|
||||
status: 302,
|
||||
headers: { location: "http://127.0.0.1/private.gif" },
|
||||
});
|
||||
if (String(url) === "https://images.example/final.gif")
|
||||
return new Response(new Uint8Array(gif([...frame(5), ...frame(15)], 0)));
|
||||
throw new Error(`Unexpected request: ${url}`);
|
||||
});
|
||||
const annotated = await annotateGifAssetMetadata(assets);
|
||||
expect(annotated[0]?.notes).toBe("animated GIF: 2 frames, 0.200s, loops forever");
|
||||
expect(annotated[1]?.notes).toBeUndefined();
|
||||
expect(
|
||||
vi
|
||||
.mocked(publicRequest)
|
||||
.mock.calls.map(([url]) => String(url))
|
||||
.sort(),
|
||||
).toEqual([
|
||||
"https://cdn.example/animated.gif",
|
||||
"https://cdn.example/private.gif",
|
||||
"https://images.example/final.gif",
|
||||
]);
|
||||
expect(canceled).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
|
||||
import type { Page } from "puppeteer-core";
|
||||
import { parseAnimatedGifMetadata } from "@hyperframes/core";
|
||||
import { safeFetch } from "./assetDownloader.js";
|
||||
import { readBoundedResponse } from "./readBoundedResponse.js";
|
||||
|
||||
export interface CatalogedAsset {
|
||||
url: string;
|
||||
@@ -276,11 +278,12 @@ function appendNote(existing: string | undefined, note: string): string {
|
||||
|
||||
async function readAssetBytes(url: string): Promise<Uint8Array | null> {
|
||||
try {
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(15_000) });
|
||||
if (!response.ok) return null;
|
||||
const contentLength = response.headers.get("content-length");
|
||||
if (contentLength && Number.parseInt(contentLength, 10) > 25 * 1024 * 1024) return null;
|
||||
return new Uint8Array(await response.arrayBuffer());
|
||||
const response = await safeFetch(url, { signal: AbortSignal.timeout(15_000) });
|
||||
if (!response?.ok) {
|
||||
await response?.body?.cancel().catch(() => {});
|
||||
return null;
|
||||
}
|
||||
return await readBoundedResponse(response, 25 * 1024 * 1024);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -22,6 +22,14 @@ import type { DesignTokens } from "./types.js";
|
||||
import type { IconCandidate } from "./faviconRanker.js";
|
||||
import { CAPTURE_USER_AGENT } from "./userAgent.js";
|
||||
import sharp from "sharp";
|
||||
import { publicRequest } from "./publicNetwork.js";
|
||||
|
||||
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
|
||||
|
||||
// Exercise production URL/redirect checks while replacing only the socket transport.
|
||||
function mockRequest(implementation: (url: string, init?: RequestInit) => Promise<Response>) {
|
||||
vi.mocked(publicRequest).mockImplementation((url, init) => implementation(String(url), init));
|
||||
}
|
||||
|
||||
describe("isPrivateUrl — SSRF denylist (security: F-003)", () => {
|
||||
it("blocks loopback, private, and metadata IPv4", () => {
|
||||
@@ -79,7 +87,7 @@ describe("isPrivateUrl — SSRF denylist (security: F-003)", () => {
|
||||
});
|
||||
|
||||
describe("safeFetch — re-validates the denylist on every redirect hop (security: F-002)", () => {
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
afterEach(() => vi.mocked(publicRequest).mockReset());
|
||||
|
||||
it("blocks a public URL that redirects to a private/metadata host", async () => {
|
||||
const fetchMock = vi.fn(async (input: string, _init?: RequestInit) => {
|
||||
@@ -92,7 +100,7 @@ describe("safeFetch — re-validates the denylist on every redirect hop (securit
|
||||
// The metadata host must NEVER be fetched.
|
||||
throw new Error(`safeFetch followed a redirect to a private host: ${input}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
|
||||
const res = await safeFetch("https://public.example/logo.png");
|
||||
expect(res).toBeNull();
|
||||
@@ -107,7 +115,7 @@ describe("safeFetch — re-validates the denylist on every redirect hop (securit
|
||||
return new Response(null, { status: 301, headers: { location: "https://b.example/y" } });
|
||||
return new Response("ok", { status: 200 });
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
|
||||
const res = await safeFetch("https://a.example/x");
|
||||
expect(res?.status).toBe(200);
|
||||
@@ -130,7 +138,7 @@ describe("safeFetch — re-validates the denylist on every redirect hop (securit
|
||||
headers: { location: target },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
expect(isPrivateUrl(target), target).toBe(true);
|
||||
expect(await safeFetch(target), target).toBeNull();
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
@@ -139,9 +147,25 @@ describe("safeFetch — re-validates the denylist on every redirect hop (securit
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels redirect bodies before validating the next hop", async () => {
|
||||
const canceled = vi.fn();
|
||||
mockRequest(
|
||||
vi.fn(
|
||||
async () =>
|
||||
new Response(new ReadableStream({ cancel: canceled }), {
|
||||
status: 302,
|
||||
headers: { location: "http://169.254.169.254/metadata" },
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(await safeFetch("https://public.example/asset")).toBeNull();
|
||||
expect(canceled).toHaveBeenCalledOnce();
|
||||
expect(publicRequest).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("returns null when the initial URL is private", async () => {
|
||||
const fetchMock = vi.fn(async () => new Response("ok"));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
const res = await safeFetch("http://169.254.169.254/");
|
||||
expect(res).toBeNull();
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
@@ -185,12 +209,12 @@ describe("toStandaloneSvg — scraped inline SVGs must survive as .svg files", (
|
||||
});
|
||||
|
||||
describe("downloadAndRewriteFonts — attempt caps", () => {
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
afterEach(() => vi.mocked(publicRequest).mockReset());
|
||||
|
||||
async function expectFailedFontAttempts(css: string, expectedAttempts: number): Promise<void> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-font-attempts-"));
|
||||
const fetchMock = vi.fn(async () => new Response("failed", { status: 503 }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
|
||||
try {
|
||||
await downloadAndRewriteFonts(css, dir);
|
||||
@@ -222,7 +246,7 @@ describe("downloadAndRewriteFonts — attempt caps", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-font-budget-"));
|
||||
const css = "@font-face { font-family: Budget; src: url(https://fonts.example/budget.woff2); }";
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
|
||||
try {
|
||||
await downloadAndRewriteFonts(css, dir, { remainingMs: () => 0 });
|
||||
@@ -269,7 +293,7 @@ const OPENAI_ICONS: IconCandidate[] = [
|
||||
];
|
||||
|
||||
describe("drop counts — why a referenced asset is not in the capture", () => {
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
afterEach(() => vi.mocked(publicRequest).mockReset());
|
||||
|
||||
/** `n` @font-face rules, each naming a DIFFERENT family, so only the global cap can bite. */
|
||||
function fontCss(n: number): string {
|
||||
@@ -285,7 +309,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
|
||||
// ever have said "some".
|
||||
await withTempDir(async (dir) => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
const { drops } = await downloadAndRewriteFonts(fontCss(4), dir, { remainingMs: () => 0 });
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(drops["budget-exhausted"]).toBe(4);
|
||||
@@ -297,10 +321,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
|
||||
it("separates the faces the global cap refused from the ones that failed", async () => {
|
||||
// 35 declared, cap 30: 30 are attempted and every attempt 503s, 5 are never reached.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response("no", { status: 503 })),
|
||||
);
|
||||
mockRequest(vi.fn(async () => new Response("no", { status: 503 })));
|
||||
const { drops } = await downloadAndRewriteFonts(fontCss(35), dir);
|
||||
expect(drops["cap-reached"]).toBe(5);
|
||||
expect(drops.unavailable).toBe(30);
|
||||
@@ -311,10 +332,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
|
||||
it("counts the faces the per-family cap refused", async () => {
|
||||
// 10 rules, all one family, per-family cap 6: 6 attempted, 4 refused.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response("no", { status: 503 })),
|
||||
);
|
||||
mockRequest(vi.fn(async () => new Response("no", { status: 503 })));
|
||||
const css = Array.from(
|
||||
{ length: 10 },
|
||||
(_, i) =>
|
||||
@@ -330,8 +348,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
|
||||
// The whole point of the tally: this page declared one face and we have it. A reader can now
|
||||
// tell this apart from a page that declared thirty and got truncated to one.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
mockRequest(
|
||||
vi.fn(
|
||||
async () =>
|
||||
new Response(
|
||||
@@ -362,8 +379,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
|
||||
it("counts an image dropped for being under the raster floor", async () => {
|
||||
// A valid small PNG is under the 10 KB floor; preserve the recorded drop reason.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
mockRequest(
|
||||
vi.fn(
|
||||
async () =>
|
||||
new Response(
|
||||
@@ -388,7 +404,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
|
||||
it("counts every catalogued image the budget never let it reach", async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
const catalog = Array.from({ length: 7 }, (_, i) => ({
|
||||
type: "Image",
|
||||
url: `https://cdn.example/img-${i}.png`,
|
||||
@@ -451,7 +467,7 @@ describe("drop counts — why a referenced asset is not in the capture", () => {
|
||||
});
|
||||
|
||||
describe("asset fetches present the same identity as the page navigation", () => {
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
afterEach(() => vi.mocked(publicRequest).mockReset());
|
||||
|
||||
/**
|
||||
* Measured against the real origin: `GET /favicon.svg` answers `403 text/html` to
|
||||
@@ -477,7 +493,7 @@ describe("asset fetches present the same identity as the page navigation", () =>
|
||||
// With a bot UA the ranker's winner 403s and the capture silently falls through to the
|
||||
// next candidate, so the file on disk is chosen by the CDN rather than by the ranker.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal("fetch", serveLikeAnAntiBotEdge());
|
||||
mockRequest(serveLikeAnAntiBotEdge());
|
||||
const { icons } = await downloadAssets(tokensWithNoSvgs(), dir, [], OPENAI_ICONS);
|
||||
expect(icons.headline?.file).toBe("assets/favicon.svg");
|
||||
expect(icons.icons[0]?.url).toBe("https://openai.example/favicon.svg");
|
||||
@@ -487,7 +503,7 @@ describe("asset fetches present the same identity as the page navigation", () =>
|
||||
it("sends one User-Agent for every asset request", async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const fetchMock = serveLikeAnAntiBotEdge();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
await downloadAssets(tokensWithNoSvgs(), dir, [], OPENAI_ICONS);
|
||||
const agents = fetchMock.mock.calls.map((call) =>
|
||||
new Headers(call[1]?.headers).get("user-agent"),
|
||||
@@ -515,7 +531,7 @@ async function testIco(): Promise<Buffer> {
|
||||
}
|
||||
|
||||
describe("declared icons — keep them all, headline the bare mark", () => {
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
afterEach(() => vi.mocked(publicRequest).mockReset());
|
||||
|
||||
/** A mark knocked out of a full-bleed rounded rect: the shape openai.com's favicon.svg has. */
|
||||
const BADGE_SVG = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 180 180">
|
||||
@@ -557,8 +573,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
|
||||
|
||||
it("writes one file per declared icon, named for its rel and sizes", async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
mockRequest(
|
||||
serve({
|
||||
"favicon.svg": BADGE_SVG,
|
||||
"favicon.ico": await testIco(),
|
||||
@@ -580,8 +595,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
|
||||
// opaque tile. openai.com declares no bare mark at all, so the headline falls back to the
|
||||
// ranker and the manifest has to say so rather than implying a preference was satisfied.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
mockRequest(
|
||||
serve({
|
||||
"favicon.svg": BADGE_SVG,
|
||||
"favicon.ico": await testIco(),
|
||||
@@ -604,10 +618,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
|
||||
// PNG does, and shape is what the brand band needs. Under the old rule this lands
|
||||
// assets/favicon.svg.
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
serve({ "favicon.svg": BADGE_SVG, "apple-icon.png": await solidPng(0) }),
|
||||
);
|
||||
mockRequest(serve({ "favicon.svg": BADGE_SVG, "apple-icon.png": await solidPng(0) }));
|
||||
const { icons } = await downloadAssets(tokensWithNoSvgs(), dir, [], [
|
||||
{ rel: "icon", href: "https://x.test/favicon.svg", sizes: null, type: "image/svg+xml" },
|
||||
{
|
||||
@@ -629,7 +640,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
|
||||
|
||||
it("headlines the SVG when it is the bare mark", async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal("fetch", serve({ "favicon.svg": BARE_SVG }));
|
||||
mockRequest(serve({ "favicon.svg": BARE_SVG }));
|
||||
const { icons } = await downloadAssets(tokensWithNoSvgs(), dir, [], [
|
||||
{ rel: "icon", href: "https://x.test/favicon.svg", sizes: null, type: "image/svg+xml" },
|
||||
] as IconCandidate[]);
|
||||
@@ -639,7 +650,7 @@ describe("declared icons — keep them all, headline the bare mark", () => {
|
||||
|
||||
it("keeps assets/favicon.<ext> so a stem match still finds the icon", async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal("fetch", serve({ "favicon.svg": BARE_SVG }));
|
||||
mockRequest(serve({ "favicon.svg": BARE_SVG }));
|
||||
const { assets } = await downloadAssets(tokensWithNoSvgs(), dir, [], [
|
||||
{ rel: "icon", href: "https://x.test/favicon.svg", sizes: null, type: "image/svg+xml" },
|
||||
] as IconCandidate[]);
|
||||
@@ -658,8 +669,7 @@ describe("capture download security boundaries", () => {
|
||||
new URL("../../../../docs/public/catalog/assets/8963f64fa28dc4ae.woff2", import.meta.url),
|
||||
);
|
||||
expect(first.equals(second)).toBe(false);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
mockRequest(
|
||||
vi.fn(
|
||||
async (url: string) =>
|
||||
new Response(new Uint8Array(url.endsWith(".ttf") ? first : second)),
|
||||
@@ -681,7 +691,7 @@ describe("capture download security boundaries", () => {
|
||||
new URL("../../../../docs/public/catalog/assets/a634cb9e7783af7e.woff2", import.meta.url),
|
||||
);
|
||||
const fetchMock = vi.fn(async () => new Response(new Uint8Array(bytes)));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockRequest(fetchMock);
|
||||
const byteBudget = { remainingBytes: bytes.length };
|
||||
await downloadAndRewriteFonts(
|
||||
"@font-face{font-family:A;src:url(https://fonts.example/site.woff2)}",
|
||||
@@ -697,16 +707,13 @@ describe("capture download security boundaries", () => {
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
afterEach(() => vi.mocked(publicRequest).mockReset());
|
||||
|
||||
it("canonicalizes an icon ADS suffix and its promoted copy while preserving SVG bytes", async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const body =
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><rect width="24" height="24"/></svg>';
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(body)),
|
||||
);
|
||||
mockRequest(vi.fn(async () => new Response(body)));
|
||||
const { icons } = await downloadAssets(
|
||||
tokensWithNoSvgs(),
|
||||
dir,
|
||||
@@ -728,10 +735,7 @@ describe("capture download security boundaries", () => {
|
||||
|
||||
it("does not publish arbitrary bytes as an OG image", async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(new Uint8Array(6000).fill(65))),
|
||||
);
|
||||
mockRequest(vi.fn(async () => new Response(new Uint8Array(6000).fill(65))));
|
||||
const tokens = tokensWithNoSvgs();
|
||||
tokens.ogImage = "https://public.example/og.jpg::$DATA";
|
||||
const { assets, drops } = await downloadAssets(tokens, dir);
|
||||
@@ -748,10 +752,7 @@ describe("capture download security boundaries", () => {
|
||||
const bytes = readFileSync(
|
||||
new URL("../../../../docs/public/catalog/assets/a634cb9e7783af7e.woff2", import.meta.url),
|
||||
);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(new Uint8Array(bytes))),
|
||||
);
|
||||
mockRequest(vi.fn(async () => new Response(new Uint8Array(bytes))));
|
||||
const source = `@font-face{font-family:Demo;src:url(https://public.example/${name})}`;
|
||||
const result = await downloadAndRewriteFonts(source, dir);
|
||||
const names = readdirSync(join(dir, "assets/fonts"));
|
||||
@@ -765,10 +766,7 @@ describe("capture download security boundaries", () => {
|
||||
|
||||
it("leaves invalid font URLs unchanged and records their rejection", async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(new Uint8Array(2048))),
|
||||
);
|
||||
mockRequest(vi.fn(async () => new Response(new Uint8Array(2048))));
|
||||
const source = "@font-face{font-family:Demo;src:url(https://public.example/font.woff2)}";
|
||||
const result = await downloadAndRewriteFonts(source, dir);
|
||||
expect(result.css).toBe(source);
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import { isBlockedNetworkHost } from "@hyperframes/engine";
|
||||
import { publicRequest } from "./publicNetwork.js";
|
||||
import { ensureCaptureDirSync, writeCaptureFileSync } from "./captureFile.js";
|
||||
import { join, extname } from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
@@ -670,10 +671,11 @@ export async function safeFetch(url: string, init?: RequestInit): Promise<Respon
|
||||
let current = url;
|
||||
for (let hop = 0; hop <= MAX_FETCH_REDIRECTS; hop++) {
|
||||
if (isPrivateUrl(current)) return null;
|
||||
const res = await fetch(current, { ...init, redirect: "manual" });
|
||||
const res = await publicRequest(current, { ...init, redirect: "manual" });
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const loc = res.headers.get("location");
|
||||
if (!loc) return res;
|
||||
await res.body?.cancel().catch(() => {});
|
||||
try {
|
||||
current = new URL(loc, current).toString();
|
||||
} catch {
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { captureBrowserArgs } from "./browserLaunchArgs.js";
|
||||
|
||||
describe("capture browser sandbox", () => {
|
||||
it.each(["darwin", "win32"] as const)("keeps Chromium's sandbox on %s", (platform) => {
|
||||
expect(captureBrowserArgs(false, 800, 600, platform)).not.toContain("--no-sandbox");
|
||||
});
|
||||
|
||||
it("retains the Linux container workaround", () => {
|
||||
expect(captureBrowserArgs(false, 800, 600, "linux")).toContain("--no-sandbox");
|
||||
});
|
||||
});
|
||||
@@ -2,9 +2,11 @@ export function captureBrowserArgs(
|
||||
disableWebgl: boolean,
|
||||
viewportWidth: number,
|
||||
viewportHeight: number,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
): string[] {
|
||||
const needsLinuxSandboxException = platform === "linux";
|
||||
return [
|
||||
"--no-sandbox",
|
||||
...(needsLinuxSandboxException ? ["--no-sandbox"] : []),
|
||||
"--disable-dev-shm-usage",
|
||||
...(disableWebgl
|
||||
? ["--disable-gpu"]
|
||||
|
||||
@@ -45,3 +45,21 @@ describe("captureWebsiteAttempt output folders", () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("captureWebsiteAttempt URL admission", () => {
|
||||
it.each([
|
||||
"file:///etc/passwd",
|
||||
"data:text/html,hello",
|
||||
"ftp://example.com/file",
|
||||
"https://user:secret@example.com/",
|
||||
])("rejects %s before creating capture folders or launching a browser", async (url) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-capture-url-"));
|
||||
onTestFinished(() => rmSync(root, { recursive: true, force: true }));
|
||||
const opts = { url, outputDir: root };
|
||||
await expect(
|
||||
captureWebsiteAttempt(opts, undefined, false, watchdog, createPartialCaptureState(opts)),
|
||||
).rejects.toThrow("HTTP or HTTPS URL without credentials");
|
||||
expect(readdirSync(root)).toEqual([]);
|
||||
expect(ensureBrowserMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { launchManagedBrowser } from "../browser/launch.js";
|
||||
import { launchCaptureBrowser } from "./launchCaptureBrowser.js";
|
||||
import { LottieDiscovery } from "./lottieDiscovery.js";
|
||||
import { createCaptureDownloadBudget } from "./readBoundedResponse.js";
|
||||
/**
|
||||
@@ -27,7 +27,7 @@ import type {
|
||||
ExtractedHtml,
|
||||
} from "./types.js";
|
||||
import type { CaptureWatchdog } from "./captureWatchdog.js";
|
||||
import { captureBrowserArgs } from "./browserLaunchArgs.js";
|
||||
|
||||
import type { PartialCaptureState } from "./partialCapture.js";
|
||||
import { runNavigationChecks } from "./navigationPhase.js";
|
||||
import { runCoreExtraction } from "./coreExtractionPhase.js";
|
||||
@@ -56,6 +56,14 @@ export async function captureWebsiteAttempt(
|
||||
onPhase,
|
||||
} = opts;
|
||||
|
||||
const initialUrl = new URL(url);
|
||||
if (
|
||||
!["http:", "https:"].includes(initialUrl.protocol) ||
|
||||
initialUrl.username ||
|
||||
initialUrl.password
|
||||
) {
|
||||
throw new Error("Capture requires a public HTTP or HTTPS URL without credentials");
|
||||
}
|
||||
const downloadByteBudget = createCaptureDownloadBudget();
|
||||
const warnings: string[] = [...state.warnings];
|
||||
state.warnings = warnings;
|
||||
@@ -111,14 +119,15 @@ export async function captureWebsiteAttempt(
|
||||
progress("browser", "Launching headless Chrome...");
|
||||
const { ensureBrowser } = await import("../browser/manager.js");
|
||||
const browser = await ensureBrowser();
|
||||
const puppeteer = await import("puppeteer-core");
|
||||
const chromeBrowser = await launchManagedBrowser(puppeteer.default, {
|
||||
headless: true,
|
||||
const captureBrowser = await launchCaptureBrowser({
|
||||
executablePath: browser.executablePath,
|
||||
protocolTimeout: captureProtocolTimeoutMs(timeout, budgetMs),
|
||||
args: captureBrowserArgs(disableWebgl, viewportWidth, viewportHeight),
|
||||
disableWebgl,
|
||||
width: viewportWidth,
|
||||
height: viewportHeight,
|
||||
});
|
||||
watchdog.registerBrowser(chromeBrowser);
|
||||
const chromeBrowser = captureBrowser.browser;
|
||||
watchdog.registerBrowser(captureBrowser);
|
||||
|
||||
let animationCatalog: CaptureResult["animationCatalog"];
|
||||
let tokens: DesignTokens = state.tokens;
|
||||
@@ -339,7 +348,7 @@ export async function captureWebsiteAttempt(
|
||||
lastPhase,
|
||||
};
|
||||
} finally {
|
||||
watchdog.unregisterBrowser(chromeBrowser);
|
||||
await chromeBrowser.close();
|
||||
watchdog.unregisterBrowser(captureBrowser);
|
||||
await captureBrowser.close();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
import { createServer, type IncomingMessage, type RequestOptions } from "node:http";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { existsSync } from "node:fs";
|
||||
import { createSocket } from "node:dgram";
|
||||
import { createServer as createTcpServer } from "node:net";
|
||||
|
||||
const fixture = vi.hoisted(() => ({
|
||||
port: 0,
|
||||
privateHits: 0,
|
||||
publicHits: 0,
|
||||
rebound: false,
|
||||
dialed: [] as string[],
|
||||
}));
|
||||
vi.mock("node:dns/promises", () => ({
|
||||
lookup: vi.fn(async (host: string) => {
|
||||
if (host === "capture-fixture.example.com") {
|
||||
return [{ address: fixture.rebound ? "127.0.0.1" : "93.184.216.34", family: 4 }];
|
||||
}
|
||||
throw new Error("Fixture has no such DNS name");
|
||||
}),
|
||||
}));
|
||||
vi.mock("node:http", async (importOriginal) => {
|
||||
const original = await importOriginal<typeof import("node:http")>();
|
||||
return {
|
||||
...original,
|
||||
request(options: RequestOptions, listener?: (response: IncomingMessage) => void) {
|
||||
if (options.hostname === "93.184.216.34") {
|
||||
fixture.dialed.push(options.hostname);
|
||||
// Simulate the checked public peer with a local server; assert the actual dial input.
|
||||
expect(Number(options.port)).toBe(fixture.port);
|
||||
options = { ...options, hostname: "127.0.0.1" };
|
||||
}
|
||||
return original.request(options, listener);
|
||||
},
|
||||
};
|
||||
});
|
||||
import { launchCaptureBrowser } from "./launchCaptureBrowser.js";
|
||||
|
||||
const chrome = process.env.HYPERFRAMES_TEST_BROWSER_PATH;
|
||||
describe.skipIf(!chrome || !existsSync(chrome))("real browser public transport", () => {
|
||||
const server = createServer((req, res) => {
|
||||
if (req.url?.startsWith("/private")) fixture.privateHits++;
|
||||
else fixture.publicHits++;
|
||||
if (req.url === "/redirect") {
|
||||
res.writeHead(302, { Location: `http://127.0.0.1:${fixture.port}/private` });
|
||||
res.end();
|
||||
} else if (req.url === "/page") {
|
||||
res.setHeader("Content-Type", "text/html");
|
||||
res.end(
|
||||
`<html><body>fixture<img src="http://127.0.0.1:${fixture.port}/private-image"><iframe src="http://127.0.0.1:${fixture.port}/private-frame"></iframe></body></html>`,
|
||||
);
|
||||
} else res.end("controlled fixture");
|
||||
});
|
||||
let capture: Awaited<ReturnType<typeof launchCaptureBrowser>>;
|
||||
beforeAll(async () => {
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
fixture.port = (server.address() as { port: number }).port;
|
||||
capture = await launchCaptureBrowser({
|
||||
executablePath: chrome!,
|
||||
protocolTimeout: 15_000,
|
||||
disableWebgl: true,
|
||||
width: 800,
|
||||
height: 600,
|
||||
});
|
||||
}, 20_000);
|
||||
afterAll(async () => {
|
||||
await capture?.close();
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
});
|
||||
|
||||
it("renders the public peer but never requests a private redirect or subresource", async () => {
|
||||
const page = await capture.browser.newPage();
|
||||
try {
|
||||
const base = `http://capture-fixture.example.com:${fixture.port}`;
|
||||
await page.goto(base + "/page", { waitUntil: "networkidle0", timeout: 10_000 });
|
||||
expect(await page.$eval("body", (body) => body.textContent)).toContain("fixture");
|
||||
await page
|
||||
.goto(base + "/redirect", { waitUntil: "domcontentloaded", timeout: 5_000 })
|
||||
.catch(() => {});
|
||||
expect(fixture.privateHits).toBe(0);
|
||||
expect(fixture.publicHits).toBeGreaterThan(0);
|
||||
expect(fixture.dialed.length).toBeGreaterThan(0);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("does not send direct WebRTC UDP to a private STUN peer", async () => {
|
||||
const udp = createSocket("udp4");
|
||||
let hits = 0;
|
||||
udp.on("message", () => {
|
||||
hits++;
|
||||
});
|
||||
await new Promise<void>((resolve) => udp.bind(0, "127.0.0.1", resolve));
|
||||
const port = udp.address().port;
|
||||
const page = await capture.browser.newPage();
|
||||
try {
|
||||
await page.evaluate(async (port) => {
|
||||
const peer = new RTCPeerConnection({ iceServers: [{ urls: `stun:127.0.0.1:${port}` }] });
|
||||
try {
|
||||
peer.createDataChannel("probe");
|
||||
await peer.setLocalDescription(await peer.createOffer());
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
} finally {
|
||||
peer.close();
|
||||
}
|
||||
}, port);
|
||||
expect(hits).toBe(0);
|
||||
} finally {
|
||||
await page.close();
|
||||
await new Promise<void>((resolve) => udp.close(() => resolve()));
|
||||
}
|
||||
}, 10_000);
|
||||
|
||||
it("does not connect directly to a private WebRTC TURN TCP peer", async () => {
|
||||
let hits = 0;
|
||||
const tcp = createTcpServer((socket) => {
|
||||
hits++;
|
||||
socket.destroy();
|
||||
});
|
||||
await new Promise<void>((resolve) => tcp.listen(0, "127.0.0.1", resolve));
|
||||
const address = tcp.address();
|
||||
if (!address || typeof address === "string") throw new Error("Missing fixture address");
|
||||
const page = await capture.browser.newPage();
|
||||
try {
|
||||
await page.evaluate(async (port) => {
|
||||
const peer = new RTCPeerConnection({
|
||||
iceServers: [
|
||||
{
|
||||
urls: `turn:127.0.0.1:${port}?transport=tcp`,
|
||||
username: "fixture",
|
||||
credential: "fixture",
|
||||
},
|
||||
],
|
||||
});
|
||||
try {
|
||||
peer.createDataChannel("probe");
|
||||
await peer.setLocalDescription(await peer.createOffer());
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
} finally {
|
||||
peer.close();
|
||||
}
|
||||
}, address.port);
|
||||
expect(hits).toBe(0);
|
||||
} finally {
|
||||
await page.close();
|
||||
await new Promise<void>((resolve) => tcp.close(() => resolve()));
|
||||
}
|
||||
}, 10_000);
|
||||
|
||||
it("rechecks DNS on a later request and rejects a private rebinding answer", async () => {
|
||||
const page = await capture.browser.newPage();
|
||||
try {
|
||||
const base = `http://capture-fixture.example.com:${fixture.port}`;
|
||||
await page.goto(base + "/first", { waitUntil: "domcontentloaded" });
|
||||
const reachedBefore = fixture.publicHits;
|
||||
fixture.rebound = true;
|
||||
await page
|
||||
.goto(base + "/second", { waitUntil: "domcontentloaded", timeout: 5_000 })
|
||||
.catch(() => {});
|
||||
expect(fixture.publicHits).toBe(reachedBefore);
|
||||
expect(fixture.privateHits).toBe(0);
|
||||
} finally {
|
||||
fixture.rebound = false;
|
||||
await page.close();
|
||||
}
|
||||
}, 15_000);
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createServer, request } from "node:http";
|
||||
import { connect } from "node:net";
|
||||
import { startCaptureProxy } from "./captureProxy.js";
|
||||
|
||||
const { resolveAddress } = vi.hoisted(() => ({ resolveAddress: vi.fn() }));
|
||||
vi.mock("./publicNetwork.js", () => ({ resolvePublicAddress: resolveAddress }));
|
||||
|
||||
describe("capture proxy", () => {
|
||||
const proxies: Awaited<ReturnType<typeof startCaptureProxy>>[] = [];
|
||||
afterEach(async () => {
|
||||
await Promise.all(proxies.splice(0).map((proxy) => proxy.close()));
|
||||
vi.restoreAllMocks();
|
||||
resolveAddress.mockReset();
|
||||
});
|
||||
|
||||
async function proxyGet(url: string): Promise<number> {
|
||||
const proxy = await startCaptureProxy();
|
||||
proxies.push(proxy);
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = request(proxy.url, { path: url }, (res) => {
|
||||
res.resume();
|
||||
res.on("end", () => resolve(res.statusCode!));
|
||||
});
|
||||
req.on("error", reject);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
it("refuses a private target before opening an upstream connection", async () => {
|
||||
resolveAddress.mockRejectedValue(new Error("Private destination"));
|
||||
expect(await proxyGet("http://127.0.0.1/private")).toBe(403);
|
||||
expect(resolveAddress).toHaveBeenCalledWith("127.0.0.1");
|
||||
});
|
||||
|
||||
it("refuses URL credentials and non-http requests", async () => {
|
||||
expect(await proxyGet("http://user:secret@example.com/")).toBe(403);
|
||||
expect(await proxyGet("file:///etc/passwd")).toBe(403);
|
||||
expect(resolveAddress).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not dial after teardown while DNS is pending", async () => {
|
||||
let hits = 0;
|
||||
const target = createServer((_req, res) => {
|
||||
hits++;
|
||||
res.end("fixture");
|
||||
});
|
||||
await new Promise<void>((resolve) => target.listen(0, "127.0.0.1", resolve));
|
||||
const address = target.address();
|
||||
if (!address || typeof address === "string") throw new Error("Missing fixture address");
|
||||
let release: (address: string) => void = () => {};
|
||||
let entered: () => void = () => {};
|
||||
const resolving = new Promise<void>((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
resolveAddress.mockImplementation(() => {
|
||||
entered();
|
||||
return new Promise<string>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
});
|
||||
const proxy = await startCaptureProxy();
|
||||
proxies.push(proxy);
|
||||
const finished = new Promise<void>((resolve) => {
|
||||
const req = request(proxy.url, { path: `http://fixture.example.com:${address.port}/late` });
|
||||
req.on("error", () => resolve());
|
||||
req.end();
|
||||
});
|
||||
try {
|
||||
await resolving;
|
||||
await proxy.close();
|
||||
release("127.0.0.1");
|
||||
await finished;
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
expect(hits).toBe(0);
|
||||
} finally {
|
||||
target.closeAllConnections();
|
||||
await new Promise<void>((resolve) => target.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
it("closes idle CONNECT clients during teardown", async () => {
|
||||
const proxy = await startCaptureProxy();
|
||||
proxies.push(proxy);
|
||||
const socket = connect(Number(new URL(proxy.url).port), "127.0.0.1");
|
||||
await new Promise<void>((resolve) => socket.once("connect", resolve));
|
||||
const closed = new Promise<void>((resolve) => socket.once("close", () => resolve()));
|
||||
await proxy.close();
|
||||
await closed;
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,161 @@
|
||||
import { createServer, request, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import { connect, type Socket } from "node:net";
|
||||
import type { Duplex } from "node:stream";
|
||||
import { resolvePublicAddress } from "./publicNetwork.js";
|
||||
|
||||
const CONNECTION_TIMEOUT_MS = 15_000;
|
||||
const HOP_HEADERS = new Set([
|
||||
"connection",
|
||||
"proxy-connection",
|
||||
"proxy-authorization",
|
||||
"proxy-authenticate",
|
||||
"keep-alive",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
]);
|
||||
|
||||
function forwardHeaders(message: IncomingMessage): IncomingMessage["headers"] {
|
||||
const headers = { ...message.headers };
|
||||
const connection = String(headers.connection ?? "").split(",");
|
||||
for (const name of [...HOP_HEADERS, ...connection]) delete headers[name.trim().toLowerCase()];
|
||||
return headers;
|
||||
}
|
||||
|
||||
function destination(raw: string, tunnel: boolean): URL {
|
||||
const url = new URL(tunnel ? `https://${raw}` : raw);
|
||||
if (
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.hash ||
|
||||
(tunnel
|
||||
? url.protocol !== "https:" || url.pathname !== "/" || !!url.search
|
||||
: url.protocol !== "http:")
|
||||
) {
|
||||
throw new Error("Unsupported capture destination");
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
function refuse(response: ServerResponse): void {
|
||||
if (!response.headersSent) response.writeHead(403, { Connection: "close" });
|
||||
response.end();
|
||||
}
|
||||
|
||||
async function forwardHttp(
|
||||
incoming: IncomingMessage,
|
||||
response: ServerResponse,
|
||||
sockets: Set<Socket>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const url = destination(incoming.url ?? "", false);
|
||||
const address = await resolvePublicAddress(url.hostname);
|
||||
if (incoming.destroyed || response.destroyed) return;
|
||||
const headers = forwardHeaders(incoming);
|
||||
headers.host = url.host;
|
||||
const upstream = request(
|
||||
{
|
||||
hostname: address,
|
||||
family: 4,
|
||||
port: url.port || 80,
|
||||
method: incoming.method,
|
||||
path: url.pathname + url.search,
|
||||
headers,
|
||||
agent: false,
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
},
|
||||
(received) => {
|
||||
response.writeHead(received.statusCode ?? 502, forwardHeaders(received));
|
||||
received.on("error", () => response.destroy());
|
||||
received.pipe(response);
|
||||
},
|
||||
);
|
||||
upstream.on("socket", (socket) => trackSocket(socket, sockets));
|
||||
upstream.setTimeout(CONNECTION_TIMEOUT_MS, () =>
|
||||
upstream.destroy(new Error("Capture request timed out")),
|
||||
);
|
||||
upstream.on("error", () => {
|
||||
if (!response.headersSent) response.writeHead(502, { Connection: "close" });
|
||||
response.end();
|
||||
});
|
||||
response.on("close", () => upstream.destroy());
|
||||
incoming.on("error", () => upstream.destroy());
|
||||
incoming.pipe(upstream);
|
||||
} catch {
|
||||
refuse(response);
|
||||
}
|
||||
}
|
||||
|
||||
function trackSocket(socket: Socket, sockets: Set<Socket>): void {
|
||||
sockets.add(socket);
|
||||
socket.once("close", () => sockets.delete(socket));
|
||||
// Teardown can destroy either half of a tunnel while its peer is still writing.
|
||||
socket.on("error", () => socket.destroy());
|
||||
}
|
||||
|
||||
async function forwardTunnel(
|
||||
incoming: IncomingMessage,
|
||||
client: Duplex,
|
||||
head: Buffer,
|
||||
sockets: Set<Socket>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const url = destination(incoming.url ?? "", true);
|
||||
const address = await resolvePublicAddress(url.hostname);
|
||||
if (client.destroyed) return;
|
||||
const upstream = connect({ host: address, family: 4, port: Number(url.port || 443) });
|
||||
trackSocket(upstream, sockets);
|
||||
const connectDeadline = setTimeout(() => upstream.destroy(), CONNECTION_TIMEOUT_MS);
|
||||
connectDeadline.unref();
|
||||
upstream.once("close", () => clearTimeout(connectDeadline));
|
||||
upstream.setTimeout(CONNECTION_TIMEOUT_MS, () => upstream.destroy());
|
||||
upstream.once("connect", () => {
|
||||
clearTimeout(connectDeadline);
|
||||
if (client.destroyed) return upstream.destroy();
|
||||
client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
|
||||
if (head.length) upstream.write(head);
|
||||
upstream.pipe(client);
|
||||
client.pipe(upstream);
|
||||
});
|
||||
upstream.once("close", () => client.destroy());
|
||||
client.once("close", () => upstream.destroy());
|
||||
} catch {
|
||||
client.end("HTTP/1.1 403 Forbidden\r\nConnection: close\r\nContent-Length: 0\r\n\r\n");
|
||||
}
|
||||
}
|
||||
|
||||
/** Every browser target uses this proxy; only a checked numeric IPv4 address is dialed. */
|
||||
export async function startCaptureProxy(): Promise<{ url: string; close(): Promise<void> }> {
|
||||
const sockets = new Set<Socket>();
|
||||
const server = createServer(
|
||||
(incoming, response) => void forwardHttp(incoming, response, sockets),
|
||||
);
|
||||
server.on("connection", (socket) => trackSocket(socket, sockets));
|
||||
server.on(
|
||||
"connect",
|
||||
(incoming, socket, head) => void forwardTunnel(incoming, socket, head, sockets),
|
||||
);
|
||||
server.on("upgrade", (_incoming, socket) => socket.destroy());
|
||||
server.maxConnections = 128;
|
||||
server.headersTimeout = CONNECTION_TIMEOUT_MS;
|
||||
server.requestTimeout = 30_000;
|
||||
server.keepAliveTimeout = 1_000;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("Capture proxy did not bind");
|
||||
let closing: Promise<void> | undefined;
|
||||
return {
|
||||
url: `http://127.0.0.1:${address.port}`,
|
||||
close() {
|
||||
closing ??= new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
for (const socket of sockets) socket.destroy();
|
||||
});
|
||||
return closing;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -7,26 +7,36 @@ import { Window } from "happy-dom";
|
||||
import type { Page } from "puppeteer-core";
|
||||
import { downloadAndRewriteFonts } from "./assetDownloader.js";
|
||||
import { extractHtml } from "./htmlExtractor.js";
|
||||
import { publicRequest } from "./publicNetwork.js";
|
||||
|
||||
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
|
||||
|
||||
const FONT = readFileSync(
|
||||
new URL("../../../../docs/public/catalog/assets/a634cb9e7783af7e.woff2", import.meta.url),
|
||||
);
|
||||
|
||||
function fakePage(window: Window): Page {
|
||||
const evaluate = async (script: unknown) => {
|
||||
if (typeof script !== "string") throw new Error("Expected a page expression");
|
||||
return runInNewContext(script, {
|
||||
const evaluate = async (script: unknown, ...args: unknown[]) => {
|
||||
const expression = typeof script === "function" ? `(${script.toString()})(...args)` : script;
|
||||
if (typeof expression !== "string") throw new Error("Expected a page expression");
|
||||
return runInNewContext(expression, {
|
||||
args,
|
||||
window,
|
||||
document: window.document,
|
||||
location: window.location,
|
||||
URL,
|
||||
});
|
||||
};
|
||||
return { evaluate, url: () => window.location.href } as unknown as Page;
|
||||
const addStyleTag = async ({ content }: { content: string }) => {
|
||||
const style = window.document.createElement("style");
|
||||
style.textContent = content;
|
||||
window.document.head.append(style);
|
||||
};
|
||||
return { evaluate, addStyleTag, url: () => window.location.href } as unknown as Page;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.mocked(publicRequest).mockReset();
|
||||
});
|
||||
|
||||
describe("extractHtml", () => {
|
||||
@@ -41,13 +51,10 @@ describe("extractHtml", () => {
|
||||
expect(extracted.headHtml).toContain("url(#glow)");
|
||||
|
||||
const fetched: string[] = [];
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (url: string) => {
|
||||
fetched.push(url);
|
||||
return new Response(new Uint8Array(FONT));
|
||||
}),
|
||||
);
|
||||
vi.mocked(publicRequest).mockImplementation(async (url) => {
|
||||
fetched.push(String(url));
|
||||
return new Response(new Uint8Array(FONT));
|
||||
});
|
||||
const fonts = await downloadAndRewriteFonts(extracted.headHtml, dir);
|
||||
expect(fetched).toEqual(["https://brand.example/fonts/Display-Medium.woff2"]);
|
||||
expect(fonts.css).toContain("assets/fonts/Display-Medium.woff2");
|
||||
@@ -58,3 +65,40 @@ describe("extractHtml", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("fetches Node stylesheets through validated redirects and refuses metadata redirects", async () => {
|
||||
const window = new Window({ url: "https://brand.example/" });
|
||||
window.document.head.innerHTML =
|
||||
'<link rel="stylesheet" href="https://cdn.example/start.css">' +
|
||||
'<link rel="stylesheet" href="https://cdn.example/private.css">';
|
||||
const canceled = vi.fn();
|
||||
vi.mocked(publicRequest).mockImplementation(async (url) => {
|
||||
if (String(url).endsWith("start.css"))
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: "https://styles.example/final.css" },
|
||||
});
|
||||
if (String(url).endsWith("private.css"))
|
||||
return new Response(new ReadableStream({ cancel: canceled }), {
|
||||
status: 302,
|
||||
headers: { location: "http://169.254.169.254/metadata" },
|
||||
});
|
||||
if (String(url) === "https://styles.example/final.css")
|
||||
return new Response("body{color:purple}");
|
||||
throw new Error(`Unexpected request: ${url}`);
|
||||
});
|
||||
try {
|
||||
const html = await extractHtml(fakePage(window), { settleTime: 0 });
|
||||
expect(html.headHtml).toContain("body{color:purple}");
|
||||
expect(window.document.querySelector('link[href$="start.css"]')).toBeNull();
|
||||
expect(window.document.querySelector('link[href$="private.css"]')).not.toBeNull();
|
||||
expect(vi.mocked(publicRequest).mock.calls.map(([url]) => String(url))).toEqual([
|
||||
"https://cdn.example/start.css",
|
||||
"https://styles.example/final.css",
|
||||
"https://cdn.example/private.css",
|
||||
]);
|
||||
expect(canceled).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
await window.happyDOM.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
import type { Page } from "puppeteer-core";
|
||||
import type { ExtractedHtml } from "./types.js";
|
||||
import { isPrivateUrl } from "./assetDownloader.js";
|
||||
import { safeFetch } from "./assetDownloader.js";
|
||||
import { CAPTURE_USER_AGENT } from "./userAgent.js";
|
||||
|
||||
const DEFAULT_SETTLE_TIME = 3000;
|
||||
@@ -33,12 +33,14 @@ export async function extractHtml(
|
||||
|
||||
for (const href of stylesheetUrls) {
|
||||
try {
|
||||
if (isPrivateUrl(href)) continue;
|
||||
const res = await fetch(href, {
|
||||
const res = await safeFetch(href, {
|
||||
signal: AbortSignal.timeout(10000),
|
||||
headers: { "User-Agent": CAPTURE_USER_AGENT },
|
||||
});
|
||||
if (!res.ok) continue;
|
||||
if (!res?.ok) {
|
||||
await res?.body?.cancel().catch(() => {});
|
||||
continue;
|
||||
}
|
||||
let css = await res.text();
|
||||
// Fix relative url() references
|
||||
// fallow-ignore-next-line complexity
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { existsSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({ launch: vi.fn(), proxyClose: vi.fn(), exit: vi.fn() }));
|
||||
vi.mock("puppeteer-core", () => ({ default: { launch: mocks.launch } }));
|
||||
vi.mock("./captureProxy.js", () => ({
|
||||
startCaptureProxy: async () => ({ url: "http://127.0.0.1:4567", close: mocks.proxyClose }),
|
||||
}));
|
||||
vi.mock("../utils/commandResult.js", () => ({ requestCliExit: mocks.exit }));
|
||||
|
||||
const originalSignals = new Map(
|
||||
(["SIGINT", "SIGTERM"] as const).map((signal) => [signal, process.listeners(signal)]),
|
||||
);
|
||||
let profile: string | undefined;
|
||||
afterEach(async () => {
|
||||
for (const [signal, original] of originalSignals) {
|
||||
for (const listener of process.listeners(signal)) {
|
||||
if (!original.includes(listener)) process.removeListener(signal, listener);
|
||||
}
|
||||
}
|
||||
if (profile) await rm(profile, { recursive: true, force: true });
|
||||
vi.resetModules();
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
it.each(["SIGINT", "SIGTERM"] as const)(
|
||||
"joins capture profile cleanup before root exit on %s without session.close",
|
||||
async (signal) => {
|
||||
const child = Object.assign(new EventEmitter(), {
|
||||
pid: process.pid,
|
||||
exitCode: null as number | null,
|
||||
signalCode: null,
|
||||
});
|
||||
const browser = Object.assign(new EventEmitter(), {
|
||||
process: () => child,
|
||||
close: vi.fn(async () => {
|
||||
child.exitCode = 0;
|
||||
child.emit("exit", 0);
|
||||
browser.emit("disconnected");
|
||||
}),
|
||||
});
|
||||
mocks.launch.mockResolvedValue(browser);
|
||||
mocks.proxyClose.mockResolvedValue(undefined);
|
||||
const { launchCaptureBrowser } = await import("./launchCaptureBrowser.js");
|
||||
const session = await launchCaptureBrowser({
|
||||
executablePath: "/chrome",
|
||||
protocolTimeout: 1000,
|
||||
disableWebgl: false,
|
||||
width: 800,
|
||||
height: 600,
|
||||
});
|
||||
profile = mocks.launch.mock.calls[0]?.[0].userDataDir;
|
||||
expect(existsSync(profile!)).toBe(true);
|
||||
const exit = new Promise<{ code: number; profileExists: boolean }>((resolve) => {
|
||||
mocks.exit.mockImplementation((code: number) => {
|
||||
resolve({ code, profileExists: existsSync(profile!) });
|
||||
});
|
||||
});
|
||||
process.emit(signal);
|
||||
expect(await exit).toEqual({ code: signal === "SIGINT" ? 130 : 143, profileExists: false });
|
||||
await session.close();
|
||||
await session.close();
|
||||
expect(browser.close).toHaveBeenCalledOnce();
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,73 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
const mocks = vi.hoisted(() => ({
|
||||
proxyClose: vi.fn(),
|
||||
browserClose: vi.fn(),
|
||||
once: vi.fn(),
|
||||
launch: vi.fn(),
|
||||
}));
|
||||
vi.mock("./captureProxy.js", () => ({
|
||||
startCaptureProxy: async () => ({ url: "http://127.0.0.1:4567", close: mocks.proxyClose }),
|
||||
}));
|
||||
vi.mock("../browser/launch.js", () => ({
|
||||
launchManagedBrowser: mocks.launch,
|
||||
closeManagedBrowser: mocks.browserClose,
|
||||
registerBrowserSessionCleanup: () => () => {},
|
||||
}));
|
||||
vi.mock("puppeteer-core", () => ({ default: {} }));
|
||||
import { launchCaptureBrowser } from "./launchCaptureBrowser.js";
|
||||
const options = {
|
||||
executablePath: "/chrome",
|
||||
protocolTimeout: 1000,
|
||||
disableWebgl: false,
|
||||
width: 800,
|
||||
height: 600,
|
||||
};
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
mocks.proxyClose.mockResolvedValue(undefined);
|
||||
mocks.browserClose.mockResolvedValue(undefined);
|
||||
mocks.launch.mockResolvedValue({ close: mocks.browserClose, once: mocks.once });
|
||||
});
|
||||
describe("capture browser transport lifecycle", () => {
|
||||
it("forces proxy for loopback and disables direct UDP routes", async () => {
|
||||
const session = await launchCaptureBrowser(options);
|
||||
expect(mocks.launch.mock.calls[0]?.[1].args).toEqual(
|
||||
expect.arrayContaining([
|
||||
"--proxy-server=http://127.0.0.1:4567",
|
||||
"--proxy-bypass-list=<-loopback>",
|
||||
"--disable-quic",
|
||||
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
|
||||
]),
|
||||
);
|
||||
const profile = mocks.launch.mock.calls[0]?.[1].userDataDir;
|
||||
expect(JSON.parse(readFileSync(join(profile, "Default", "Preferences"), "utf8"))).toEqual({
|
||||
webrtc: { ip_handling_policy: "disable_non_proxied_udp" },
|
||||
});
|
||||
await session.close();
|
||||
expect(existsSync(profile)).toBe(false);
|
||||
expect(mocks.proxyClose.mock.invocationCallOrder[0] ?? 0).toBeLessThan(
|
||||
mocks.browserClose.mock.invocationCallOrder[0] ?? 0,
|
||||
);
|
||||
});
|
||||
it("closes proxy when launch fails", async () => {
|
||||
mocks.launch.mockRejectedValue(new Error("launch failed"));
|
||||
await expect(launchCaptureBrowser(options)).rejects.toThrow("launch failed");
|
||||
expect(mocks.proxyClose).toHaveBeenCalledOnce();
|
||||
});
|
||||
it("closes proxy before a failing browser shutdown", async () => {
|
||||
const session = await launchCaptureBrowser(options);
|
||||
mocks.browserClose.mockRejectedValue(new Error("close failed"));
|
||||
await expect(session.close()).rejects.toThrow("close failed");
|
||||
expect(mocks.proxyClose).toHaveBeenCalledOnce();
|
||||
});
|
||||
it("closes proxy on unexpected browser disconnect", async () => {
|
||||
const session = await launchCaptureBrowser(options);
|
||||
const [event, callback] = mocks.once.mock.calls[0] ?? [];
|
||||
expect(event).toBe("disconnected");
|
||||
callback();
|
||||
expect(mocks.proxyClose).toHaveBeenCalledOnce();
|
||||
await session.close();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { Browser } from "puppeteer-core";
|
||||
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
launchManagedBrowser,
|
||||
closeManagedBrowser,
|
||||
registerBrowserSessionCleanup,
|
||||
} from "../browser/launch.js";
|
||||
import { captureBrowserArgs } from "./browserLaunchArgs.js";
|
||||
import { startCaptureProxy } from "./captureProxy.js";
|
||||
|
||||
/** The launch-wide proxy covers pages, popups, workers and both extraction passes. */
|
||||
export async function launchCaptureBrowser(options: {
|
||||
executablePath: string;
|
||||
protocolTimeout: number;
|
||||
disableWebgl: boolean;
|
||||
width: number;
|
||||
height: number;
|
||||
}): Promise<{ browser: Browser; close(): Promise<void> }> {
|
||||
const proxy = await startCaptureProxy();
|
||||
let profile: string | undefined;
|
||||
let browser: Browser | undefined;
|
||||
let closing: Promise<void> | undefined;
|
||||
let unregister: (() => void) | undefined;
|
||||
const close = () => {
|
||||
closing ??= (async () => {
|
||||
try {
|
||||
await proxy.close();
|
||||
if (browser) await closeManagedBrowser(browser);
|
||||
} finally {
|
||||
try {
|
||||
if (profile) await rm(profile, { recursive: true, force: true });
|
||||
} finally {
|
||||
unregister?.();
|
||||
}
|
||||
}
|
||||
})();
|
||||
return closing;
|
||||
};
|
||||
try {
|
||||
const profilePath = await mkdtemp(join(tmpdir(), "hf-capture-profile-"));
|
||||
profile = profilePath;
|
||||
await mkdir(join(profile, "Default"));
|
||||
await writeFile(
|
||||
join(profile, "Default", "Preferences"),
|
||||
JSON.stringify({
|
||||
webrtc: { ip_handling_policy: "disable_non_proxied_udp" },
|
||||
}),
|
||||
);
|
||||
unregister = registerBrowserSessionCleanup(close);
|
||||
const puppeteer = await import("puppeteer-core");
|
||||
browser = await launchManagedBrowser(puppeteer.default, {
|
||||
headless: true,
|
||||
userDataDir: profile,
|
||||
executablePath: options.executablePath,
|
||||
protocolTimeout: options.protocolTimeout,
|
||||
// Linux containers may lack the user namespaces Chromium's sandbox requires.
|
||||
args: [
|
||||
...captureBrowserArgs(options.disableWebgl, options.width, options.height),
|
||||
`--proxy-server=${proxy.url}`,
|
||||
"--proxy-bypass-list=<-loopback>",
|
||||
"--disable-quic",
|
||||
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
|
||||
],
|
||||
});
|
||||
browser.once("disconnected", () => void proxy.close());
|
||||
return {
|
||||
browser,
|
||||
close,
|
||||
};
|
||||
} catch (error) {
|
||||
await close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { publicRequest } from "./publicNetwork.js";
|
||||
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { discoverLottieResponse, LottieDiscovery } from "./lottieDiscovery.js";
|
||||
|
||||
@@ -14,7 +16,7 @@ describe("bounded Lottie discovery", () => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.mocked(publicRequest).mockImplementation(fetchMock);
|
||||
const budget = { remainingBytes: 1000 };
|
||||
let settled = false;
|
||||
const pending = discovery
|
||||
@@ -44,7 +46,7 @@ describe("bounded Lottie discovery", () => {
|
||||
headers: () => ({ "content-type": "application/json" }),
|
||||
});
|
||||
const fetchMock = vi.fn(async (_url: string | URL | Request) => new Response("{}"));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.mocked(publicRequest).mockImplementation(fetchMock);
|
||||
const budget = { remainingBytes: 1000 };
|
||||
expect(await discovery.run(budget, () => 10000)).toEqual([]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(32);
|
||||
@@ -60,14 +62,14 @@ describe("bounded Lottie discovery", () => {
|
||||
});
|
||||
discovery.collect({ url: () => "https://public.example/anim.lottie", headers: () => ({}) });
|
||||
const fetchMock = vi.fn(async (_url: string | URL | Request) => new Response("{}"));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.mocked(publicRequest).mockImplementation(fetchMock);
|
||||
await discovery.run({ remainingBytes: 1000 }, () => 10000);
|
||||
const urls = fetchMock.mock.calls.map((call) => call[0]);
|
||||
expect(urls).toContain("https://public.example/real.json");
|
||||
expect(urls).not.toContain("https://public.example/api/0");
|
||||
expect(urls).toHaveLength(31);
|
||||
});
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
afterEach(() => vi.mocked(publicRequest).mockReset());
|
||||
it.each([undefined, "1"])(
|
||||
"does not trust intercepted Content-Length %s or materialize Puppeteer bodies",
|
||||
async (length) => {
|
||||
@@ -83,8 +85,7 @@ describe("bounded Lottie discovery", () => {
|
||||
buffer,
|
||||
};
|
||||
const cancel = vi.fn();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.mocked(publicRequest).mockImplementation(
|
||||
vi.fn(
|
||||
async () =>
|
||||
new Response(
|
||||
@@ -106,10 +107,7 @@ describe("bounded Lottie discovery", () => {
|
||||
it("retains valid discovered JSON with its existing byte accounting", async () => {
|
||||
const data = { v: "5.12.2", w: 100, h: 100, layers: [], fr: 30, ip: 0, op: 30 };
|
||||
const body = JSON.stringify(data);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(body)),
|
||||
);
|
||||
vi.mocked(publicRequest).mockImplementation(vi.fn(async () => new Response(body)));
|
||||
const budget = { remainingBytes: body.length };
|
||||
const found = await discoverLottieResponse(
|
||||
{ url: () => "https://public.example/animation.json?version=1", headers: () => ({}) },
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { publicRequest } from "./publicNetwork.js";
|
||||
vi.mock("./publicNetwork.js", () => ({ publicRequest: vi.fn() }));
|
||||
import {
|
||||
existsSync,
|
||||
lstatSync,
|
||||
@@ -31,7 +33,7 @@ function tempDir(): string {
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
vi.mocked(publicRequest).mockReset();
|
||||
for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true });
|
||||
tempDirs.length = 0;
|
||||
});
|
||||
@@ -44,7 +46,7 @@ describe("Lottie capture budget", () => {
|
||||
remainingMs = 0;
|
||||
return new Response(JSON.stringify({ w: 100, h: 100, layers: [] }), { status: 200 });
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.mocked(publicRequest).mockImplementation(fetchMock);
|
||||
|
||||
const saved = await saveLottieAnimations(
|
||||
[{ url: "https://one.example/a.json" }, { url: "https://two.example/b.json" }],
|
||||
@@ -173,7 +175,7 @@ describe("video capture live budget", () => {
|
||||
headers: { "content-type": "video/mp4" },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.mocked(publicRequest).mockImplementation(fetchMock);
|
||||
|
||||
const capture = captureVideoManifest(page, dir, () => {}, {
|
||||
sampleMs: 10_000,
|
||||
@@ -205,10 +207,7 @@ describe("remainingVideoDownloadTimeoutMs", () => {
|
||||
describe("Lottie capture rejects unsafe persistence", () => {
|
||||
it("does not write corrupt archives as raw Lottie files", async () => {
|
||||
const dir = tempDir();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response("not a zip")),
|
||||
);
|
||||
vi.mocked(publicRequest).mockImplementation(vi.fn(async () => new Response("not a zip")));
|
||||
expect(
|
||||
await saveLottieAnimations([{ url: "https://public.example/bad.lottie" }], dir, dir),
|
||||
).toBe(0);
|
||||
@@ -216,8 +215,7 @@ describe("Lottie capture rejects unsafe persistence", () => {
|
||||
});
|
||||
it("does not publish truthy non-array layers", async () => {
|
||||
const dir = tempDir();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.mocked(publicRequest).mockImplementation(
|
||||
vi.fn(async () => new Response('{"w":100,"h":100,"layers":true}')),
|
||||
);
|
||||
expect(await saveLottieAnimations([{ url: "https://public.example/bad.json" }], dir, dir)).toBe(
|
||||
@@ -292,8 +290,7 @@ describe("media previews replace pre-planted links", () => {
|
||||
typeof expression === "function" ? { x: 0, y: 0, width: 640, height: 360 } : [descriptor],
|
||||
);
|
||||
const page = { evaluate, screenshot: vi.fn(async () => PNG) } as unknown as Page;
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.mocked(publicRequest).mockImplementation(
|
||||
vi.fn(async () => new Response(null, { status: 404 })),
|
||||
);
|
||||
|
||||
@@ -327,8 +324,7 @@ describe("media folders refuse a planted directory link (#4304)", () => {
|
||||
const { dir, outside } = plantedCapture(link);
|
||||
const screenshot = vi.fn();
|
||||
const page = { evaluate: vi.fn(async () => [descriptor]), screenshot } as unknown as Page;
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.mocked(publicRequest).mockImplementation(
|
||||
vi.fn(async () => new Response(null, { status: 404 })),
|
||||
);
|
||||
|
||||
@@ -364,7 +360,7 @@ describe("media folders refuse a planted directory link (#4304)", () => {
|
||||
async () => {
|
||||
const { dir, outside } = plantedCapture("assets/lottie");
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
vi.mocked(publicRequest).mockImplementation(fetchMock);
|
||||
|
||||
await expect(
|
||||
saveLottieAnimations(
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import {
|
||||
createServer,
|
||||
type IncomingMessage,
|
||||
type RequestOptions,
|
||||
type ServerResponse,
|
||||
} from "node:http";
|
||||
import { createServer as createHttpsServer, type RequestOptions as HttpsOptions } from "node:https";
|
||||
import { createServer as createTcpServer, type Socket } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { TLSSocket } from "node:tls";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const transport = vi.hoisted(() => ({
|
||||
lookup: vi.fn(),
|
||||
dials: [] as HttpsOptions[],
|
||||
certificate: "",
|
||||
trustCertificate: true,
|
||||
}));
|
||||
|
||||
vi.mock("node:dns/promises", () => ({ lookup: transport.lookup }));
|
||||
|
||||
// Only the already-checked numeric peer is mapped to an isolated loopback server.
|
||||
// The real Node HTTP/TLS transport, headers and certificate checks remain active.
|
||||
vi.mock("node:http", async (importOriginal) => {
|
||||
const original = await importOriginal<typeof import("node:http")>();
|
||||
return {
|
||||
...original,
|
||||
request(options: RequestOptions, callback: (response: IncomingMessage) => void) {
|
||||
if (options.hostname !== "93.184.216.34") throw new Error("Unexpected fixture peer");
|
||||
transport.dials.push(options);
|
||||
return original.request({ ...options, hostname: "127.0.0.1" }, callback);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("node:https", async (importOriginal) => {
|
||||
const original = await importOriginal<typeof import("node:https")>();
|
||||
return {
|
||||
...original,
|
||||
request(options: HttpsOptions, callback: (response: IncomingMessage) => void) {
|
||||
if (options.hostname !== "93.184.216.34") throw new Error("Unexpected fixture peer");
|
||||
transport.dials.push(options);
|
||||
return original.request(
|
||||
{
|
||||
...options,
|
||||
hostname: "127.0.0.1",
|
||||
ca: transport.trustCertificate ? transport.certificate : undefined,
|
||||
},
|
||||
callback,
|
||||
);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { publicRequest } from "./publicNetwork.js";
|
||||
|
||||
type RequestReceipt = {
|
||||
host: string | undefined;
|
||||
path: string | undefined;
|
||||
servername: string | undefined;
|
||||
};
|
||||
const receipts: RequestReceipt[] = [];
|
||||
const sockets = new Set<Socket>();
|
||||
const streams = new Set<ServerResponse>();
|
||||
let fixtureDirectory: string;
|
||||
let httpPort: number;
|
||||
let httpsPort: number;
|
||||
let malformedPort: number;
|
||||
|
||||
function handleRequest(request: IncomingMessage, response: ServerResponse) {
|
||||
const servername = (request.socket as TLSSocket).servername;
|
||||
receipts.push({
|
||||
host: request.headers.host,
|
||||
path: request.url,
|
||||
servername: typeof servername === "string" ? servername : undefined,
|
||||
});
|
||||
if (request.url === "/slow-transfer") {
|
||||
streams.add(response);
|
||||
response.writeHead(200, { "content-type": "video/mp4", "content-length": 35 * 1024 * 1024 });
|
||||
response.flushHeaders();
|
||||
let remaining = 35;
|
||||
const interval = setInterval(() => {
|
||||
response.write(Buffer.alloc(1024 * 1024));
|
||||
if (--remaining === 0) response.end();
|
||||
}, 1000);
|
||||
response.once("close", () => {
|
||||
clearInterval(interval);
|
||||
streams.delete(response);
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (request.url === "/stream") {
|
||||
streams.add(response);
|
||||
response.writeHead(200, { "content-type": "application/octet-stream" });
|
||||
response.write("first chunk");
|
||||
const interval = setInterval(() => response.write("next chunk"), 10);
|
||||
response.once("close", () => {
|
||||
clearInterval(interval);
|
||||
streams.delete(response);
|
||||
});
|
||||
return;
|
||||
}
|
||||
response.end("fixture response");
|
||||
}
|
||||
|
||||
const httpServer = createServer(handleRequest);
|
||||
const httpsServer = createHttpsServer({}, handleRequest);
|
||||
const malformedServer = createTcpServer((socket) => socket.end("not a TLS handshake\r\n"));
|
||||
const servers = [httpServer, httpsServer, malformedServer];
|
||||
|
||||
async function listen(server: (typeof servers)[number]): Promise<number> {
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("Missing fixture port");
|
||||
return address.port;
|
||||
}
|
||||
|
||||
async function waitForStreamClose() {
|
||||
const deadline = Date.now() + 2000;
|
||||
while (streams.size && Date.now() < deadline) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(streams.size).toBe(0);
|
||||
}
|
||||
|
||||
describe("publicRequest controlled socket integration", () => {
|
||||
beforeAll(async () => {
|
||||
fixtureDirectory = await mkdtemp(join(tmpdir(), "capture-public-network-"));
|
||||
const key = join(fixtureDirectory, "key.pem");
|
||||
const certificate = join(fixtureDirectory, "certificate.pem");
|
||||
execFileSync(
|
||||
"openssl",
|
||||
[
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
key,
|
||||
"-out",
|
||||
certificate,
|
||||
"-days",
|
||||
"1",
|
||||
"-subj",
|
||||
"/CN=assets.capture-fixture.com",
|
||||
"-addext",
|
||||
"subjectAltName=DNS:assets.capture-fixture.com",
|
||||
],
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
transport.certificate = await readFile(certificate, "utf8");
|
||||
httpsServer.setSecureContext({ key: await readFile(key), cert: transport.certificate });
|
||||
httpsServer.on("tlsClientError", () => {});
|
||||
for (const server of servers) {
|
||||
server.on("connection", (socket) => {
|
||||
sockets.add(socket);
|
||||
socket.once("close", () => sockets.delete(socket));
|
||||
});
|
||||
}
|
||||
[httpPort, httpsPort, malformedPort] = await Promise.all([
|
||||
listen(httpServer),
|
||||
listen(httpsServer),
|
||||
listen(malformedServer),
|
||||
]);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
receipts.length = 0;
|
||||
transport.dials.length = 0;
|
||||
transport.trustCertificate = true;
|
||||
transport.lookup.mockReset();
|
||||
transport.lookup.mockResolvedValue([{ address: "127.0.0.1", family: 4 }]);
|
||||
transport.lookup.mockResolvedValueOnce([{ address: "93.184.216.34", family: 4 }]);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
for (const socket of sockets) socket.destroy();
|
||||
await Promise.all(
|
||||
servers.map(
|
||||
(server) =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
}),
|
||||
),
|
||||
);
|
||||
await rm(fixtureDirectory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("pins the checked address without a second DNS lookup and preserves HTTP Host/path", async () => {
|
||||
const response = await publicRequest(`http://assets.capture-fixture.com:${httpPort}/asset?q=1`);
|
||||
expect(await response.text()).toBe("fixture response");
|
||||
expect(transport.lookup).toHaveBeenCalledTimes(1);
|
||||
expect(transport.lookup).toHaveBeenCalledWith("assets.capture-fixture.com", {
|
||||
all: true,
|
||||
family: 4,
|
||||
verbatim: true,
|
||||
});
|
||||
expect(transport.dials).toHaveLength(1);
|
||||
expect(transport.dials[0]?.hostname).toBe("93.184.216.34");
|
||||
expect(receipts).toEqual([
|
||||
{
|
||||
host: `assets.capture-fixture.com:${httpPort}`,
|
||||
path: "/asset?q=1",
|
||||
servername: undefined,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves HTTPS Host and SNI while verifying the trusted fixture certificate", async () => {
|
||||
const response = await publicRequest(`https://assets.capture-fixture.com:${httpsPort}/secure`);
|
||||
expect(await response.text()).toBe("fixture response");
|
||||
expect(transport.dials[0]).toMatchObject({
|
||||
hostname: "93.184.216.34",
|
||||
servername: "assets.capture-fixture.com",
|
||||
rejectUnauthorized: true,
|
||||
});
|
||||
expect(receipts).toEqual([
|
||||
{
|
||||
host: `assets.capture-fixture.com:${httpsPort}`,
|
||||
path: "/secure",
|
||||
servername: "assets.capture-fixture.com",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects an untrusted certificate before any HTTP request reaches the target", async () => {
|
||||
transport.trustCertificate = false;
|
||||
await expect(
|
||||
publicRequest(`https://assets.capture-fixture.com:${httpsPort}/secure`),
|
||||
).rejects.toThrow(/self.signed|certificate/i);
|
||||
expect(transport.dials[0]?.rejectUnauthorized).toBe(true);
|
||||
expect(receipts).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("rejects a trusted certificate with the wrong hostname", async () => {
|
||||
await expect(
|
||||
publicRequest(`https://other.capture-fixture.com:${httpsPort}/secure`),
|
||||
).rejects.toThrow(/hostname|altnames|certificate/i);
|
||||
expect(transport.dials[0]).toMatchObject({
|
||||
servername: "other.capture-fixture.com",
|
||||
rejectUnauthorized: true,
|
||||
});
|
||||
expect(receipts).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("rejects a malformed TLS handshake", async () => {
|
||||
await expect(
|
||||
publicRequest(`https://assets.capture-fixture.com:${malformedPort}/secure`),
|
||||
).rejects.toThrow(/ssl|tls|version|record/i);
|
||||
expect(receipts).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("rejects private DNS answers before dialing or reaching a target", async () => {
|
||||
transport.lookup.mockReset().mockResolvedValue([{ address: "127.0.0.1", family: 4 }]);
|
||||
await expect(
|
||||
publicRequest(`http://assets.capture-fixture.com:${httpPort}/asset`),
|
||||
).rejects.toThrow(/non-public/);
|
||||
expect(transport.dials).toHaveLength(0);
|
||||
expect(receipts).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("cancels a real streaming response without uncaught teardown errors", async () => {
|
||||
const response = await publicRequest(`http://assets.capture-fixture.com:${httpPort}/stream`);
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) throw new Error("Missing fixture stream");
|
||||
expect((await reader.read()).value?.byteLength).toBeGreaterThan(0);
|
||||
await reader.cancel();
|
||||
await waitForStreamClose();
|
||||
expect(receipts).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("finishes a 35-second HTTPS body within the caller's 120-second deadline", async () => {
|
||||
const response = await publicRequest(
|
||||
`https://assets.capture-fixture.com:${httpsPort}/slow-transfer`,
|
||||
{ signal: AbortSignal.timeout(120_000) },
|
||||
);
|
||||
expect((await response.arrayBuffer()).byteLength).toBe(35 * 1024 * 1024);
|
||||
await waitForStreamClose();
|
||||
}, 60_000);
|
||||
|
||||
it.each([true, false])(
|
||||
"uses the caller deadline when supplied, otherwise the default (%s)",
|
||||
async (explicitDeadline) => {
|
||||
const deadline = new AbortController();
|
||||
const caller = new AbortController();
|
||||
const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(deadline.signal);
|
||||
let reader: ReadableStreamDefaultReader<Uint8Array> | undefined;
|
||||
try {
|
||||
const response = await publicRequest(
|
||||
`https://assets.capture-fixture.com:${httpsPort}/stream`,
|
||||
explicitDeadline ? { signal: caller.signal } : {},
|
||||
);
|
||||
reader = response.body!.getReader();
|
||||
await reader.read();
|
||||
// Advance only the transport's default deadline; real TLS/body I/O stays active.
|
||||
deadline.abort(new Error("default deadline reached"));
|
||||
if (explicitDeadline) {
|
||||
expect((await reader.read()).value?.byteLength).toBeGreaterThan(0);
|
||||
caller.abort();
|
||||
await expect(reader.read()).rejects.toThrow();
|
||||
} else {
|
||||
await expect(reader.read()).rejects.toThrow();
|
||||
}
|
||||
} finally {
|
||||
timeout.mockRestore();
|
||||
await reader?.cancel().catch(() => {});
|
||||
await waitForStreamClose();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("propagates caller abort after HTTPS response headers and closes the socket", async () => {
|
||||
const controller = new AbortController();
|
||||
const response = await publicRequest(`https://assets.capture-fixture.com:${httpsPort}/stream`, {
|
||||
signal: controller.signal,
|
||||
});
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) throw new Error("Missing fixture stream");
|
||||
await reader.read();
|
||||
controller.abort();
|
||||
await expect(reader.read()).rejects.toThrow();
|
||||
await waitForStreamClose();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,177 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { PassThrough } from "node:stream";
|
||||
const mocks = vi.hoisted(() => ({ lookup: vi.fn(), request: vi.fn() }));
|
||||
vi.mock("node:dns/promises", () => ({ lookup: mocks.lookup }));
|
||||
vi.mock("node:http", () => ({ request: mocks.request }));
|
||||
vi.mock("node:https", () => ({ request: mocks.request }));
|
||||
import { isPublicIPv4, publicRequest, resolvePublicAddress } from "./publicNetwork.js";
|
||||
|
||||
beforeEach(() => vi.resetAllMocks());
|
||||
describe("public address policy", () => {
|
||||
it.each([
|
||||
"127.0.0.1",
|
||||
"10.0.0.1",
|
||||
"172.16.0.1",
|
||||
"192.168.0.1",
|
||||
"169.254.169.254",
|
||||
"100.64.0.1",
|
||||
"0.1.2.3",
|
||||
"192.0.0.9",
|
||||
"192.0.2.1",
|
||||
"198.18.0.1",
|
||||
"198.51.100.1",
|
||||
"203.0.113.1",
|
||||
"224.0.0.1",
|
||||
"240.0.0.1",
|
||||
"::1",
|
||||
"::ffff:8.8.8.8",
|
||||
"8.8.8.999",
|
||||
])("rejects %s", (ip) => expect(isPublicIPv4(ip)).toBe(false));
|
||||
it("accepts public literals without DNS", async () => {
|
||||
await expect(resolvePublicAddress("8.8.8.8")).resolves.toBe("8.8.8.8");
|
||||
expect(mocks.lookup).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([
|
||||
"localhost",
|
||||
"localhost.",
|
||||
"foo.localhost",
|
||||
"router.local",
|
||||
"metadata.google.internal",
|
||||
"[::1]",
|
||||
])("blocks alias/literal %s before DNS", async (host) => {
|
||||
await expect(resolvePublicAddress(host)).rejects.toThrow();
|
||||
expect(mocks.lookup).not.toHaveBeenCalled();
|
||||
});
|
||||
it("rejects mixed DNS before opening a socket", async () => {
|
||||
mocks.lookup.mockResolvedValue([
|
||||
{ address: "8.8.8.8", family: 4 },
|
||||
{ address: "127.0.0.1", family: 4 },
|
||||
]);
|
||||
await expect(publicRequest("http://example.com/")).rejects.toThrow();
|
||||
expect(mocks.request).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([{ answers: [] }, { answers: [{ address: "8.8.8.8", family: 6 }] }])(
|
||||
"rejects empty or malformed DNS",
|
||||
async ({ answers }) => {
|
||||
mocks.lookup.mockResolvedValue(answers);
|
||||
await expect(resolvePublicAddress("example.com")).rejects.toThrow();
|
||||
},
|
||||
);
|
||||
it("propagates DNS errors", async () => {
|
||||
mocks.lookup.mockRejectedValue(new Error("DNS failed"));
|
||||
await expect(resolvePublicAddress("example.com")).rejects.toThrow("DNS failed");
|
||||
});
|
||||
});
|
||||
|
||||
function fakeRequest() {
|
||||
const req = Object.assign(new EventEmitter(), {
|
||||
end: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
setTimeout: vi.fn(),
|
||||
});
|
||||
mocks.request.mockImplementation((_options, callback) => {
|
||||
req.end.mockImplementation(() => {
|
||||
const incoming = Object.assign(new PassThrough(), {
|
||||
statusCode: 302,
|
||||
statusMessage: "Found",
|
||||
headers: { location: "http://127.0.0.1/" },
|
||||
});
|
||||
callback(incoming);
|
||||
incoming.end("redirect");
|
||||
});
|
||||
return req;
|
||||
});
|
||||
return req;
|
||||
}
|
||||
describe("pinned request", () => {
|
||||
it("dials the checked numeric IP only, preserving TLS identity and manual redirects", async () => {
|
||||
mocks.lookup
|
||||
.mockResolvedValueOnce([{ address: "8.8.8.8", family: 4 }])
|
||||
.mockResolvedValue([{ address: "127.0.0.1", family: 4 }]);
|
||||
fakeRequest();
|
||||
const response = await publicRequest("https://example.com:8443/path?q=1", {
|
||||
headers: { host: "evil.test", "accept-encoding": "gzip" },
|
||||
});
|
||||
expect(mocks.lookup).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.request).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.request.mock.calls[0]?.[0]).toMatchObject({
|
||||
hostname: "8.8.8.8",
|
||||
servername: "example.com",
|
||||
rejectUnauthorized: true,
|
||||
port: "8443",
|
||||
path: "/path?q=1",
|
||||
agent: false,
|
||||
headers: { host: "example.com:8443", "accept-encoding": "identity" },
|
||||
});
|
||||
expect(response.status).toBe(302);
|
||||
expect(await response.text()).toBe("redirect");
|
||||
});
|
||||
it.each(["file:///tmp/a", "http://user:secret@example.com"])(
|
||||
"rejects unsafe URL %s",
|
||||
async (url) => {
|
||||
await expect(publicRequest(url)).rejects.toThrow();
|
||||
expect(mocks.request).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
it("does not dial after abort during DNS", async () => {
|
||||
const controller = new AbortController();
|
||||
mocks.lookup.mockImplementation(async () => {
|
||||
controller.abort();
|
||||
return [{ address: "8.8.8.8", family: 4 }];
|
||||
});
|
||||
await expect(
|
||||
publicRequest("https://example.com", { signal: controller.signal }),
|
||||
).rejects.toThrow();
|
||||
expect(mocks.request).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("transport lifecycle", () => {
|
||||
it("bounds stalled DNS without dialing", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
mocks.lookup.mockImplementation(() => new Promise(() => {}));
|
||||
const result = expect(publicRequest("https://example.com")).rejects.toThrow("DNS timed out");
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
await result;
|
||||
expect(mocks.request).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
it("cancels the upstream stream and socket when the body is cancelled", async () => {
|
||||
mocks.lookup.mockResolvedValue([{ address: "8.8.8.8", family: 4 }]);
|
||||
const incoming = Object.assign(new PassThrough(), { statusCode: 200, headers: {} });
|
||||
const req = Object.assign(new EventEmitter(), { end: vi.fn(), destroy: vi.fn() });
|
||||
mocks.request.mockImplementation((_options, callback) => {
|
||||
req.end.mockImplementation(() => callback(incoming));
|
||||
return req;
|
||||
});
|
||||
const response = await publicRequest("https://example.com");
|
||||
await response.body?.cancel();
|
||||
expect(incoming.destroyed).toBe(true);
|
||||
expect(req.destroy).toHaveBeenCalled();
|
||||
// Socket teardown may race already queued terminal/data notifications.
|
||||
expect(() => {
|
||||
incoming.emit("aborted");
|
||||
incoming.emit("error", new Error("late socket error"));
|
||||
incoming.emit("data", Buffer.from("late bytes"));
|
||||
incoming.emit("end");
|
||||
}).not.toThrow();
|
||||
});
|
||||
it("fails unsupported compressed responses instead of returning corrupted asset bytes", async () => {
|
||||
mocks.lookup.mockResolvedValue([{ address: "8.8.8.8", family: 4 }]);
|
||||
const incoming = Object.assign(new PassThrough(), {
|
||||
statusCode: 200,
|
||||
headers: { "content-encoding": "gzip" },
|
||||
});
|
||||
const req = Object.assign(new EventEmitter(), { end: vi.fn(), destroy: vi.fn() });
|
||||
mocks.request.mockImplementation((_options, callback) => {
|
||||
req.end.mockImplementation(() => callback(incoming));
|
||||
return req;
|
||||
});
|
||||
await expect(publicRequest("https://example.com")).rejects.toThrow("content encoding");
|
||||
expect(incoming.destroyed).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,195 @@
|
||||
import { lookup } from "node:dns/promises";
|
||||
import { isIP } from "node:net";
|
||||
import { request as httpRequest, type IncomingMessage, type ClientRequest } from "node:http";
|
||||
import { request as httpsRequest } from "node:https";
|
||||
|
||||
function ipv4Number(address: string): number {
|
||||
return address.split(".").reduce((value, octet) => value * 256 + Number(octet), 0);
|
||||
}
|
||||
|
||||
// Conservatively exclude special-use networks, including all of 192.0/16.
|
||||
const NON_PUBLIC_IPV4: ReadonlyArray<readonly [string, number]> = [
|
||||
["0.0.0.0", 8],
|
||||
["10.0.0.0", 8],
|
||||
["100.64.0.0", 10],
|
||||
["127.0.0.0", 8],
|
||||
["169.254.0.0", 16],
|
||||
["172.16.0.0", 12],
|
||||
["192.0.0.0", 16],
|
||||
["192.88.99.0", 24],
|
||||
["192.168.0.0", 16],
|
||||
["198.18.0.0", 15],
|
||||
["198.51.100.0", 24],
|
||||
["203.0.113.0", 24],
|
||||
["224.0.0.0", 3],
|
||||
];
|
||||
const NON_PUBLIC_RANGES = NON_PUBLIC_IPV4.map(([network, prefix]) => ({
|
||||
start: ipv4Number(network),
|
||||
end: ipv4Number(network) + 2 ** (32 - prefix),
|
||||
}));
|
||||
|
||||
/** Conservative IPv4-only policy: special-use addresses fail closed. */
|
||||
export function isPublicIPv4(address: string): boolean {
|
||||
if (isIP(address) !== 4) return false;
|
||||
const value = ipv4Number(address);
|
||||
return !NON_PUBLIC_RANGES.some(({ start, end }) => value >= start && value < end);
|
||||
}
|
||||
|
||||
/** Validate every A answer, then return a numeric peer that callers must dial directly. */
|
||||
export async function resolvePublicAddress(hostname: string): Promise<string> {
|
||||
const host = hostname.toLowerCase().replace(/\.$/, "");
|
||||
if (isIP(host) === 4) {
|
||||
if (!isPublicIPv4(host)) throw new Error("Capture requires a public IPv4 address");
|
||||
return host;
|
||||
}
|
||||
if (
|
||||
!host.includes(".") ||
|
||||
!/^[a-z0-9.-]+$/.test(host) ||
|
||||
/(?:^|\.)(?:localhost|local|internal|home|lan|test|invalid|example)$/.test(host) ||
|
||||
host.split(".").some((label) => !label || label.startsWith("-") || label.endsWith("-"))
|
||||
) {
|
||||
throw new Error("Capture requires a public hostname");
|
||||
}
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const answers = await Promise.race([
|
||||
lookup(host, { all: true, family: 4, verbatim: true }),
|
||||
new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new Error("Capture DNS timed out")), 10_000);
|
||||
timer.unref();
|
||||
}),
|
||||
]);
|
||||
if (
|
||||
!answers[0] ||
|
||||
answers.some((answer) => answer.family !== 4 || !isPublicIPv4(answer.address))
|
||||
) {
|
||||
throw new Error("Capture DNS returned a non-public address");
|
||||
}
|
||||
const first = answers[0];
|
||||
if (!first) throw new Error("Capture DNS returned no address");
|
||||
return first.address;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/** Pinned asset transport; redirects are returned for caller validation. */
|
||||
export async function publicRequest(url: string | URL, init: RequestInit = {}): Promise<Response> {
|
||||
const target = new URL(url);
|
||||
if (!["http:", "https:"].includes(target.protocol) || target.username || target.password) {
|
||||
throw new Error("Capture requires an HTTP(S) URL without credentials");
|
||||
}
|
||||
const method = (init.method ?? "GET").toUpperCase();
|
||||
if (!["GET", "HEAD"].includes(method) || init.body != null) {
|
||||
throw new Error("Capture asset requests support only GET and HEAD without a body");
|
||||
}
|
||||
// Video callers budget the entire body transfer; a default must not shorten it.
|
||||
const signal = init.signal ?? AbortSignal.timeout(30_000);
|
||||
signal.throwIfAborted();
|
||||
const address = await new Promise<string>((resolve, reject) => {
|
||||
const onAbort = () => reject(signal.reason);
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
resolvePublicAddress(target.hostname)
|
||||
.then(resolve, reject)
|
||||
.finally(() => signal.removeEventListener("abort", onAbort));
|
||||
});
|
||||
signal.throwIfAborted();
|
||||
const headers = new Headers(init.headers);
|
||||
for (const name of [
|
||||
"connection",
|
||||
"proxy-authorization",
|
||||
"proxy-connection",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
"content-length",
|
||||
])
|
||||
headers.delete(name);
|
||||
headers.set("host", target.host);
|
||||
headers.set("accept-encoding", "identity");
|
||||
return new Promise<Response>((resolve, reject) => {
|
||||
const request = target.protocol === "https:" ? httpsRequest : httpRequest;
|
||||
const req = request(
|
||||
{
|
||||
protocol: target.protocol,
|
||||
hostname: address,
|
||||
port: target.port || undefined,
|
||||
path: `${target.pathname}${target.search}`,
|
||||
method,
|
||||
headers: Object.fromEntries(headers),
|
||||
agent: false,
|
||||
servername: target.hostname,
|
||||
rejectUnauthorized: true,
|
||||
signal,
|
||||
},
|
||||
(incoming) => {
|
||||
try {
|
||||
resolve(assetResponse(incoming, req, method));
|
||||
} catch (error) {
|
||||
incoming.destroy();
|
||||
reject(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
req.once("error", reject);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function responseHeaders(incoming: IncomingMessage): Headers {
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(incoming.headers)) {
|
||||
if (value === undefined) continue;
|
||||
for (const item of Array.isArray(value) ? value : [value]) headers.append(name, item);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
function assetResponse(incoming: IncomingMessage, req: ClientRequest, method: string): Response {
|
||||
// Even discarded/null bodies can emit errors during socket teardown.
|
||||
incoming.on("error", () => {});
|
||||
const headers = responseHeaders(incoming);
|
||||
const encoding = headers.get("content-encoding")?.toLowerCase();
|
||||
if (encoding && encoding !== "identity") {
|
||||
throw new Error("Capture server ignored identity content encoding");
|
||||
}
|
||||
const status = incoming.statusCode ?? 502;
|
||||
if (method === "HEAD" || [204, 205, 304].includes(status)) {
|
||||
incoming.resume();
|
||||
return new Response(null, { status, headers });
|
||||
}
|
||||
return new Response(responseBody(incoming, req), { status, headers });
|
||||
}
|
||||
|
||||
function responseBody(incoming: IncomingMessage, req: ClientRequest): ReadableStream<Uint8Array> {
|
||||
let finished = false;
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
incoming.on("data", (chunk: Buffer) => {
|
||||
if (finished) return;
|
||||
controller.enqueue(chunk);
|
||||
incoming.pause();
|
||||
});
|
||||
incoming.once("end", () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
controller.close();
|
||||
});
|
||||
const fail = (error: Error) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
controller.error(error);
|
||||
};
|
||||
incoming.once("error", fail);
|
||||
incoming.once("aborted", () => fail(new Error("Capture response aborted")));
|
||||
incoming.pause();
|
||||
},
|
||||
pull() {
|
||||
incoming.resume();
|
||||
},
|
||||
cancel() {
|
||||
finished = true;
|
||||
incoming.destroy();
|
||||
req.destroy();
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -125,24 +125,121 @@ describe("generateProjectScaffold metadata", () => {
|
||||
});
|
||||
|
||||
describe("loadEnvFile", () => {
|
||||
const keys = ["hyperframes_feedback_email", "HyperFrames_Feedback_Email", "HF_SCAFFOLD_TEST_KEY"];
|
||||
const keys = ["hyperframes_feedback_email", "HyperFrames_Feedback_Email", "GEMINI_API_KEY"];
|
||||
let dir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = fs.mkdtempSync(join(tmpdir(), "hf-scaffold-env-"));
|
||||
vi.stubEnv("GEMINI_API_KEY", undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const key of keys) delete process.env[key];
|
||||
vi.unstubAllEnvs();
|
||||
for (const key of keys.slice(0, 2)) delete process.env[key];
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("never takes the feedback email from a project file, in any letter case", () => {
|
||||
fs.writeFileSync(
|
||||
join(dir, ".env"),
|
||||
'hyperframes_feedback_email=a@example.com\nHyperFrames_Feedback_Email=b@example.com\nHF_SCAFFOLD_TEST_KEY="kept"\n',
|
||||
'hyperframes_feedback_email=a@example.com\nHyperFrames_Feedback_Email=b@example.com\nGEMINI_API_KEY="kept"\n',
|
||||
);
|
||||
loadEnvFile(join(dir, "out"));
|
||||
expect(keys.map((key) => process.env[key])).toEqual([undefined, undefined, "kept"]);
|
||||
});
|
||||
|
||||
it("never accepts browser executables from an output ancestor", () => {
|
||||
const browserKeys = [
|
||||
"HYPERFRAMES_BROWSER_PATH",
|
||||
"PRODUCER_HEADLESS_SHELL_PATH",
|
||||
"hyperframes_browser_path",
|
||||
];
|
||||
const before = browserKeys.map((key) => process.env[key]);
|
||||
try {
|
||||
for (const key of browserKeys) delete process.env[key];
|
||||
fs.writeFileSync(
|
||||
join(dir, ".env"),
|
||||
browserKeys.map((key) => `${key}=/untrusted/browser`).join("\n"),
|
||||
);
|
||||
loadEnvFile(join(dir, "out"));
|
||||
expect(browserKeys.map((key) => process.env[key])).toEqual([undefined, undefined, undefined]);
|
||||
} finally {
|
||||
browserKeys.forEach((key, index) => {
|
||||
const value = before[index];
|
||||
if (value === undefined) delete process.env[key];
|
||||
else process.env[key] = value;
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
"PATH",
|
||||
"Path",
|
||||
"LOCALAPPDATA",
|
||||
"PROGRAMFILES",
|
||||
"PROGRAMFILES(X86)",
|
||||
"HOME",
|
||||
"LD_PRELOAD",
|
||||
"ld_preload",
|
||||
"LD_LIBRARY_PATH",
|
||||
"DYLD_INSERT_LIBRARIES",
|
||||
"NODE_OPTIONS",
|
||||
"Node_Options",
|
||||
"HYPERFRAMES_BROWSER_PATH",
|
||||
"PRODUCER_HEADLESS_SHELL_PATH",
|
||||
"localappdata",
|
||||
"UNRECOGNIZED_FUTURE_SETTING",
|
||||
])("does not import launcher or unknown setting %s", (key) => {
|
||||
vi.stubEnv(key, undefined);
|
||||
fs.writeFileSync(join(dir, ".env"), `${key}=/untrusted/value\nGEMINI_API_KEY=allowed\n`);
|
||||
loadEnvFile(join(dir, "out"));
|
||||
expect(process.env[key]).toBeUndefined();
|
||||
expect(process.env.GEMINI_API_KEY).toBe("allowed");
|
||||
});
|
||||
|
||||
it("does not import a supported provider setting with the wrong letter case", () => {
|
||||
fs.writeFileSync(join(dir, ".env"), "gemini_api_key=/untrusted/value\n");
|
||||
loadEnvFile(join(dir, "out"));
|
||||
expect(process.env.gemini_api_key).toBeUndefined();
|
||||
expect(process.env.GEMINI_API_KEY).toBeUndefined();
|
||||
});
|
||||
|
||||
it("loads supported vision provider settings without replacing inherited credentials", () => {
|
||||
const settings = {
|
||||
OPENROUTER_API_KEY: "router-key",
|
||||
GOOGLE_API_KEY: "google-key",
|
||||
HYPERFRAMES_VERTEX_PROJECT_ID: "example-project",
|
||||
HYPERFRAMES_VERTEX_SERVICE_ACCOUNT: '{"client_email":"test@example.com"}',
|
||||
HYPERFRAMES_VERTEX_LOCATION: "us-central1",
|
||||
HYPERFRAMES_VERTEX_MODEL: "vertex-model",
|
||||
HYPERFRAMES_OPENROUTER_MODEL: "router-model",
|
||||
HYPERFRAMES_GEMINI_MODEL: "gemini-model",
|
||||
HYPERFRAMES_VISION_TIMEOUT_MS: "2000",
|
||||
};
|
||||
for (const key of Object.keys(settings)) vi.stubEnv(key, undefined);
|
||||
vi.stubEnv("GEMINI_API_KEY", "inherited-key");
|
||||
fs.writeFileSync(
|
||||
join(dir, ".env"),
|
||||
[
|
||||
"GEMINI_API_KEY=project-key",
|
||||
...Object.entries(settings).map(([key, value]) => `${key}=${value}`),
|
||||
].join("\n"),
|
||||
);
|
||||
loadEnvFile(join(dir, "out"));
|
||||
expect(process.env.GEMINI_API_KEY).toBe("inherited-key");
|
||||
for (const [key, value] of Object.entries(settings)) expect(process.env[key]).toBe(value);
|
||||
});
|
||||
|
||||
it("can skip loading the output's dotenv file entirely", () => {
|
||||
const previous = process.env.HYPERFRAMES_NO_CAPTURE_DOTENV;
|
||||
try {
|
||||
process.env.HYPERFRAMES_NO_CAPTURE_DOTENV = "1";
|
||||
fs.writeFileSync(join(dir, ".env"), "GEMINI_API_KEY=untrusted\n");
|
||||
loadEnvFile(join(dir, "out"));
|
||||
expect(process.env.GEMINI_API_KEY).toBeUndefined();
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.HYPERFRAMES_NO_CAPTURE_DOTENV;
|
||||
else process.env.HYPERFRAMES_NO_CAPTURE_DOTENV = previous;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,22 +7,23 @@
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { applyDotEnv } from "../utils/dotEnv.js";
|
||||
import { applyDotEnv, CAPTURE_ENV_KEYS } from "../utils/dotEnv.js";
|
||||
import { writeNewFileSync } from "../utils/writeNewFile.js";
|
||||
import type { CatalogedAsset } from "./assetCataloger.js";
|
||||
import type { CaptureResult, DesignTokens } from "./types.js";
|
||||
|
||||
/**
|
||||
* Load .env file by walking up from startDir (up to 5 levels).
|
||||
* Sets process.env keys that are not already set. Best-effort — never throws.
|
||||
* Load supported vision settings from .env by walking up from startDir (up to 5 levels).
|
||||
* Keeps inherited settings. Best-effort — never throws.
|
||||
*/
|
||||
export function loadEnvFile(startDir: string): void {
|
||||
if (process.env.HYPERFRAMES_NO_CAPTURE_DOTENV === "1") return;
|
||||
try {
|
||||
let dir = resolve(startDir);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const envPath = resolve(dir, ".env");
|
||||
try {
|
||||
applyDotEnv(readFileSync(envPath, "utf-8"), process.env);
|
||||
applyDotEnv(readFileSync(envPath, "utf-8"), process.env, CAPTURE_ENV_KEYS);
|
||||
break;
|
||||
} catch {
|
||||
dir = resolve(dir, "..");
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { mockTelemetry } from "./cliDispatchTestUtils.js";
|
||||
|
||||
const originalArgv = [...process.argv];
|
||||
const originalExitCode = process.exitCode;
|
||||
const dirs: string[] = [];
|
||||
const eventNames = ["exit", "uncaughtException", "unhandledRejection"] as const;
|
||||
const listeners = eventNames.map((event) => [event, process.listeners(event)] as const);
|
||||
const streams = [process.stdout, process.stderr].map(
|
||||
(stream) => [stream, stream.listeners("error")] as const,
|
||||
);
|
||||
afterEach(() => {
|
||||
process.argv = [...originalArgv];
|
||||
process.exitCode = originalExitCode;
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
for (const path of [
|
||||
"./commands/capture.js",
|
||||
"./commands/init.js",
|
||||
"./telemetry/index.js",
|
||||
"./telemetry/events.js",
|
||||
])
|
||||
vi.doUnmock(path);
|
||||
vi.resetModules();
|
||||
for (const [event, original] of listeners) {
|
||||
for (const listener of process.listeners(event)) {
|
||||
if (!original.includes(listener)) process.removeListener(event, listener);
|
||||
}
|
||||
}
|
||||
for (const [stream, original] of streams) {
|
||||
for (const listener of stream.listeners("error")) {
|
||||
if (!original.includes(listener)) stream.removeListener("error", listener);
|
||||
}
|
||||
}
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it.each(
|
||||
(
|
||||
[
|
||||
["capture", false, undefined, "fixture-key", undefined],
|
||||
["capture", true, undefined, undefined, undefined],
|
||||
["init", false, process.execPath, "fixture-key", undefined],
|
||||
["init", true, process.execPath, "fixture-key", undefined],
|
||||
["capture", false, process.execPath, "fixture-key", process.execPath],
|
||||
] as const
|
||||
).flatMap((test) =>
|
||||
[[], ["--json"], ["--json", "--verbose"]].map((prefix) => [...test, prefix] as const),
|
||||
),
|
||||
)(
|
||||
"applies CWD dotenv policy at real CLI startup: %s, opt-out=%s",
|
||||
async (command, skip, browser, key, inheritedBrowser, prefix) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-cli-dotenv-"));
|
||||
dirs.push(dir);
|
||||
writeFileSync(
|
||||
join(dir, ".env"),
|
||||
`HYPERFRAMES_BROWSER_PATH=${process.execPath}\nNODE_OPTIONS=--no-warnings\nGEMINI_API_KEY=fixture-key\n`,
|
||||
);
|
||||
vi.spyOn(process, "cwd").mockReturnValue(dir);
|
||||
vi.stubEnv("HYPERFRAMES_BROWSER_PATH", inheritedBrowser);
|
||||
vi.stubEnv("NODE_OPTIONS", undefined);
|
||||
vi.stubEnv("GEMINI_API_KEY", undefined);
|
||||
vi.stubEnv("HYPERFRAMES_NO_CAPTURE_DOTENV", skip ? "1" : undefined);
|
||||
let observed: unknown;
|
||||
const definition = {
|
||||
default: {
|
||||
meta: { name: command },
|
||||
args: {},
|
||||
run: () => {
|
||||
observed = [
|
||||
process.env.HYPERFRAMES_BROWSER_PATH,
|
||||
process.env.NODE_OPTIONS,
|
||||
process.env.GEMINI_API_KEY,
|
||||
];
|
||||
},
|
||||
},
|
||||
};
|
||||
vi.doMock("./commands/capture.js", () => definition);
|
||||
vi.doMock("./commands/init.js", () => definition);
|
||||
mockTelemetry();
|
||||
process.argv = ["node", "cli.ts", ...prefix, command];
|
||||
await import("./cli.js");
|
||||
expect(observed).toEqual([browser, command === "capture" ? undefined : "--no-warnings", key]);
|
||||
},
|
||||
);
|
||||
+20
-11
@@ -60,17 +60,26 @@ if (rootVersionRequested) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── Load .env from CWD ─────────────────────────────────────────────────────
|
||||
// Agents run from the project directory where .env holds API keys (Gemini,
|
||||
// HeyGen, ElevenLabs). Load it automatically so they don't need `source .env`.
|
||||
const { applyDotEnv } = await import("./utils/dotEnv.js");
|
||||
try {
|
||||
const { readFileSync } = await import("node:fs");
|
||||
const { resolve } = await import("node:path");
|
||||
const envPath = resolve(process.cwd(), ".env");
|
||||
applyDotEnv(readFileSync(envPath, "utf-8"), process.env);
|
||||
} catch {
|
||||
/* .env not present — fine, env vars may be set another way */
|
||||
// Capture treats CWD as project data too: its early loader must use the same
|
||||
// allowlist/opt-out as output ancestors, before browser selection can observe it.
|
||||
const { applyDotEnv, CAPTURE_ENV_KEYS } = await import("./utils/dotEnv.js");
|
||||
// The root has no value-taking options: Citty skips flags up to the first
|
||||
// positional command, and stops subcommand selection at the -- separator.
|
||||
const separator = argv.indexOf("--");
|
||||
const commandArgs = separator < 0 ? argv : argv.slice(0, separator);
|
||||
const captureEnv = commandArgs.find((arg) => !arg.startsWith("-")) === "capture";
|
||||
if (!captureEnv || process.env.HYPERFRAMES_NO_CAPTURE_DOTENV !== "1") {
|
||||
try {
|
||||
const { readFileSync } = await import("node:fs");
|
||||
const envPath = resolve(process.cwd(), ".env");
|
||||
applyDotEnv(
|
||||
readFileSync(envPath, "utf-8"),
|
||||
process.env,
|
||||
captureEnv ? CAPTURE_ENV_KEYS : undefined,
|
||||
);
|
||||
} catch {
|
||||
/* .env not present — fine, env vars may be set another way */
|
||||
}
|
||||
}
|
||||
|
||||
// ── Lazy imports ────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -22,4 +22,26 @@ describe("applyDotEnv", () => {
|
||||
);
|
||||
expect(env).toEqual({ OTHER: "1" });
|
||||
});
|
||||
|
||||
it("copies only exact allowed names when a caller limits the imported settings", () => {
|
||||
const env: NodeJS.ProcessEnv = { GEMINI_API_KEY: "inherited" };
|
||||
applyDotEnv(
|
||||
"GEMINI_API_KEY=project\nGOOGLE_API_KEY=allowed\ngoogle_api_key=other\nPATH=/untrusted\nLD_PRELOAD=payload",
|
||||
env,
|
||||
["GEMINI_API_KEY", "GOOGLE_API_KEY"],
|
||||
);
|
||||
expect(env).toEqual({ GEMINI_API_KEY: "inherited", GOOGLE_API_KEY: "allowed" });
|
||||
});
|
||||
|
||||
it("an allowlist cannot opt the feedback email back in", () => {
|
||||
const env: NodeJS.ProcessEnv = {};
|
||||
applyDotEnv("HyperFrames_Feedback_Email=private", env, ["HyperFrames_Feedback_Email"]);
|
||||
expect(env).toEqual({});
|
||||
});
|
||||
|
||||
it("an empty allowlist imports nothing", () => {
|
||||
const env: NodeJS.ProcessEnv = {};
|
||||
applyDotEnv("GEMINI_API_KEY=project", env, []);
|
||||
expect(env).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,18 @@
|
||||
import { FEEDBACK_EMAIL_ENV } from "./feedbackSource.js";
|
||||
|
||||
export const CAPTURE_ENV_KEYS = [
|
||||
"OPENROUTER_API_KEY",
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_API_KEY",
|
||||
"HYPERFRAMES_VERTEX_PROJECT_ID",
|
||||
"HYPERFRAMES_VERTEX_SERVICE_ACCOUNT",
|
||||
"HYPERFRAMES_VERTEX_LOCATION",
|
||||
"HYPERFRAMES_VERTEX_MODEL",
|
||||
"HYPERFRAMES_OPENROUTER_MODEL",
|
||||
"HYPERFRAMES_GEMINI_MODEL",
|
||||
"HYPERFRAMES_VISION_TIMEOUT_MS",
|
||||
];
|
||||
|
||||
// A quoted value ends at its closing quote; unquoted, `#` starts a comment only after whitespace, so `pass#word` stays.
|
||||
function valueOf(raw: string): string {
|
||||
const quote = raw.charAt(0);
|
||||
@@ -22,10 +35,22 @@ function entryOf(rawLine: string): [string, string] | null {
|
||||
|
||||
// Never over a key already set, and never the feedback email in any letter case (Windows env names are case-blind):
|
||||
// only the launching app may attach it, and an agent can write a project `.env`.
|
||||
export function applyDotEnv(content: string, env: NodeJS.ProcessEnv): void {
|
||||
// An optional exact-name allowlist narrows callers that load less-trusted project settings.
|
||||
export function applyDotEnv(
|
||||
content: string,
|
||||
env: NodeJS.ProcessEnv,
|
||||
allowedKeys?: readonly string[],
|
||||
): void {
|
||||
const allowed = allowedKeys === undefined ? undefined : new Set(allowedKeys);
|
||||
for (const entry of content.split("\n").map(entryOf)) {
|
||||
if (!entry) continue;
|
||||
const [key, value] = entry;
|
||||
if (key && key.toUpperCase() !== FEEDBACK_EMAIL_ENV && !(key in env)) env[key] = value;
|
||||
if (
|
||||
key &&
|
||||
key.toUpperCase() !== FEEDBACK_EMAIL_ENV &&
|
||||
(allowed === undefined || allowed.has(key)) &&
|
||||
!(key in env)
|
||||
)
|
||||
env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user