diff --git a/docs/packages/producer.mdx b/docs/packages/producer.mdx index 8e1bcfed17..aa4dafe81c 100644 --- a/docs/packages/producer.mdx +++ b/docs/packages/producer.mdx @@ -230,6 +230,11 @@ address a name resolves to and connects to the address it checked. A refused req fails like any other unreachable asset. Leave it unset for local renders that load assets from your own machine or network. +Where the only way out is an egress proxy, set `HYPERFRAMES_CAPTURE_PROXY` to its +`http://` URL, as for `capture`. The local proxy then sends every browser request to +it by hostname, and the egress proxy must refuse private and internal destinations +itself. + The setting accepts `true`, `1`, `yes` or `on` to turn the restriction on and `false`, `0`, `no` or `off` to leave it off, in any letter case. Any other value turns it on and logs a warning, so a misspelled value never leaves a render service open. diff --git a/packages/engine/src/services/browserManager.publicOnly.test.ts b/packages/engine/src/services/browserManager.publicOnly.test.ts new file mode 100644 index 0000000000..103b565ccc --- /dev/null +++ b/packages/engine/src/services/browserManager.publicOnly.test.ts @@ -0,0 +1,55 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createServer, request, type Server } from "node:http"; +import { _publicOnlyNetworkForTests } from "./browserManager.js"; + +const { resolveAddress } = vi.hoisted(() => ({ resolveAddress: vi.fn() })); +vi.mock("./publicNetwork/publicNetwork.js", async (actual) => ({ + ...(await actual()), + resolvePublicAddress: resolveAddress, +})); + +describe("render page network", () => { + const servers: Server[] = []; + afterEach(async () => { + vi.unstubAllEnvs(); + resolveAddress.mockReset(); + await Promise.all(servers.splice(0).map((s) => new Promise((r) => s.close(r)))); + }); + + it("tunnels page traffic to the egress proxy named by HYPERFRAMES_CAPTURE_PROXY", async () => { + resolveAddress.mockRejectedValue(new Error("dialled directly")); + const tunnels: string[] = []; + const upstream = createServer(); + upstream.on("connect", (req, socket) => { + tunnels.push(req.url ?? ""); + socket.end("HTTP/1.1 200 Connection Established\r\n\r\n"); + }); + servers.push(upstream); + await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve)); + vi.stubEnv( + "HYPERFRAMES_CAPTURE_PROXY", + `http://127.0.0.1:${(upstream.address() as { port: number }).port}`, + ); + + const network = await _publicOnlyNetworkForTests("127.0.0.1:4321"); + try { + const proxyUrl = network.args + .find((a) => a.startsWith("--proxy-server="))! + .slice("--proxy-server=".length); + const status = await new Promise((resolve, reject) => { + const req = request(proxyUrl, { method: "CONNECT", path: "example.com:443" }); + req.on("connect", (res, socket) => { + socket.destroy(); + resolve(res.statusCode!); + }); + req.on("error", reject); + req.end(); + }); + expect(status).toBe(200); + expect(tunnels).toEqual(["example.com:443"]); + expect(resolveAddress).not.toHaveBeenCalled(); + } finally { + await network.close(); + } + }); +}); diff --git a/packages/engine/src/services/browserManager.ts b/packages/engine/src/services/browserManager.ts index 1e35539bcd..3689c93720 100644 --- a/packages/engine/src/services/browserManager.ts +++ b/packages/engine/src/services/browserManager.ts @@ -877,6 +877,7 @@ function createBrowserLaunchFingerprint( /** Test-only export for launch-argument/capture-mode agreement. */ export const _createBrowserLaunchFingerprintForTests = createBrowserLaunchFingerprint; +export const _publicOnlyNetworkForTests = publicOnlyNetwork; export async function acquireBrowser( chromeArgs: string[], @@ -900,7 +901,7 @@ async function launchInsideSandbox(ppt: PuppeteerNode, options: LaunchOptions): } async function publicOnlyNetwork(pageServerHost: string | undefined) { - const proxy = await startCaptureProxy(null, logRefusal); + const proxy = await startCaptureProxy(undefined, logRefusal); try { const userDataDir = await createPublicOnlyProfile("hf-render-profile-"); return { diff --git a/packages/engine/src/services/publicNetwork/captureProxy.test.ts b/packages/engine/src/services/publicNetwork/captureProxy.test.ts index bf79890107..e6f928d4cb 100644 --- a/packages/engine/src/services/publicNetwork/captureProxy.test.ts +++ b/packages/engine/src/services/publicNetwork/captureProxy.test.ts @@ -99,6 +99,16 @@ describe("capture proxy", () => { expect(refused).toEqual([]); }); + it("listens on the fixed loopback port it is given", async () => { + const free = createServer(); + await new Promise((resolve) => free.listen(0, "127.0.0.1", resolve)); + const port = (free.address() as { port: number }).port; + await new Promise((resolve) => free.close(() => resolve())); + const proxy = await startCaptureProxy(null, undefined, port); + proxies.push(proxy); + expect(proxy.url).toBe(`http://127.0.0.1:${port}`); + }); + it("refuses URL credentials and non-http requests", async () => { expect(await proxyGet("http://user:secret@example.com/")).toBe(403); expect(await proxyGet("file:///etc/passwd")).toBe(403); diff --git a/packages/engine/src/services/publicNetwork/captureProxy.ts b/packages/engine/src/services/publicNetwork/captureProxy.ts index 8cd735877a..6cb01e87e7 100644 --- a/packages/engine/src/services/publicNetwork/captureProxy.ts +++ b/packages/engine/src/services/publicNetwork/captureProxy.ts @@ -207,6 +207,7 @@ async function forwardTunnel( export async function startCaptureProxy( upstream: UpstreamProxy | null = captureUpstreamProxy() ?? null, onRefused?: OnRefused, + fixedLoopbackPort?: number, ): Promise<{ url: string; close(): Promise }> { const proxy = upstream ?? undefined; const sockets = new Set(); @@ -224,7 +225,12 @@ export async function startCaptureProxy( server.headersTimeout = CONNECTION_TIMEOUT_MS; server.requestTimeout = 30_000; server.keepAliveTimeout = 1_000; - await listenOnBrowserSafePort(server, "127.0.0.1"); + if (fixedLoopbackPort === undefined) await listenOnBrowserSafePort(server, "127.0.0.1"); + else + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(fixedLoopbackPort, "127.0.0.1", () => resolve()); + }); const address = server.address(); if (!address || typeof address === "string") throw new Error("Capture proxy did not bind"); let closing: Promise | undefined; diff --git a/packages/engine/src/utils/urlDownloader.restrictedNetwork.test.ts b/packages/engine/src/utils/urlDownloader.restrictedNetwork.test.ts index c4e107a0b2..338b98ce8e 100644 --- a/packages/engine/src/utils/urlDownloader.restrictedNetwork.test.ts +++ b/packages/engine/src/utils/urlDownloader.restrictedNetwork.test.ts @@ -1,6 +1,15 @@ import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { EventEmitter } from "node:events"; -import { createServer, type IncomingHttpHeaders, type Server } from "node:http"; +import { execFileSync } from "node:child_process"; +import { + createServer, + type IncomingHttpHeaders, + type IncomingMessage, + type Server, + type ServerResponse, +} from "node:http"; +import { createServer as createHttpsServer } from "node:https"; +import { getCACertificates, setDefaultCACertificates } from "node:tls"; import { connect, isIP, type AddressInfo, type LookupFunction } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -70,15 +79,17 @@ let received: Array<{ url: string; headers: IncomingHttpHeaders }> = []; const BODY = "\u0000\u0001asset-bytes"; let server: Server; +function serveRoute(req: IncomingMessage, res: ServerResponse): void { + const url = `https://${req.headers.host}${req.url}`; + received.push({ url, headers: req.headers }); + const found = routes[url]; + if (found === "drop") return void req.socket.destroy(); + if (!found) return void res.writeHead(599).end(); + res.writeHead(found.status, found.headers).end(found.body ?? ""); +} + beforeAll(async () => { - server = createServer((req, res) => { - const url = `https://${req.headers.host}${req.url}`; - received.push({ url, headers: req.headers }); - const found = routes[url]; - if (found === "drop") return req.socket.destroy(); - if (!found) return res.writeHead(599).end(); - res.writeHead(found.status, found.headers).end(found.body ?? ""); - }); + server = createServer(serveRoute); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); mocks.serverPort = (server.address() as AddressInfo).port; }); @@ -279,3 +290,121 @@ describe("downloads with the render-service setting off", () => { expect(mocks.resolve4).not.toHaveBeenCalled(); }); }); + +describe("downloads on a render service through the egress proxy in HYPERFRAMES_CAPTURE_PROXY", () => { + const HOST = "cdn.egress-assets.com"; + const tunnels: string[] = []; + const defaultCAs = getCACertificates("default"); + let origin: Server; + let egress: Server; + + beforeAll(async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-egress-tls-")); + dirs.push(dir); + execFileSync( + "openssl", + [ + ...["req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes"], + ...["-keyout", join(dir, "key.pem"), "-out", join(dir, "cert.pem"), "-days", "1"], + ...["-subj", `/CN=${HOST}`, "-addext", `subjectAltName=DNS:${HOST}`], + ], + { stdio: "ignore" }, + ); + const cert = readFileSync(join(dir, "cert.pem"), "utf8"); + setDefaultCACertificates([...defaultCAs, cert]); + origin = createHttpsServer({ key: readFileSync(join(dir, "key.pem")), cert }, serveRoute); + await new Promise((resolve) => origin.listen(0, "127.0.0.1", resolve)); + const originPort = (origin.address() as AddressInfo).port; + egress = createServer(); + egress.on("connect", (request, client) => { + tunnels.push(request.url ?? ""); + const upstream = connect(originPort, "127.0.0.1", () => { + client.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + upstream.pipe(client).pipe(upstream); + }); + upstream.on("error", () => client.destroy()); + client.on("error", () => upstream.destroy()); + }); + await new Promise((resolve) => egress.listen(0, "127.0.0.1", resolve)); + }); + afterAll(async () => { + setDefaultCACertificates(defaultCAs); + egress.closeAllConnections(); + origin.closeAllConnections(); + await Promise.all([egress, origin].map((s) => new Promise((resolve) => s.close(resolve)))); + }); + beforeEach(() => { + tunnels.length = 0; + vi.stubEnv("PRODUCER_RESTRICT_PAGE_NETWORK", "true"); + vi.stubEnv( + "HYPERFRAMES_CAPTURE_PROXY", + `http://127.0.0.1:${(egress.address() as AddressInfo).port}`, + ); + }); + + it("downloads by hostname through the proxy, with the usual headers, resolving no names", async () => { + routes[`https://${HOST}/a.png`] = { status: 200, body: BODY }; + const path = await downloadToTemp(`https://${HOST}/a.png`, tempDir()); + expect(readFileSync(path, "utf8")).toBe(BODY); + expect(tunnels).toEqual([`${HOST}:443`]); + expect(received[0]?.headers["user-agent"]).toBeTruthy(); + expect(received[0]?.headers.accept).toBe("*/*"); + expect(mocks.resolve4).not.toHaveBeenCalled(); + expect(mocks.lookup).not.toHaveBeenCalled(); + }); + + it("unpacks a gzip text response", async () => { + routes[`https://${HOST}/app.js`] = { + status: 200, + headers: { "content-encoding": "gzip" }, + body: gzipSync("console.log('hello');"), + }; + await expect(fetchPublicHttpsText(`https://${HOST}/app.js`, { maxBytes: 1024 })).resolves.toBe( + "console.log('hello');", + ); + }); + + it("stops a gzip text response whose unpacked size passes the cap", async () => { + routes[`https://${HOST}/bomb.js`] = { + status: 200, + headers: { "content-encoding": "gzip" }, + body: gzipSync(Buffer.alloc(8 * 1024 * 1024)), + }; + await expect( + fetchPublicHttpsText(`https://${HOST}/bomb.js`, { maxBytes: 64 * 1024 }), + ).rejects.toMatchObject({ kind: "length_mismatch", retryable: false }); + }); + + it("refuses a compressed asset download", async () => { + routes[`https://${HOST}/z.png`] = { + status: 200, + headers: { "content-encoding": "gzip" }, + body: gzipSync(BODY), + }; + await expect(downloadToTemp(`https://${HOST}/z.png`, tempDir())).rejects.toMatchObject({ + kind: "http_rejected", + retryable: false, + }); + }); + + it("retries a 5xx like any 5xx", async () => { + routes[`https://${HOST}/busy.png`] = { status: 503, body: "busy" }; + await expect(downloadToTemp(`https://${HOST}/busy.png`, tempDir())).rejects.toMatchObject({ + kind: "http_transient", + retryable: true, + }); + expect(received).toHaveLength(2); + }); + + it("still refuses a redirect to a local-looking name before contacting the proxy", async () => { + routes[`https://${HOST}/hop.png`] = { + status: 302, + headers: { location: "https://metadata.internal/b.png" }, + }; + await expect(downloadToTemp(`https://${HOST}/hop.png`, tempDir())).rejects.toMatchObject({ + kind: "http_rejected", + message: "Download redirect target is not permitted", + }); + expect(received.map((request) => request.url)).toEqual([`https://${HOST}/hop.png`]); + }); +}); diff --git a/packages/engine/src/utils/urlDownloader.ts b/packages/engine/src/utils/urlDownloader.ts index 6d76c073c1..d0cd0d8c2e 100644 --- a/packages/engine/src/utils/urlDownloader.ts +++ b/packages/engine/src/utils/urlDownloader.ts @@ -27,6 +27,10 @@ import { NonPublicHostError, resolvePublicAddress, } from "../services/publicNetwork/publicNetwork.js"; +import { + captureUpstreamProxy, + type UpstreamProxy, +} from "../services/publicNetwork/upstreamProxy.js"; import { dirname, extname, join } from "path"; import { Readable, Transform } from "stream"; import { pipeline } from "stream/promises"; @@ -509,6 +513,23 @@ function undiciTransport(lookup: LookupFunction): Promise { return transport; } +const proxyTransports = new Map>(); + +function egressProxyTransport(proxy: UpstreamProxy): Promise { + const uri = `http://${proxy.host.includes(":") ? `[${proxy.host}]` : proxy.host}:${proxy.port}`; + const key = `${uri} ${proxy.authorization ?? ""}`; + let transport = proxyTransports.get(key); + if (!transport) { + transport = importUndiciPackageEvenUnderBun().then(({ ProxyAgent, fetch: undiciFetch }) => { + const dispatcher = new ProxyAgent({ uri, token: proxy.authorization }); + return (target, options) => + undiciFetch(target, { ...(options as object), dispatcher }) as unknown as Promise; + }); + proxyTransports.set(key, transport); + } + return transport; +} + const UNRESOLVED_HOST_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "EAI_FAIL"]); /** Behind an env proxy, refuses a host this machine resolves to a non-public address; the proxy resolves the rest. */ @@ -532,7 +553,11 @@ function assertProxiedHostResolvesPublic(url: string, signal?: AbortSignal | nul } const fetchPublic: DownloadFetch = async (url, init) => { - if (resolveRestrictPageNetwork()) return (await undiciTransport(capturePolicyLookup))(url, init); + if (resolveRestrictPageNetwork()) { + const upstream = captureUpstreamProxy(); + if (upstream) return (await egressProxyTransport(upstream))(url, init); + return (await undiciTransport(capturePolicyLookup))(url, init); + } if (fetchGoesThroughEnvProxy()) { await assertProxiedHostResolvesPublic(url, init.signal); return fetch(url, init);