From 5fcce21a645a2bad0fec0e2dc62e08a9af596430 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 17 Nov 2026 06:09:42 +0000 Subject: [PATCH] fix(engine): check proxied download hosts; keep raw GIF downloads private (#8040) * fix(engine): check proxied download hosts against local DNS; keep raw GIF downloads private Behind an env proxy (HTTPS_PROXY with NODE_USE_ENV_PROXY=1 or --use-env-proxy), composition downloads skipped the resolved-address check and relied on the literal-host check alone. An internal DNS name passed, and NO_PROXY hosts were dialled directly. Now the downloader resolves the host on this machine first and refuses it when any answer is non-public. A name this machine cannot resolve is still left to the proxy, so proxy-only networks keep working. Studio's remote GIF prep now picks only URLs whose path ends in .gif, discards a downloaded body that does not start with a GIF signature, and the raw downloads folder is no longer served to the page; only the converted output is. * fix(cli): read the downloaded GIF's header through a bounded read stream CodeQL read the openSync call as a temp-file creation. The file sits in the project's .hyperframes folder and is only read, but a six-byte read stream does the same check without a file descriptor of our own. --- packages/cli/src/server/studioServer.test.ts | 65 +++++++++++++++++-- packages/cli/src/server/studioServer.ts | 28 +++++++- .../utils/urlDownloader.publicAddress.test.ts | 61 +++++++++++++---- packages/engine/src/utils/urlDownloader.ts | 24 ++++++- packages/parsers/src/privatePath.ts | 6 ++ .../src/helpers/safePath.test.ts | 7 ++ 6 files changed, 168 insertions(+), 23 deletions(-) diff --git a/packages/cli/src/server/studioServer.test.ts b/packages/cli/src/server/studioServer.test.ts index 547f04e4a9..57d0c1f3e7 100644 --- a/packages/cli/src/server/studioServer.test.ts +++ b/packages/cli/src/server/studioServer.test.ts @@ -65,11 +65,31 @@ vi.mock("../../../producer/src/services/heifImagePrep.js", () => ({ preparedImages: [], }), })); +// Remote GIF sources: the stand-in download writes `body` and the GIF step records what it was handed. +const remoteGif = vi.hoisted(() => ({ + body: "GIF89a", + requested: [] as string[], + sourceAssets: undefined as Map | undefined, +})); +vi.mock("../../../producer/src/utils/urlDownloader.js", () => ({ + downloadToTemp: async (url: string, destDir: string) => { + const { mkdirSync, writeFileSync } = await import("node:fs"); + remoteGif.requested.push(url); + mkdirSync(destDir, { recursive: true }); + const path = `${destDir}/download_${remoteGif.requested.length}.gif`; + writeFileSync(path, remoteGif.body); + return path; + }, + writeUrlDownloadTelemetry: () => {}, +})); vi.mock("../../../producer/src/services/animatedGifPrep.js", () => ({ - prepareAnimatedGifInputs: async (html: string, options: { outputDir: string }) => ({ - ...(await imagePrep.convert(html, options)), - preparedGifs: [], - }), + prepareAnimatedGifInputs: async ( + html: string, + options: { outputDir: string; sourceAssets?: Map }, + ) => { + remoteGif.sourceAssets = options.sourceAssets; + return { ...(await imagePrep.convert(html, options)), preparedGifs: [] }; + }, })); const engineState = vi.hoisted(() => ({ acquireBrowser: async (..._args: unknown[]): Promise => { @@ -1330,6 +1350,43 @@ describe("createStudioServer preview image preparation", () => { expect(prepared(projectDir)).toEqual(["heif", "gif"]); }); + async function prepareRemoteGif(body: string, src: string) { + remoteGif.body = body; + remoteGif.requested = []; + remoteGif.sourceAssets = undefined; + const projectDir = tmpProject(); + server = createStudioServer({ projectDir }); + await server.adapter.transformPreviewHtml?.({ + html: `
`, + project: { id: "p", dir: projectDir }, + activeCompositionPath: "index.html", + }); + return projectDir; + } + + it("hands a downloaded GIF to the GIF step", async () => { + await prepareRemoteGif("GIF89a-frames", "https://cdn.example.test/loop.gif?v=2"); + expect(remoteGif.requested).toEqual(["https://cdn.example.test/loop.gif?v=2"]); + expect([...(remoteGif.sourceAssets?.keys() ?? [])]).toEqual([ + "https://cdn.example.test/loop.gif?v=2", + ]); + }); + + it("discards a downloaded body that is not a GIF", async () => { + const projectDir = await prepareRemoteGif('{"secret":1}', "https://cdn.example.test/a.gif"); + expect(remoteGif.sourceAssets?.size).toBe(0); + const downloads = join(projectDir, ".hyperframes", "prepared-assets", "downloads"); + expect(readdirSync(downloads)).toEqual([]); + }); + + it.each(["https://intranet.example.test/api?.gif", "https://intranet.example.test/api#.gif"])( + "does not download %s, whose path does not end in .gif", + async (src) => { + await prepareRemoteGif("GIF89a", src); + expect(remoteGif.requested).toEqual([]); + }, + ); + it.skipIf(process.platform === "win32")( "serves images unprepared when .hyperframes is a link, and writes nothing where it points", async () => { diff --git a/packages/cli/src/server/studioServer.ts b/packages/cli/src/server/studioServer.ts index 34e2f86459..13cf8e828f 100644 --- a/packages/cli/src/server/studioServer.ts +++ b/packages/cli/src/server/studioServer.ts @@ -10,7 +10,15 @@ import type { HttpBindings } from "@hono/node-server"; import { Hono, type Context } from "hono"; import { streamSSE } from "hono/streaming"; import { realpath, terminalText } from "@hyperframes/core"; -import { existsSync, readFileSync, statSync, writeFileSync, unlinkSync } from "node:fs"; +import { + createReadStream, + existsSync, + readFileSync, + rmSync, + statSync, + writeFileSync, + unlinkSync, +} from "node:fs"; import { replaceFileAtomically } from "@hyperframes/core/atomic-file"; import { createHash } from "node:crypto"; import { fileURLToPath } from "node:url"; @@ -121,7 +129,7 @@ const LINT_WORKER_HEAP_MB = 1024; const IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable"; const REMOTE_GIF_IMG_SRC_RE = - /]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+\.gif(?:[?#][^"']*)?)["'][^>]*>/gi; + /]*?\bsrc\s*=\s*["'](https?:\/\/[^"'?#]+\.gif(?:[?#][^"']*)?)["'][^>]*>/gi; async function loadStudioProducer() { if (!isDevMode()) return await import("@hyperframes/producer"); @@ -240,6 +248,15 @@ function collectRemoteGifImageSources(html: string): string[] { return [...urls]; } +async function startsWithGifSignature(path: string): Promise { + const chunks: Buffer[] = []; + for await (const chunk of createReadStream(path, { start: 0, end: 5 })) { + chunks.push(Buffer.from(chunk)); + } + const signature = Buffer.concat(chunks).toString("latin1"); + return signature === "GIF87a" || signature === "GIF89a"; +} + async function downloadRemoteGifImageSources( html: string, downloadDir: string, @@ -249,7 +266,12 @@ async function downloadRemoteGifImageSources( await Promise.all( collectRemoteGifImageSources(html).map(async (url) => { try { - sourceAssets.set(url, await downloadToTemp(url, downloadDir)); + const path = await downloadToTemp(url, downloadDir); + if (!(await startsWithGifSignature(path))) { + rmSync(path, { force: true }); + throw new Error("downloaded file is not a GIF"); + } + sourceAssets.set(url, path); } catch (err) { console.warn( "[Studio] Remote animated GIF prep skipped:", diff --git a/packages/engine/src/utils/urlDownloader.publicAddress.test.ts b/packages/engine/src/utils/urlDownloader.publicAddress.test.ts index 32e02af5f5..bcf93f98ab 100644 --- a/packages/engine/src/utils/urlDownloader.publicAddress.test.ts +++ b/packages/engine/src/utils/urlDownloader.publicAddress.test.ts @@ -3,13 +3,19 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -const answers = vi.hoisted(() => ({ list: [] as Array<{ address: string; family: number }> })); +const answers = vi.hoisted(() => ({ + list: [] as Array<{ address: string; family: number }>, + errorCode: undefined as string | undefined, +})); vi.mock("node:dns", () => { const lookup = ( _host: string, _options: unknown, - callback: (error: null, addresses: typeof answers.list) => void, - ) => callback(null, answers.list); + callback: (error: NodeJS.ErrnoException | null, addresses: typeof answers.list) => void, + ) => + answers.errorCode + ? callback(Object.assign(new Error("lookup failed"), { code: answers.errorCode }), []) + : callback(null, answers.list); return { default: { lookup }, lookup }; }); @@ -17,6 +23,9 @@ import { downloadToTemp, fetchPublicHttpsText, UrlDownloadError } from "./urlDow const dirs: string[] = []; afterEach(() => { + answers.errorCode = undefined; + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); @@ -49,20 +58,44 @@ describe("downloads refuse hosts that resolve to private addresses", () => { }); }); - it("keeps Node's own fetch when it goes through an env proxy, which resolves the name", async () => { - answers.list = [{ address: "10.1.2.3", family: 4 }]; + function useEnvProxy() { vi.stubEnv("HTTPS_PROXY", "http://proxy.test:3128"); vi.stubEnv("NODE_USE_ENV_PROXY", "1"); const viaProxy = vi.fn(async () => new Response("")); vi.stubGlobal("fetch", viaProxy); - try { - await expect( - fetchPublicHttpsText("https://assets.example.test/x.html", { maxBytes: 1024 }), - ).resolves.toBe(""); - expect(viaProxy).toHaveBeenCalledOnce(); - } finally { - vi.unstubAllGlobals(); - vi.unstubAllEnvs(); - } + return viaProxy; + } + + it.each([ + ["a NO_PROXY host", { NO_PROXY: "corp.example.test" }], + ["a proxied host", {}], + ])("refuses %s that resolves to a private address behind an env proxy", async (_, env) => { + answers.list = [{ address: "10.1.2.3", family: 4 }]; + const viaProxy = useEnvProxy(); + for (const [key, value] of Object.entries(env)) vi.stubEnv(key, value); + const dir = mkdtempSync(join(tmpdir(), "hf-public-address-")); + dirs.push(dir); + const error = await downloadToTemp("https://jira.corp.example.test/a.gif", dir).catch((e) => e); + expect(error).toBeInstanceOf(UrlDownloadError); + expect(error).toMatchObject({ kind: "http_rejected", retryable: false }); + expect(viaProxy).not.toHaveBeenCalled(); + }); + + it("goes through the env proxy for a host that resolves to a public address", async () => { + answers.list = [{ address: "93.184.216.34", family: 4 }]; + const viaProxy = useEnvProxy(); + await expect( + fetchPublicHttpsText("https://assets.example.test/x.html", { maxBytes: 1024 }), + ).resolves.toBe(""); + expect(viaProxy).toHaveBeenCalledOnce(); + }); + + it("leaves a name this machine cannot resolve to the env proxy", async () => { + answers.errorCode = "ENOTFOUND"; + const viaProxy = useEnvProxy(); + await expect( + fetchPublicHttpsText("https://assets.example.test/x.html", { maxBytes: 1024 }), + ).resolves.toBe(""); + expect(viaProxy).toHaveBeenCalledOnce(); }); }); diff --git a/packages/engine/src/utils/urlDownloader.ts b/packages/engine/src/utils/urlDownloader.ts index 0d7949d8f9..8f4d787685 100644 --- a/packages/engine/src/utils/urlDownloader.ts +++ b/packages/engine/src/utils/urlDownloader.ts @@ -509,10 +509,30 @@ function undiciTransport(lookup: LookupFunction): Promise { return transport; } +const UNRESOLVED_HOST_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "ENODATA", "ESERVFAIL"]); + +/** Behind an env proxy, refuses a host this machine resolves to a non-public address; the proxy resolves the rest. */ +function assertProxiedHostResolvesPublic(url: string): Promise { + const hostname = new URL(url).hostname.replace(/^\[|\]$/g, ""); + return new Promise((resolvePromise, reject) => { + dns.lookup(hostname, { all: true }, (error, answers) => { + if (error) { + return UNRESOLVED_HOST_CODES.has(error.code ?? "") ? resolvePromise() : reject(error); + } + if (answers.some((a) => isBlockedNetworkHost(a.address.split("%")[0]!))) { + return reject(nonPublicAddressError()); + } + resolvePromise(); + }); + }); +} + const fetchPublic: DownloadFetch = async (url, init) => { if (resolveRestrictPageNetwork()) return (await undiciTransport(capturePolicyLookup))(url, init); - // Behind an env proxy the proxy resolves the name; the URL and redirect checks are the guard then. - if (fetchGoesThroughEnvProxy()) return fetch(url, init); + if (fetchGoesThroughEnvProxy()) { + await assertProxiedHostResolvesPublic(url); + return fetch(url, init); + } return (await undiciTransport(publicOnlyLookup))(url, init); }; diff --git a/packages/parsers/src/privatePath.ts b/packages/parsers/src/privatePath.ts index 137e330f2a..9711b7b6a5 100644 --- a/packages/parsers/src/privatePath.ts +++ b/packages/parsers/src/privatePath.ts @@ -47,6 +47,7 @@ const STUDIO_FILES = new Set([ ".hyperframes/studio-manual-edits.json", ]); const STUDIO_FOLDERS = [".hyperframes/prepared-assets", ".hyperframes/requests"]; +const UNCONVERTED_PREVIEW_DOWNLOADS = ".hyperframes/prepared-assets/downloads"; // Credentials and coding-tool state that projects often sit next to: registry tokens, MCP and agent settings, cloud and // SSH keys. No composition loads one, so no file server serves one and no upload ships one. @@ -121,6 +122,11 @@ const LOADED_ASSET_TYPE = /.\.(?:png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|heic|heif|mp4|m4v|mov|webm|mkv|mxf|avi|mp3|wav|m4a|aac|ogg|oga|opus|flac|woff2?|ttf|otf|css|m?js|cjs)$/; function isStudioOwnFile(path: string): boolean { + if ( + path === UNCONVERTED_PREVIEW_DOWNLOADS || + path.startsWith(`${UNCONVERTED_PREVIEW_DOWNLOADS}/`) + ) + return false; return ( STUDIO_FILES.has(path) || STUDIO_FOLDERS.some((folder) => path === folder || path.startsWith(`${folder}/`)) diff --git a/packages/studio-server/src/helpers/safePath.test.ts b/packages/studio-server/src/helpers/safePath.test.ts index 8fb11f352f..2ed69bf770 100644 --- a/packages/studio-server/src/helpers/safePath.test.ts +++ b/packages/studio-server/src/helpers/safePath.test.ts @@ -120,6 +120,13 @@ describe("isPrivateProjectFile", () => { expect(isPrivateProjectFile("/p", "/p/.hyperframes/prepared-assets")).toBe(false); expect(isPrivateProjectFile("/p", "/p/.hyperframes/requests/1/before.png")).toBe(false); expect(isPrivateProjectFile("/p", "/p/.hyperframes/requests-old/x")).toBe(true); + expect(isPrivateProjectFile("/p", "/p/.hyperframes/prepared-assets/downloads/a.gif")).toBe( + true, + ); + expect(isPrivateProjectFile("/p", "/p/.hyperframes/Prepared-Assets/DOWNLOADS/a.gif")).toBe( + true, + ); + expect(isPrivateProjectFile("/p", "/p/.hyperframes/prepared-assets/downloads")).toBe(true); expect(isPrivateProjectFile("/p", "/p/agent-handoff.json")).toBe(false); });