mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(engine): check proxied download hosts; keep raw GIF downloads private (#8040)
* fix(engine): check proxied download hosts against local DNS; keep raw GIF downloads private Behind an env proxy (HTTPS_PROXY with NODE_USE_ENV_PROXY=1 or --use-env-proxy), composition downloads skipped the resolved-address check and relied on the literal-host check alone. An internal DNS name passed, and NO_PROXY hosts were dialled directly. Now the downloader resolves the host on this machine first and refuses it when any answer is non-public. A name this machine cannot resolve is still left to the proxy, so proxy-only networks keep working. Studio's remote GIF prep now picks only URLs whose path ends in .gif, discards a downloaded body that does not start with a GIF signature, and the raw downloads folder is no longer served to the page; only the converted output is. * fix(cli): read the downloaded GIF's header through a bounded read stream CodeQL read the openSync call as a temp-file creation. The file sits in the project's .hyperframes folder and is only read, but a six-byte read stream does the same check without a file descriptor of our own.
This commit is contained in:
@@ -65,11 +65,31 @@ vi.mock("../../../producer/src/services/heifImagePrep.js", () => ({
|
||||
preparedImages: [],
|
||||
}),
|
||||
}));
|
||||
// Remote GIF sources: the stand-in download writes `body` and the GIF step records what it was handed.
|
||||
const remoteGif = vi.hoisted(() => ({
|
||||
body: "GIF89a",
|
||||
requested: [] as string[],
|
||||
sourceAssets: undefined as Map<string, string> | undefined,
|
||||
}));
|
||||
vi.mock("../../../producer/src/utils/urlDownloader.js", () => ({
|
||||
downloadToTemp: async (url: string, destDir: string) => {
|
||||
const { mkdirSync, writeFileSync } = await import("node:fs");
|
||||
remoteGif.requested.push(url);
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
const path = `${destDir}/download_${remoteGif.requested.length}.gif`;
|
||||
writeFileSync(path, remoteGif.body);
|
||||
return path;
|
||||
},
|
||||
writeUrlDownloadTelemetry: () => {},
|
||||
}));
|
||||
vi.mock("../../../producer/src/services/animatedGifPrep.js", () => ({
|
||||
prepareAnimatedGifInputs: async (html: string, options: { outputDir: string }) => ({
|
||||
...(await imagePrep.convert(html, options)),
|
||||
preparedGifs: [],
|
||||
}),
|
||||
prepareAnimatedGifInputs: async (
|
||||
html: string,
|
||||
options: { outputDir: string; sourceAssets?: Map<string, string> },
|
||||
) => {
|
||||
remoteGif.sourceAssets = options.sourceAssets;
|
||||
return { ...(await imagePrep.convert(html, options)), preparedGifs: [] };
|
||||
},
|
||||
}));
|
||||
const engineState = vi.hoisted(() => ({
|
||||
acquireBrowser: async (..._args: unknown[]): Promise<unknown> => {
|
||||
@@ -1330,6 +1350,43 @@ describe("createStudioServer preview image preparation", () => {
|
||||
expect(prepared(projectDir)).toEqual(["heif", "gif"]);
|
||||
});
|
||||
|
||||
async function prepareRemoteGif(body: string, src: string) {
|
||||
remoteGif.body = body;
|
||||
remoteGif.requested = [];
|
||||
remoteGif.sourceAssets = undefined;
|
||||
const projectDir = tmpProject();
|
||||
server = createStudioServer({ projectDir });
|
||||
await server.adapter.transformPreviewHtml?.({
|
||||
html: `<div data-composition-id="main"><img src="${src}"></div>`,
|
||||
project: { id: "p", dir: projectDir },
|
||||
activeCompositionPath: "index.html",
|
||||
});
|
||||
return projectDir;
|
||||
}
|
||||
|
||||
it("hands a downloaded GIF to the GIF step", async () => {
|
||||
await prepareRemoteGif("GIF89a-frames", "https://cdn.example.test/loop.gif?v=2");
|
||||
expect(remoteGif.requested).toEqual(["https://cdn.example.test/loop.gif?v=2"]);
|
||||
expect([...(remoteGif.sourceAssets?.keys() ?? [])]).toEqual([
|
||||
"https://cdn.example.test/loop.gif?v=2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("discards a downloaded body that is not a GIF", async () => {
|
||||
const projectDir = await prepareRemoteGif('{"secret":1}', "https://cdn.example.test/a.gif");
|
||||
expect(remoteGif.sourceAssets?.size).toBe(0);
|
||||
const downloads = join(projectDir, ".hyperframes", "prepared-assets", "downloads");
|
||||
expect(readdirSync(downloads)).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["https://intranet.example.test/api?.gif", "https://intranet.example.test/api#.gif"])(
|
||||
"does not download %s, whose path does not end in .gif",
|
||||
async (src) => {
|
||||
await prepareRemoteGif("GIF89a", src);
|
||||
expect(remoteGif.requested).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"serves images unprepared when .hyperframes is a link, and writes nothing where it points",
|
||||
async () => {
|
||||
|
||||
@@ -10,7 +10,15 @@ import type { HttpBindings } from "@hono/node-server";
|
||||
import { Hono, type Context } from "hono";
|
||||
import { streamSSE } from "hono/streaming";
|
||||
import { realpath, terminalText } from "@hyperframes/core";
|
||||
import { existsSync, readFileSync, statSync, writeFileSync, unlinkSync } from "node:fs";
|
||||
import {
|
||||
createReadStream,
|
||||
existsSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
unlinkSync,
|
||||
} from "node:fs";
|
||||
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
|
||||
import { createHash } from "node:crypto";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -121,7 +129,7 @@ const LINT_WORKER_HEAP_MB = 1024;
|
||||
const IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable";
|
||||
|
||||
const REMOTE_GIF_IMG_SRC_RE =
|
||||
/<img\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+\.gif(?:[?#][^"']*)?)["'][^>]*>/gi;
|
||||
/<img\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"'?#]+\.gif(?:[?#][^"']*)?)["'][^>]*>/gi;
|
||||
|
||||
async function loadStudioProducer() {
|
||||
if (!isDevMode()) return await import("@hyperframes/producer");
|
||||
@@ -240,6 +248,15 @@ function collectRemoteGifImageSources(html: string): string[] {
|
||||
return [...urls];
|
||||
}
|
||||
|
||||
async function startsWithGifSignature(path: string): Promise<boolean> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of createReadStream(path, { start: 0, end: 5 })) {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
}
|
||||
const signature = Buffer.concat(chunks).toString("latin1");
|
||||
return signature === "GIF87a" || signature === "GIF89a";
|
||||
}
|
||||
|
||||
async function downloadRemoteGifImageSources(
|
||||
html: string,
|
||||
downloadDir: string,
|
||||
@@ -249,7 +266,12 @@ async function downloadRemoteGifImageSources(
|
||||
await Promise.all(
|
||||
collectRemoteGifImageSources(html).map(async (url) => {
|
||||
try {
|
||||
sourceAssets.set(url, await downloadToTemp(url, downloadDir));
|
||||
const path = await downloadToTemp(url, downloadDir);
|
||||
if (!(await startsWithGifSignature(path))) {
|
||||
rmSync(path, { force: true });
|
||||
throw new Error("downloaded file is not a GIF");
|
||||
}
|
||||
sourceAssets.set(url, path);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
"[Studio] Remote animated GIF prep skipped:",
|
||||
|
||||
@@ -3,13 +3,19 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const answers = vi.hoisted(() => ({ list: [] as Array<{ address: string; family: number }> }));
|
||||
const answers = vi.hoisted(() => ({
|
||||
list: [] as Array<{ address: string; family: number }>,
|
||||
errorCode: undefined as string | undefined,
|
||||
}));
|
||||
vi.mock("node:dns", () => {
|
||||
const lookup = (
|
||||
_host: string,
|
||||
_options: unknown,
|
||||
callback: (error: null, addresses: typeof answers.list) => void,
|
||||
) => callback(null, answers.list);
|
||||
callback: (error: NodeJS.ErrnoException | null, addresses: typeof answers.list) => void,
|
||||
) =>
|
||||
answers.errorCode
|
||||
? callback(Object.assign(new Error("lookup failed"), { code: answers.errorCode }), [])
|
||||
: callback(null, answers.list);
|
||||
return { default: { lookup }, lookup };
|
||||
});
|
||||
|
||||
@@ -17,6 +23,9 @@ import { downloadToTemp, fetchPublicHttpsText, UrlDownloadError } from "./urlDow
|
||||
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => {
|
||||
answers.errorCode = undefined;
|
||||
vi.unstubAllGlobals();
|
||||
vi.unstubAllEnvs();
|
||||
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
@@ -49,20 +58,44 @@ describe("downloads refuse hosts that resolve to private addresses", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps Node's own fetch when it goes through an env proxy, which resolves the name", async () => {
|
||||
answers.list = [{ address: "10.1.2.3", family: 4 }];
|
||||
function useEnvProxy() {
|
||||
vi.stubEnv("HTTPS_PROXY", "http://proxy.test:3128");
|
||||
vi.stubEnv("NODE_USE_ENV_PROXY", "1");
|
||||
const viaProxy = vi.fn(async () => new Response("<html></html>"));
|
||||
vi.stubGlobal("fetch", viaProxy);
|
||||
try {
|
||||
await expect(
|
||||
fetchPublicHttpsText("https://assets.example.test/x.html", { maxBytes: 1024 }),
|
||||
).resolves.toBe("<html></html>");
|
||||
expect(viaProxy).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
vi.unstubAllGlobals();
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
return viaProxy;
|
||||
}
|
||||
|
||||
it.each([
|
||||
["a NO_PROXY host", { NO_PROXY: "corp.example.test" }],
|
||||
["a proxied host", {}],
|
||||
])("refuses %s that resolves to a private address behind an env proxy", async (_, env) => {
|
||||
answers.list = [{ address: "10.1.2.3", family: 4 }];
|
||||
const viaProxy = useEnvProxy();
|
||||
for (const [key, value] of Object.entries(env)) vi.stubEnv(key, value);
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-public-address-"));
|
||||
dirs.push(dir);
|
||||
const error = await downloadToTemp("https://jira.corp.example.test/a.gif", dir).catch((e) => e);
|
||||
expect(error).toBeInstanceOf(UrlDownloadError);
|
||||
expect(error).toMatchObject({ kind: "http_rejected", retryable: false });
|
||||
expect(viaProxy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("goes through the env proxy for a host that resolves to a public address", async () => {
|
||||
answers.list = [{ address: "93.184.216.34", family: 4 }];
|
||||
const viaProxy = useEnvProxy();
|
||||
await expect(
|
||||
fetchPublicHttpsText("https://assets.example.test/x.html", { maxBytes: 1024 }),
|
||||
).resolves.toBe("<html></html>");
|
||||
expect(viaProxy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("leaves a name this machine cannot resolve to the env proxy", async () => {
|
||||
answers.errorCode = "ENOTFOUND";
|
||||
const viaProxy = useEnvProxy();
|
||||
await expect(
|
||||
fetchPublicHttpsText("https://assets.example.test/x.html", { maxBytes: 1024 }),
|
||||
).resolves.toBe("<html></html>");
|
||||
expect(viaProxy).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -509,10 +509,30 @@ function undiciTransport(lookup: LookupFunction): Promise<DownloadFetch> {
|
||||
return transport;
|
||||
}
|
||||
|
||||
const UNRESOLVED_HOST_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "ENODATA", "ESERVFAIL"]);
|
||||
|
||||
/** Behind an env proxy, refuses a host this machine resolves to a non-public address; the proxy resolves the rest. */
|
||||
function assertProxiedHostResolvesPublic(url: string): Promise<void> {
|
||||
const hostname = new URL(url).hostname.replace(/^\[|\]$/g, "");
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
dns.lookup(hostname, { all: true }, (error, answers) => {
|
||||
if (error) {
|
||||
return UNRESOLVED_HOST_CODES.has(error.code ?? "") ? resolvePromise() : reject(error);
|
||||
}
|
||||
if (answers.some((a) => isBlockedNetworkHost(a.address.split("%")[0]!))) {
|
||||
return reject(nonPublicAddressError());
|
||||
}
|
||||
resolvePromise();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const fetchPublic: DownloadFetch = async (url, init) => {
|
||||
if (resolveRestrictPageNetwork()) return (await undiciTransport(capturePolicyLookup))(url, init);
|
||||
// Behind an env proxy the proxy resolves the name; the URL and redirect checks are the guard then.
|
||||
if (fetchGoesThroughEnvProxy()) return fetch(url, init);
|
||||
if (fetchGoesThroughEnvProxy()) {
|
||||
await assertProxiedHostResolvesPublic(url);
|
||||
return fetch(url, init);
|
||||
}
|
||||
return (await undiciTransport(publicOnlyLookup))(url, init);
|
||||
};
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ const STUDIO_FILES = new Set([
|
||||
".hyperframes/studio-manual-edits.json",
|
||||
]);
|
||||
const STUDIO_FOLDERS = [".hyperframes/prepared-assets", ".hyperframes/requests"];
|
||||
const UNCONVERTED_PREVIEW_DOWNLOADS = ".hyperframes/prepared-assets/downloads";
|
||||
|
||||
// Credentials and coding-tool state that projects often sit next to: registry tokens, MCP and agent settings, cloud and
|
||||
// SSH keys. No composition loads one, so no file server serves one and no upload ships one.
|
||||
@@ -121,6 +122,11 @@ const LOADED_ASSET_TYPE =
|
||||
/.\.(?:png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|heic|heif|mp4|m4v|mov|webm|mkv|mxf|avi|mp3|wav|m4a|aac|ogg|oga|opus|flac|woff2?|ttf|otf|css|m?js|cjs)$/;
|
||||
|
||||
function isStudioOwnFile(path: string): boolean {
|
||||
if (
|
||||
path === UNCONVERTED_PREVIEW_DOWNLOADS ||
|
||||
path.startsWith(`${UNCONVERTED_PREVIEW_DOWNLOADS}/`)
|
||||
)
|
||||
return false;
|
||||
return (
|
||||
STUDIO_FILES.has(path) ||
|
||||
STUDIO_FOLDERS.some((folder) => path === folder || path.startsWith(`${folder}/`))
|
||||
|
||||
@@ -120,6 +120,13 @@ describe("isPrivateProjectFile", () => {
|
||||
expect(isPrivateProjectFile("/p", "/p/.hyperframes/prepared-assets")).toBe(false);
|
||||
expect(isPrivateProjectFile("/p", "/p/.hyperframes/requests/1/before.png")).toBe(false);
|
||||
expect(isPrivateProjectFile("/p", "/p/.hyperframes/requests-old/x")).toBe(true);
|
||||
expect(isPrivateProjectFile("/p", "/p/.hyperframes/prepared-assets/downloads/a.gif")).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isPrivateProjectFile("/p", "/p/.hyperframes/Prepared-Assets/DOWNLOADS/a.gif")).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isPrivateProjectFile("/p", "/p/.hyperframes/prepared-assets/downloads")).toBe(true);
|
||||
expect(isPrivateProjectFile("/p", "/p/agent-handoff.json")).toBe(false);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user