mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
fix(catalog): confine hosted preview asset downloads (#3843)
* fix(catalog): confine hosted preview asset downloads * test(catalog): pin credential and port refusal with finite streams
This commit is contained in:
James Russo
+1
-1
@@ -48,7 +48,7 @@
|
||||
"player:perf": "bun run --filter @hyperframes/player perf",
|
||||
"format:check": "oxfmt --check .",
|
||||
"knip": "knip",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/",
|
||||
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/",
|
||||
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
|
||||
"test:skills": "node --test 'skills/**/*.test.mjs'",
|
||||
"generate:previews": "tsx scripts/generate-template-previews.ts",
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { fetchHostedFiles } from "./catalog-hosted-files.js";
|
||||
|
||||
function fixture(url = "https://static.heygen.ai/asset", path = "asset.bin") {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-hosted-test-"));
|
||||
writeFileSync(join(root, "registry-item.json"), JSON.stringify({ files: [{ url, path }] }));
|
||||
return root;
|
||||
}
|
||||
|
||||
test("downloads exact bytes through a relative CDN redirect", async (t) => {
|
||||
const root = fixture();
|
||||
const calls: string[] = [];
|
||||
t.mock.method(
|
||||
globalThis,
|
||||
"fetch",
|
||||
async (url: Parameters<typeof fetch>[0], options?: RequestInit) => {
|
||||
calls.push(String(url));
|
||||
assert.equal(options?.redirect, "manual");
|
||||
assert.ok(options?.signal);
|
||||
return calls.length === 1
|
||||
? new Response(null, { status: 302, headers: { location: "/final" } })
|
||||
: new Response(new Uint8Array([0, 255, 42]));
|
||||
},
|
||||
);
|
||||
try {
|
||||
await fetchHostedFiles(root);
|
||||
assert.deepEqual(calls, ["https://static.heygen.ai/asset", "https://static.heygen.ai/final"]);
|
||||
assert.deepEqual(readFileSync(join(root, "asset.bin")), Buffer.from([0, 255, 42]));
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
for (const location of [
|
||||
"http://169.254.169.254/meta",
|
||||
"https://localhost/x",
|
||||
"https://static.heygen.ai.evil.test/x",
|
||||
"http://static.heygen.ai/x",
|
||||
]) {
|
||||
test(`rejects redirect to ${location} before a second request`, async (t) => {
|
||||
const root = fixture();
|
||||
const mock = t.mock.method(
|
||||
globalThis,
|
||||
"fetch",
|
||||
async () => new Response(null, { status: 302, headers: { location } }),
|
||||
);
|
||||
try {
|
||||
await assert.rejects(fetchHostedFiles(root), /HTTPS CDN/);
|
||||
assert.equal(mock.mock.callCount(), 1);
|
||||
assert.deepEqual(readdirSync(root), ["registry-item.json"]);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const url of [
|
||||
"https://127.0.0.1/x",
|
||||
"https://user:pw@static.heygen.ai/x",
|
||||
"https://static.heygen.ai:8443/x",
|
||||
]) {
|
||||
test(`rejects disallowed initial URL ${url} without a request`, async (t) => {
|
||||
const root = fixture(url);
|
||||
const mock = t.mock.method(globalThis, "fetch", async () => assert.fail("unexpected request"));
|
||||
try {
|
||||
await assert.rejects(fetchHostedFiles(root), /HTTPS CDN/);
|
||||
assert.equal(mock.mock.callCount(), 0);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("enforces actual streaming bytes even when Content-Length understates them", async (t) => {
|
||||
const root = fixture();
|
||||
let cancelled = false;
|
||||
let chunks = 0;
|
||||
const chunk = new Uint8Array(129 * 1024 * 1024);
|
||||
t.mock.method(
|
||||
globalThis,
|
||||
"fetch",
|
||||
async () =>
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
pull(controller) {
|
||||
if (chunks++ < 3) controller.enqueue(chunk);
|
||||
else controller.close();
|
||||
},
|
||||
cancel() {
|
||||
cancelled = true;
|
||||
},
|
||||
}),
|
||||
{ headers: { "content-length": "1" } },
|
||||
),
|
||||
);
|
||||
try {
|
||||
await assert.rejects(fetchHostedFiles(root), /256 MiB/);
|
||||
assert.equal(cancelled, true);
|
||||
assert.deepEqual(readdirSync(root), ["registry-item.json"]);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
for (const mode of ["declared oversize", "loop", "HTTP error", "stream error"]) {
|
||||
test(`${mode} preserves the existing asset`, async (t) => {
|
||||
const root = fixture();
|
||||
writeFileSync(join(root, "asset.bin"), "previous");
|
||||
let calls = 0;
|
||||
let cancelled = false;
|
||||
t.mock.method(globalThis, "fetch", async () => {
|
||||
calls++;
|
||||
const body = new ReadableStream({
|
||||
start(controller) {
|
||||
if (mode === "stream error") controller.error(new Error("broken stream"));
|
||||
},
|
||||
cancel() {
|
||||
cancelled = true;
|
||||
},
|
||||
});
|
||||
if (mode === "loop")
|
||||
return new Response(body, { status: 302, headers: { location: "/asset" } });
|
||||
if (mode === "HTTP error") return new Response(body, { status: 500 });
|
||||
return new Response(body, {
|
||||
headers: {
|
||||
"content-length": mode === "declared oversize" ? String(257 * 1024 * 1024) : "1",
|
||||
},
|
||||
});
|
||||
});
|
||||
try {
|
||||
await assert.rejects(fetchHostedFiles(root), /256 MiB|redirect limit|HTTP 500|broken stream/);
|
||||
assert.equal(readFileSync(join(root, "asset.bin"), "utf8"), "previous");
|
||||
assert.equal(calls, mode === "loop" ? 6 : 1);
|
||||
if (mode !== "stream error") assert.equal(cancelled, true);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("skips manifest paths outside the copied project before fetching", async (t) => {
|
||||
const root = fixture("https://static.heygen.ai/asset", "../escape.bin");
|
||||
const mock = t.mock.method(globalThis, "fetch", async () => assert.fail("unexpected request"));
|
||||
try {
|
||||
await fetchHostedFiles(root);
|
||||
assert.equal(mock.mock.callCount(), 0);
|
||||
assert.deepEqual(readdirSync(root), ["registry-item.json"]);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
import { existsSync, readFileSync, realpathSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { isContainedIn } from "./registry-target-paths.mjs";
|
||||
|
||||
// All published registry assets use this CDN. Keep every redirect on the same
|
||||
// HTTPS origin; a contributor manifest must not select a runner-local service.
|
||||
const ASSET_ORIGIN = "https://static.heygen.ai";
|
||||
// Match the media-freeze ceiling to allow future video assets while bounding runner memory.
|
||||
const MAX_ASSET_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
function assertCdnUrl(url: URL): void {
|
||||
if (url.origin !== ASSET_ORIGIN || url.username || url.password) {
|
||||
throw new Error("Hosted asset URL must use the registry HTTPS CDN");
|
||||
}
|
||||
}
|
||||
|
||||
async function readHostedBody(response: Response): Promise<Uint8Array> {
|
||||
await validateResponse(response);
|
||||
const chunks: Uint8Array[] = [];
|
||||
let bytes = 0;
|
||||
if (response.body) {
|
||||
for await (const chunk of response.body) {
|
||||
bytes += chunk.byteLength;
|
||||
if (bytes > MAX_ASSET_BYTES) throw new Error("Hosted asset exceeds 256 MiB");
|
||||
chunks.push(chunk);
|
||||
}
|
||||
}
|
||||
return Buffer.concat(chunks, bytes);
|
||||
}
|
||||
|
||||
async function validateResponse(response: Response): Promise<void> {
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error(`Hosted asset fetch failed: HTTP ${response.status}`);
|
||||
}
|
||||
if (Number(response.headers.get("content-length")) > MAX_ASSET_BYTES) {
|
||||
await response.body?.cancel();
|
||||
throw new Error("Hosted asset exceeds 256 MiB");
|
||||
}
|
||||
}
|
||||
|
||||
async function redirectTarget(response: Response, current: URL): Promise<URL> {
|
||||
await response.body?.cancel();
|
||||
const location = response.headers.get("location");
|
||||
if (!location) throw new Error("Hosted asset redirect has no Location");
|
||||
return new URL(location, current);
|
||||
}
|
||||
|
||||
async function fetchHostedAsset(url: string): Promise<Uint8Array> {
|
||||
const signal = AbortSignal.timeout(120_000);
|
||||
let current = new URL(url);
|
||||
for (let hop = 0; hop <= 5; hop++) {
|
||||
assertCdnUrl(current);
|
||||
const response = await fetch(current.href, { redirect: "manual", signal });
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
current = await redirectTarget(response, current);
|
||||
} else {
|
||||
return readHostedBody(response);
|
||||
}
|
||||
}
|
||||
throw new Error("Hosted asset exceeded redirect limit");
|
||||
}
|
||||
|
||||
interface HostedFile {
|
||||
path: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
function hasHostedFields(file: { path?: string; url?: string }): file is HostedFile {
|
||||
return typeof file.path === "string" && typeof file.url === "string";
|
||||
}
|
||||
|
||||
function hostedFilesOf(root: string): HostedFile[] {
|
||||
const manifestPath = join(root, "registry-item.json");
|
||||
if (!existsSync(manifestPath)) return [];
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
|
||||
files?: { path?: string; url?: string }[];
|
||||
};
|
||||
return (manifest.files ?? [])
|
||||
.filter(hasHostedFields)
|
||||
.filter((file) => file.url.startsWith("https://") && isContainedIn(root, file.path));
|
||||
}
|
||||
|
||||
/** Materialize manifest-hosted bytes only inside the copied preview project. */
|
||||
export async function fetchHostedFiles(projectDir: string): Promise<void> {
|
||||
const root = realpathSync(projectDir);
|
||||
for (const file of hostedFilesOf(root)) {
|
||||
const bytes = await fetchHostedAsset(file.url);
|
||||
const destination = resolve(root, file.path);
|
||||
mkdirSync(dirname(destination), { recursive: true });
|
||||
writeFileSync(destination, bytes);
|
||||
}
|
||||
}
|
||||
@@ -42,7 +42,8 @@ import {
|
||||
executeRenderJob,
|
||||
} from "../packages/producer/src/index.js";
|
||||
import { compileForRender } from "../packages/producer/src/services/htmlCompiler.js";
|
||||
import { isContainedIn, resolveContainedCopies } from "./registry-target-paths.mjs";
|
||||
import { resolveContainedCopies } from "./registry-target-paths.mjs";
|
||||
import { fetchHostedFiles } from "./catalog-hosted-files.js";
|
||||
import { withHostedDefaults } from "./registry-hosted-assets.ts";
|
||||
import type { RegistryItem } from "../packages/core/src/index.js";
|
||||
import { openOpaqueCapture } from "./preview-capture.js";
|
||||
@@ -132,45 +133,6 @@ function outputDir(kind: ItemKind): string {
|
||||
return resolve(repoRoot, "docs/images/catalog", typeDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the item's CDN-hosted files into the copied project.
|
||||
*
|
||||
* `cpSync` only carries what is committed. A file declaring `url` deliberately
|
||||
* is not, so without this the preview renders the composition with every image
|
||||
* missing and the failure looks like a layout bug rather than an absent file.
|
||||
*
|
||||
* Same containment rule as the target mirror, and for the same reason: this
|
||||
* runs on `pull_request`, so `files[].path` is a contributor's string deciding
|
||||
* where bytes land on the runner.
|
||||
*/
|
||||
function hostedFilesOf(projectDir: string): { path: string; url: string }[] {
|
||||
const manifestPath = join(projectDir, "registry-item.json");
|
||||
if (!existsSync(manifestPath)) return [];
|
||||
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as {
|
||||
files?: { path?: string; url?: string }[];
|
||||
};
|
||||
return (manifest.files ?? []).filter(
|
||||
(file): file is { path: string; url: string } =>
|
||||
typeof file.path === "string" &&
|
||||
typeof file.url === "string" &&
|
||||
file.url.startsWith("https://") &&
|
||||
isContainedIn(projectDir, file.path),
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchHostedFiles(projectDir: string): Promise<void> {
|
||||
for (const file of hostedFilesOf(projectDir)) {
|
||||
const res = await fetch(file.url);
|
||||
if (!res.ok) {
|
||||
throw new Error(`Hosted asset fetch failed: ${file.url} — HTTP ${res.status}`);
|
||||
}
|
||||
const destPath = resolve(projectDir, file.path);
|
||||
mkdirSync(dirname(destPath), { recursive: true });
|
||||
writeFileSync(destPath, new Uint8Array(await res.arrayBuffer()));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite the composition's variable defaults from local names to CDN URLs.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user