diff --git a/package.json b/package.json index 99d9b751db..d3c3fce3a0 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "player:perf": "bun run --filter @hyperframes/player perf", "format:check": "oxfmt --check .", "knip": "knip", - "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/", + "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts && vitest run scripts/catalog/", "typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json", "test:skills": "node --test 'skills/**/*.test.mjs'", "generate:previews": "tsx scripts/generate-template-previews.ts", diff --git a/scripts/catalog-hosted-files.test.ts b/scripts/catalog-hosted-files.test.ts new file mode 100644 index 0000000000..bbec485f42 --- /dev/null +++ b/scripts/catalog-hosted-files.test.ts @@ -0,0 +1,155 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fetchHostedFiles } from "./catalog-hosted-files.js"; + +function fixture(url = "https://static.heygen.ai/asset", path = "asset.bin") { + const root = mkdtempSync(join(tmpdir(), "hf-hosted-test-")); + writeFileSync(join(root, "registry-item.json"), JSON.stringify({ files: [{ url, path }] })); + return root; +} + +test("downloads exact bytes through a relative CDN redirect", async (t) => { + const root = fixture(); + const calls: string[] = []; + t.mock.method( + globalThis, + "fetch", + async (url: Parameters[0], options?: RequestInit) => { + calls.push(String(url)); + assert.equal(options?.redirect, "manual"); + assert.ok(options?.signal); + return calls.length === 1 + ? new Response(null, { status: 302, headers: { location: "/final" } }) + : new Response(new Uint8Array([0, 255, 42])); + }, + ); + try { + await fetchHostedFiles(root); + assert.deepEqual(calls, ["https://static.heygen.ai/asset", "https://static.heygen.ai/final"]); + assert.deepEqual(readFileSync(join(root, "asset.bin")), Buffer.from([0, 255, 42])); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +for (const location of [ + "http://169.254.169.254/meta", + "https://localhost/x", + "https://static.heygen.ai.evil.test/x", + "http://static.heygen.ai/x", +]) { + test(`rejects redirect to ${location} before a second request`, async (t) => { + const root = fixture(); + const mock = t.mock.method( + globalThis, + "fetch", + async () => new Response(null, { status: 302, headers: { location } }), + ); + try { + await assert.rejects(fetchHostedFiles(root), /HTTPS CDN/); + assert.equal(mock.mock.callCount(), 1); + assert.deepEqual(readdirSync(root), ["registry-item.json"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +} + +for (const url of [ + "https://127.0.0.1/x", + "https://user:pw@static.heygen.ai/x", + "https://static.heygen.ai:8443/x", +]) { + test(`rejects disallowed initial URL ${url} without a request`, async (t) => { + const root = fixture(url); + const mock = t.mock.method(globalThis, "fetch", async () => assert.fail("unexpected request")); + try { + await assert.rejects(fetchHostedFiles(root), /HTTPS CDN/); + assert.equal(mock.mock.callCount(), 0); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +} + +test("enforces actual streaming bytes even when Content-Length understates them", async (t) => { + const root = fixture(); + let cancelled = false; + let chunks = 0; + const chunk = new Uint8Array(129 * 1024 * 1024); + t.mock.method( + globalThis, + "fetch", + async () => + new Response( + new ReadableStream({ + pull(controller) { + if (chunks++ < 3) controller.enqueue(chunk); + else controller.close(); + }, + cancel() { + cancelled = true; + }, + }), + { headers: { "content-length": "1" } }, + ), + ); + try { + await assert.rejects(fetchHostedFiles(root), /256 MiB/); + assert.equal(cancelled, true); + assert.deepEqual(readdirSync(root), ["registry-item.json"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +for (const mode of ["declared oversize", "loop", "HTTP error", "stream error"]) { + test(`${mode} preserves the existing asset`, async (t) => { + const root = fixture(); + writeFileSync(join(root, "asset.bin"), "previous"); + let calls = 0; + let cancelled = false; + t.mock.method(globalThis, "fetch", async () => { + calls++; + const body = new ReadableStream({ + start(controller) { + if (mode === "stream error") controller.error(new Error("broken stream")); + }, + cancel() { + cancelled = true; + }, + }); + if (mode === "loop") + return new Response(body, { status: 302, headers: { location: "/asset" } }); + if (mode === "HTTP error") return new Response(body, { status: 500 }); + return new Response(body, { + headers: { + "content-length": mode === "declared oversize" ? String(257 * 1024 * 1024) : "1", + }, + }); + }); + try { + await assert.rejects(fetchHostedFiles(root), /256 MiB|redirect limit|HTTP 500|broken stream/); + assert.equal(readFileSync(join(root, "asset.bin"), "utf8"), "previous"); + assert.equal(calls, mode === "loop" ? 6 : 1); + if (mode !== "stream error") assert.equal(cancelled, true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +} + +test("skips manifest paths outside the copied project before fetching", async (t) => { + const root = fixture("https://static.heygen.ai/asset", "../escape.bin"); + const mock = t.mock.method(globalThis, "fetch", async () => assert.fail("unexpected request")); + try { + await fetchHostedFiles(root); + assert.equal(mock.mock.callCount(), 0); + assert.deepEqual(readdirSync(root), ["registry-item.json"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/catalog-hosted-files.ts b/scripts/catalog-hosted-files.ts new file mode 100644 index 0000000000..7fc76702e9 --- /dev/null +++ b/scripts/catalog-hosted-files.ts @@ -0,0 +1,93 @@ +import { existsSync, readFileSync, realpathSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { isContainedIn } from "./registry-target-paths.mjs"; + +// All published registry assets use this CDN. Keep every redirect on the same +// HTTPS origin; a contributor manifest must not select a runner-local service. +const ASSET_ORIGIN = "https://static.heygen.ai"; +// Match the media-freeze ceiling to allow future video assets while bounding runner memory. +const MAX_ASSET_BYTES = 256 * 1024 * 1024; + +function assertCdnUrl(url: URL): void { + if (url.origin !== ASSET_ORIGIN || url.username || url.password) { + throw new Error("Hosted asset URL must use the registry HTTPS CDN"); + } +} + +async function readHostedBody(response: Response): Promise { + await validateResponse(response); + const chunks: Uint8Array[] = []; + let bytes = 0; + if (response.body) { + for await (const chunk of response.body) { + bytes += chunk.byteLength; + if (bytes > MAX_ASSET_BYTES) throw new Error("Hosted asset exceeds 256 MiB"); + chunks.push(chunk); + } + } + return Buffer.concat(chunks, bytes); +} + +async function validateResponse(response: Response): Promise { + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`Hosted asset fetch failed: HTTP ${response.status}`); + } + if (Number(response.headers.get("content-length")) > MAX_ASSET_BYTES) { + await response.body?.cancel(); + throw new Error("Hosted asset exceeds 256 MiB"); + } +} + +async function redirectTarget(response: Response, current: URL): Promise { + await response.body?.cancel(); + const location = response.headers.get("location"); + if (!location) throw new Error("Hosted asset redirect has no Location"); + return new URL(location, current); +} + +async function fetchHostedAsset(url: string): Promise { + const signal = AbortSignal.timeout(120_000); + let current = new URL(url); + for (let hop = 0; hop <= 5; hop++) { + assertCdnUrl(current); + const response = await fetch(current.href, { redirect: "manual", signal }); + if (response.status >= 300 && response.status < 400) { + current = await redirectTarget(response, current); + } else { + return readHostedBody(response); + } + } + throw new Error("Hosted asset exceeded redirect limit"); +} + +interface HostedFile { + path: string; + url: string; +} + +function hasHostedFields(file: { path?: string; url?: string }): file is HostedFile { + return typeof file.path === "string" && typeof file.url === "string"; +} + +function hostedFilesOf(root: string): HostedFile[] { + const manifestPath = join(root, "registry-item.json"); + if (!existsSync(manifestPath)) return []; + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { + files?: { path?: string; url?: string }[]; + }; + return (manifest.files ?? []) + .filter(hasHostedFields) + .filter((file) => file.url.startsWith("https://") && isContainedIn(root, file.path)); +} + +/** Materialize manifest-hosted bytes only inside the copied preview project. */ +export async function fetchHostedFiles(projectDir: string): Promise { + const root = realpathSync(projectDir); + for (const file of hostedFilesOf(root)) { + const bytes = await fetchHostedAsset(file.url); + const destination = resolve(root, file.path); + mkdirSync(dirname(destination), { recursive: true }); + writeFileSync(destination, bytes); + } +} diff --git a/scripts/generate-catalog-previews.ts b/scripts/generate-catalog-previews.ts index 03d01b8827..002aa3ea05 100644 --- a/scripts/generate-catalog-previews.ts +++ b/scripts/generate-catalog-previews.ts @@ -42,7 +42,8 @@ import { executeRenderJob, } from "../packages/producer/src/index.js"; import { compileForRender } from "../packages/producer/src/services/htmlCompiler.js"; -import { isContainedIn, resolveContainedCopies } from "./registry-target-paths.mjs"; +import { resolveContainedCopies } from "./registry-target-paths.mjs"; +import { fetchHostedFiles } from "./catalog-hosted-files.js"; import { withHostedDefaults } from "./registry-hosted-assets.ts"; import type { RegistryItem } from "../packages/core/src/index.js"; import { openOpaqueCapture } from "./preview-capture.js"; @@ -132,45 +133,6 @@ function outputDir(kind: ItemKind): string { return resolve(repoRoot, "docs/images/catalog", typeDir); } -/** - * Download the item's CDN-hosted files into the copied project. - * - * `cpSync` only carries what is committed. A file declaring `url` deliberately - * is not, so without this the preview renders the composition with every image - * missing and the failure looks like a layout bug rather than an absent file. - * - * Same containment rule as the target mirror, and for the same reason: this - * runs on `pull_request`, so `files[].path` is a contributor's string deciding - * where bytes land on the runner. - */ -function hostedFilesOf(projectDir: string): { path: string; url: string }[] { - const manifestPath = join(projectDir, "registry-item.json"); - if (!existsSync(manifestPath)) return []; - - const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as { - files?: { path?: string; url?: string }[]; - }; - return (manifest.files ?? []).filter( - (file): file is { path: string; url: string } => - typeof file.path === "string" && - typeof file.url === "string" && - file.url.startsWith("https://") && - isContainedIn(projectDir, file.path), - ); -} - -async function fetchHostedFiles(projectDir: string): Promise { - for (const file of hostedFilesOf(projectDir)) { - const res = await fetch(file.url); - if (!res.ok) { - throw new Error(`Hosted asset fetch failed: ${file.url} — HTTP ${res.status}`); - } - const destPath = resolve(projectDir, file.path); - mkdirSync(dirname(destPath), { recursive: true }); - writeFileSync(destPath, new Uint8Array(await res.arrayBuffer())); - } -} - /** * Rewrite the composition's variable defaults from local names to CDN URLs. *