fix(cli): bound and validate registry installation (#3814)

* fix(cli): bound and validate registry installation

* fix(cli): preserve registry install compatibility
This commit is contained in:
James Russo
2026-09-09 12:41:48 -04:00
committed by GitHub
parent b8328f9573
commit caa72de355
13 changed files with 666 additions and 74 deletions
@@ -0,0 +1,17 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { expect, it } from "vitest";
import { readBoundedRegistryFile } from "./boundedFile.js";
it("reads regular files up to the bound and rejects oversized or non-file cache entries", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-cache-bound-"));
try {
const path = join(dir, "cache");
writeFileSync(path, "1234");
expect(readBoundedRegistryFile(path, 4).toString()).toBe("1234");
expect(() => readBoundedRegistryFile(path, 3)).toThrow(/limit/);
expect(() => readBoundedRegistryFile(dir, 100)).toThrow();
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
+22
View File
@@ -0,0 +1,22 @@
import { closeSync, fstatSync, openSync, readSync } from "node:fs";
/** Read one regular file through a fixed descriptor without trusting a prior path stat. */
export function readBoundedRegistryFile(path: string, maxBytes: number): Buffer {
const fd = openSync(path, "r");
try {
const stat = fstatSync(fd);
if (!stat.isFile() || stat.size > maxBytes) throw new Error("Registry file exceeds read limit");
const chunks: Buffer[] = [];
let total = 0;
while (total <= maxBytes) {
const chunk = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes + 1 - total));
const count = readSync(fd, chunk);
if (count === 0) return Buffer.concat(chunks, total);
total += count;
chunks.push(chunk.subarray(0, count));
}
throw new Error("Registry file exceeds read limit");
} finally {
closeSync(fd);
}
}
+77 -9
View File
@@ -1,19 +1,24 @@
import { createHash } from "node:crypto";
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import {
mkdirSync,
mkdtempSync,
readFileSync,
writeFileSync,
symlinkSync,
rmSync,
existsSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { join } from "node:path";
import { describe, expect, it, vi } from "vitest";
import type { RegistryItem } from "@hyperframes/core";
// The installer fetches over the network; the point of these tests is what it
// does to files on disk, so the fetch is replaced by a local write.
// does to files on disk, so the fetch returns controlled bytes.
const remote = vi.hoisted(() => ({ contents: "REGISTRY VERSION\n" }));
vi.mock("./remote.js", () => ({
DEFAULT_REGISTRY_URL: "https://example.test/r",
fetchItemFile: vi.fn(async (_item: unknown, _file: unknown, destPath: string): Promise<void> => {
mkdirSync(dirname(destPath), { recursive: true });
writeFileSync(destPath, remote.contents, "utf-8");
}),
fetchItemFile: vi.fn(async () => Buffer.from(remote.contents)),
}));
const { hasLocalEdits, installItem } = await import("./installer.js");
@@ -24,9 +29,11 @@ function project(): string {
const item = {
name: "data-chart",
title: "Data chart",
description: "Chart component",
type: "hyperframes:component",
files: [
{ path: "data-chart.html", target: "components/data-chart.html", type: "hyperframes:file" },
{ path: "data-chart.html", target: "components/data-chart.html", type: "hyperframes:snippet" },
],
} as unknown as RegistryItem;
@@ -52,6 +59,61 @@ describe("hasLocalEdits", () => {
});
describe("installItem", () => {
it("rejects a target directory symlink that escapes the project even with force", async () => {
const dir = project();
const outside = project();
try {
symlinkSync(outside, join(dir, "components"), "junction");
await expect(installItem(item, { destDir: dir, force: true })).rejects.toThrow(
/Unsafe target/,
);
expect(existsSync(join(outside, "data-chart.html"))).toBe(false);
} finally {
rmSync(dir, { recursive: true, force: true });
rmSync(outside, { recursive: true, force: true });
}
});
it("preserves directory symlinks whose destination stays inside the project", async () => {
const dir = project();
try {
mkdirSync(join(dir, "actual"));
symlinkSync(join(dir, "actual"), join(dir, "components"), "junction");
await installItem(item, { destDir: dir, force: true });
expect(readFileSync(join(dir, "actual/data-chart.html"), "utf8")).toContain(
"REGISTRY VERSION",
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it.each(["darwin", "win32"] as const)(
"rejects absent case and Unicode aliases on %s before downloads",
async (platform) => {
vi.spyOn(process, "platform", "get").mockReturnValue(platform);
try {
for (const names of [
["Foo.html", "foo.html"],
["Café.html", "Cafe\u0301.html"],
]) {
const dir = project();
try {
const conflicting = {
...item,
files: names.map((name) => ({ ...item.files[0]!, target: `components/${name}` })),
};
await expect(installItem(conflicting, { destDir: dir, force: true })).rejects.toThrow(
/duplicate/,
);
expect(existsSync(join(dir, "components"))).toBe(false);
} finally {
rmSync(dir, { recursive: true, force: true });
}
}
} finally {
vi.restoreAllMocks();
}
},
);
it("records what it installed, so a later install can tell", async () => {
const dir = project();
const result = await installItem(item, { destDir: dir });
@@ -113,6 +175,10 @@ describe("installItem", () => {
// the pre-marker bytes would make every reinstall look like an edit.
const block = {
name: "hero",
title: "Hero",
description: "Hero block",
dimensions: { width: 100, height: 100 },
duration: 1,
type: "hyperframes:block",
files: [{ path: "hero.html", target: "blocks/hero.html", type: "hyperframes:composition" }],
} as unknown as RegistryItem;
@@ -129,12 +195,14 @@ describe("installItem", () => {
describe("installing several items, as a dependency plan does", () => {
const other = {
name: "shared-caption",
title: "Shared caption",
description: "Shared component",
type: "hyperframes:component",
files: [
{
path: "shared-caption.html",
target: "components/shared-caption.html",
type: "hyperframes:file",
type: "hyperframes:snippet",
},
],
} as unknown as RegistryItem;
+53 -20
View File
@@ -1,3 +1,6 @@
import { validRegistryItem } from "./validation.js";
import { registryRoot, registryTargetPath, publishRegistryFile } from "./publication.js";
import type { DownloadByteBudget } from "../capture/readBoundedResponse.js";
/**
* Registry installer — copies item files into a destination project.
*
@@ -7,7 +10,7 @@
*/
import { createHash } from "node:crypto";
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { existsSync, readFileSync } from "node:fs";
import { resolve, relative, isAbsolute } from "node:path";
import type { FileTarget, RegistryItem } from "@hyperframes/core";
import { fetchItemFile, DEFAULT_REGISTRY_URL } from "./remote.js";
@@ -57,7 +60,7 @@ function digest(contents: Buffer | string): string {
}
function readInstallRecord(destDir: string): InstallRecord {
const path = resolve(destDir, INSTALL_RECORD);
const path = registryTargetPath(destDir, INSTALL_RECORD);
if (!existsSync(path)) return {};
try {
const parsed: unknown = JSON.parse(readFileSync(path, "utf-8"));
@@ -77,7 +80,7 @@ function readInstallRecord(destDir: string): InstallRecord {
function writeInstallRecord(destDir: string, record: InstallRecord): void {
const sorted = Object.fromEntries(Object.entries(record).sort(([a], [b]) => a.localeCompare(b)));
writeFileSync(resolve(destDir, INSTALL_RECORD), `${JSON.stringify(sorted, null, 2)}\n`, "utf-8");
publishRegistryFile(destDir, INSTALL_RECORD, `${JSON.stringify(sorted, null, 2)}\n`);
}
/**
@@ -157,8 +160,9 @@ async function installOneFile(
baseUrl: string,
record: InstallRecord,
options: InstallOptions,
budget: DownloadByteBudget,
): Promise<FileOutcome> {
const destPath = resolve(destDir, file.target);
const destPath = registryTargetPath(destDir, file.target);
// Decided before fetching rather than after: a file we are going to keep
// should never be overwritten and then put back, because a crash in
@@ -171,27 +175,23 @@ async function installOneFile(
return { destPath, target: file.target, preserved: true, hash: null, vars: null };
}
await fetchItemFile(item, file, destPath, baseUrl);
let bytes = await fetchItemFile(item, file, baseUrl, budget);
if (isInstalledRegistryBlockComposition(item, file)) {
const source = readFileSync(destPath, "utf-8");
writeFileSync(destPath, addRegistryItemMarker(source, item), "utf-8");
bytes = Buffer.from(addRegistryItemMarker(bytes.toString("utf8"), item));
}
// A component has no mount element to hang values on, so the chosen
// values go into its own declaration or they go nowhere. See
// variableDefaults.ts for why that is the only surviving home.
let vars: ApplyResult | null = null;
if (options.variableValues && isInstalledComponentSnippet(item, file)) {
const source = readFileSync(destPath, "utf-8");
vars = applyVariableDefaults(source, options.variableValues);
if (vars.applied.length > 0) writeFileSync(destPath, vars.html, "utf-8");
vars = applyVariableDefaults(bytes.toString("utf8"), options.variableValues);
if (vars.applied.length > 0) bytes = Buffer.from(vars.html);
}
publishRegistryFile(destDir, file.target, bytes);
// Hash what actually landed, marker and baked defaults included, or the
// next install reads its own output as the project's edit.
return {
destPath,
target: file.target,
preserved: false,
hash: digest(readFileSync(destPath)),
hash: digest(bytes),
vars,
};
}
@@ -204,6 +204,7 @@ export async function installItem(
item: RegistryItem,
options: InstallOptions,
): Promise<InstallResult> {
if (!validRegistryItem(item, item.name, item.type)) throw new Error("Invalid registry item");
const baseUrl = options.baseUrl ?? DEFAULT_REGISTRY_URL;
const destDir = resolve(options.destDir);
@@ -212,12 +213,13 @@ export async function installItem(
assertSafeTarget(destDir, file.target);
}
const record = readInstallRecord(destDir);
const root = registryRoot(destDir);
validatePhysicalTargets(root, item.files);
const record = readInstallRecord(root);
const budget = { remainingBytes: 512 * 1024 * 1024 };
const outcomes = await Promise.all(
item.files.map((file: FileTarget) =>
installOneFile(item, file, destDir, baseUrl, record, options),
),
const outcomes = await installFileBatches(item.files, (file) =>
installOneFile(item, file, root, baseUrl, record, options, budget),
);
const written = outcomes.filter((o) => !o.preserved).map((o) => o.destPath);
@@ -227,7 +229,7 @@ export async function installItem(
for (const outcome of outcomes) {
if (outcome.hash) record[outcome.target] = outcome.hash;
}
writeInstallRecord(destDir, record);
writeInstallRecord(root, record);
}
const vars = outcomes.map((o) => o.vars).filter((v): v is ApplyResult => v !== null);
@@ -246,3 +248,34 @@ export async function installItem(
variablesInvalid: vars.flatMap((v) => v.invalid),
};
}
function validatePhysicalTargets(root: string, files: FileTarget[]): void {
const targets = new Set<string>();
const aliasKey = (path: string): string =>
process.platform === "win32" || process.platform === "darwin"
? path.normalize("NFC").toLowerCase()
: path;
const recordKey = aliasKey(registryTargetPath(root, INSTALL_RECORD));
for (const file of files) {
const path = registryTargetPath(root, file.target);
const key = aliasKey(path);
if (targets.has(key) || key === recordKey)
throw new Error("Unsafe target: duplicate or reserved install record");
targets.add(key);
}
}
async function installFileBatches(
files: FileTarget[],
install: (file: FileTarget) => Promise<FileOutcome>,
): Promise<FileOutcome[]> {
const outcomes: FileOutcome[] = [];
for (let at = 0; at < files.length; at += 4) {
const batch = await Promise.allSettled(files.slice(at, at + 4).map(install));
for (const result of batch) {
if (result.status === "rejected") throw result.reason;
outcomes.push(result.value);
}
}
return outcomes;
}
@@ -0,0 +1,105 @@
import {
mkdtempSync,
mkdirSync,
readFileSync,
writeFileSync,
symlinkSync,
lstatSync,
chmodSync,
linkSync,
rmSync,
readdirSync,
} from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { expect, it } from "vitest";
import { publishRegistryFile, registryRoot, registryTargetPath } from "./publication.js";
it("rejects traversal, Windows aliases and streams before publication", () => {
const root = registryRoot(mkdtempSync(join(tmpdir(), "hf-publish-")));
try {
for (const target of [
"../outside",
"/absolute",
"C:alias",
"file:stream",
"CON.txt",
"nul",
"folder/LPT1.txt",
"trailing.",
"trailing ",
"a//b",
"a/./b",
]) {
expect(() => registryTargetPath(root, target), target).toThrow(/Unsafe target/);
}
expect(readdirSync(root)).toEqual([]);
} finally {
rmSync(root, { recursive: true, force: true });
}
});
it("protects external leaf symlinks including the install record", () => {
const root = registryRoot(mkdtempSync(join(tmpdir(), "hf-publish-")));
const outside = mkdtempSync(join(tmpdir(), "hf-outside-"));
try {
const victim = join(outside, "victim.json");
writeFileSync(victim, "original");
for (const target of ["file.html", "hyperframes.lock.json"]) {
symlinkSync(victim, join(root, target), "file");
expect(() => publishRegistryFile(root, target, "attack")).toThrow(/Unsafe target/);
expect(readFileSync(victim, "utf8")).toBe("original");
}
} finally {
rmSync(root, { recursive: true, force: true });
rmSync(outside, { recursive: true, force: true });
}
});
it("atomically replaces a hard-linked leaf without altering its other name", () => {
const root = registryRoot(mkdtempSync(join(tmpdir(), "hf-publish-")));
try {
writeFileSync(join(root, "original"), "old");
linkSync(join(root, "original"), join(root, "target"));
publishRegistryFile(root, "target", "new");
expect(readFileSync(join(root, "original"), "utf8")).toBe("old");
expect(readFileSync(join(root, "target"), "utf8")).toBe("new");
mkdirSync(join(root, "directory"));
expect(() => publishRegistryFile(root, "directory", "cannot replace directory")).toThrow();
expect(readdirSync(root).some((name) => name.startsWith(".hf-install-"))).toBe(false);
} finally {
rmSync(root, { recursive: true, force: true });
}
});
it("publishes through an internal leaf alias to its physical target", () => {
const root = registryRoot(mkdtempSync(join(tmpdir(), "hf-publish-")));
try {
mkdirSync(join(root, "nested"));
writeFileSync(join(root, "nested/physical"), "old");
symlinkSync(join(root, "nested/physical"), join(root, "alias"), "file");
publishRegistryFile(root, "alias", "new");
expect(lstatSync(join(root, "alias")).isSymbolicLink()).toBe(true);
expect(readFileSync(join(root, "nested/physical"), "utf8")).toBe("new");
} finally {
rmSync(root, { recursive: true, force: true });
}
});
it.skipIf(process.platform === "win32")(
"preserves executable mode when replacing an installed file",
() => {
const root = registryRoot(mkdtempSync(join(tmpdir(), "hf-mode-")));
try {
const path = join(root, "script.sh");
writeFileSync(path, "old");
chmodSync(path, 0o755);
publishRegistryFile(root, "script.sh", "new");
expect(lstatSync(path).mode & 0o777).toBe(0o755);
symlinkSync(path, join(root, "alias"), "file");
publishRegistryFile(root, "alias", "via alias");
expect(lstatSync(path).mode & 0o777).toBe(0o755);
expect(lstatSync(join(root, "alias")).isSymbolicLink()).toBe(true);
} finally {
rmSync(root, { recursive: true, force: true });
}
},
);
+71
View File
@@ -0,0 +1,71 @@
import {
lstatSync,
chmodSync,
mkdirSync,
mkdtempSync,
realpathSync,
renameSync,
rmSync,
writeFileSync,
} from "node:fs";
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
export function registryRoot(directory: string): string {
mkdirSync(directory, { recursive: true });
return realpathSync(directory);
}
export function registryTargetPath(root: string, target: string): string {
const parts = target.split(/[\\/]/);
if (isAbsolute(target) || parts.some(unsafeSegment)) throw new Error(`Unsafe target "${target}"`);
let path = root;
for (const part of parts) {
path = join(path, part);
if (lstatSync(path, { throwIfNoEntry: false })) path = realpathSync(path);
assertContained(root, path);
}
return path;
}
function unsafeSegment(part: string): boolean {
return (
!part ||
part === "." ||
part === ".." ||
/[<>:"|?*]/.test(part) ||
Array.from(part).some((char) => char.charCodeAt(0) < 32) ||
/[. ]$/.test(part) ||
/^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part)
);
}
function assertContained(root: string, path: string): void {
const rel = relative(root, path);
if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
throw new Error(`Unsafe target outside registry destination: ${path}`);
}
}
/** Ancestor directories are trusted against concurrent local replacement during publication. */
export function publishRegistryFile(
root: string,
target: string,
bytes: Uint8Array | string,
): string {
const path = registryTargetPath(root, target);
mkdirSync(dirname(path), { recursive: true });
const parent = realpathSync(dirname(path));
assertContained(root, parent);
const destination = resolve(parent, basename(path));
const stage = mkdtempSync(join(parent, ".hf-install-"));
try {
const stagedFile = join(stage, "file");
writeFileSync(stagedFile, bytes, { flag: "wx" });
const previous = lstatSync(destination, { throwIfNoEntry: false });
if (previous?.isFile()) chmodSync(stagedFile, previous.mode & 0o777);
renameSync(stagedFile, destination);
return destination;
} finally {
rmSync(stage, { recursive: true, force: true });
}
}
+106 -17
View File
@@ -1,6 +1,6 @@
import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { MockInstance } from "vitest";
import { mkdtempSync, rmSync } from "node:fs";
import { mkdtempSync, rmSync, readdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -21,13 +21,23 @@ const {
DEFAULT_REGISTRY_URL,
} = await import("./remote.js");
const MANIFEST = { name: "hyperframes", items: [{ name: "count-up" }] };
const ITEM = { name: "count-up", type: "hyperframes:component", files: [] };
const MANIFEST = {
name: "hyperframes",
homepage: "https://hyperframes.heygen.com",
items: [{ name: "count-up", type: "hyperframes:component" }],
};
const ITEM = {
name: "count-up",
type: "hyperframes:component",
title: "Count up",
description: "Counter",
files: [{ path: "count.html", target: "count.html", type: "hyperframes:snippet" }],
};
const ONE_DAY_MS = 24 * 60 * 60 * 1000;
function ok(body: unknown): Response {
return { ok: true, status: 200, json: async () => body } as unknown as Response;
return new Response(JSON.stringify(body));
}
/**
@@ -64,6 +74,21 @@ afterAll(() => {
});
describe("fetchRegistryManifest", () => {
it("rejects poisoned stale cache data instead of returning it on network failure", async () => {
vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(ok(MANIFEST));
await fetchRegistryManifest(DEFAULT_REGISTRY_URL);
const cache = join(scratchHome, ".hyperframes/cache");
const file = readdirSync(cache)[0]!;
writeFileSync(
join(cache, file),
JSON.stringify({
fetchedAt: 0,
data: { ...MANIFEST, items: [{ name: "../../outside", type: "hyperframes:component" }] },
}),
);
vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("offline"));
await expect(fetchRegistryManifest(DEFAULT_REGISTRY_URL)).resolves.toBeUndefined();
});
it("serves a fresh cache without touching the network", async () => {
const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(ok(MANIFEST));
await fetchRegistryManifest(DEFAULT_REGISTRY_URL);
@@ -123,7 +148,13 @@ describe("fetchRegistryManifest", () => {
vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(ok(MANIFEST));
await fetchRegistryManifest(DEFAULT_REGISTRY_URL);
const fresher = { ...MANIFEST, items: [{ name: "count-up" }, { name: "push-in" }] };
const fresher = {
...MANIFEST,
items: [
{ name: "count-up", type: "hyperframes:component" },
{ name: "push-in", type: "hyperframes:component" },
],
};
vi.spyOn(globalThis, "fetch").mockResolvedValue(ok(fresher));
await expect(fetchRegistryManifest(DEFAULT_REGISTRY_URL, { skipCache: true })).resolves.toEqual(
@@ -133,6 +164,30 @@ describe("fetchRegistryManifest", () => {
});
describe("fetchItemManifest", () => {
it("rejects a cache escape name before fetching", async () => {
const fetchSpy = vi.spyOn(globalThis, "fetch");
await expect(
fetchItemManifest("x/../../../.config/tool", "hyperframes:component"),
).rejects.toThrow(/Invalid registry item/);
expect(fetchSpy).not.toHaveBeenCalled();
});
it("rejects a response for a different requested item", async () => {
vi.spyOn(globalThis, "fetch").mockResolvedValue(ok(ITEM));
await expect(fetchItemManifest("other", "hyperframes:component")).rejects.toThrow(
/Invalid registry manifest/,
);
});
it("cancels oversized manifest responses without materializing their bodies", async () => {
const cancel = vi.fn();
const response = new Response(new ReadableStream({ cancel }), {
headers: { "content-length": "20000000" },
});
vi.spyOn(globalThis, "fetch").mockResolvedValue(response);
await expect(fetchItemManifest("count-up", "hyperframes:component")).rejects.toThrow(
/download limit/,
);
expect(cancel).toHaveBeenCalledOnce();
});
it("serves the expired cache when the item fetch fails", async () => {
const fetchSpy = await staleAfterPriming(ITEM, () =>
fetchItemManifest("count-up", "hyperframes:component", DEFAULT_REGISTRY_URL),
@@ -245,10 +300,46 @@ describe("assetSourceUrl", () => {
});
});
describe("bounded registry file downloads", () => {
it.each([undefined, "1"])(
"cancels bodies exceeding the shared budget despite length %s",
async (length) => {
const cancel = vi.fn();
vi.spyOn(globalThis, "fetch").mockResolvedValue(
new Response(
new ReadableStream({
pull(controller) {
controller.enqueue(new Uint8Array(4));
},
cancel,
}),
{ headers: length ? { "content-length": length } : {} },
),
);
const budget = { remainingBytes: 5 };
await expect(
fetchItemFile(ITEM as never, ITEM.files[0] as never, DEFAULT_REGISTRY_URL, budget),
).rejects.toThrow(/budget/);
expect(budget.remainingBytes).toBe(0);
expect(cancel).toHaveBeenCalledOnce();
},
);
it("shares the byte budget across concurrent files", async () => {
vi.spyOn(globalThis, "fetch").mockImplementation(async () => new Response("12345678"));
const budget = { remainingBytes: 10 };
const outcomes = await Promise.allSettled(
[1, 2].map(() =>
fetchItemFile(ITEM as never, ITEM.files[0] as never, DEFAULT_REGISTRY_URL, budget),
),
);
expect(outcomes.filter((result) => result.status === "fulfilled")).toHaveLength(1);
expect(budget.remainingBytes).toBe(0);
});
});
describe("fetchItemFile retries", () => {
const item = { name: "blur-in", type: "hyperframes:component" } as never;
const file = { path: "blur-in.html", target: "compositions/components/blur-in.html" } as never;
const dest = () => join(scratchHome, `dl-${Math.random().toString(36).slice(2)}.html`);
it("recovers from a transient blip instead of failing the whole install", async () => {
// Item files are the one uncached path, so a single blip used to kill the
@@ -256,13 +347,11 @@ describe("fetchItemFile retries", () => {
const fetchSpy = vi
.spyOn(globalThis, "fetch")
.mockRejectedValueOnce(new Error("fetch failed", { cause: new Error("ECONNRESET") }))
.mockResolvedValueOnce({
ok: true,
status: 200,
arrayBuffer: async () => new TextEncoder().encode("<div>ok</div>").buffer,
} as unknown as Response);
.mockResolvedValueOnce(new Response("<div>ok</div>"));
await expect(fetchItemFile(item, file, dest(), DEFAULT_REGISTRY_URL)).resolves.toBeUndefined();
await expect(fetchItemFile(item, file, DEFAULT_REGISTRY_URL)).resolves.toEqual(
Buffer.from("<div>ok</div>"),
);
expect(fetchSpy).toHaveBeenCalledTimes(2);
});
@@ -275,7 +364,7 @@ describe("fetchItemFile retries", () => {
}),
);
await expect(fetchItemFile(item, file, dest(), DEFAULT_REGISTRY_URL)).rejects.toThrow(
await expect(fetchItemFile(item, file, DEFAULT_REGISTRY_URL)).rejects.toThrow(
/self-signed certificate/,
);
expect(fetchSpy).toHaveBeenCalledTimes(1);
@@ -286,9 +375,9 @@ describe("fetchItemFile retries", () => {
new Error("fetch failed", { cause: new Error("self-signed certificate in chain") }),
);
await expect(
fetchItemFile(item, file, dest(), "https://private.example/registry"),
).rejects.toThrow(/https:\/\/private\.example\/registry\/components\/blur-in\/blur-in\.html/);
await expect(fetchItemFile(item, file, "https://private.example/registry")).rejects.toThrow(
/https:\/\/private\.example\/registry\/components\/blur-in\/blur-in\.html/,
);
});
it("gives up after a bounded number of attempts", async () => {
@@ -296,7 +385,7 @@ describe("fetchItemFile retries", () => {
.spyOn(globalThis, "fetch")
.mockRejectedValue(new Error("fetch failed", { cause: new Error("ECONNRESET") }));
await expect(fetchItemFile(item, file, dest(), DEFAULT_REGISTRY_URL)).rejects.toThrow();
await expect(fetchItemFile(item, file, DEFAULT_REGISTRY_URL)).rejects.toThrow();
expect(fetchSpy).toHaveBeenCalledTimes(3);
});
});
+46 -27
View File
@@ -1,3 +1,8 @@
import { readBoundedRegistryFile } from "./boundedFile.js";
import { publishRegistryFile, registryRoot } from "./publication.js";
import { fetchRegistryHttps, registryHttpsUrl, registryPathUrl } from "./transport.js";
import { readBoundedResponse, type DownloadByteBudget } from "../capture/readBoundedResponse.js";
import { validRegistryName, validRegistryManifest, validRegistryItem } from "./validation.js";
/**
* Remote Registry Fetching
*
@@ -12,8 +17,7 @@
* `<type-dir>` comes from ITEM_TYPE_DIRS in @hyperframes/core.
*/
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join, dirname } from "node:path";
import { join, basename } from "node:path";
import { homedir } from "node:os";
import {
ITEM_TYPE_DIRS,
@@ -27,6 +31,7 @@ export const DEFAULT_REGISTRY_URL =
"https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry";
const FETCH_TIMEOUT_MS = 10_000;
const MAX_MANIFEST_BYTES = 10 * 1024 * 1024;
// ── Caching ─────────────────────────────────────────────────────────────────
// 24h TTL on manifest fetches so the interactive picker stays snappy offline.
@@ -54,17 +59,23 @@ function cachePath(baseUrl: string, key: string): string {
* host reported the entire catalog as unreachable and sent authors off to
* hand-write what they already had on disk.
*/
function readCacheEntry<T>(path: string): CacheEntry<T> | undefined {
function readCacheEntry<T>(
path: string,
validate: (value: unknown) => value is T,
): CacheEntry<T> | undefined {
try {
const entry = JSON.parse(readFileSync(path, "utf-8")) as CacheEntry<T>;
if (typeof entry.fetchedAt !== "number") return undefined;
const entry = JSON.parse(
readBoundedRegistryFile(path, MAX_MANIFEST_BYTES).toString("utf8"),
) as CacheEntry<unknown>;
if (!entry || typeof entry.fetchedAt !== "number" || !Number.isFinite(entry.fetchedAt))
return undefined;
// The callers now test the entry rather than the payload, so an empty
// payload would satisfy them: `null` data would short-circuit the fetch
// and be handed back as a RegistryItem. Rejecting it here keeps the miss
// failing toward "go ask the network" rather than toward "the catalog is
// empty", which is the whole point of the change around it.
if (entry.data === undefined || entry.data === null) return undefined;
return entry;
if (!validate(entry.data)) return undefined;
return { fetchedAt: entry.fetchedAt, data: entry.data };
} catch {
// Missing file or corrupt JSON → cache miss.
return undefined;
@@ -77,9 +88,9 @@ function isFresh<T>(entry: CacheEntry<T>): boolean {
function writeCache<T>(path: string, data: T): void {
try {
mkdirSync(dirname(path), { recursive: true });
const root = registryRoot(CACHE_DIR);
const entry: CacheEntry<T> = { fetchedAt: Date.now(), data };
writeFileSync(path, JSON.stringify(entry), "utf-8");
publishRegistryFile(root, basename(path), JSON.stringify(entry));
} catch {
// Cache writes are opportunistic. A read-only home directory or sandboxed
// environment should not make the registry appear unreachable.
@@ -88,12 +99,16 @@ function writeCache<T>(path: string, data: T): void {
// ── Fetchers ────────────────────────────────────────────────────────────────
async function fetchJson<T>(url: string): Promise<T> {
const res = await fetch(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) });
async function fetchJson<T>(url: string, validate: (value: unknown) => value is T): Promise<T> {
const res = await fetchRegistryHttps(url, AbortSignal.timeout(FETCH_TIMEOUT_MS));
if (!res.ok) {
throw new Error(`Registry fetch failed: ${url} — HTTP ${res.status}`);
}
return (await res.json()) as T;
const bytes = await readBoundedResponse(res, MAX_MANIFEST_BYTES);
if (!bytes) throw new Error("Registry manifest exceeds download limit");
const data: unknown = JSON.parse(bytes.toString("utf8"));
if (!validate(data)) throw new Error("Invalid registry manifest");
return data;
}
/**
@@ -110,12 +125,13 @@ export async function fetchRegistryManifest(
baseUrl: string = DEFAULT_REGISTRY_URL,
options?: { skipCache?: boolean },
): Promise<RegistryManifest | undefined> {
const url = registryPathUrl(baseUrl, "registry.json");
const cacheFile = cachePath(baseUrl, "registry");
const cached = readCacheEntry<RegistryManifest>(cacheFile);
const cached = readCacheEntry(cacheFile, validRegistryManifest);
if (!options?.skipCache && cached && isFresh(cached)) return cached.data;
try {
const manifest = await fetchJson<RegistryManifest>(`${baseUrl}/registry.json`);
const manifest = await fetchJson(url, validRegistryManifest);
writeCache(cacheFile, manifest);
return manifest;
} catch {
@@ -134,14 +150,17 @@ export async function fetchItemManifest(
type: ItemType,
baseUrl: string = DEFAULT_REGISTRY_URL,
): Promise<RegistryItem> {
if (!validRegistryName(name) || !Object.hasOwn(ITEM_TYPE_DIRS, type))
throw new Error("Invalid registry item name or type");
const dir = ITEM_TYPE_DIRS[type];
const validate = (value: unknown): value is RegistryItem => validRegistryItem(value, name, type);
const url = registryPathUrl(baseUrl, dir, name, "registry-item.json");
const cacheFile = cachePath(baseUrl, `${dir}__${name}`);
const cached = readCacheEntry<RegistryItem>(cacheFile);
const cached = readCacheEntry(cacheFile, validate);
if (cached && isFresh(cached)) return cached.data;
const url = `${baseUrl}/${dir}/${name}/registry-item.json`;
try {
const item = await fetchJson<RegistryItem>(url);
const item = await fetchJson(url, validate);
writeCache(cacheFile, item);
return item;
} catch (err) {
@@ -202,9 +221,10 @@ function isRetryableTransport(err: unknown): boolean {
*/
async function fetchWithRetry(url: string, attempts = 3): Promise<Response> {
let lastErr: unknown;
const signal = AbortSignal.timeout(FETCH_TIMEOUT_MS);
for (let attempt = 1; attempt <= attempts; attempt++) {
try {
return await fetch(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) });
return await fetchRegistryHttps(url, signal);
} catch (err) {
lastErr = err;
if (attempt === attempts || !isRetryableTransport(err)) break;
@@ -235,26 +255,25 @@ export function assetSourceUrl(
baseUrl: string = DEFAULT_REGISTRY_URL,
): string {
if (file.url === undefined) {
return `${baseUrl}/${ITEM_TYPE_DIRS[item.type]}/${item.name}/${file.path}`;
return registryPathUrl(baseUrl, ITEM_TYPE_DIRS[item.type], item.name, file.path);
}
if (!file.url.startsWith("https://")) {
throw new Error(
`Unsafe file.url "${file.url}" for "${item.name}/${file.path}": must be an absolute https:// URL.`,
);
}
return file.url;
return registryHttpsUrl(file.url).href;
}
/**
* Download a single file referenced by an item to a local destination.
* Caller is responsible for target-path validation (see installer.ts).
* Read bounded item bytes; the installer owns transforms and safe publication.
*/
export async function fetchItemFile(
item: RegistryItem,
file: FileTarget,
destPath: string,
baseUrl: string = DEFAULT_REGISTRY_URL,
): Promise<void> {
budget: DownloadByteBudget = { remainingBytes: 512 * 1024 * 1024 },
): Promise<Buffer> {
// Reject path-traversal in file.path (mirrors assertSafeTarget for file.target).
if (/(^|[/\\])\.\.([/\\]|$)/.test(file.path)) {
throw new Error(`Unsafe file.path "${file.path}": path segments may not contain "..".`);
@@ -274,7 +293,7 @@ export async function fetchItemFile(
if (!res.ok) {
throw new Error(`File fetch failed: ${url} — HTTP ${res.status}`);
}
const buf = new Uint8Array(await res.arrayBuffer());
mkdirSync(dirname(destPath), { recursive: true });
writeFileSync(destPath, buf);
const bytes = await readBoundedResponse(res, 128 * 1024 * 1024, budget);
if (!bytes) throw new Error("Registry file exceeds download budget");
return bytes;
}
@@ -0,0 +1,42 @@
import { afterEach, expect, it, vi } from "vitest";
import { registryPathUrl, registryHttpsUrl, fetchRegistryHttps } from "./transport.js";
afterEach(() => vi.unstubAllGlobals());
it("preserves private registries and signed CDN URLs while encoding path segments", () => {
expect(
registryPathUrl("https://registry.internal/base/", "blocks", "demo", "my file#.html"),
).toBe("https://registry.internal/base/blocks/demo/my%20file%23.html");
expect(registryHttpsUrl("https://cdn.example/a?signature=abc").search).toBe("?signature=abc");
expect(() => registryPathUrl("https://registry.internal/?secret=1", "registry.json")).toThrow();
expect(() => registryPathUrl("https://registry.internal", "../escape")).toThrow();
});
it.each([
"http://private.example/file",
"https://user:pass@private.example/file",
"file:///tmp/file",
])("rejects unsafe redirect %s before issuing its request", async (location) => {
const cancel = vi.fn();
const fetchMock = vi.fn(
async () =>
new Response(new ReadableStream({ cancel }), { status: 302, headers: { location } }),
);
vi.stubGlobal("fetch", fetchMock);
await expect(
fetchRegistryHttps("https://registry.internal/file", AbortSignal.timeout(1000)),
).rejects.toThrow();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(cancel).toHaveBeenCalledOnce();
});
it("follows a relative HTTPS redirect", async () => {
const fetchMock = vi
.fn()
.mockResolvedValueOnce(new Response(null, { status: 302, headers: { location: "../asset" } }))
.mockResolvedValueOnce(new Response("ok"));
vi.stubGlobal("fetch", fetchMock);
expect(
await (
await fetchRegistryHttps("https://private.example/base/file", AbortSignal.timeout(1000))
).text(),
).toBe("ok");
expect(fetchMock.mock.calls[1]?.[0]).toBe("https://private.example/asset");
});
+43
View File
@@ -0,0 +1,43 @@
/** Registry hosts may be private; authenticate the HTTPS transport, not public IP reachability. */
export function registryHttpsUrl(value: string): URL {
const url = new URL(value);
if (url.protocol !== "https:" || url.username || url.password || url.hash) {
throw new Error(
"Registry URLs must be an absolute https:// URL without credentials or fragments",
);
}
return url;
}
export function registryPathUrl(base: string, ...parts: string[]): string {
const url = registryHttpsUrl(base);
if (url.search) throw new Error("Registry base URL must not contain a query");
const segments = parts.flatMap((part) => part.split("/"));
if (
segments.some(
(part) =>
!part ||
part === "." ||
part === ".." ||
part.includes("\\") ||
Array.from(part).some((char) => char.charCodeAt(0) < 32),
)
) {
throw new Error("Unsafe registry source path");
}
url.pathname = url.pathname.replace(/\/$/, "") + "/" + segments.map(encodeURIComponent).join("/");
return url.href;
}
export async function fetchRegistryHttps(value: string, signal: AbortSignal): Promise<Response> {
let url = registryHttpsUrl(value);
for (let redirects = 0; redirects <= 5; redirects++) {
const response = await fetch(url.href, { signal, redirect: "manual" });
if (![301, 302, 303, 307, 308].includes(response.status)) return response;
const location = response.headers.get("location");
await response.body?.cancel();
if (!location) throw new Error("Registry redirect has no location");
url = registryHttpsUrl(new URL(location, url).href);
}
throw new Error("Too many registry redirects");
}
@@ -0,0 +1,42 @@
import { describe, expect, it } from "vitest";
import { validRegistryName, validRegistryManifest, validRegistryItem } from "./validation.js";
const item = {
name: "demo",
type: "hyperframes:component",
title: "Demo",
description: "Example",
files: [{ path: "demo.html", target: "components/demo.html", type: "hyperframes:snippet" }],
};
describe("registry manifest boundary", () => {
it("rejects cache-key traversal and marker breakouts", () => {
for (const name of ["x/../../../.config/tool", "demo -->", "a".repeat(129)])
expect(validRegistryName(name)).toBe(false);
expect(validRegistryName("demo-2")).toBe(true);
});
it("validates the published item schema and requested identity", () => {
expect(validRegistryItem(item, "demo", "hyperframes:component")).toBe(true);
expect(validRegistryItem(item, "other", "hyperframes:component")).toBe(false);
expect(validRegistryItem(item, "demo", "hyperframes:block")).toBe(false);
expect(
validRegistryItem(
{ ...item, files: [{ ...item.files[0], target: "../outside" }] },
"demo",
"hyperframes:component",
),
).toBe(false);
});
it("validates cached and remote top-level manifest shape and cardinality", () => {
const manifest = {
name: "registry",
homepage: "https://registry.example",
items: [{ name: "demo", type: "hyperframes:component" }],
};
expect(validRegistryManifest(manifest)).toBe(true);
expect(validRegistryManifest({ ...manifest, items: [{ name: "../outside" }] })).toBe(false);
expect(
validRegistryManifest({ ...manifest, items: Array(10_001).fill(manifest.items[0]) }),
).toBe(false);
});
});
+34
View File
@@ -0,0 +1,34 @@
import { Ajv2020 } from "ajv/dist/2020.js";
import registrySchema from "@hyperframes/core/schemas/registry.json";
import itemSchema from "@hyperframes/core/schemas/registry-item.json";
import type { RegistryManifest, RegistryItem, ItemType } from "@hyperframes/core";
const ajv = new Ajv2020({ strict: false, validateFormats: false });
const manifestShape = ajv.compile<RegistryManifest>(registrySchema);
const itemShape = ajv.compile<RegistryItem>(itemSchema);
export function validRegistryName(name: string): boolean {
return name.length <= 128 && /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(name);
}
export function validRegistryManifest(value: unknown): value is RegistryManifest {
return (
manifestShape(value) &&
value.items.length <= 10_000 &&
value.items.every((item) => validRegistryName(item.name))
);
}
export function validRegistryItem(
value: unknown,
name: string,
type: ItemType,
): value is RegistryItem {
return (
itemShape(value) &&
value.name === name &&
value.type === type &&
validRegistryName(value.name) &&
value.files.length <= 1_000
);
}
+8 -1
View File
@@ -109,7 +109,8 @@ const server = createServer((req, res) => {
res.end(hit.body);
});
await new Promise((r) => server.listen(0, "127.0.0.1", r));
const registryUrl = `http://127.0.0.1:${server.address().port}`;
const fixtureTransportUrl = `http://127.0.0.1:${server.address().port}`;
const registryUrl = `https://registry-${server.address().port}.fixture.invalid`;
const sandbox = mkdtempSync(join(tmpdir(), "hf-telemetry-e2e-"));
const hookPath = join(sandbox, "capture-hook.cjs");
@@ -125,6 +126,12 @@ globalThis.fetch = async function (input, init) {
appendFileSync(OUT, (init && init.body) + "\\n");
return new Response('{"status":1}', { status: 200 });
}
// Test-only transport mapping: exercise the HTTPS registry contract without
// provisioning TLS. Redirect enforcement has separate transport unit tests.
const parsed = new URL(url);
if (parsed.origin === ${JSON.stringify(registryUrl)}) {
return realFetch(${JSON.stringify(fixtureTransportUrl)} + parsed.pathname + parsed.search, init);
}
return realFetch(input, init);
};
`,