diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e91e452423..a8e953597d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,19 +99,21 @@ before merge, listed at the end. 2. For components: include a `demo.html` 3. Run `npx hyperframes lint` and `npx hyperframes validate` on your HTML 4. Test the install flow: `hyperframes add --dir /tmp/test-project` -5. Regenerate the manifest: `npx tsx scripts/generate-registry-items.ts` - -`registry/registry.json` is generated from the item directories, so edit it with -that script rather than by hand. An entry added by hand survives until the next -regeneration and then disappears; entries left behind for directories that no -longer exist are worse, because `hyperframes add ` resolves the name and -then fails on missing files. +5. Generate and validate the catalog: `bun run check:catalog-drift` ### Generated catalog files -Catalog pull requests commit only the item's source files. Do not commit the -registry index, search vectors, generated docs pages, preview payloads, gallery -data, or Catalog navigation. CI builds and validates them from your sources. +Catalog additions and edits commit only the item's source files. Do not commit +search vectors, generated docs pages, preview payloads, gallery data, or Catalog +navigation. CI builds and validates them from your sources. + +Deleting or renaming an item has one exception: remove its old entry from +`registry/registry.json` in the same PR that deletes the complete item directory. +The CLI reads item files directly from main, so leaving that entry would offer an +item whose files no longer exist. Keep every other index entry and all metadata +unchanged. Do not add the new entry for a rename or an add-plus-delete PR; +automation publishes additions. CI rejects missing removals, added entries, +metadata edits, and removals whose item directories still exist. To generate the complete catalog locally after installing dependencies and building the workspace packages: @@ -122,13 +124,15 @@ bun run generate:catalog This command downloads the pinned embedding model when needed. It updates the registry index, vectors, payloads, pages, gallery and navigation in dependency -order. Restore those generated files before committing an item-source PR. +order. Restore generated files before committing, preserving only the required +index-entry removals when you delete item directories. After source changes merge, automation updates one standing publication PR on `bot/catalog-publish`, titled `chore(catalog): publish generated catalog`. A maintainer approves any waiting workflow runs, reviews it, and merges it after -checks pass. Until that PR merges, the published catalog remains at its previous -snapshot. Existing CLI and docs URLs remain unchanged. +checks pass. New items become discoverable after publication; edits to existing +CLI item sources take effect on main immediately. Generated docs and search +artifacts update when the publication PR merges. Existing URLs remain unchanged. Example manifests are authored source. To deliberately scaffold them, use `scripts/scaffold-example-manifests.ts`; indexing never rewrites them. diff --git a/package.json b/package.json index 4db1037c24..cb9ef40c72 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "player:perf": "bun run --filter @hyperframes/player perf", "format:check": "oxfmt --check .", "knip": "knip", - "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", + "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts", "typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json", "test:skills": "node --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'", "generate:previews": "tsx scripts/generate-template-previews.ts", diff --git a/scripts/catalog-publication.test.mjs b/scripts/catalog-publication.test.mjs index c7b233c2fa..22d49938d6 100644 --- a/scripts/catalog-publication.test.mjs +++ b/scripts/catalog-publication.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { test } from "node:test"; @@ -160,6 +160,7 @@ catch (error) { }, ); return { + root, base, run, calls: () => @@ -202,3 +203,13 @@ test("a failed staging cleanup preserves the publication failure as well", (t) = false, ); }); + +test("publication refuses symlinks instead of reading outside the generated artifact", (t) => { + const fixture = publicationFixture(t, false); + const artifact = join(fixture.root, "registry/registry.json"); + const unrelated = join(fixture.root, "private.txt"); + writeFileSync(unrelated, "must not be published"); + rmSync(artifact); + symlinkSync(unrelated, artifact); + assert.throws(() => catalogChanges(fixture.root, "HEAD"), /ELOOP|symbolic link/); +}); diff --git a/scripts/check-catalog-source-pr.mjs b/scripts/check-catalog-source-pr.mjs index cd35e27867..32643a231f 100644 --- a/scripts/check-catalog-source-pr.mjs +++ b/scripts/check-catalog-source-pr.mjs @@ -1,26 +1,74 @@ import { execFileSync } from "node:child_process"; +import { isDeepStrictEqual } from "node:util"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { isGeneratedCatalogPath } from "./catalog-generated-paths.mjs"; -export function committedCatalogOutputs(paths) { - if (!paths.some((path) => /^registry\/(blocks|components|examples)\/[^/]+\//.test(path))) - return []; - return paths.filter(isGeneratedCatalogPath); +const ITEM_FILE = /^registry\/(blocks|components|examples)\/[^/]+\//; +const ITEM_MANIFEST = /^registry\/[^/]+\/[^/]+\/registry-item\.json$/; +const INDEX = "registry/registry.json"; + +export function committedCatalogOutputs(paths, removalOnlyIndex = false) { + if (!paths.some((path) => ITEM_FILE.test(path))) return []; + return paths.filter(isGeneratedCatalogPath).filter((path) => path !== INDEX || !removalOnlyIndex); } -if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - const base = process.argv[2] ?? "origin/main"; - const paths = execFileSync("git", ["diff", "--name-only", "-z", `${base}...HEAD`], { - encoding: "utf8", - }) - .split("\0") - .filter(Boolean); - const outputs = committedCatalogOutputs(paths); - if (outputs.length > 0) { - console.error( - `Catalog PRs commit item sources only. Restore these generated files from the base branch:\n${outputs.join("\n")}`, - ); - process.exitCode = 1; - } else console.log("Catalog source PR contains no derived catalog files."); +function git(...args) { + return execFileSync("git", args, { encoding: "utf8" }); } + +function sourceFiles(ref) { + return git("ls-tree", "-r", "--name-only", "-z", ref, "--", "registry") + .split("\0") + .filter((path) => ITEM_FILE.test(path)); +} + +function removedItems(base) { + const before = sourceFiles(base); + const after = new Set(sourceFiles("HEAD")); + const remainingDirectories = new Set([...after].map((path) => ITEM_FILE.exec(path)[0])); + return before + .filter((path) => ITEM_MANIFEST.test(path) && !after.has(path)) + .map((path) => { + const directory = ITEM_FILE.exec(path)[0]; + if (remainingDirectories.has(directory)) + throw new Error( + `Remove the complete item directory when deleting its manifest: ${directory}`, + ); + return JSON.parse(git("show", `${base}:${path}`)); + }); +} + +function checkIndexRemovals(base, indexChanged) { + const removed = new Set(removedItems(base).map((item) => `${item.type}/${item.name}`)); + const before = JSON.parse(git("show", `${base}:${INDEX}`)); + const after = JSON.parse(git("show", `HEAD:${INDEX}`)); + const items = before.items.filter((item) => !removed.has(`${item.type}/${item.name}`)); + const hasRemovals = items.length < before.items.length; + if (!isDeepStrictEqual(after, { ...before, items })) + throw new Error( + "Source PRs must remove deleted items from registry/registry.json in the same commit. " + + "Do not add entries or change index metadata; automation publishes those changes.", + ); + if (indexChanged && !hasRemovals) + throw new Error( + "Only removal of entries whose item directories are deleted may edit registry/registry.json.", + ); + return hasRemovals; +} + +function main() { + const requestedBase = process.argv[2] ?? "origin/main"; + const base = git("merge-base", requestedBase, "HEAD").trim(); + const paths = git("diff", "--name-only", "-z", base, "HEAD").split("\0").filter(Boolean); + if (!paths.some((path) => ITEM_FILE.test(path))) return; + const removalOnly = checkIndexRemovals(base, paths.includes(INDEX)); + const outputs = committedCatalogOutputs(paths, removalOnly); + if (outputs.length > 0) + throw new Error( + `Catalog PRs commit item sources only, with a removal-only index exception for deleted items. Restore these generated files:\n${outputs.join("\n")}`, + ); + console.log("Catalog source PR contains only item sources and any required index removals."); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main(); diff --git a/scripts/check-catalog-source-pr.test.mjs b/scripts/check-catalog-source-pr.test.mjs new file mode 100644 index 0000000000..23718558fa --- /dev/null +++ b/scripts/check-catalog-source-pr.test.mjs @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; + +const guard = fileURLToPath(new URL("./check-catalog-source-pr.mjs", import.meta.url)); +const old = { name: "old", type: "hyperframes:component" }; +const kept = { name: "kept", type: "hyperframes:block" }; + +function fixture(t) { + const root = mkdtempSync(join(tmpdir(), "catalog-source-pr-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim(); + const write = (path, contents) => { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), contents); + }; + const item = (directory, entry) => { + write(`registry/${directory}/${entry.name}/registry-item.json`, JSON.stringify(entry)); + write(`registry/${directory}/${entry.name}/index.html`, "
item
"); + }; + const index = (items, revision = "published") => + write( + "registry/registry.json", + JSON.stringify({ name: "catalog", items, vectorRevision: revision }), + ); + const commit = () => { + git("add", "-A"); + git( + "-c", + "user.name=Test", + "-c", + "user.email=test@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + "commit", + "-qm", + "fixture", + ); + }; + git("init", "-q"); + item("components", old); + write("registry/components/old/assets/registry-item.json", "asset metadata"); + item("blocks", kept); + index([old, kept]); + commit(); + const base = git("rev-parse", "HEAD"); + return { + root, + item, + index, + removeOld: () => rmSync(join(root, "registry/components/old"), { recursive: true }), + check: () => { + commit(); + return spawnSync(process.execPath, [guard, base], { cwd: root, encoding: "utf8" }); + }, + }; +} + +test("a deletion removes its index entry in the same source PR", (t) => { + const repo = fixture(t); + repo.removeOld(); + repo.index([kept]); + const result = repo.check(); + assert.equal(result.status, 0, result.stderr); +}); + +test("a rename unpublishes the old name and leaves the new name for automation", (t) => { + const repo = fixture(t); + repo.removeOld(); + repo.item("components", { ...old, name: "renamed" }); + repo.index([kept]); + const result = repo.check(); + assert.equal(result.status, 0, result.stderr); +}); + +test("an add-plus-delete PR carries only the old entry removal", (t) => { + const repo = fixture(t); + repo.removeOld(); + repo.item("blocks", { name: "new", type: "hyperframes:block" }); + repo.index([kept]); + const result = repo.check(); + assert.equal(result.status, 0, result.stderr); +}); + +test("a deletion cannot leave a dangling index entry", (t) => { + const repo = fixture(t); + repo.removeOld(); + const result = repo.check(); + assert.equal(result.status, 1); + assert.match(result.stderr, /must remove deleted items/); +}); + +test("the removal exception cannot publish an added entry", (t) => { + const repo = fixture(t); + repo.removeOld(); + const added = { name: "new", type: "hyperframes:block" }; + repo.item("blocks", added); + repo.index([kept, added]); + const result = repo.check(); + assert.equal(result.status, 1); + assert.match(result.stderr, /Do not add entries/); +}); + +test("the removal exception cannot unpublish a directory that still exists", (t) => { + const repo = fixture(t); + repo.item("blocks", { name: "new", type: "hyperframes:block" }); + repo.index([kept]); + const result = repo.check(); + assert.equal(result.status, 1); + assert.match(result.stderr, /must remove deleted items/); +}); + +test("the removal exception preserves index metadata", (t) => { + const repo = fixture(t); + repo.removeOld(); + repo.index([kept], "changed"); + const result = repo.check(); + assert.equal(result.status, 1); + assert.match(result.stderr, /change index metadata/); +}); + +test("deleting only the manifest does not qualify as deleting an item directory", (t) => { + const repo = fixture(t); + rmSync(join(repo.root, "registry/components/old/registry-item.json")); + repo.index([kept]); + const result = repo.check(); + assert.equal(result.status, 1); + assert.match(result.stderr, /complete item directory/); +}); + +test("deleting nested asset metadata does not unpublish its item", (t) => { + const repo = fixture(t); + rmSync(join(repo.root, "registry/components/old/assets/registry-item.json")); + const result = repo.check(); + assert.equal(result.status, 0, result.stderr); +}); diff --git a/scripts/publish-catalog.mjs b/scripts/publish-catalog.mjs index 5ed14e933a..3b770f8d31 100644 --- a/scripts/publish-catalog.mjs +++ b/scripts/publish-catalog.mjs @@ -1,5 +1,13 @@ import { execFileSync } from "node:child_process"; -import { lstatSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { + closeSync, + constants, + fstatSync, + mkdtempSync, + openSync, + readFileSync, + rmSync, +} from "node:fs"; import { join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; @@ -48,13 +56,21 @@ export function catalogChanges(root, base) { .sort() .map((path) => { if (removed.has(path)) return { path, kind: "delete" }; - const absolute = resolve(root, path); - if (!lstatSync(absolute).isFile()) - throw new Error(`Generated artifact must be a regular file: ${path}`); - return { path, kind: "write", contents: readFileSync(absolute).toString("base64") }; + return { path, kind: "write", contents: artifactContents(resolve(root, path)) }; }); } +function artifactContents(path) { + const descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + if (!fstatSync(descriptor).isFile()) + throw new Error(`Generated artifact must be a regular file: ${path}`); + return readFileSync(descriptor).toString("base64"); + } finally { + closeSync(descriptor); + } +} + function batchEntry(change) { const size = Buffer.byteLength(JSON.stringify(change)); if (size > MAX_BATCH_BYTES)