From f85301cbec101be81f3eb77bdab15c208ad876e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sun, 9 Aug 2026 17:23:45 +0000 Subject: [PATCH] ci: fail when the search index stops covering the registry The catalog vector artifact is regenerated by hand. Nothing in CI, in package.json or in a hook rebuilds it, because embedding needs the 32 MB model. So adding a registry item silently makes it invisible to meaning search until someone remembers to regenerate. The failure is asymmetric, which is what makes it easy to miss. Removing an item is self-healing: the ranker still scores the dead vector, then filters the name before display, so a user is never offered something they cannot install. Adding one is not: the item is absent from the candidate set entirely, not ranked low. Comparing the two name lists needs neither the model nor a network call, so the gate runs in seconds. CI checks rather than fixes, for the same reason it cannot regenerate. Scoped to blocks and components. Examples are starter projects a user scaffolds, never something catalog ranks, and the artifact carries no vector for them, so demanding one would keep this gate permanently red and it would be ignored within a week. Verified in both directions rather than assumed: adding an unindexed item exits 1 and names it, restoring the registry exits 0. --- .github/workflows/ci.yml | 20 ++++++ scripts/catalog/check-artifact-coverage.ts | 77 ++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 scripts/catalog/check-artifact-coverage.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d91f5d4e99..5c3aa42956 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: code: ${{ steps.filter.outputs.code }} cli: ${{ steps.filter.outputs.cli }} skills: ${{ steps.filter.outputs.skills }} + catalog_index: ${{ steps.filter.outputs.catalog_index }} codex_plugin: ${{ steps.filter.outputs.codex_plugin }} gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }} studio: ${{ steps.filter.outputs.studio }} @@ -49,6 +50,10 @@ jobs: with: token: "" filters: | + catalog_index: + - "registry/registry.json" + - "registry/catalog-artifact/**" + - "scripts/catalog/check-artifact-coverage.ts" code: - "packages/**" - "scripts/**" @@ -360,6 +365,21 @@ jobs: # `gen:skills-manifest --check`, which compares per-skill content hashes; the # manifest carries no version/timestamp, so it only fails on real content # drift. bun runs the TS script directly, no install needed. + catalog-index-coverage: + name: "Catalog: search index covers the registry" + needs: changes + if: needs.changes.outputs.catalog_index == 'true' + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + # Comparing two name lists needs neither the embedding model nor a + # network call, so this stays a seconds-long gate. Regenerating the + # vectors does need the model, which is why CI checks rather than fixes. + - name: Verify every searchable registry item has a vector + run: bun scripts/catalog/check-artifact-coverage.ts + skills-manifest: name: "Skills: manifest in sync" needs: changes diff --git a/scripts/catalog/check-artifact-coverage.ts b/scripts/catalog/check-artifact-coverage.ts new file mode 100644 index 0000000000..9c45589b2f --- /dev/null +++ b/scripts/catalog/check-artifact-coverage.ts @@ -0,0 +1,77 @@ +/** + * Fails when the committed catalog vector artifact no longer covers the + * registry it is meant to search. + * + * The artifact is regenerated by hand: nothing in CI or in a hook rebuilds it, + * because embedding needs the 32 MB model. So adding a registry item silently + * makes it invisible to meaning search until someone remembers. Removing one is + * self-healing (the ranker filters names it cannot install), which is why the + * asymmetry is easy to miss. + * + * Comparing the two name lists needs no model and no network, so it is cheap + * enough to run on every change that touches either side. + */ +import { readFileSync } from "node:fs"; + +type RegistryItem = { name: string; type?: string }; +type Registry = { items: RegistryItem[] }; +type Artifact = { model?: string; dimensions?: number; names?: string[] }; + +const REGISTRY = "registry/registry.json"; +const ARTIFACT = "registry/catalog-artifact/local-vectors.json"; + +function read(path: string): T { + try { + return JSON.parse(readFileSync(path, "utf8")) as T; + } catch (error) { + console.error(`Could not read ${path}: ${(error as Error).message}`); + process.exit(1); + } +} + +const registry = read(REGISTRY); +const artifact = read(ARTIFACT); + +// Only blocks and components are searchable moves. Examples are starter +// projects a user scaffolds, never something `catalog` ranks, so the artifact +// deliberately carries no vector for them and demanding one would keep this +// gate permanently red. +const SEARCHABLE = new Set(["hyperframes:block", "hyperframes:component"]); +const registryNames = new Set( + registry.items.filter((i) => SEARCHABLE.has(i.type ?? "")).map((i) => i.name), +); +const artifactNames = new Set(artifact.names ?? []); + +const unindexed = [...registryNames].filter((n) => !artifactNames.has(n)).sort(); +const dropped = [...artifactNames].filter((n) => !registryNames.has(n)).sort(); + +const show = (names: string[]) => + names + .slice(0, 10) + .map((n) => ` ${n}`) + .join("\n") + (names.length > 10 ? `\n ... and ${names.length - 10} more` : ""); + +console.log(`registry: ${registryNames.size} searchable items (blocks + components)`); +console.log(`artifact: ${artifactNames.size} vectors (${artifact.model ?? "unknown model"})`); + +if (dropped.length > 0) { + // Not fatal: the CLI filters these before a user ever sees them. + console.log(`\nnote: ${dropped.length} vector(s) name items the registry no longer has.`); + console.log(show(dropped)); + console.log(" These are filtered at search time, so they cost space, not correctness."); +} + +if (unindexed.length === 0) { + console.log("\nEvery registry item has a vector. Meaning search can see the whole catalog."); + process.exit(0); +} + +console.error(`\n${unindexed.length} registry item(s) have no vector:`); +console.error(show(unindexed)); +console.error( + "\nMeaning search cannot return these at all: they are absent from the candidate\n" + + "set, not ranked low. Word search still finds them.\n\n" + + "Regenerate and commit the artifact:\n" + + " bun scripts/catalog/build-local-vectors.ts\n", +); +process.exit(1);