diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d91f5d4e99..5c3aa42956 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: code: ${{ steps.filter.outputs.code }} cli: ${{ steps.filter.outputs.cli }} skills: ${{ steps.filter.outputs.skills }} + catalog_index: ${{ steps.filter.outputs.catalog_index }} codex_plugin: ${{ steps.filter.outputs.codex_plugin }} gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }} studio: ${{ steps.filter.outputs.studio }} @@ -49,6 +50,10 @@ jobs: with: token: "" filters: | + catalog_index: + - "registry/registry.json" + - "registry/catalog-artifact/**" + - "scripts/catalog/check-artifact-coverage.ts" code: - "packages/**" - "scripts/**" @@ -360,6 +365,21 @@ jobs: # `gen:skills-manifest --check`, which compares per-skill content hashes; the # manifest carries no version/timestamp, so it only fails on real content # drift. bun runs the TS script directly, no install needed. + catalog-index-coverage: + name: "Catalog: search index covers the registry" + needs: changes + if: needs.changes.outputs.catalog_index == 'true' + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + # Comparing two name lists needs neither the embedding model nor a + # network call, so this stays a seconds-long gate. Regenerating the + # vectors does need the model, which is why CI checks rather than fixes. + - name: Verify every searchable registry item has a vector + run: bun scripts/catalog/check-artifact-coverage.ts + skills-manifest: name: "Skills: manifest in sync" needs: changes diff --git a/scripts/catalog/check-artifact-coverage.ts b/scripts/catalog/check-artifact-coverage.ts new file mode 100644 index 0000000000..9c45589b2f --- /dev/null +++ b/scripts/catalog/check-artifact-coverage.ts @@ -0,0 +1,77 @@ +/** + * Fails when the committed catalog vector artifact no longer covers the + * registry it is meant to search. + * + * The artifact is regenerated by hand: nothing in CI or in a hook rebuilds it, + * because embedding needs the 32 MB model. So adding a registry item silently + * makes it invisible to meaning search until someone remembers. Removing one is + * self-healing (the ranker filters names it cannot install), which is why the + * asymmetry is easy to miss. + * + * Comparing the two name lists needs no model and no network, so it is cheap + * enough to run on every change that touches either side. + */ +import { readFileSync } from "node:fs"; + +type RegistryItem = { name: string; type?: string }; +type Registry = { items: RegistryItem[] }; +type Artifact = { model?: string; dimensions?: number; names?: string[] }; + +const REGISTRY = "registry/registry.json"; +const ARTIFACT = "registry/catalog-artifact/local-vectors.json"; + +function read(path: string): T { + try { + return JSON.parse(readFileSync(path, "utf8")) as T; + } catch (error) { + console.error(`Could not read ${path}: ${(error as Error).message}`); + process.exit(1); + } +} + +const registry = read(REGISTRY); +const artifact = read(ARTIFACT); + +// Only blocks and components are searchable moves. Examples are starter +// projects a user scaffolds, never something `catalog` ranks, so the artifact +// deliberately carries no vector for them and demanding one would keep this +// gate permanently red. +const SEARCHABLE = new Set(["hyperframes:block", "hyperframes:component"]); +const registryNames = new Set( + registry.items.filter((i) => SEARCHABLE.has(i.type ?? "")).map((i) => i.name), +); +const artifactNames = new Set(artifact.names ?? []); + +const unindexed = [...registryNames].filter((n) => !artifactNames.has(n)).sort(); +const dropped = [...artifactNames].filter((n) => !registryNames.has(n)).sort(); + +const show = (names: string[]) => + names + .slice(0, 10) + .map((n) => ` ${n}`) + .join("\n") + (names.length > 10 ? `\n ... and ${names.length - 10} more` : ""); + +console.log(`registry: ${registryNames.size} searchable items (blocks + components)`); +console.log(`artifact: ${artifactNames.size} vectors (${artifact.model ?? "unknown model"})`); + +if (dropped.length > 0) { + // Not fatal: the CLI filters these before a user ever sees them. + console.log(`\nnote: ${dropped.length} vector(s) name items the registry no longer has.`); + console.log(show(dropped)); + console.log(" These are filtered at search time, so they cost space, not correctness."); +} + +if (unindexed.length === 0) { + console.log("\nEvery registry item has a vector. Meaning search can see the whole catalog."); + process.exit(0); +} + +console.error(`\n${unindexed.length} registry item(s) have no vector:`); +console.error(show(unindexed)); +console.error( + "\nMeaning search cannot return these at all: they are absent from the candidate\n" + + "set, not ranked low. Word search still finds them.\n\n" + + "Regenerate and commit the artifact:\n" + + " bun scripts/catalog/build-local-vectors.ts\n", +); +process.exit(1);