diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6463d8c9e1..d91f5d4e99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -259,6 +259,7 @@ jobs: - run: bash scripts/ci/install-workspace-dependencies.sh - run: bun run build - run: bun run --filter '*' typecheck + - run: bun run typecheck:scripts test: name: Test diff --git a/bun.lock b/bun.lock index af5e10ccec..2b56c070fc 100644 --- a/bun.lock +++ b/bun.lock @@ -7,7 +7,9 @@ "devDependencies": { "@commitlint/cli": "^20.5.0", "@commitlint/config-conventional": "^20.5.0", + "@hyperframes/core": "workspace:*", "@hyperframes/player": "workspace:*", + "@hyperframes/producer": "workspace:*", "@types/node": "^25.0.10", "concurrently": "^8.2.0", "fallow": "^2.75.0", @@ -18,12 +20,13 @@ "oxlint": "^1.56.0", "tsx": "^4.21.0", "typescript": "^5.0.0", + "vitest": "^3.2.4", "yaml": "^2.9.0", }, }, "packages/aws-lambda": { "name": "@hyperframes/aws-lambda", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/client-sfn": "^3.700.0", @@ -56,7 +59,7 @@ }, "packages/cli": { "name": "@hyperframes/cli", - "version": "0.7.78", + "version": "0.7.97", "bin": { "hyperframes": "./bin/hyperframes.mjs", }, @@ -106,7 +109,7 @@ }, "packages/core": { "name": "@hyperframes/core", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@chenglou/pretext": "^0.0.5", "@hyperframes/lint": "workspace:*", @@ -131,7 +134,7 @@ }, "packages/engine": { "name": "@hyperframes/engine", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@hono/node-server": "^2.0.5", "@hyperframes/core": "workspace:^", @@ -150,7 +153,7 @@ }, "packages/gcp-cloud-run": { "name": "@hyperframes/gcp-cloud-run", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@google-cloud/storage": "^7.14.0", "@google-cloud/workflows": "^4.2.0", @@ -171,7 +174,7 @@ }, "packages/lint": { "name": "@hyperframes/lint", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@hyperframes/parsers": "workspace:*", "htmlparser2": "^10.1.0", @@ -189,7 +192,7 @@ }, "packages/parsers": { "name": "@hyperframes/parsers", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@babel/parser": "^7.27.0", "acorn": "^8.17.0", @@ -209,7 +212,7 @@ }, "packages/player": { "name": "@hyperframes/player", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@hyperframes/core": "workspace:*", }, @@ -224,7 +227,7 @@ }, "packages/producer": { "name": "@hyperframes/producer", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@fontsource/archivo-black": "^5.2.8", "@fontsource/eb-garamond": "^5.2.7", @@ -269,7 +272,7 @@ }, "packages/sdk": { "name": "@hyperframes/sdk", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@hyperframes/core": "workspace:*", "@hyperframes/parsers": "workspace:*", @@ -299,7 +302,7 @@ }, "packages/shader-transitions": { "name": "@hyperframes/shader-transitions", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "html2canvas": "^1.4.1", }, @@ -311,7 +314,7 @@ }, "packages/studio": { "name": "@hyperframes/studio", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@codemirror/autocomplete": "^6.20.1", "@codemirror/commands": "^6.10.3", @@ -361,7 +364,7 @@ }, "packages/studio-server": { "name": "@hyperframes/studio-server", - "version": "0.7.78", + "version": "0.7.97", "dependencies": { "@hyperframes/core": "workspace:*", "@hyperframes/parsers": "workspace:*", diff --git a/lefthook.yml b/lefthook.yml index 3477e0df44..8d42077b37 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -19,7 +19,7 @@ pre-commit: run: bun packages/cli/scripts/gen-skills-manifest.ts && git add skills-manifest.json typecheck: glob: "*.{ts,tsx}" - run: cd packages/core && bunx tsc --noEmit && cd ../studio && bunx tsc --noEmit + run: cd packages/core && bunx tsc --noEmit && cd ../studio && bunx tsc --noEmit && cd ../.. && bunx tsc --noEmit -p scripts/tsconfig.json # Mirrors the CI gate (same `--base origin/main` so the local hook can't # pass on a branch CI would fail). Audits the working tree, which means # unstaged WIP in `packages/**` is part of the diff — stash before diff --git a/package.json b/package.json index 46fb3ceb50..8ed1c928fb 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "changelog:weekly": "tsx scripts/changelog-weekly.ts", "sync-schemas": "tsx scripts/sync-schemas.ts", "sync-schemas:check": "tsx scripts/sync-schemas.ts --check", + "typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json", "sync:package-subpaths": "node scripts/package-subpaths.mjs --write", "check:package-subpaths": "node scripts/package-subpaths.mjs", "lint": "bun run check:docs-snippet-motion && bun run check:tracked-artifacts && bun run check:workspace-contracts && bun run check:gcp-cloud-run-dockerfile && bun run check:package-cycles && bun run check:package-subpaths && bun run check:cli-process-ownership && oxlint . && tsx scripts/lint-skills.ts && node scripts/check-skill-mirror.mjs", @@ -48,7 +49,7 @@ "player:perf": "bun run --filter @hyperframes/player perf", "format:check": "oxfmt --check .", "knip": "knip", - "test:scripts": "node --import tsx --test scripts/check-tracked-artifacts.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs", + "test:scripts": "node --import tsx --test scripts/check-tracked-artifacts.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs && vitest run scripts/catalog", "test:skills": "node --test 'skills/**/*.test.mjs'", "generate:previews": "tsx scripts/generate-template-previews.ts", "generate:catalog-previews": "tsx scripts/generate-catalog-previews.ts", @@ -60,7 +61,9 @@ "devDependencies": { "@commitlint/cli": "^20.5.0", "@commitlint/config-conventional": "^20.5.0", + "@hyperframes/core": "workspace:*", "@hyperframes/player": "workspace:*", + "@hyperframes/producer": "workspace:*", "@types/node": "^25.0.10", "concurrently": "^8.2.0", "fallow": "^2.75.0", @@ -71,6 +74,7 @@ "oxlint": "^1.56.0", "tsx": "^4.21.0", "typescript": "^5.0.0", + "vitest": "^3.2.4", "yaml": "^2.9.0" }, "resolutions": { diff --git a/packages/cli/src/commands/catalog.test.ts b/packages/cli/src/commands/catalog.test.ts index 2f57b1bd3e..b7681a5ad2 100644 --- a/packages/cli/src/commands/catalog.test.ts +++ b/packages/cli/src/commands/catalog.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; -import { pickByName } from "./catalog.js"; +import { countUnindexed, pickByName } from "./catalog.js"; /** The whole registry, which is what "in this registry" has to be measured against. */ const registryNames = new Set(["fade-through", "whip-pan", "count-up"]); @@ -44,3 +44,203 @@ describe("pickByName", () => { expect(missing).toBe(0); }); }); + +describe("countUnindexed", () => { + it("counts the registry moves the on-device index holds no vector for", () => { + // The move published after the artifact was fetched. Meaning search cannot + // rank it at all, which is the failure this number exists to expose. + expect(countUnindexed(registryNames, ["fade-through"])).toBe(2); + }); + + it("reports nothing when the index covers the registry", () => { + expect(countUnindexed(registryNames, ["count-up", "whip-pan", "fade-through"])).toBe(0); + }); + + it("does not let names the registry dropped paper over a gap", () => { + // The artifact holds two names this registry cannot install and is missing + // two it can. Comparing sizes rather than membership would call that even. + expect(countUnindexed(registryNames, ["fade-through", "accordion", "alert-dialog"])).toBe(2); + }); +}); + +// ── The command envelope ──────────────────────────────────────────────────── +// The JSON envelope is the surface an agent reads, so under-coverage and the +// score are pinned where they are actually published rather than only at the +// helper that computes them. + +const state = vi.hoisted(() => ({ + registry: [] as Array<{ name: string; type: string }>, + ranking: null as Array<{ name: string; score: number }> | null, + indexed: [] as string[], +})); + +vi.mock("../registry/resolver.js", () => ({ + listRegistryItems: async () => state.registry, + loadAllItems: async (entries: Array<{ name: string; type: string }>) => + entries.map((entry) => ({ + name: entry.name, + type: entry.type, + title: entry.name, + description: `${entry.name} description`, + tags: [], + })), +})); + +vi.mock("../registry/localModel.js", () => ({ + // "ready" is a user who opted into the on-device tier at some point. Every + // later search takes that tier with no flag, which is how a frozen artifact + // goes on answering forever. + localModelStatus: () => ({ status: "ready" }), + localModelConsent: () => true, + ensureLocalModel: async () => true, + recordLocalModelConsent: () => {}, + downloadOfferMessage: () => "offer", + nonInteractiveConsentMessage: () => "consent", +})); + +vi.mock("../registry/localSemantic.js", () => ({ + localSemanticRanking: async () => state.ranking, + localVectorNames: () => state.indexed, + hasLocalVectors: () => true, + fetchLocalVectors: async () => true, +})); + +const block = (name: string): { name: string; type: string } => ({ + name, + type: "hyperframes:block", +}); +const component = (name: string): { name: string; type: string } => ({ + name, + type: "hyperframes:component", +}); + +interface Envelope { + tier: string; + dropped: number; + unindexed: number; + top_score?: number; + shown: number; +} + +async function runCatalog(args: Record): Promise { + const command = (await import("./catalog.js")).default as unknown as { + run: (context: { args: Record }) => Promise; + }; + const lines: string[] = []; + const log = vi.spyOn(console, "log").mockImplementation((...parts: unknown[]) => { + lines.push(parts.map(String).join(" ")); + }); + try { + await command.run({ args }); + } finally { + log.mockRestore(); + } + // Colour is decoration; assertions are about the words. The escape byte is + // built rather than written: as a literal or as \u001B it is a control + // character in the source, which the lint rules reject either way. + const ansi = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g"); + return lines.join("\n").replace(ansi, ""); +} + +async function runEnvelope(args: Record): Promise { + return JSON.parse(await runCatalog({ json: true, ...args })) as Envelope; +} + +beforeEach(() => { + state.registry = [block("count-up"), block("fade-through"), component("whip-pan")]; + state.indexed = ["count-up", "fade-through", "whip-pan"]; + state.ranking = [ + { name: "count-up", score: 0.71 }, + { name: "whip-pan", score: 0.42 }, + { name: "fade-through", score: 0.31 }, + ]; +}); + +describe("catalog --json meaning search", () => { + it("reports the registry moves meaning search cannot see", async () => { + // Published after the user's artifact was fetched: in the registry, absent + // from the index, and therefore unreturnable by any query. + state.registry.push(block("split-screen")); + + const envelope = await runEnvelope({ query: "make a number count up" }); + + expect(envelope.tier).toBe("on-device"); + expect(envelope.unindexed).toBe(1); + }); + + it("reports nothing unindexed when the artifact matches the registry", async () => { + const envelope = await runEnvelope({ query: "make a number count up" }); + + expect(envelope.unindexed).toBe(0); + }); + + it("measures unindexed against the unfiltered registry under --type", async () => { + // The missing move is a component; the user asked for blocks. Their filter + // is not the index being stale, and it must not hide a stale index either. + state.registry.push(component("push-in")); + + const envelope = await runEnvelope({ query: "make a number count up", type: "block" }); + + expect(envelope.shown).toBe(2); + expect(envelope.unindexed).toBe(1); + }); + + it("keeps dropped counted against the unfiltered registry under --type", async () => { + // accordion is the only name the registry genuinely lacks. whip-pan is + // installable and merely filtered out, so it is not a drop. + state.ranking = [ + { name: "count-up", score: 0.71 }, + { name: "whip-pan", score: 0.62 }, + { name: "accordion", score: 0.55 }, + { name: "fade-through", score: 0.31 }, + ]; + + const envelope = await runEnvelope({ query: "make a number count up", type: "block" }); + + expect(envelope.dropped).toBe(1); + expect(envelope.shown).toBe(2); + }); + + it("carries the score of the best result it actually showed", async () => { + // The top-ranked name is not installable here, so reporting the ranking's + // own head would describe a row the caller never received. + state.ranking = [ + { name: "accordion", score: 0.93 }, + { name: "count-up", score: 0.71 }, + { name: "whip-pan", score: 0.42 }, + { name: "fade-through", score: 0.31 }, + ]; + + const envelope = await runEnvelope({ query: "make a number count up" }); + + expect(envelope.top_score).toBeCloseTo(0.71); + }); + + it("omits the score on the word tier, whose scale is not the same one", async () => { + state.ranking = null; + + const envelope = await runEnvelope({ query: "count up" }); + + expect(envelope.tier).toBe("words"); + expect(envelope.top_score).toBeUndefined(); + // Word matching ranks the live registry listing, so it is never stale. + expect(envelope.unindexed).toBe(0); + }); +}); + +describe("catalog meaning search, on a terminal", () => { + it("says how much is missing and what to run about it", async () => { + state.registry.push(block("split-screen")); + + const output = await runCatalog({ query: "make a number count up" }); + + expect(output).toContain("1 of 4 moves are missing from the on-device index"); + expect(output).toContain("Re-run with --on-device to refresh it."); + }); + + it("says nothing when the index covers the registry", async () => { + const output = await runCatalog({ query: "make a number count up" }); + + expect(output).not.toContain("missing from the on-device index"); + }); +}); diff --git a/packages/cli/src/commands/catalog.ts b/packages/cli/src/commands/catalog.ts index 5336141c34..bc71691cb9 100644 --- a/packages/cli/src/commands/catalog.ts +++ b/packages/cli/src/commands/catalog.ts @@ -31,6 +31,7 @@ import { fetchLocalVectors, hasLocalVectors, localSemanticRanking, + localVectorNames, } from "../registry/localSemantic.js"; /** @@ -47,6 +48,7 @@ import { async function prepareOnDeviceTier(opts: { assumedYes: boolean; registry: string; + registryNames: ReadonlySet; }): Promise { const warnings: string[] = []; const warn = (message: string): void => { @@ -81,7 +83,20 @@ async function prepareOnDeviceTier(opts: { recordLocalModelConsent(true); const model = await ensureLocalModel(); - const vectors = hasLocalVectors() || (await fetchLocalVectors(opts.registry)); + // The vectors are fetched once and nothing ever invalidates them, so an index + // that no longer covers the registry is the same situation as one that is not + // there at all: meaning search cannot reach part of the catalog. Both take the + // path this flag already takes. Deciding it costs a local read of the + // artifact's own name list; the only fetch is the one --on-device already + // performs when the vectors are absent. + if (!hasLocalVectors() || countUnindexed(opts.registryNames, localVectorNames()) > 0) { + await fetchLocalVectors(opts.registry); + } + // Deliberately not the fetch's own answer. A refresh that fails still leaves + // the previous vectors on disk, and those still rank: reporting the tier + // unavailable there would be false, and the search that follows says what is + // actually wrong with them. + const vectors = hasLocalVectors(); if (!model || !vectors) { warn( `on-device search unavailable: ${!model ? "model" : "catalog vectors"} could not be fetched`, @@ -127,7 +142,8 @@ export default defineCommand({ // Consent for the model download is otherwise only reachable through a // prompt that fires on a TTY, which leaves every agent and CI run unable // to opt in at all. - description: "Use on-device meaning search, downloading the model if needed", + description: + "Use on-device meaning search, downloading the model and refreshing a stale index if needed", }, }, // one flag-parsing entry point feeding three output paths (json, interactive, table); splitting those is its own change @@ -177,6 +193,7 @@ export default defineCommand({ ? await prepareOnDeviceTier({ assumedYes: args.yes === true || !process.stdout.isTTY || json, registry: config.registry, + registryNames, }) : []; const searched = query ? await applySearch(tagged, query, registryNames) : null; @@ -194,6 +211,8 @@ export default defineCommand({ tier: tierToken(searched), tier_detail: tierDetail(searched), dropped: searched?.missing ?? 0, + unindexed: searched?.unindexed ?? 0, + ...(searched?.topScore != null ? { top_score: searched.topScore } : {}), shown: 0, total: tagged.length, ...(warnings.length ? { warnings } : {}), @@ -240,6 +259,8 @@ export default defineCommand({ tier: tierToken(searched), tier_detail: tierDetail(searched), dropped: searched?.missing ?? 0, + unindexed: searched?.unindexed ?? 0, + ...(searched?.topScore != null ? { top_score: searched.topScore } : {}), shown: output.length, total: tagged.length, ...(warnings.length ? { warnings } : {}), @@ -261,6 +282,22 @@ export default defineCommand({ ? ` · ${searched.missing} ranked ${searched.missing === 1 ? "move" : "moves"} not in this registry` : ""; console.log(c.dim(` ${matching.length} of ${tagged.length} moves · ${how}${unshowable}`)); + if (searched.unindexed > 0) { + // The costly direction, so it gets a line of its own rather than a + // suffix: these moves cannot come back from meaning search at any rank, + // for any query, and a reader has to learn what to run about it. + // Silent when the index covers the registry. + const remedy = + args["on-device"] === true + ? "The published index is behind this registry." + : "Re-run with --on-device to refresh it."; + console.log( + c.warn( + ` ${searched.unindexed} of ${registryNames.size} moves are missing from the ` + + `on-device index, so meaning search cannot return them. ${remedy}`, + ), + ); + } if (searched.localMode === "words") { // Suppressed when on-device was asked for and refused: telling someone // to pass a flag one line after explaining that flag cannot work here @@ -358,20 +395,81 @@ export function pickByName( return { ranked, missing: names.filter((name) => !registryNames.has(name)).length }; } +/** + * Registry moves the on-device index holds no vector for. + * + * The counterpart to `pickByName`'s `missing`, and the direction that costs + * something. `missing` is over-coverage: names the artifact ranks that this + * registry cannot install, which only wastes a rank. This is under-coverage: + * moves the registry has that were never embedded, so meaning search cannot + * return them at any rank, for any query, and until now nothing in the output + * said so. The vectors are fetched once and never invalidated, so every move + * published since that fetch lands here. + * + * Measured against the unfiltered registry, matching `missing`: a --type or + * --tag filter removing a move is the user narrowing their own search, not an + * index that cannot see it. + */ +export function countUnindexed( + registryNames: ReadonlySet, + indexedNames: Iterable, +): number { + const indexed = new Set(indexedNames); + let count = 0; + for (const name of registryNames) { + if (!indexed.has(name)) count += 1; + } + return count; +} + +interface SearchOutcome { + items: T[]; + localMode: LocalMode; + /** Over-coverage: ranked names this registry has no item for. */ + missing: number; + /** + * Under-coverage: registry moves the tier that answered could not see. + * + * Always zero for word matching, which ranks the live registry listing and + * so cannot be stale. Non-zero only on the on-device tier, whose vectors are + * a separately published artifact that drifts from the registry. + */ + unindexed: number; + /** + * Similarity of the best result actually shown, when the answering tier + * produces one. Null for word matching, whose score is a different and + * non-comparable scale. Deliberately not a threshold: it is the signal a + * caller needs to judge a ranker that ships without one. + */ + topScore: number | null; +} + async function applySearch( items: T[], query: string, registryNames: ReadonlySet, -): Promise<{ items: T[]; localMode: LocalMode; missing: number }> { +): Promise> { // On-device meaning search, when the user opted into the model. Free and // offline, and it answers phrasings word matching cannot reach. if (localModelStatus().status === "ready") { try { const ranking = await localSemanticRanking(query); if (ranking) { - const { ranked, missing } = pickByName(items, ranking, registryNames); - if (ranked.length > 0) { - return { items: ranked.slice(0, 25), localMode: "local-model", missing }; + const { ranked, missing } = pickByName( + items, + ranking.map((entry) => entry.name), + registryNames, + ); + const best = ranked[0]; + if (best) { + const scoreByName = new Map(ranking.map((entry) => [entry.name, entry.score])); + return { + items: ranked.slice(0, 25), + localMode: "local-model", + missing, + unindexed: countUnindexed(registryNames, localVectorNames()), + topScore: scoreByName.get(best.name) ?? null, + }; } } } catch (error) { @@ -393,7 +491,7 @@ async function applySearch `${item.name} ${item.title} ${item.description}`, ); // Word matching ranks the items in hand, so nothing can go missing. - return { items: words, localMode: "words", missing: 0 }; + return { items: words, localMode: "words", missing: 0, unindexed: 0, topScore: null }; } /** diff --git a/packages/cli/src/registry/localSemantic.ts b/packages/cli/src/registry/localSemantic.ts index 505593dcfd..e8dc4702ca 100644 --- a/packages/cli/src/registry/localSemantic.ts +++ b/packages/cli/src/registry/localSemantic.ts @@ -50,11 +50,17 @@ export async function fetchLocalVectors( const base = registryBaseUrl.replace(/\/+$/, ""); try { mkdirSync(directory, { recursive: true }); + // Downloaded in full before anything is written. The two files have to + // agree on how many rows there are, so a fetch that fails halfway through + // must leave the previous pair intact rather than pairing a new name list + // with an old matrix, which loads as an error instead of as stale data. + const fetched: Array<[string, Buffer]> = []; for (const file of ["local-vectors.json", "local-vectors.bin"] as const) { const response = await fetch(`${base}/catalog-artifact/${file}`); if (!response.ok) return false; - writeFileSync(join(directory, file), Buffer.from(await response.arrayBuffer())); + fetched.push([file, Buffer.from(await response.arrayBuffer())]); } + for (const [file, bytes] of fetched) writeFileSync(join(directory, file), bytes); return hasLocalVectors(directory); } catch { return false; @@ -90,11 +96,40 @@ function loadLocalVectors(directory = localVectorDirectory()): LocalVectorSet { return { names: meta.names, dimensions: meta.dimensions, vectors }; } -/** Names ranked best first. Returns null when local semantic search is not available. */ +/** + * The names the on-device index holds, without loading the vector matrix. + * + * Callers compare this against the live registry to see what meaning search + * cannot reach. Reads the metadata file only, so it costs a small JSON parse + * rather than the whole matrix, and answers empty rather than throwing: an + * absent or unreadable artifact covers nothing, which is a coverage answer + * rather than a reason to fail the search that asked. + */ +export function localVectorNames(directory = localVectorDirectory()): string[] { + if (!hasLocalVectors(directory)) return []; + try { + const meta = JSON.parse(readFileSync(join(directory, "local-vectors.json"), "utf-8")) as { + names?: string[]; + }; + return meta.names ?? []; + } catch { + return []; + } +} + +/** + * Names ranked best first, each with the similarity that placed it there. + * Returns null when local semantic search is not available. + * + * The score rides along because this ranker has no relevance threshold: every + * query returns the whole set in some order, so a caller holding names alone + * cannot tell a real match from the least-bad row of a query that matched + * nothing at all. + */ export async function localSemanticRanking( query: string, directory = localVectorDirectory(), -): Promise { +): Promise | null> { if (!isLocalModelReady() || !hasLocalVectors(directory)) return null; const set = loadLocalVectors(directory); @@ -112,5 +147,5 @@ export async function localSemanticRanking( // Ties break on descending name, matching every other ranking in this system. scored.sort((a, b) => b.score - a.score || b.name.localeCompare(a.name)); - return scored.map((entry) => entry.name); + return scored; } diff --git a/registry/catalog-artifact/README.md b/registry/catalog-artifact/README.md new file mode 100644 index 0000000000..f32de26331 --- /dev/null +++ b/registry/catalog-artifact/README.md @@ -0,0 +1,45 @@ +# Catalog artifact + +Two files ship from here, and only these two. The CLI fetches them over HTTP +when a user opts into offline catalog search (`catalog --query ... --on-device`), +so they are served from the registry rather than bundled in the package. + +| File | What it is | +| -------------------- | ----------------------------------------------------------------------- | +| `local-vectors.json` | `{ model, dimensions, names }`. `names` is the row order of the binary. | +| `local-vectors.bin` | Float32, row-major, `names.length * dimensions` values, no header. | + +Currently 168 rows at 384 dimensions: 258,048 bytes, one row per installable +registry item. + +## Provenance + +No digest, revision or build timestamp is recorded in either file, and the +runtime only checks that `dimensions` matches the model it loaded. `model` is a +label the build wrote, not a proof. So the only real provenance check is to +rebuild the rows and compare them, which works because both inputs are in this +repository: + +- the text each row was embedded from is `itemRetrievalText(registry-item.json)` + (title, description, tags, joined by newlines, name deliberately excluded), + over `registry/blocks/*` and `registry/components/*` sorted by name; +- the model is the pinned quantized `bge-small-en-v1.5` ONNX build that + `packages/cli/src/registry/localModel.ts` downloads. + +Re-embedding in batches of 16, the batch size the build uses, reproduces the +shipped rows exactly (cosine 1.000000). Batch size matters: the same text +embedded alone differs at cosine 0.9969, because padding within a batch changes +the quantized result. A rebuild that does not match this way was not built from +this registry, or not with this model. + +## Rebuilding + +`scripts/catalog/build-local-vectors.ts` reads `catalog.json` from this +directory, and that file is gitignored and absent. Nothing in `scripts/` writes +it: `catalogFromRegistry` in `scripts/catalog/catalog-artifact.ts` produces +exactly the right shape but has no caller, so the first step is currently manual. + +`build-catalog-artifact.ts` does **not** produce these files. It builds the +3072-dimension hosted artifact from an external shelf file, for the hosted search +tier that this repository does not ship. Its `--shelf`, `manifest.json` and +`text-embedding-3-large` have nothing to do with `local-vectors.*`. diff --git a/scripts/backfill-block-previews.ts b/scripts/backfill-block-previews.ts index 1069c81a3b..c33b89c694 100644 --- a/scripts/backfill-block-previews.ts +++ b/scripts/backfill-block-previews.ts @@ -10,7 +10,7 @@ * npx tsx scripts/backfill-block-previews.ts --dry-run # preview changes only */ -import { readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { readdirSync, readFileSync, writeFileSync, type Dirent } from "node:fs"; import { join, resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; @@ -44,7 +44,7 @@ function writeManifest(manifestPath: string, manifest: Record): function backfillBlocks() { const blocksDir = join(registryDir, "blocks"); - let entries: ReturnType>; + let entries: Dirent[]; try { entries = readdirSync(blocksDir, { withFileTypes: true }); } catch { @@ -81,7 +81,7 @@ function backfillBlocks() { function normalizeComponents() { const componentsDir = join(registryDir, "components"); - let entries: ReturnType>; + let entries: Dirent[]; try { entries = readdirSync(componentsDir, { withFileTypes: true }); } catch { diff --git a/scripts/catalog/build-catalog-artifact.ts b/scripts/catalog/build-catalog-artifact.ts index 6075e60d7c..93b59088e2 100644 --- a/scripts/catalog/build-catalog-artifact.ts +++ b/scripts/catalog/build-catalog-artifact.ts @@ -16,6 +16,7 @@ import { buildArtifact, manifestBytes, movesMissingFromRegistry, + parseShelf, verifyArtifact, type Embedder, } from "./catalog-artifact.js"; @@ -112,17 +113,17 @@ async function main(): Promise { // the burden of discovering it on the consumer at request time. verifyArtifact(built); - // A move the registry cannot serve ranks well and then cannot be installed, - // which looks like a bad search rather than a bad build. Reported, not - // fatal: the shelf legitimately leads the registry while moves land. - const known = registryNames(arg("registry-index") ?? DEFAULT_REGISTRY_INDEX); - if (known === undefined) { - console.warn("warning registry index unreadable; coverage not checked"); + // What is left to report is the gap, not a defect in the artifact: buildArtifact + // already dropped these, so the published catalog never contains a move the + // registry cannot serve. Reported, not fatal, because the shelf legitimately + // leads the registry while moves land. + if (installableNames === undefined) { + console.warn("warning registry index unreadable; published every shelf move unchecked"); } else { - const missing = movesMissingFromRegistry(Object.keys(built.catalog), known); - if (missing.length > 0) { + const dropped = movesMissingFromRegistry([...parseShelf(shelfText).keys()], installableNames); + if (dropped.length > 0) { console.warn( - `warning ${missing.length} shelf moves have no registry item: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ", ..." : ""}`, + `warning ${dropped.length} shelf moves dropped, no registry item: ${dropped.slice(0, 5).join(", ")}${dropped.length > 5 ? ", ..." : ""}`, ); } } diff --git a/scripts/changelog-weekly.ts b/scripts/changelog-weekly.ts index 1c208347ca..dbcc8435a1 100644 --- a/scripts/changelog-weekly.ts +++ b/scripts/changelog-weekly.ts @@ -531,7 +531,7 @@ function capitalize(value: string) { if (!value) { return value; } - return value[0].toUpperCase() + value.slice(1); + return value.charAt(0).toUpperCase() + value.slice(1); } function git(args: string[]) { diff --git a/scripts/cli-options.ts b/scripts/cli-options.ts index 5983cdcda5..0cf95e535c 100644 --- a/scripts/cli-options.ts +++ b/scripts/cli-options.ts @@ -27,7 +27,7 @@ export function parseMappedArgument< parsed: Parsed, config: ParserConfig, ) { - const arg = args[index]; + const arg = args[index] ?? config.fail(`No argument at index ${index}`); if (applyInlineValueOption(arg, parsed, config.inlineValueOptions)) { return index; diff --git a/scripts/draft-changelog.ts b/scripts/draft-changelog.ts index a57cccede8..cc902f7349 100644 --- a/scripts/draft-changelog.ts +++ b/scripts/draft-changelog.ts @@ -273,12 +273,8 @@ export function parseConventionalSubject(subject: string): ParsedSubject { }; } - return { - type: match[1], - scope: match[2], - summary: match[4], - breaking: match[3] === "!", - }; + const [, type = "other", scope, breaking, summary = subject] = match; + return { type, scope, summary, breaking: breaking === "!" }; } function extractPrNumber(subject: string) { @@ -444,7 +440,7 @@ function capitalize(value: string) { if (!value) { return value; } - return value[0].toUpperCase() + value.slice(1); + return value.charAt(0).toUpperCase() + value.slice(1); } function renderTagsLiteral(tags: string[]) { diff --git a/scripts/generate-catalog-pages.ts b/scripts/generate-catalog-pages.ts index ef7dac0375..6fd2deea35 100644 --- a/scripts/generate-catalog-pages.ts +++ b/scripts/generate-catalog-pages.ts @@ -234,7 +234,7 @@ function textureGroupsFor(manifest: RegistryItem): TextureGroup[] { if (!group || typeof group !== "object") return false; if (!("title" in group) || typeof group.title !== "string") return false; if (!("items" in group) || !Array.isArray(group.items)) return false; - return group.items.every((item) => typeof item === "string"); + return group.items.every((item: unknown) => typeof item === "string"); }); } @@ -403,7 +403,7 @@ function generateTextureAnimationExample( function generateTexturePreview(manifest: RegistryItem, textureGroups: TextureGroup[]): string[] { const sampleItems = textureGroups .map((group) => group.items[0]) - .filter(Boolean) + .filter((item): item is string => Boolean(item)) .slice(0, 6); const lines: string[] = ['
']; @@ -411,8 +411,8 @@ function generateTexturePreview(manifest: RegistryItem, textureGroups: TextureGr const maskPath = textureMaskUrlFor(manifest, item); lines.push( `
`, - `
${textureLabel(item!)}
`, - `
${textureSampleWord(item!)}
`, + `
${textureLabel(item)}
`, + `
${textureSampleWord(item)}
`, "
", ); } diff --git a/scripts/generate-catalog-previews.ts b/scripts/generate-catalog-previews.ts index d8a262994b..35c332fd01 100644 --- a/scripts/generate-catalog-previews.ts +++ b/scripts/generate-catalog-previews.ts @@ -277,20 +277,21 @@ async function prepareProjectDir(item: CatalogItem): Promise { return tmpDir; } +/** Pull a `data-` pixel value out of the wrapper markup, or fall back. */ +function wrapperDimension(html: string, attr: "width" | "height", fallback: number): number { + const match = html.match(new RegExp(`data-${attr}="(\\d+)"`))?.[1]; + return match ? parseInt(match, 10) : fallback; +} + async function generateThumbnail(item: CatalogItem, projectDir: string): Promise { const outDir = outputDir(item.kind); mkdirSync(outDir, { recursive: true }); // Read dimensions from the wrapper index.html (which may differ from native // dimensions for portrait overlays that are scaled to fit landscape). - let width = 1920; - let height = 1080; - const wrapperPath = join(projectDir, "index.html"); - const wrapperHtml = readFileSync(wrapperPath, "utf-8"); - const wMatch = wrapperHtml.match(/data-width="(\d+)"/); - const hMatch = wrapperHtml.match(/data-height="(\d+)"/); - if (wMatch) width = parseInt(wMatch[1], 10); - if (hMatch) height = parseInt(hMatch[1], 10); + const wrapperHtml = readFileSync(join(projectDir, "index.html"), "utf-8"); + const width = wrapperDimension(wrapperHtml, "width", 1920); + const height = wrapperDimension(wrapperHtml, "height", 1080); const framesDir = join(projectDir, "_thumb_frames"); mkdirSync(framesDir, { recursive: true }); diff --git a/scripts/lint-skills.ts b/scripts/lint-skills.ts index 8aa912d9da..d03b26b97b 100644 --- a/scripts/lint-skills.ts +++ b/scripts/lint-skills.ts @@ -225,7 +225,7 @@ export function lintFrontmatter(content: string): LineViolation[] { violation(1, `Missing SKILL.md YAML frontmatter (must start with '---').`, ""), ]; } - const parsed = parseFrontmatterYaml(match[1]); + const parsed = parseFrontmatterYaml(match[1] ?? ""); const preflightError = parsedDataError(parsed); if (preflightError) return [preflightError]; return validateShape(parsed.data as Record); diff --git a/scripts/set-version.ts b/scripts/set-version.ts index ce176429f6..150d3e1db7 100644 --- a/scripts/set-version.ts +++ b/scripts/set-version.ts @@ -226,7 +226,9 @@ function assertTagMonotonicity(version: string) { process.exit(1); } -export function releaseRequiresChangelog(options: ReleaseOptions) { +export function releaseRequiresChangelog( + options: Pick, +) { return !options.skipTag && !options.skipChangelogCheck && !isPrerelease(options.version); } @@ -267,7 +269,7 @@ export function unreviewedChangelogArtifacts(version: string) { } function artifactExists(artifact: string) { - const [path, marker] = artifact.split("#"); + const [path = artifact, marker] = artifact.split("#"); const absolutePath = join(ROOT, path); if (!existsSync(absolutePath)) { @@ -277,7 +279,7 @@ function artifactExists(artifact: string) { } function artifactHasGeneratedTodo(artifact: string) { - const [path, marker] = artifact.split("#"); + const [path = artifact, marker] = artifact.split("#"); const content = readFileSync(join(ROOT, path), "utf-8"); if (!marker) { return hasGeneratedChangelogTodo(content); diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json new file mode 100644 index 0000000000..d3044f3b7b --- /dev/null +++ b/scripts/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true, + "allowJs": true, + "allowImportingTsExtensions": true + }, + "include": ["**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/skills-manifest.json b/skills-manifest.json index cb7dbe3e27..a7fa1bfa4e 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -26,7 +26,7 @@ "files": 121 }, "hyperframes-cli": { - "hash": "3eab4c5fc1ae8e26", + "hash": "17e2300803951ed7", "files": 11 }, "hyperframes-core": { @@ -42,7 +42,7 @@ "files": 3 }, "hyperframes-registry": { - "hash": "3a864992e765b9bf", + "hash": "21dc40bf1261b90d", "files": 10 }, "media-use": { diff --git a/skills/hyperframes-cli/SKILL.md b/skills/hyperframes-cli/SKILL.md index 180a7bbe95..949594dac8 100644 --- a/skills/hyperframes-cli/SKILL.md +++ b/skills/hyperframes-cli/SKILL.md @@ -16,7 +16,7 @@ Run commands as `npx hyperframes ...` unless project instructions provide a wrap ## Development loop 1. **Scaffold:** `npx hyperframes init ` or capture a site. In non-TTY mode, pass `--non-interactive --example=`. -2. **Find the move:** before authoring motion by hand, search for a primitive that already does it: `npx hyperframes catalog --query "reveal a headline one line at a time" --smart`. Ask for the effect you want rather than the mechanism you have in mind. Install with `npx hyperframes add ` (see `/hyperframes-registry`). Author by hand only once nothing fits. +2. **Find the move:** before authoring motion by hand, search for a primitive that already does it: `npx hyperframes catalog --query "reveal a headline one line at a time"`. Ask for the effect you want rather than the mechanism you have in mind. Install with `npx hyperframes add ` (see `/hyperframes-registry`). Author by hand only once nothing fits. 3. **Author:** write the composition using `/hyperframes-core`. 4. **Get fast feedback while editing:** run `npx hyperframes lint` after the first HTML pass and after structural changes. 5. **Run the final gate:** run `npx hyperframes check`; it reruns lint before opening the browser. Do not prepend a redundant standalone lint invocation. Add `--snapshots` for annotated overview frames and finding crops. @@ -62,10 +62,10 @@ Treat tiny unstyled content, canvas-sized icons, missing hero elements, or timel ## Agent conventions -- **Search the catalog by meaning before writing motion by hand.** `npx hyperframes catalog --query "" --smart`. `--smart` is a per-run opt-in that needs no prompt, so it works non-interactively; it is also the only part of the catalog flow that sends anything to HeyGen (the query text), and it is free for signed-in users. Without it the command falls back to matching words the description happens to share, which misses any phrasing that does not reuse the catalog's own vocabulary. -- **Read which tier answered; never infer it from results appearing.** With `--json` the envelope carries `tier` (`meaning`, `on-device` or `words`), `top_score`, `dropped` and `warnings`. A weak result on the weakest tier is expected; the same result on the best tier is a bug. `top_score` is reported, not judged: real short queries and gibberish overlap on this catalog, so there is no "nothing matched" and you should not invent one without your own evidence. -- **`dropped` above zero means the registry is a different generation from the search index.** The strongest matches are the ones being lost, so refresh the registry rather than rewording the query. -- **Offer the offline tier; never enable it silently.** A one-time ~33 MB model plus catalog vectors, cached under `~/.hyperframes`, not added to the project or any package. It ranks by meaning with nothing sent. Say the size out loud and let the person decide, then pass `--on-device` (with `-y` to skip the prompt) once they agree. The CLI only offers it on a TTY, so in an agent or CI run the offer never appears and the user has to be told. +- **Search the catalog before writing motion by hand.** `npx hyperframes catalog --query ""`. Search is entirely local: there is no hosted tier, no account, and the query text is never sent anywhere. By default it ranks on vocabulary shared with the item's name, title and description, which misses any phrasing that does not reuse the catalog's own wording. Add `--on-device` to rank by meaning instead (see the offline tier below). +- **Read which tier answered; never infer it from results appearing.** With `--json` the envelope carries `query`, `tier` (`on-device` or `words`), `tier_detail`, `dropped`, `unindexed`, `shown`, `total` and `results`, plus `top_score` when the answering tier produces one and `warnings` when a tier was asked for and could not run. A weak result on `words` is expected; the same result on `on-device` is a bug. `top_score` is on-device only and has no threshold behind it: the ranker returns the whole catalog in some order for every query, so read it as evidence rather than as a pass or fail. +- **`dropped` and `unindexed` are opposite skews between the registry and the on-device index, and rewording the query fixes neither.** `dropped` counts ranked names this registry cannot install, so the strongest matches are the ones being lost. `unindexed` counts registry moves the index cannot see at all, which no query can ever return. Refreshing the registry is not the answer to either: its manifest carries a 24h TTL and heals itself, while the vectors are a separately published artifact fetched into `~/.hyperframes/catalog/`. Re-running with `--on-device` refetches that index when `unindexed` is above zero, so that is the remedy to hand the user. A pure over-coverage skew (`dropped` above zero while `unindexed` is zero) does not trigger the refetch; clearing `~/.hyperframes/catalog/` is the only way out of that one. Both counts are of names rather than of results, so either can exceed `total`. +- **Offer the offline tier; never enable it silently.** A one-time ~33 MB download (a quantized ONNX build of `bge-small-en-v1.5` plus its tokenizer, pinned to a fixed revision) and the catalog vectors from the registry, both cached under `~/.hyperframes/`, neither added to the project or any package. Once cached it ranks by meaning with nothing sent. Say the size out loud and let the person decide, then pass `--on-device` (with `-y` to skip the prompt) once they agree. The interactive offer only fires on a TTY, and under `--json` nothing about it is printed at all, so in an agent run you have to raise it with the user yourself. - Prefer `--json` for agent and CI calls. Server-mode `render`, `preview`, and `play` do not provide ordinary JSON output; `preview --selection --json` and `preview --context --json` are query-mode exceptions. - `doctor --json` always exits zero. Gate on its payload: diff --git a/skills/hyperframes-registry/SKILL.md b/skills/hyperframes-registry/SKILL.md index 7fd60fd81c..57c2f6cad2 100644 --- a/skills/hyperframes-registry/SKILL.md +++ b/skills/hyperframes-registry/SKILL.md @@ -85,15 +85,17 @@ See [wiring-components.md](./references/wiring-components.md) for full details. ## Discovery -Use the CLI as the primary discovery surface. **Search by intent before browsing:** the registry holds hundreds of items, so listing them and matching on names or tags is the slow path, and it fails whenever the author's wording differs from yours. +Use the CLI as the primary discovery surface. **Search by intent before browsing:** the registry holds more items than you can scan by eye, so listing them and matching on names or tags is the slow path, and it fails whenever the author's wording differs from yours. ```bash # Rank the whole catalog against what the beat should do -npx hyperframes catalog --query "reveal a headline one line at a time" --smart -npx hyperframes add line-by-line-slide +npx hyperframes catalog --query "reveal a headline one line at a time" +npx hyperframes add caption-clip-wipe ``` -`--smart` ranks by meaning and is free for signed-in HeyGen users; it sends the query text and nothing else. Without it the command falls back to word matching, which only finds items whose description happens to reuse your words. With `--json` the envelope names which tier answered, so check that rather than assuming a ranking happened. Only installable items are ranked. See `/hyperframes-cli` for the offline tier and the consent gates. +Search is local and sends nothing. By default it ranks on vocabulary shared with the item's name, title and description, so it only finds items that reuse your words; `--on-device` ranks by meaning instead, after a one-time model download. With `--json` the envelope names which tier answered, so check that rather than assuming a ranking happened. + +Installability is applied after ranking, not before it: a name the vectors carry but this registry cannot serve is dropped from the results and counted in `dropped`, so a non-zero `dropped` means the two are different generations. See `/hyperframes-cli` for the offline tier, the consent gates, and how to refresh a stale index. To browse or filter instead of search: