From fde415553dc75ee97c4274514ddfe429bccd3dfa Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 20 Sep 2026 15:05:55 -0400 Subject: [PATCH] feat(cli): index media-use assets for catalog search --- .github/workflows/ci.yml | 1 + packages/cli/src/commands/catalog.test.ts | 27 +++++- packages/cli/src/commands/catalog.ts | 79 +++++++++++++++ packages/cli/src/registry/localSemantic.ts | 106 +++++++++++++++++++-- scripts/catalog/build-local-vectors.ts | 86 +++++++++++++---- scripts/catalog/check-artifact-coverage.ts | 33 ++++++- skills/hyperframes-cli/SKILL.md | 4 +- 7 files changed, 306 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd3250dba4..c6387faaaf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: catalog_index: - "registry/registry.json" - "registry/catalog-artifact/**" + - "skills/media-use/audio/assets/sfx/**" - "scripts/catalog/check-artifact-coverage.ts" docs_catalog: - "docs/**" diff --git a/packages/cli/src/commands/catalog.test.ts b/packages/cli/src/commands/catalog.test.ts index 1b11016554..a23df786da 100644 --- a/packages/cli/src/commands/catalog.test.ts +++ b/packages/cli/src/commands/catalog.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { countUnindexed, pickByName, searchMissCommand } from "./catalog.js"; +import { countUnindexed, pickByName, rankMediaRows, searchMissCommand } from "./catalog.js"; /** The whole registry, which is what "in this registry" has to be measured against. */ const registryNames = new Set(["fade-through", "whip-pan", "count-up"]); @@ -63,6 +63,31 @@ describe("countUnindexed", () => { }); }); +describe("rankMediaRows", () => { + it("ranks the click SFX first for a click query", () => { + const rows = [ + { + id: "whoosh", + kind: "sfx", + title: "whoosh", + description: "Fast scene transition accent", + tags: ["sfx"], + file: "skills/media-use/audio/assets/sfx/whoosh.mp3", + }, + { + id: "click", + kind: "sfx", + title: "click", + description: "Crisp UI click for a button press or selection", + tags: ["sfx"], + file: "skills/media-use/audio/assets/sfx/click.mp3", + }, + ]; + + expect(rankMediaRows("click", rows).map((row) => row.id)).toEqual(["click"]); + }); +}); + // ── The command envelope ──────────────────────────────────────────────────── // The JSON envelope is the surface an agent reads, so under-coverage and the // score are pinned where they are actually published rather than only at the diff --git a/packages/cli/src/commands/catalog.ts b/packages/cli/src/commands/catalog.ts index bfdf9b9fc0..4b66c450ce 100644 --- a/packages/cli/src/commands/catalog.ts +++ b/packages/cli/src/commands/catalog.ts @@ -30,12 +30,73 @@ import { import { ensureLocalRuntime } from "../registry/localEmbedder.js"; import { cachedLocalVectorRevision, + fetchMediaVectors, fetchLocalVectors, hasLocalVectors, localSemanticRanking, localVectorNames, + mediaSemanticRanking, + mediaVectorRows, } from "../registry/localSemantic.js"; +export interface MediaCatalogRow { + id: string; + kind: string; + title: string; + description: string; + tags: string[]; + file: string; + duration?: number; + dimensions?: { width: number; height: number }; +} + +export function rankMediaRows(query: string, rows: MediaCatalogRow[]): MediaCatalogRow[] { + return searchByWords(query, rows, (row) => ({ + strong: `${row.id} ${row.title}`, + weak: `${row.description} ${row.tags.join(" ")} ${row.kind}`, + })); +} + +async function runMediaCatalog( + query: string, + registry: string, +): Promise { + const fetched = await fetchMediaVectors(registry); + const rows = mediaVectorRows() as MediaCatalogRow[]; + const warnings: string[] = []; + if (!fetched && rows.length === 0) { + warnings.push("media search unavailable: the media index could not be fetched"); + } + + let matching = rankMediaRows(query, rows); + let tier: "words" | "on-device" = "words"; + let tierDetail = "local word match"; + if (matching.length === 0 && localModelStatus().status === "ready") { + const ranking = await mediaSemanticRanking(query); + if (ranking) { + matching = ranking.map(({ row }) => row as MediaCatalogRow); + tier = "on-device"; + tierDetail = "on-device meaning search"; + } + } + + console.log( + JSON.stringify( + { + query, + tier, + tier_detail: tierDetail, + shown: matching.length, + total: rows.length, + ...(warnings.length ? { warnings } : {}), + results: matching, + }, + null, + 2, + ), + ); +} + /** * Get the offline tier ready, and report every reason it could not be. * @@ -155,6 +216,10 @@ export default defineCommand({ description: "Search by meaning when the on-device model is on, otherwise by name, title, description and tags", }, + media: { + type: "boolean", + description: "Search the media-use asset corpus instead of installable registry items", + }, yes: { type: "boolean", alias: "y", @@ -177,6 +242,20 @@ export default defineCommand({ const dir = resolve(process.cwd()); const config = loadProjectConfig(dir) ?? DEFAULT_PROJECT_CONFIG; + if (args.media === true) { + if (!json) { + console.error("--media requires --json because media rows are not installable registry items."); + finishCommand(1); + } + const query = typeof args.query === "string" ? args.query.trim() : ""; + if (!query) { + console.error("--media requires --query."); + finishCommand(1); + } + await runMediaCatalog(query, config.registry); + return; + } + let typeFilter: ItemType | undefined; if (args.type === "block") typeFilter = "hyperframes:block"; else if (args.type === "component") typeFilter = "hyperframes:component"; diff --git a/packages/cli/src/registry/localSemantic.ts b/packages/cli/src/registry/localSemantic.ts index 42a444a720..4618884b17 100644 --- a/packages/cli/src/registry/localSemantic.ts +++ b/packages/cli/src/registry/localSemantic.ts @@ -29,11 +29,24 @@ interface LocalVectorMetadata { names: string[]; dimensions: number; revision?: string; + rows?: MediaVectorRow[]; +} + +export interface MediaVectorRow { + id: string; + kind: string; + title: string; + description: string; + tags: string[]; + file: string; + duration?: number; + dimensions?: { width: number; height: number }; } export interface FetchLocalVectorOptions { directory?: string; expectedRevision?: string; + artifactBasename?: "local-vectors" | "media-vectors"; } const CATALOG_ARTIFACT_TIMEOUT_MS = 30_000; @@ -63,9 +76,12 @@ function localVectorDirectory(): string { * contract. A pair that fails it is a truncated download or a different * model, never something worth caching. */ -function vectorPairAgrees(fetched: Array<[string, Buffer]>): boolean { - const meta = fetched.find(([file]) => file === "local-vectors.json")?.[1]; - const bin = fetched.find(([file]) => file === "local-vectors.bin")?.[1]; +function vectorPairAgrees( + fetched: Array<[string, Buffer]>, + artifactBasename: "local-vectors" | "media-vectors", +): boolean { + const meta = fetched.find(([file]) => file === `${artifactBasename}.json`)?.[1]; + const bin = fetched.find(([file]) => file === `${artifactBasename}.bin`)?.[1]; if (!meta || !bin) return false; try { const parsed = JSON.parse(meta.toString("utf-8")) as { @@ -83,9 +99,10 @@ function vectorPairAgrees(fetched: Array<[string, Buffer]>): boolean { function vectorRevisionAgrees( fetched: Array<[string, Buffer]>, expectedRevision?: string, + artifactBasename: "local-vectors" | "media-vectors" = "local-vectors", ): boolean { if (expectedRevision === undefined) return true; - const meta = fetched.find(([file]) => file === "local-vectors.json")?.[1]; + const meta = fetched.find(([file]) => file === `${artifactBasename}.json`)?.[1]; if (!meta) return false; try { const parsed = JSON.parse(meta.toString("utf-8")) as { revision?: unknown }; @@ -100,6 +117,7 @@ export async function fetchLocalVectors( options: FetchLocalVectorOptions = {}, ): Promise { const directory = options.directory ?? localVectorDirectory(); + const artifactBasename = options.artifactBasename ?? "local-vectors"; const base = registryBaseUrl.replace(/\/+$/, ""); try { mkdirSync(directory, { recursive: true, mode: 0o700 }); @@ -108,7 +126,7 @@ export async function fetchLocalVectors( // must leave the previous pair intact rather than pairing a new name list // with an old matrix, which loads as an error instead of as stale data. const fetched: Array<[string, Buffer]> = []; - for (const file of ["local-vectors.json", "local-vectors.bin"] as const) { + for (const file of [`${artifactBasename}.json`, `${artifactBasename}.bin`] as const) { const response = await fetch(`${base}/catalog-artifact/${file}`, { signal: AbortSignal.timeout(CATALOG_ARTIFACT_TIMEOUT_MS), }); @@ -119,7 +137,10 @@ export async function fetchLocalVectors( // discovering the mismatch at load time leaves a cache that fails every // subsequent search until someone deletes it by hand, and it is the only // point where a truncated or wrong-model response can still be refused. - if (!vectorPairAgrees(fetched) || !vectorRevisionAgrees(fetched, options.expectedRevision)) { + if ( + !vectorPairAgrees(fetched, artifactBasename) || + !vectorRevisionAgrees(fetched, options.expectedRevision, artifactBasename) + ) { return false; } // 0o600: the cache is this user's, and the directory may be world-writable @@ -127,12 +148,21 @@ export async function fetchLocalVectors( for (const [file, bytes] of fetched) { writeFileSync(join(directory, file), bytes, { mode: 0o600 }); } - return hasLocalVectors(directory); + return artifactBasename === "media-vectors" + ? hasMediaVectors(directory) + : hasLocalVectors(directory); } catch { return false; } } +export function fetchMediaVectors( + registryBaseUrl: string, + options: Omit = {}, +): Promise { + return fetchLocalVectors(registryBaseUrl, { ...options, artifactBasename: "media-vectors" }); +} + export function hasLocalVectors(directory = localVectorDirectory()): boolean { return ( existsSync(join(directory, "local-vectors.bin")) && @@ -140,6 +170,25 @@ export function hasLocalVectors(directory = localVectorDirectory()): boolean { ); } +export function hasMediaVectors(directory = localVectorDirectory()): boolean { + return ( + existsSync(join(directory, "media-vectors.bin")) && + existsSync(join(directory, "media-vectors.json")) + ); +} + +export function mediaVectorRows(directory = localVectorDirectory()): MediaVectorRow[] { + if (!hasMediaVectors(directory)) return []; + try { + const metadata = JSON.parse( + readFileSync(join(directory, "media-vectors.json"), "utf-8"), + ) as LocalVectorMetadata; + return metadata.rows ?? []; + } catch { + return []; + } +} + function loadLocalVectors(directory = localVectorDirectory()): LocalVectorSet { const meta = JSON.parse( readFileSync(join(directory, "local-vectors.json"), "utf-8"), @@ -161,6 +210,29 @@ function loadLocalVectors(directory = localVectorDirectory()): LocalVectorSet { return { names: meta.names, dimensions: meta.dimensions, vectors }; } +function loadMediaVectors(directory = localVectorDirectory()): LocalVectorSet & { + rows: MediaVectorRow[]; +} { + const meta = JSON.parse( + readFileSync(join(directory, "media-vectors.json"), "utf-8"), + ) as LocalVectorMetadata; + const buffer = readFileSync(join(directory, "media-vectors.bin")); + const vectors = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4); + const rows = meta.rows ?? []; + if (rows.length !== meta.names.length) { + throw new Error(`media vectors hold ${rows.length} rows, expected ${meta.names.length}`); + } + if (vectors.length !== rows.length * meta.dimensions) { + throw new Error(`media vectors hold ${vectors.length} floats, expected ${rows.length * meta.dimensions}`); + } + if (meta.dimensions !== LOCAL_MODEL_DIMENSIONS) { + throw new Error( + `media vectors are ${meta.dimensions}-dimension, model produces ${LOCAL_MODEL_DIMENSIONS}`, + ); + } + return { names: meta.names, dimensions: meta.dimensions, vectors, rows }; +} + /** * The names the on-device index holds, without loading the vector matrix. * @@ -227,3 +299,23 @@ export async function localSemanticRanking( scored.sort((a, b) => b.score - a.score || b.name.localeCompare(a.name)); return scored; } + +export async function mediaSemanticRanking( + query: string, + directory = localVectorDirectory(), +): Promise | null> { + if (!isLocalModelReady() || !hasMediaVectors(directory)) return null; + const set = loadMediaVectors(directory); + const embedder = await loadLocalEmbedder(); + const [queryVector] = await embedder.embed([query], { isQuery: true }); + if (!queryVector) return null; + return set.rows + .map((row, index) => ({ + row, + score: cosine( + queryVector, + Array.from(set.vectors.subarray(index * set.dimensions, (index + 1) * set.dimensions)), + ), + })) + .sort((a, b) => b.score - a.score || b.row.id.localeCompare(a.row.id)); +} diff --git a/scripts/catalog/build-local-vectors.ts b/scripts/catalog/build-local-vectors.ts index a13104a966..394ec8e52c 100644 --- a/scripts/catalog/build-local-vectors.ts +++ b/scripts/catalog/build-local-vectors.ts @@ -32,6 +32,17 @@ import { localVectorRevision, } from "./catalog-artifact.js"; +export interface MediaVectorRow { + id: string; + kind: string; + title: string; + description: string; + tags: string[]; + file: string; + duration?: number; + dimensions?: { width: number; height: number }; +} + /** Distinct from 1 so the pre-commit hook can tell "cannot" from "failed". */ const EXIT_NO_MODEL = 3; @@ -40,6 +51,36 @@ function arg(name: string): string | undefined { return index === -1 ? undefined : process.argv[index + 1]; } +function mediaRows(manifestPath: string): MediaVectorRow[] { + const parsed = JSON.parse(readFileSync(manifestPath, "utf8")) as + | MediaVectorRow[] + | Record & { tags?: string[] }>; + const bundledSfx = manifestPath.endsWith("skills/media-use/audio/assets/sfx/manifest.json"); + const rootRelativeFile = (file: string): string => + bundledSfx && !file.startsWith("skills/") + ? `skills/media-use/audio/assets/sfx/${file}` + : file; + const rows = Array.isArray(parsed) + ? parsed.map((row) => ({ ...row, file: rootRelativeFile(row.file) })) + : Object.entries(parsed).map(([id, entry]) => ({ + id, + kind: "sfx", + title: id, + description: entry.description, + tags: entry.tags ?? ["sfx"], + file: rootRelativeFile(entry.file), + ...(entry.duration === undefined ? {} : { duration: entry.duration }), + ...(entry.dimensions === undefined ? {} : { dimensions: entry.dimensions }), + })); + if (rows.length === 0) throw new Error(`media manifest contains no rows: ${manifestPath}`); + for (const row of rows) { + if (!row.id || !row.file || !row.title || !row.description) { + throw new Error(`media manifest row is missing id, title, description, or file`); + } + } + return [...rows].sort((left, right) => left.id.localeCompare(right.id)); +} + /** * Embed in batches, in `names` order. * @@ -81,19 +122,24 @@ function packVectors(names: string[], vectors: number[][]): Float32Array { async function main(): Promise { const dir = arg("artifact") ?? "registry/catalog-artifact"; const registryDir = arg("registry") ?? "registry"; + const manifestPath = arg("manifest"); + const basename = arg("output") ?? (manifestPath ? "media-vectors" : "local-vectors"); // Read the corpus straight from the registry rather than from a catalog.json // built by the hosted-tier script. That file is not in the repo, so the // documented regeneration command used to fail on a missing path, which is // the whole reason the index was allowed to drift. - const catalogMap = catalogFromRegistry( - registryDir, - (path) => readFileSync(path, "utf-8"), - (path) => - readdirSync(path, { withFileTypes: true }) - .filter((e) => e.isDirectory()) - .map((e) => e.name), - ); + const rows = manifestPath ? mediaRows(manifestPath) : undefined; + const catalogMap = rows + ? new Map(rows.map((row) => [row.id, `${row.title}\n${row.description}\n${row.tags.join(" ")}\n${row.kind}`])) + : catalogFromRegistry( + registryDir, + (path) => readFileSync(path, "utf-8"), + (path) => + readdirSync(path, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => e.name), + ); const catalog = Object.fromEntries(catalogMap); const names = Object.keys(catalog).sort(); if (names.length === 0) throw new Error(`no registry items found under ${registryDir}`); @@ -122,25 +168,27 @@ async function main(): Promise { const vectors = await embedInBatches(names, catalog, embedder); const flat = packVectors(names, vectors); - writeFileSync(join(dir, "local-vectors.bin"), Buffer.from(flat.buffer)); + writeFileSync(join(dir, `${basename}.bin`), Buffer.from(flat.buffer)); writeFileSync( - join(dir, "local-vectors.json"), - `${JSON.stringify({ model: LOCAL_MODEL_ID, modelRevision: LOCAL_MODEL_REVISION, dimensions: LOCAL_MODEL_DIMENSIONS, revision, names }, null, 2)}\n`, - ); - const registryPath = join(registryDir, "registry.json"); - const registry = JSON.parse(readFileSync(registryPath, "utf-8")) as RegistryManifest; - writeFileSync( - registryPath, - `${JSON.stringify({ ...registry, catalogArtifact: { revision } }, null, 2)}\n`, + join(dir, `${basename}.json`), + `${JSON.stringify({ model: LOCAL_MODEL_ID, modelRevision: LOCAL_MODEL_REVISION, dimensions: LOCAL_MODEL_DIMENSIONS, revision, names, ...(rows ? { rows } : {}) }, null, 2)}\n`, ); + if (!rows) { + const registryPath = join(registryDir, "registry.json"); + const registry = JSON.parse(readFileSync(registryPath, "utf-8")) as RegistryManifest; + writeFileSync( + registryPath, + `${JSON.stringify({ ...registry, catalogArtifact: { revision } }, null, 2)}\n`, + ); + } const megabytes = (flat.byteLength / 1024 / 1024).toFixed(2); - console.log(`moves ${names.length}`); + console.log(`${rows ? "media" : "moves"} ${names.length}`); console.log(`model ${LOCAL_MODEL_ID}`); console.log(`dimensions ${LOCAL_MODEL_DIMENSIONS}`); console.log(`revision ${revision}`); console.log(`payload ${megabytes} MB`); - console.log(`written ${dir}`); + console.log(`written ${dir}/${basename}.{json,bin}`); } await main(); diff --git a/scripts/catalog/check-artifact-coverage.ts b/scripts/catalog/check-artifact-coverage.ts index 6188d15317..227c338765 100644 --- a/scripts/catalog/check-artifact-coverage.ts +++ b/scripts/catalog/check-artifact-coverage.ts @@ -23,9 +23,12 @@ import { catalogFromRegistry, localVectorRevision } from "./catalog-artifact.js" type RegistryItem = { name: string; type?: string }; type Registry = { items: RegistryItem[]; catalogArtifact?: { revision?: string } }; type Artifact = { model?: string; dimensions?: number; revision?: string; names?: string[] }; +type MediaArtifact = { rows?: Array<{ id?: string; file?: string }> }; const REGISTRY = "registry/registry.json"; const ARTIFACT = "registry/catalog-artifact/local-vectors.json"; +const MEDIA_MANIFEST = "skills/media-use/audio/assets/sfx/manifest.json"; +const MEDIA_ARTIFACT = "registry/catalog-artifact/media-vectors.json"; function read(path: string): T { try { @@ -67,6 +70,23 @@ const expectedRevision = localVectorRevision( ); const artifactRevisionMatches = artifact.revision === expectedRevision; const registryRevisionMatches = registry.catalogArtifact?.revision === expectedRevision; +const mediaSource = read>(MEDIA_MANIFEST); +const mediaArtifact = read(MEDIA_ARTIFACT); +const mediaRows = new Map( + (mediaArtifact.rows ?? []) + .filter( + (row): row is { id: string; file: string } => + typeof row.id === "string" && typeof row.file === "string", + ) + .map((row) => [row.id, row.file]), +); +const missingMediaRows = Object.entries(mediaSource) + .filter( + ([id, source]) => + mediaRows.get(id) !== `skills/media-use/audio/assets/sfx/${source.file ?? ""}`, + ) + .map(([id]) => id) + .sort(); const show = (names: string[]) => names @@ -76,6 +96,12 @@ const show = (names: string[]) => console.log(`registry: ${registryNames.size} searchable items (blocks + components)`); console.log(`artifact: ${artifactNames.size} vectors (${artifact.model ?? "unknown model"})`); +console.log(`media: ${mediaRows.size} rows for ${Object.keys(mediaSource).length} bundled SFX files`); + +if (missingMediaRows.length > 0) { + console.error(`\n${missingMediaRows.length} bundled SFX file(s) have no matching media row:`); + console.error(show(missingMediaRows)); +} if (dropped.length > 0) { // Not fatal: the CLI filters these before a user ever sees them. @@ -96,7 +122,12 @@ if (!artifactRevisionMatches || !registryRevisionMatches) { console.error(` registry: ${registry.catalogArtifact?.revision ?? "missing"}`); } -if (unindexed.length > 0 || !artifactRevisionMatches || !registryRevisionMatches) { +if ( + unindexed.length > 0 || + !artifactRevisionMatches || + !registryRevisionMatches || + missingMediaRows.length > 0 +) { console.error( "\nMeaning search is stale. Word search still uses the live registry.\n\n" + "If you have the embedding model, regenerate and commit the artifact:\n" + diff --git a/skills/hyperframes-cli/SKILL.md b/skills/hyperframes-cli/SKILL.md index 0e296a27e7..990587d960 100644 --- a/skills/hyperframes-cli/SKILL.md +++ b/skills/hyperframes-cli/SKILL.md @@ -16,7 +16,7 @@ Run commands as `npx hyperframes ...` unless project instructions provide a wrap ## Development loop 1. **Scaffold:** `npx hyperframes init ` (centered blank). Or capture a site. Pass `--example=` only to start from a named example. -2. **Find the move:** before authoring motion by hand, search for a primitive that already does it: `npx hyperframes catalog --query "reveal a headline one line at a time"`. Ask for the effect you want rather than the mechanism you have in mind. Install with `npx hyperframes add ` (see `/hyperframes-registry`). Author by hand only once nothing fits. +2. **Find the move:** if the request names an asset, sound, image, voice or fast visual edit, resolve it through `/media-use` before proposing a plan. Otherwise, before authoring motion by hand, search for a primitive that already does it: `npx hyperframes catalog --query "reveal a headline one line at a time"`. Ask for the effect you want rather than the mechanism you have in mind. Install with `npx hyperframes add ` (see `/hyperframes-registry`). Author by hand only once nothing fits. 3. **Author:** write the composition using `/hyperframes-core`. To know what is on a project's timeline (tracks, clips, starts, ends, what plays), run `npx hyperframes timeline --json` instead of reading `index.html` and every sub-composition file: nested rows carry absolute main-timeline `absStart`/`absEnd` and their owning `file`, not just their local, per-sub-composition time. Prefer `--json` over the text form; it costs fewer tokens for the same or better correctness. See `references/upgrade-info-misc.md` for one-liners that answer common questions without reading the whole output. 4. **Get fast feedback while editing:** run `npx hyperframes lint` after the first HTML pass and after structural changes. 5. **Run the final gate:** run `npx hyperframes check`; it reruns lint before opening the browser. Do not prepend a redundant standalone lint invocation. Add `--snapshots` for annotated overview frames and finding crops. @@ -36,7 +36,7 @@ Run commands as `npx hyperframes ...` unless project instructions provide a wrap - For fade-in/fade-out, crossfade, track gain, volume automation, ducking, voiceover carve, or FX on placed audio, read `/hyperframes-audio`. Load core alongside it when clip placement or picture timing also changes. -- Use `/media-use` only to source/generate media or preprocess a derived asset. +- A request naming an asset, sound, image, voice or fast visual edit resolves through `/media-use` before a plan is proposed. Copy creator edit markup from `/hyperframes-core` → `references/creator-editing-recipes.md`. ```bash