From 7c79aba9505f13ffcafbaa15ae868f8b9b8f56df Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 20 Sep 2026 18:46:23 -0400 Subject: [PATCH] feat(cli): route media search through media-use --- packages/cli/scripts/build-copy.mjs | 2 + packages/cli/src/commands/catalog.test.ts | 38 +-------- packages/cli/src/commands/catalog.ts | 78 ------------------- .../media-use/lib/bundled-sfx-provider.mjs | 20 +++-- packages/cli/src/media-use/lib/registry.mjs | 2 +- .../cli/src/media-use/lib/registry.test.mjs | 12 +-- packages/cli/src/media-use/resolve.mjs | 42 +++++++++- packages/cli/src/registry/mediaSearch.ts | 23 ++++++ 8 files changed, 88 insertions(+), 129 deletions(-) create mode 100644 packages/cli/src/registry/mediaSearch.ts diff --git a/packages/cli/scripts/build-copy.mjs b/packages/cli/scripts/build-copy.mjs index 83e0fbc5c3..091e3289fd 100644 --- a/packages/cli/scripts/build-copy.mjs +++ b/packages/cli/scripts/build-copy.mjs @@ -102,6 +102,8 @@ async function main() { join(mediaEngine, "resolve.mjs"), join(DIST, "skills", "media-use", "scripts", "resolve.mjs"), ); + mkdirSync(join(DIST, "skills", "registry"), { recursive: true }); + copyDirContents(join(DIST, "registry"), join(DIST, "skills", "registry")); const dockerfile = join(CLI_ROOT, "src", "docker", "Dockerfile.render"); if (existsSync(dockerfile)) { diff --git a/packages/cli/src/commands/catalog.test.ts b/packages/cli/src/commands/catalog.test.ts index dad034c9c4..6440bb4f2d 100644 --- a/packages/cli/src/commands/catalog.test.ts +++ b/packages/cli/src/commands/catalog.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { countUnindexed, pickByName, rankMediaRows, searchMissCommand } from "./catalog.js"; +import { countUnindexed, pickByName, searchMissCommand } from "./catalog.js"; /** The whole registry, which is what "in this registry" has to be measured against. */ const registryNames = new Set(["fade-through", "whip-pan", "count-up"]); @@ -63,42 +63,6 @@ describe("countUnindexed", () => { }); }); -describe("rankMediaRows", () => { - it("ranks the click SFX first for a click query", () => { - const rows = [ - { - id: "whoosh", - kind: "sfx", - title: "whoosh", - description: "Fast scene transition accent", - tags: ["sfx"], - file: "skills/media-use/audio/assets/sfx/whoosh.mp3", - }, - { - id: "click", - kind: "sfx", - title: "click", - description: "Crisp UI click for a button press or selection", - tags: ["sfx"], - file: "skills/media-use/audio/assets/sfx/click.mp3", - }, - ]; - - expect(rankMediaRows("click", rows).map((row) => row.id)).toEqual(["click"]); - }); -}); - -describe("catalog --media", () => { - it("returns JSON media rows without entering the installable registry path", async () => { - const output = await runCatalog({ media: true, json: true, query: "click" }); - expect(JSON.parse(output)).toMatchObject({ - tier: "words", - total: 1, - results: [{ id: "click", file: "skills/media-use/audio/assets/sfx/click.mp3" }], - }); - }); -}); - // ── The command envelope ──────────────────────────────────────────────────── // The JSON envelope is the surface an agent reads, so under-coverage and the // score are pinned where they are actually published rather than only at the diff --git a/packages/cli/src/commands/catalog.ts b/packages/cli/src/commands/catalog.ts index 4295da55a1..bfdf9b9fc0 100644 --- a/packages/cli/src/commands/catalog.ts +++ b/packages/cli/src/commands/catalog.ts @@ -30,70 +30,12 @@ import { import { ensureLocalRuntime } from "../registry/localEmbedder.js"; import { cachedLocalVectorRevision, - fetchMediaVectors, fetchLocalVectors, hasLocalVectors, localSemanticRanking, localVectorNames, - mediaSemanticRanking, - mediaVectorRows, } from "../registry/localSemantic.js"; -export interface MediaCatalogRow { - id: string; - kind: string; - title: string; - description: string; - tags: string[]; - file: string; - duration?: number; - dimensions?: { width: number; height: number }; -} - -export function rankMediaRows(query: string, rows: MediaCatalogRow[]): MediaCatalogRow[] { - return searchByWords(query, rows, (row) => ({ - strong: `${row.id} ${row.title}`, - weak: `${row.description} ${row.tags.join(" ")} ${row.kind}`, - })); -} - -async function runMediaCatalog(query: string, registry: string): Promise { - const fetched = await fetchMediaVectors(registry); - const rows = mediaVectorRows() as MediaCatalogRow[]; - const warnings: string[] = []; - if (!fetched && rows.length === 0) { - warnings.push("media search unavailable: the media index could not be fetched"); - } - - let matching = rankMediaRows(query, rows); - let tier: "words" | "on-device" = "words"; - let tierDetail = "local word match"; - if (matching.length === 0 && localModelStatus().status === "ready") { - const ranking = await mediaSemanticRanking(query); - if (ranking) { - matching = ranking.map(({ row }) => row as MediaCatalogRow); - tier = "on-device"; - tierDetail = "on-device meaning search"; - } - } - - console.log( - JSON.stringify( - { - query, - tier, - tier_detail: tierDetail, - shown: matching.length, - total: rows.length, - ...(warnings.length ? { warnings } : {}), - results: matching, - }, - null, - 2, - ), - ); -} - /** * Get the offline tier ready, and report every reason it could not be. * @@ -213,10 +155,6 @@ export default defineCommand({ description: "Search by meaning when the on-device model is on, otherwise by name, title, description and tags", }, - media: { - type: "boolean", - description: "Search the media-use asset corpus instead of installable registry items", - }, yes: { type: "boolean", alias: "y", @@ -239,22 +177,6 @@ export default defineCommand({ const dir = resolve(process.cwd()); const config = loadProjectConfig(dir) ?? DEFAULT_PROJECT_CONFIG; - if (args.media === true) { - if (!json) { - console.error( - "--media requires --json because media rows are not installable registry items.", - ); - finishCommand(1); - } - const query = typeof args.query === "string" ? args.query.trim() : ""; - if (!query) { - console.error("--media requires --query."); - finishCommand(1); - } - await runMediaCatalog(query, config.registry); - return; - } - let typeFilter: ItemType | undefined; if (args.type === "block") typeFilter = "hyperframes:block"; else if (args.type === "component") typeFilter = "hyperframes:component"; diff --git a/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs b/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs index e7903c3ef6..5287a11f41 100644 --- a/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs +++ b/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs @@ -1,5 +1,6 @@ import { existsSync, readFileSync } from "node:fs"; import { extname, join } from "node:path"; +import { rankMediaRows } from "../../registry/mediaSearch.js"; const LIB_DIR = process.env.HYPERFRAMES_MEDIA_USE_SFX_DIR || @@ -101,11 +102,20 @@ export const bundledSfxProvider = { if (!health.ok) throw new BundledSfxAssetsError(health); const manifest = JSON.parse(readFileSync(join(libraryDir, "manifest.json"), "utf8")); - const ranked = Object.entries(manifest) - .map(([key, entry]) => ({ key, entry, score: score(intent, key, entry) })) - .filter(({ entry, score }) => entry?.file && score > 0) - .sort((a, b) => b.score - a.score || a.key.localeCompare(b.key)); - const best = ranked[0]; + const rows = Object.entries(manifest).map(([key, entry]) => ({ + id: key, + kind: "sfx", + title: key, + description: entry?.description || key, + tags: ["sfx"], + file: entry?.file || "", + duration: entry?.duration, + })); + const bestRow = rankMediaRows(intent, rows)[0]; + const best = + bestRow && manifest[bestRow.id]?.file + ? { key: bestRow.id, entry: manifest[bestRow.id] } + : null; if (!best) return null; const localPath = join(libraryDir, best.entry.file); diff --git a/packages/cli/src/media-use/lib/registry.mjs b/packages/cli/src/media-use/lib/registry.mjs index 8faa919209..eb6a0beadd 100644 --- a/packages/cli/src/media-use/lib/registry.mjs +++ b/packages/cli/src/media-use/lib/registry.mjs @@ -51,8 +51,8 @@ const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // rem const REGISTRY = { bgm: [N("heygen.audio.sounds", { search: bgmProvider.search })], sfx: [ - N("heygen.audio.sounds", { search: sfxProvider.search }), A("bundled.sfx", { search: bundledSfxProvider.search }), + N("heygen.audio.sounds", { search: sfxProvider.search }), ], image: [ N("heygen.asset.search", { search: imageProvider.search }), diff --git a/packages/cli/src/media-use/lib/registry.test.mjs b/packages/cli/src/media-use/lib/registry.test.mjs index 08f30351c3..07bd1c8c8a 100644 --- a/packages/cli/src/media-use/lib/registry.test.mjs +++ b/packages/cli/src/media-use/lib/registry.test.mjs @@ -84,13 +84,13 @@ test("video cascade: HeyGen first, LTX local fallback, generate-only", async () assert.equal(await runCapability("video", "search", "x", {}), null); }); -test("sfx cascade: HeyGen catalog first, bundled library remains the local fallback", () => { +test("sfx cascade: bundled library first, HeyGen catalog remains the network fallback", () => { const ps = getProviders("sfx"); - assert.equal(ps[0].name, "heygen.audio.sounds"); - assert.ok(ps[0].network, "HeyGen SFX catalog is network-only"); - assert.equal(ps[1].name, "bundled.sfx"); - assert.equal(typeof ps[1].search, "function"); - assert.ok(!ps[1].network, "bundled SFX remain available offline"); + assert.equal(ps[0].name, "bundled.sfx"); + assert.equal(typeof ps[0].search, "function"); + assert.ok(!ps[0].network, "bundled SFX remain available offline"); + assert.equal(ps[1].name, "heygen.audio.sounds"); + assert.ok(ps[1].network, "HeyGen SFX catalog is network-only"); }); test("ctx.provider forces one generator (e.g. 'make an image WITH codex')", async () => { diff --git a/packages/cli/src/media-use/resolve.mjs b/packages/cli/src/media-use/resolve.mjs index b7874ae892..17fac5cd64 100644 --- a/packages/cli/src/media-use/resolve.mjs +++ b/packages/cli/src/media-use/resolve.mjs @@ -49,6 +49,8 @@ import { versionLessThan, } from "./lib/heygen-cli.mjs"; import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs"; +import { rankMediaRowsWithVectors } from "../registry/mediaSearch.js"; +import { fetchMediaVectors, mediaVectorRows } from "../registry/localSemantic.js"; const INGEST_TYPES = listTypes(); const DEFAULT_EXT = { @@ -408,11 +410,47 @@ async function run() { return resolveColor(type, intent, { projectDir }); } - // 3. provider search — registry tries providers in order (heygen-CLI first) + // 3. SFX is local-first: bundled library, committed local index, then HeyGen. let searchResult = null; let providerFailure = null; try { - searchResult = await runCapability(type, "search", intent, ctx); + if (type === "sfx" && !args.provider) { + searchResult = await runCapability(type, "search", intent, { + ...ctx, + provider: "bundled.sfx", + }); + if (!searchResult) { + const registry = + process.env.HYPERFRAMES_REGISTRY || + "https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry"; + await fetchMediaVectors(registry); + const ranked = await rankMediaRowsWithVectors( + intent, + mediaVectorRows().filter((row) => row.kind === "sfx"), + ); + const row = ranked.rows[0]; + const candidates = row + ? [resolve(row.file), join(import.meta.dirname, "..", "..", "..", row.file)] + : []; + const localPath = candidates.find((candidate) => existsSync(candidate)); + if (row && localPath) { + searchResult = { + localPath, + ext: extname(localPath), + source: "local-index", + metadata: { + description: row.description, + duration: row.duration ?? null, + provider: "catalog.local", + provenance: { library_key: row.id, tier: ranked.tier }, + }, + }; + } + } + if (!searchResult) searchResult = await runCapability(type, "search", intent, ctx); + } else { + searchResult = await runCapability(type, "search", intent, ctx); + } } catch (error) { providerFailure = error; // search failed, try generate diff --git a/packages/cli/src/registry/mediaSearch.ts b/packages/cli/src/registry/mediaSearch.ts new file mode 100644 index 0000000000..64bfd086c5 --- /dev/null +++ b/packages/cli/src/registry/mediaSearch.ts @@ -0,0 +1,23 @@ +import { searchByWords } from "./localSearch.js"; +import { mediaSemanticRanking, type MediaVectorRow } from "./localSemantic.js"; + +export type MediaSearchRow = MediaVectorRow; + +export function rankMediaRows(query: string, rows: MediaSearchRow[]): MediaSearchRow[] { + return searchByWords(query, rows, (row) => ({ + strong: `${row.id} ${row.title}`, + weak: `${row.description} ${row.tags.join(" ")} ${row.kind}`, + })); +} + +export async function rankMediaRowsWithVectors( + query: string, + rows: MediaSearchRow[], +): Promise<{ rows: MediaSearchRow[]; tier: "words" | "on-device" }> { + const words = rankMediaRows(query, rows); + if (words.length > 0) return { rows: words, tier: "words" }; + const semantic = await mediaSemanticRanking(query); + return semantic + ? { rows: semantic.map(({ row }) => row), tier: "on-device" } + : { rows: [], tier: "words" }; +}