diff --git a/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs b/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs index 5287a11f41..a0d09b77d6 100644 --- a/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs +++ b/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs @@ -1,6 +1,6 @@ import { existsSync, readFileSync } from "node:fs"; import { extname, join } from "node:path"; -import { rankMediaRows } from "../../registry/mediaSearch.js"; +import { rankMediaRows } from "../../registry/mediaSearch.mjs"; const LIB_DIR = process.env.HYPERFRAMES_MEDIA_USE_SFX_DIR || @@ -76,25 +76,10 @@ export function inspectBundledSfxAssets(libraryDir = LIB_DIR) { }; } -const normalize = (value) => - String(value) - .toLowerCase() - .replace(/[^a-z0-9]+/g, " ") - .trim(); - export function extensionForBundledSfxFile(filename) { return extname(filename) || ".mp3"; } -function score(intent, key, entry) { - const query = normalize(intent); - const name = normalize(key); - if (query === name) return 100; - if (query.includes(name) || name.includes(query)) return 50; - const haystack = new Set(normalize(`${key} ${entry.description || ""}`).split(/\s+/)); - return query.split(/\s+/).filter((token) => token && haystack.has(token)).length; -} - export const bundledSfxProvider = { async search(intent, ctx = {}) { const libraryDir = ctx.libraryDir || LIB_DIR; diff --git a/packages/cli/src/media-use/resolve.mjs b/packages/cli/src/media-use/resolve.mjs index e8c18b5016..c129018aa2 100644 --- a/packages/cli/src/media-use/resolve.mjs +++ b/packages/cli/src/media-use/resolve.mjs @@ -49,8 +49,6 @@ import { versionLessThan, } from "./lib/heygen-cli.mjs"; import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs"; -import { rankMediaRowsWithVectors } from "../registry/mediaSearch.js"; -import { fetchMediaVectors, mediaVectorRows } from "../registry/localSemantic.js"; const INGEST_TYPES = listTypes(); const DEFAULT_EXT = { @@ -422,6 +420,8 @@ async function run() { const registry = process.env.HYPERFRAMES_REGISTRY || "https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry"; + const { fetchMediaVectors, mediaVectorRows } = await import("../registry/localSemantic.js"); + const { rankMediaRowsWithVectors } = await import("../registry/mediaSearch.js"); await fetchMediaVectors(registry); const ranked = await rankMediaRowsWithVectors( intent, diff --git a/packages/cli/src/registry/mediaSearch.mjs b/packages/cli/src/registry/mediaSearch.mjs new file mode 100644 index 0000000000..39077951ef --- /dev/null +++ b/packages/cli/src/registry/mediaSearch.mjs @@ -0,0 +1,84 @@ +const STOP = new Set( + ( + "the a an and or of to in on at is are be it its for with that this as by from into " + + "one two must not no all over under across while when where which who whom whose they " + + "them their we our you your he she his her but if then than so such can may might will " + + "would should each other another same both few more most some any every" + ).split(" "), +); + +const PLURAL_RULES = [ + [/([^aeiou])ies$/, "$1y"], + [/(ch|sh|ss|x)es$/, "$1"], + [/(ss|us|is)$/, "$&"], + [/s$/, ""], +]; +const STRONG_FIELD_WEIGHT = 3; +const INFERRED_TOKEN_WEIGHT = 0.35; + +function singularize(word) { + if (word.length <= 3) return word; + for (const [pattern, replacement] of PLURAL_RULES) { + if (pattern.test(word)) return word.replace(pattern, replacement); + } + return word; +} + +function tokenize(text) { + const words = + String(text) + .toLowerCase() + .match(/[a-z]+/g) || []; + return words.filter((word) => word.length > 2 && !STOP.has(word)).map(singularize); +} + +export function rankMediaRows(query, rows) { + const asked = tokenize(query); + const want = new Map(asked.map((token) => [token, 1])); + if (want.size === 0) return []; + + const parsed = rows.map((row) => { + const strongTokens = new Set(tokenize(`${row.id} ${row.title}`)); + const allTokens = new Set([ + ...strongTokens, + ...tokenize(`${row.description} ${row.tags.join(" ")} ${row.kind}`), + ]); + return { row, strongTokens, allTokens }; + }); + const vocabulary = new Set(parsed.flatMap(({ allTokens }) => [...allTokens])); + for (const token of asked) { + if (token.length < 6) continue; + for (let cut = 3; cut <= token.length - 3; cut++) { + const head = token.slice(0, cut); + const tail = token.slice(cut); + if (vocabulary.has(head) && vocabulary.has(tail)) { + want.set(head, INFERRED_TOKEN_WEIGHT); + want.set(tail, INFERRED_TOKEN_WEIGHT); + break; + } + } + } + for (let index = 0; index < asked.length - 1; index++) { + const joined = `${asked[index]}${asked[index + 1]}`; + if (vocabulary.has(joined)) want.set(joined, INFERRED_TOKEN_WEIGHT); + } + + const idf = new Map(); + for (const token of want.keys()) { + const documentFrequency = parsed.filter(({ allTokens }) => allTokens.has(token)).length; + idf.set(token, Math.log((parsed.length + 1) / (documentFrequency + 1)) + 1); + } + return parsed + .map(({ row, strongTokens, allTokens }) => { + let score = 0; + for (const [token, asking] of want) { + const weight = (idf.get(token) || 1) * asking; + if (strongTokens.has(token)) score += STRONG_FIELD_WEIGHT * weight; + else if (allTokens.has(token)) score += weight; + } + return { row, score }; + }) + .filter(({ score }) => score > 0) + .sort((a, b) => b.score - a.score) + .map(({ row }) => row); +} diff --git a/packages/cli/src/registry/mediaSearch.ts b/packages/cli/src/registry/mediaSearch.ts index 64bfd086c5..de30ae4609 100644 --- a/packages/cli/src/registry/mediaSearch.ts +++ b/packages/cli/src/registry/mediaSearch.ts @@ -1,13 +1,10 @@ -import { searchByWords } from "./localSearch.js"; import { mediaSemanticRanking, type MediaVectorRow } from "./localSemantic.js"; +import { rankMediaRows as rankMediaRowsRuntime } from "./mediaSearch.mjs"; export type MediaSearchRow = MediaVectorRow; export function rankMediaRows(query: string, rows: MediaSearchRow[]): MediaSearchRow[] { - return searchByWords(query, rows, (row) => ({ - strong: `${row.id} ${row.title}`, - weak: `${row.description} ${row.tags.join(" ")} ${row.kind}`, - })); + return rankMediaRowsRuntime(query, rows) as MediaSearchRow[]; } export async function rankMediaRowsWithVectors( diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts index 50365fb20f..9473139afd 100644 --- a/packages/cli/tsup.config.ts +++ b/packages/cli/tsup.config.ts @@ -17,6 +17,8 @@ export default defineConfig({ runtimeVersion: "src/runtimeVersion.ts", renderSetupWorker: "src/renderSetupWorker.ts", shaderTransitionWorker: "../producer/src/services/shaderTransitionWorker.ts", + "registry/mediaSearch": "src/registry/mediaSearch.ts", + "registry/localSemantic": "src/registry/localSemantic.ts", }, format: ["esm"], outDir: "dist", diff --git a/registry/catalog-artifact/media-vectors.json b/registry/catalog-artifact/media-vectors.json index 4059ded2a6..faa379345d 100644 --- a/registry/catalog-artifact/media-vectors.json +++ b/registry/catalog-artifact/media-vectors.json @@ -30,9 +30,7 @@ "kind": "sfx", "title": "chime", "description": "Soft melodic chime — gentle positive beat: success/confirmation or a lighthearted transition. Sync to the visual moment.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/chime.mp3", "duration": 2.5 }, @@ -41,9 +39,7 @@ "kind": "sfx", "title": "click", "description": "Crisp UI click — button press, toggle, selection. Short accent, sync exactly to the on-screen action.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/click.mp3", "duration": 0.37 }, @@ -52,9 +48,7 @@ "kind": "sfx", "title": "click-soft", "description": "Quiet short click — low-key UI tap / soft selection. Short accent, sync exactly to the on-screen action.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/click-soft.mp3", "duration": 0.37 }, @@ -63,9 +57,7 @@ "kind": "sfx", "title": "error", "description": "Negative / error tone — failure state, a 'wrong' beat, or a glitchy interruption.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/error.mp3", "duration": 1.62 }, @@ -74,9 +66,7 @@ "kind": "sfx", "title": "glitch-1", "description": "Punchy digital glitch — hard-cut accent or sudden reveal. Trigger on the hit; let the decay bleed into the next shot (J-cut).", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/glitch-1.mp3", "duration": 2.64 }, @@ -85,9 +75,7 @@ "kind": "sfx", "title": "glitch-2", "description": "Harsh, longer glitch — chaotic / jarring transition or a distorted reveal.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/glitch-2.mp3", "duration": 3.5 }, @@ -96,9 +84,7 @@ "kind": "sfx", "title": "glitch-3", "description": "Low-key glitch texture — subtle digital shift, minimal transition that sits under other audio.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/glitch-3.mp3", "duration": 3.1 }, @@ -107,9 +93,7 @@ "kind": "sfx", "title": "impact-bass-1", "description": "Bass impact hit — logo/hero snap, headline slam. Trigger on the visual landing; decay carries into the next shot (J-cut).", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/impact-bass-1.mp3", "duration": 2.12 }, @@ -118,9 +102,7 @@ "kind": "sfx", "title": "impact-bass-2", "description": "Bass impact with a short swell — brief anticipation then a deep hit. Place so the peak lands on the reveal.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/impact-bass-2.mp3", "duration": 2.59 }, @@ -129,9 +111,7 @@ "kind": "sfx", "title": "key-press", "description": "Single key press — one keystroke / terminal-input beat. Short accent, sync to the typed character.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/key-press.mp3", "duration": 0.4 }, @@ -140,9 +120,7 @@ "kind": "sfx", "title": "notification", "description": "Notification chime — alert, message-in, toast/badge appears. Sync to the element entering.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/notification.mp3", "duration": 2.46 }, @@ -151,9 +129,7 @@ "kind": "sfx", "title": "ping", "description": "Sharp electronic ping — punchy accent on a key reveal or data point. Sync to the beat.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/ping.mp3", "duration": 1.32 }, @@ -162,9 +138,7 @@ "kind": "sfx", "title": "pop", "description": "Quick pop — element appear/spawn, chip/tag/badge in. Small precise accent, sync to the pop-in.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/pop.mp3", "duration": 0.72 }, @@ -173,9 +147,7 @@ "kind": "sfx", "title": "riser", "description": "Long cinematic riser (~10s build, peak at the end). Trigger at (climax_time − 10.03s) so it crests exactly on the reveal.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/riser.mp3", "duration": 10.03 }, @@ -184,9 +156,7 @@ "kind": "sfx", "title": "sparkle", "description": "Bright sparkle / shimmer — magical reveal or 'shine' highlight on a hero element. Sync to the highlight.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/sparkle.mp3", "duration": 1.8 }, @@ -195,9 +165,7 @@ "kind": "sfx", "title": "typing", "description": "Typing burst (~1.5s of keys) — keyboard / code typing reveal, text-being-typed beat. Start as the text begins typing.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/typing.mp3", "duration": 1.5 }, @@ -206,9 +174,7 @@ "kind": "sfx", "title": "whoosh", "description": "Punchy whoosh/impact — fast reveal or hard transition accent. Sync to the motion.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/whoosh.mp3", "duration": 0.57 }, @@ -217,9 +183,7 @@ "kind": "sfx", "title": "whoosh-cinematic", "description": "Cinematic whoosh build (~5.5s) — sweeping scene transition. Align so the swell peaks on the cut.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/whoosh-cinematic.mp3", "duration": 5.54 }, @@ -228,9 +192,7 @@ "kind": "sfx", "title": "whoosh-short", "description": "Short whoosh — quick swipe/slide accent, fast element move, snappy transition. Sync to the motion.", - "tags": [ - "sfx" - ], + "tags": ["sfx"], "file": "skills/media-use/audio/assets/sfx/whoosh-short.mp3", "duration": 0.57 }