mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 15:24:19 +00:00
fix(cli): make media search runtime-safe
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { extname, join } from "node:path";
|
||||
import { rankMediaRows } from "../../registry/mediaSearch.js";
|
||||
import { rankMediaRows } from "../../registry/mediaSearch.mjs";
|
||||
|
||||
const LIB_DIR =
|
||||
process.env.HYPERFRAMES_MEDIA_USE_SFX_DIR ||
|
||||
@@ -76,25 +76,10 @@ export function inspectBundledSfxAssets(libraryDir = LIB_DIR) {
|
||||
};
|
||||
}
|
||||
|
||||
const normalize = (value) =>
|
||||
String(value)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.trim();
|
||||
|
||||
export function extensionForBundledSfxFile(filename) {
|
||||
return extname(filename) || ".mp3";
|
||||
}
|
||||
|
||||
function score(intent, key, entry) {
|
||||
const query = normalize(intent);
|
||||
const name = normalize(key);
|
||||
if (query === name) return 100;
|
||||
if (query.includes(name) || name.includes(query)) return 50;
|
||||
const haystack = new Set(normalize(`${key} ${entry.description || ""}`).split(/\s+/));
|
||||
return query.split(/\s+/).filter((token) => token && haystack.has(token)).length;
|
||||
}
|
||||
|
||||
export const bundledSfxProvider = {
|
||||
async search(intent, ctx = {}) {
|
||||
const libraryDir = ctx.libraryDir || LIB_DIR;
|
||||
|
||||
@@ -49,8 +49,6 @@ import {
|
||||
versionLessThan,
|
||||
} from "./lib/heygen-cli.mjs";
|
||||
import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs";
|
||||
import { rankMediaRowsWithVectors } from "../registry/mediaSearch.js";
|
||||
import { fetchMediaVectors, mediaVectorRows } from "../registry/localSemantic.js";
|
||||
|
||||
const INGEST_TYPES = listTypes();
|
||||
const DEFAULT_EXT = {
|
||||
@@ -422,6 +420,8 @@ async function run() {
|
||||
const registry =
|
||||
process.env.HYPERFRAMES_REGISTRY ||
|
||||
"https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry";
|
||||
const { fetchMediaVectors, mediaVectorRows } = await import("../registry/localSemantic.js");
|
||||
const { rankMediaRowsWithVectors } = await import("../registry/mediaSearch.js");
|
||||
await fetchMediaVectors(registry);
|
||||
const ranked = await rankMediaRowsWithVectors(
|
||||
intent,
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
const STOP = new Set(
|
||||
(
|
||||
"the a an and or of to in on at is are be it its for with that this as by from into " +
|
||||
"one two must not no all over under across while when where which who whom whose they " +
|
||||
"them their we our you your he she his her but if then than so such can may might will " +
|
||||
"would should each other another same both few more most some any every"
|
||||
).split(" "),
|
||||
);
|
||||
|
||||
const PLURAL_RULES = [
|
||||
[/([^aeiou])ies$/, "$1y"],
|
||||
[/(ch|sh|ss|x)es$/, "$1"],
|
||||
[/(ss|us|is)$/, "$&"],
|
||||
[/s$/, ""],
|
||||
];
|
||||
const STRONG_FIELD_WEIGHT = 3;
|
||||
const INFERRED_TOKEN_WEIGHT = 0.35;
|
||||
|
||||
function singularize(word) {
|
||||
if (word.length <= 3) return word;
|
||||
for (const [pattern, replacement] of PLURAL_RULES) {
|
||||
if (pattern.test(word)) return word.replace(pattern, replacement);
|
||||
}
|
||||
return word;
|
||||
}
|
||||
|
||||
function tokenize(text) {
|
||||
const words =
|
||||
String(text)
|
||||
.toLowerCase()
|
||||
.match(/[a-z]+/g) || [];
|
||||
return words.filter((word) => word.length > 2 && !STOP.has(word)).map(singularize);
|
||||
}
|
||||
|
||||
export function rankMediaRows(query, rows) {
|
||||
const asked = tokenize(query);
|
||||
const want = new Map(asked.map((token) => [token, 1]));
|
||||
if (want.size === 0) return [];
|
||||
|
||||
const parsed = rows.map((row) => {
|
||||
const strongTokens = new Set(tokenize(`${row.id} ${row.title}`));
|
||||
const allTokens = new Set([
|
||||
...strongTokens,
|
||||
...tokenize(`${row.description} ${row.tags.join(" ")} ${row.kind}`),
|
||||
]);
|
||||
return { row, strongTokens, allTokens };
|
||||
});
|
||||
const vocabulary = new Set(parsed.flatMap(({ allTokens }) => [...allTokens]));
|
||||
for (const token of asked) {
|
||||
if (token.length < 6) continue;
|
||||
for (let cut = 3; cut <= token.length - 3; cut++) {
|
||||
const head = token.slice(0, cut);
|
||||
const tail = token.slice(cut);
|
||||
if (vocabulary.has(head) && vocabulary.has(tail)) {
|
||||
want.set(head, INFERRED_TOKEN_WEIGHT);
|
||||
want.set(tail, INFERRED_TOKEN_WEIGHT);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (let index = 0; index < asked.length - 1; index++) {
|
||||
const joined = `${asked[index]}${asked[index + 1]}`;
|
||||
if (vocabulary.has(joined)) want.set(joined, INFERRED_TOKEN_WEIGHT);
|
||||
}
|
||||
|
||||
const idf = new Map();
|
||||
for (const token of want.keys()) {
|
||||
const documentFrequency = parsed.filter(({ allTokens }) => allTokens.has(token)).length;
|
||||
idf.set(token, Math.log((parsed.length + 1) / (documentFrequency + 1)) + 1);
|
||||
}
|
||||
return parsed
|
||||
.map(({ row, strongTokens, allTokens }) => {
|
||||
let score = 0;
|
||||
for (const [token, asking] of want) {
|
||||
const weight = (idf.get(token) || 1) * asking;
|
||||
if (strongTokens.has(token)) score += STRONG_FIELD_WEIGHT * weight;
|
||||
else if (allTokens.has(token)) score += weight;
|
||||
}
|
||||
return { row, score };
|
||||
})
|
||||
.filter(({ score }) => score > 0)
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.map(({ row }) => row);
|
||||
}
|
||||
@@ -1,13 +1,10 @@
|
||||
import { searchByWords } from "./localSearch.js";
|
||||
import { mediaSemanticRanking, type MediaVectorRow } from "./localSemantic.js";
|
||||
import { rankMediaRows as rankMediaRowsRuntime } from "./mediaSearch.mjs";
|
||||
|
||||
export type MediaSearchRow = MediaVectorRow;
|
||||
|
||||
export function rankMediaRows(query: string, rows: MediaSearchRow[]): MediaSearchRow[] {
|
||||
return searchByWords(query, rows, (row) => ({
|
||||
strong: `${row.id} ${row.title}`,
|
||||
weak: `${row.description} ${row.tags.join(" ")} ${row.kind}`,
|
||||
}));
|
||||
return rankMediaRowsRuntime(query, rows) as MediaSearchRow[];
|
||||
}
|
||||
|
||||
export async function rankMediaRowsWithVectors(
|
||||
|
||||
@@ -17,6 +17,8 @@ export default defineConfig({
|
||||
runtimeVersion: "src/runtimeVersion.ts",
|
||||
renderSetupWorker: "src/renderSetupWorker.ts",
|
||||
shaderTransitionWorker: "../producer/src/services/shaderTransitionWorker.ts",
|
||||
"registry/mediaSearch": "src/registry/mediaSearch.ts",
|
||||
"registry/localSemantic": "src/registry/localSemantic.ts",
|
||||
},
|
||||
format: ["esm"],
|
||||
outDir: "dist",
|
||||
|
||||
@@ -30,9 +30,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "chime",
|
||||
"description": "Soft melodic chime — gentle positive beat: success/confirmation or a lighthearted transition. Sync to the visual moment.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/chime.mp3",
|
||||
"duration": 2.5
|
||||
},
|
||||
@@ -41,9 +39,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "click",
|
||||
"description": "Crisp UI click — button press, toggle, selection. Short accent, sync exactly to the on-screen action.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/click.mp3",
|
||||
"duration": 0.37
|
||||
},
|
||||
@@ -52,9 +48,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "click-soft",
|
||||
"description": "Quiet short click — low-key UI tap / soft selection. Short accent, sync exactly to the on-screen action.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/click-soft.mp3",
|
||||
"duration": 0.37
|
||||
},
|
||||
@@ -63,9 +57,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "error",
|
||||
"description": "Negative / error tone — failure state, a 'wrong' beat, or a glitchy interruption.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/error.mp3",
|
||||
"duration": 1.62
|
||||
},
|
||||
@@ -74,9 +66,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "glitch-1",
|
||||
"description": "Punchy digital glitch — hard-cut accent or sudden reveal. Trigger on the hit; let the decay bleed into the next shot (J-cut).",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/glitch-1.mp3",
|
||||
"duration": 2.64
|
||||
},
|
||||
@@ -85,9 +75,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "glitch-2",
|
||||
"description": "Harsh, longer glitch — chaotic / jarring transition or a distorted reveal.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/glitch-2.mp3",
|
||||
"duration": 3.5
|
||||
},
|
||||
@@ -96,9 +84,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "glitch-3",
|
||||
"description": "Low-key glitch texture — subtle digital shift, minimal transition that sits under other audio.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/glitch-3.mp3",
|
||||
"duration": 3.1
|
||||
},
|
||||
@@ -107,9 +93,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "impact-bass-1",
|
||||
"description": "Bass impact hit — logo/hero snap, headline slam. Trigger on the visual landing; decay carries into the next shot (J-cut).",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/impact-bass-1.mp3",
|
||||
"duration": 2.12
|
||||
},
|
||||
@@ -118,9 +102,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "impact-bass-2",
|
||||
"description": "Bass impact with a short swell — brief anticipation then a deep hit. Place so the peak lands on the reveal.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/impact-bass-2.mp3",
|
||||
"duration": 2.59
|
||||
},
|
||||
@@ -129,9 +111,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "key-press",
|
||||
"description": "Single key press — one keystroke / terminal-input beat. Short accent, sync to the typed character.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/key-press.mp3",
|
||||
"duration": 0.4
|
||||
},
|
||||
@@ -140,9 +120,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "notification",
|
||||
"description": "Notification chime — alert, message-in, toast/badge appears. Sync to the element entering.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/notification.mp3",
|
||||
"duration": 2.46
|
||||
},
|
||||
@@ -151,9 +129,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "ping",
|
||||
"description": "Sharp electronic ping — punchy accent on a key reveal or data point. Sync to the beat.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/ping.mp3",
|
||||
"duration": 1.32
|
||||
},
|
||||
@@ -162,9 +138,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "pop",
|
||||
"description": "Quick pop — element appear/spawn, chip/tag/badge in. Small precise accent, sync to the pop-in.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/pop.mp3",
|
||||
"duration": 0.72
|
||||
},
|
||||
@@ -173,9 +147,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "riser",
|
||||
"description": "Long cinematic riser (~10s build, peak at the end). Trigger at (climax_time − 10.03s) so it crests exactly on the reveal.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/riser.mp3",
|
||||
"duration": 10.03
|
||||
},
|
||||
@@ -184,9 +156,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "sparkle",
|
||||
"description": "Bright sparkle / shimmer — magical reveal or 'shine' highlight on a hero element. Sync to the highlight.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/sparkle.mp3",
|
||||
"duration": 1.8
|
||||
},
|
||||
@@ -195,9 +165,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "typing",
|
||||
"description": "Typing burst (~1.5s of keys) — keyboard / code typing reveal, text-being-typed beat. Start as the text begins typing.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/typing.mp3",
|
||||
"duration": 1.5
|
||||
},
|
||||
@@ -206,9 +174,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "whoosh",
|
||||
"description": "Punchy whoosh/impact — fast reveal or hard transition accent. Sync to the motion.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/whoosh.mp3",
|
||||
"duration": 0.57
|
||||
},
|
||||
@@ -217,9 +183,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "whoosh-cinematic",
|
||||
"description": "Cinematic whoosh build (~5.5s) — sweeping scene transition. Align so the swell peaks on the cut.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/whoosh-cinematic.mp3",
|
||||
"duration": 5.54
|
||||
},
|
||||
@@ -228,9 +192,7 @@
|
||||
"kind": "sfx",
|
||||
"title": "whoosh-short",
|
||||
"description": "Short whoosh — quick swipe/slide accent, fast element move, snappy transition. Sync to the motion.",
|
||||
"tags": [
|
||||
"sfx"
|
||||
],
|
||||
"tags": ["sfx"],
|
||||
"file": "skills/media-use/audio/assets/sfx/whoosh-short.mp3",
|
||||
"duration": 0.57
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user