diff --git a/packages/cli/scripts/build-copy.mjs b/packages/cli/scripts/build-copy.mjs index 0709e05713..cc11e373bb 100644 --- a/packages/cli/scripts/build-copy.mjs +++ b/packages/cli/scripts/build-copy.mjs @@ -155,6 +155,12 @@ async function main() { join(mediaEngine, "resolve.mjs"), join(DIST, "skills", "media-use", "scripts", "resolve.mjs"), ); + // The SFX media index ships with the release whose bundled SFX files it names. + mkdirSync(join(DIST, "catalog-artifact"), { recursive: true }); + cpSync( + join(REPO_ROOT, "registry", "catalog-artifact", "media-vectors.json"), + join(DIST, "catalog-artifact", "media-vectors.json"), + ); const dockerfile = join(CLI_ROOT, "src", "docker", "Dockerfile.render"); if (existsSync(dockerfile)) { diff --git a/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs b/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs index f30edce9e3..47d673771e 100644 --- a/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs +++ b/packages/cli/src/media-use/lib/bundled-sfx-provider.mjs @@ -1,5 +1,5 @@ -import { existsSync, readFileSync } from "node:fs"; -import { extname, join } from "node:path"; +import { existsSync, readFileSync, realpathSync, statSync } from "node:fs"; +import { extname, isAbsolute, join, relative } from "node:path"; import { rankMediaRows } from "./media-search.mjs"; const LIB_DIR = @@ -80,6 +80,31 @@ export function extensionForBundledSfxFile(filename) { return extname(filename) || ".mp3"; } +const INDEX_FILE_PREFIX = "skills/media-use/audio/assets/sfx/"; +const AUDIO_FILE_NAME = + /^[A-Za-z0-9][A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*\.(?:mp3|wav|ogg|m4a|aac|flac)$/i; + +/** + * Map a media index row's `file` to an audio file inside the bundled SFX folder, or null. + * The only accepted form is `skills/media-use/audio/assets/sfx/` with a plain audio + * file name; the result must still be a regular file inside the folder after realpath. + */ +export function bundledSfxAssetPath(file, libraryDir = LIB_DIR) { + if (typeof file !== "string" || !file.startsWith(INDEX_FILE_PREFIX)) return null; + const name = file.slice(INDEX_FILE_PREFIX.length); + if (!AUDIO_FILE_NAME.test(name)) return null; + try { + const root = realpathSync(libraryDir); + const asset = realpathSync(join(root, name)); + const inside = relative(root, asset); + if (!inside || inside.startsWith("..") || isAbsolute(inside)) return null; + if (!AUDIO_FILE_NAME.test(inside) || !statSync(asset).isFile()) return null; + return asset; + } catch { + return null; + } +} + export const bundledSfxProvider = { async search(intent, ctx = {}) { const libraryDir = ctx.libraryDir || LIB_DIR; diff --git a/packages/cli/src/media-use/lib/bundled-sfx-provider.test.mjs b/packages/cli/src/media-use/lib/bundled-sfx-provider.test.mjs index 7609e57549..9eb619c136 100644 --- a/packages/cli/src/media-use/lib/bundled-sfx-provider.test.mjs +++ b/packages/cli/src/media-use/lib/bundled-sfx-provider.test.mjs @@ -1,11 +1,12 @@ import { strict as assert } from "node:assert"; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; import { BUNDLED_SFX_RECOVERY_COMMAND, BundledSfxAssetsError, + bundledSfxAssetPath, bundledSfxProvider, extensionForBundledSfxFile, inspectBundledSfxAssets, @@ -137,3 +138,41 @@ test("literal ids outrank distinct ids with the same stem", () => { ["cats", "cat"], ); }); + +test("media index files map only to regular audio files inside the bundled SFX folder", (t) => { + const root = mkdtempSync(join(tmpdir(), "media-use-sfx-paths-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const libraryDir = join(root, "sfx"); + mkdirSync(join(libraryDir, "folder.mp3"), { recursive: true }); + writeFileSync(join(libraryDir, "chime.mp3"), "chime"); + writeFileSync(join(libraryDir, "notes.txt"), "notes"); + writeFileSync(join(root, "outside.mp3"), "outside"); + symlinkSync(join(root, "outside.mp3"), join(libraryDir, "escape.mp3")); + const prefix = "skills/media-use/audio/assets/sfx/"; + + assert.equal( + bundledSfxAssetPath(`${prefix}chime.mp3`, libraryDir), + realpathSync(join(libraryDir, "chime.mp3")), + ); + for (const file of [ + "/etc/hosts", + "../../.ssh/id_ed25519", + "chime.mp3", + "./chime.mp3", + "skills/media-use/audio/assets/sfx", + `${prefix}../../../../.ssh/id_ed25519`, + `${prefix}../outside.mp3`, + `${prefix}/chime.mp3`, + `${prefix}.\\chime.mp3`, + `${prefix}sub/chime.mp3`, + `${prefix}notes.txt`, + `${prefix}missing.mp3`, + `${prefix}folder.mp3`, + `${prefix}escape.mp3`, + `${prefix}${join(libraryDir, "chime.mp3")}`, + "skills\\media-use\\audio\\assets\\sfx\\chime.mp3", + 7, + ]) { + assert.equal(bundledSfxAssetPath(file, libraryDir), null, String(file)); + } +}); diff --git a/packages/cli/src/media-use/lib/local-media-search.mjs b/packages/cli/src/media-use/lib/local-media-search.mjs index a910ee00c3..cd097e0c53 100644 --- a/packages/cli/src/media-use/lib/local-media-search.mjs +++ b/packages/cli/src/media-use/lib/local-media-search.mjs @@ -1,36 +1,65 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { homedir } from "node:os"; -import { join, resolve } from "node:path"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; -const DEFAULT_DIRECTORY = join(homedir(), ".hyperframes", "catalog"); +const MEDIA_VECTOR_DIMENSIONS = 384; -function directory() { - const cache = process.env.HYPERFRAMES_CACHE_DIR; +// The index ships inside the CLI (dist/catalog-artifact, or registry/catalog-artifact in a +// source checkout), so it always describes the SFX files of this same release. +const BUNDLED_MEDIA_VECTORS = [ + join(import.meta.dirname, "..", "..", "..", "..", "catalog-artifact", "media-vectors.json"), + join( + import.meta.dirname, + "..", + "..", + "..", + "..", + "..", + "registry", + "catalog-artifact", + "media-vectors.json", + ), +].find((candidate) => existsSync(candidate)); + +function isMediaVectorRow(row) { return ( - process.env.HYPERFRAMES_CATALOG_ARTIFACT_DIR || - (cache ? join(resolve(cache), "catalog") : DEFAULT_DIRECTORY) + !!row && + typeof row === "object" && + typeof row.id === "string" && + typeof row.kind === "string" && + typeof row.title === "string" && + typeof row.description === "string" && + Array.isArray(row.tags) && + row.tags.every((tag) => typeof tag === "string") && + typeof row.file === "string" && + (row.duration === undefined || + (typeof row.duration === "number" && Number.isFinite(row.duration) && row.duration >= 0)) ); } -export async function fetchMediaVectors(registryBaseUrl, options = {}) { - const target = options.directory || directory(); - const base = registryBaseUrl.replace(/\/+$/, ""); - mkdirSync(target, { recursive: true, mode: 0o700 }); - const fetched = []; - for (const file of ["media-vectors.json", "media-vectors.bin"]) { - const response = await fetch(`${base}/catalog-artifact/${file}`); - if (!response.ok) return false; - fetched.push([file, Buffer.from(await response.arrayBuffer())]); +function indexProblem(metadata) { + if (!metadata || typeof metadata !== "object") return "not an object"; + if (metadata.dimensions !== MEDIA_VECTOR_DIMENSIONS) { + return `expected ${MEDIA_VECTOR_DIMENSIONS} dimensions, got ${metadata.dimensions}`; } - for (const [file, bytes] of fetched) writeFileSync(join(target, file), bytes, { mode: 0o600 }); - return true; + const { names, rows } = metadata; + if (!Array.isArray(names) || !Array.isArray(rows) || names.length !== rows.length) { + return "names and rows do not line up"; + } + if (rows.some((row, index) => !isMediaVectorRow(row) || row.id !== names[index])) { + return "a row is malformed or out of order"; + } + const revision = createHash("sha256").update(JSON.stringify(rows)).digest("hex"); + if (metadata.metadataRevision !== revision) return "rows do not match metadataRevision"; + return null; } -export function mediaVectorRows(target = directory()) { - const metadataPath = join(target, "media-vectors.json"); - if (!existsSync(metadataPath)) return []; +export function mediaVectorRows(metadataPath = BUNDLED_MEDIA_VECTORS) { + if (!metadataPath || !existsSync(metadataPath)) return []; const metadata = JSON.parse(readFileSync(metadataPath, "utf8")); - return Array.isArray(metadata.rows) ? metadata.rows : []; + const problem = indexProblem(metadata); + if (problem) throw new Error(`media index ${metadataPath} is invalid: ${problem}`); + return metadata.rows; } export { rankMediaRowsWithVectors } from "./media-search.mjs"; diff --git a/packages/cli/src/media-use/lib/local-media-search.test.mjs b/packages/cli/src/media-use/lib/local-media-search.test.mjs index c8b68e954b..f82dcd5c1f 100644 --- a/packages/cli/src/media-use/lib/local-media-search.test.mjs +++ b/packages/cli/src/media-use/lib/local-media-search.test.mjs @@ -1,11 +1,49 @@ import { strict as assert } from "node:assert"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; +import { bundledSfxAssetPath } from "./bundled-sfx-provider.mjs"; import { mediaVectorRows } from "./local-media-search.mjs"; -test("media vectors are read from the cache folder's catalog when HYPERFRAMES_CACHE_DIR is set", (t) => { +const SHIPPED_INDEX = join( + import.meta.dirname, + "..", + "..", + "..", + "..", + "..", + "registry", + "catalog-artifact", + "media-vectors.json", +); + +function shippedIndex() { + return JSON.parse(readFileSync(SHIPPED_INDEX, "utf8")); +} + +function writeIndex(t, metadata) { + const dir = mkdtempSync(join(tmpdir(), "hf-media-index-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const path = join(dir, "media-vectors.json"); + writeFileSync(path, JSON.stringify(metadata)); + return path; +} + +function withRevision(metadata) { + const metadataRevision = createHash("sha256").update(JSON.stringify(metadata.rows)).digest("hex"); + return { ...metadata, metadataRevision }; +} + +test("the bundled media index validates and every row names a bundled SFX file", () => { + const rows = mediaVectorRows(); + assert.equal(rows.length, shippedIndex().rows.length); + assert.ok(rows.length > 0); + for (const row of rows) assert.ok(bundledSfxAssetPath(row.file), row.file); +}); + +test("a media index in the user's cache folder is never read", (t) => { const cache = mkdtempSync(join(tmpdir(), "hf-cache-")); const prior = process.env.HYPERFRAMES_CACHE_DIR; process.env.HYPERFRAMES_CACHE_DIR = cache; @@ -17,7 +55,38 @@ test("media vectors are read from the cache folder's catalog when HYPERFRAMES_CA mkdirSync(join(cache, "catalog")); writeFileSync( join(cache, "catalog", "media-vectors.json"), - JSON.stringify({ rows: [{ id: "a" }] }), + JSON.stringify({ + rows: [{ id: "hosts", kind: "sfx", file: "/etc/hosts" }], + }), ); - assert.deepEqual(mediaVectorRows(), [{ id: "a" }]); + assert.deepEqual(mediaVectorRows(), shippedIndex().rows); +}); + +test("an index whose rows do not match its metadata revision is refused", (t) => { + const metadata = shippedIndex(); + metadata.rows[0] = { ...metadata.rows[0], file: "/etc/hosts" }; + assert.throws(() => mediaVectorRows(writeIndex(t, metadata)), /metadataRevision/); +}); + +test("an index with the wrong dimensions or malformed rows is refused", (t) => { + const base = shippedIndex(); + assert.throws( + () => mediaVectorRows(writeIndex(t, { ...base, dimensions: 3072 })), + /384 dimensions/, + ); + const badRow = withRevision({ + ...base, + rows: [{ ...base.rows[0], file: 7 }, ...base.rows.slice(1)], + }); + assert.throws(() => mediaVectorRows(writeIndex(t, badRow)), /malformed or out of order/); + const reordered = withRevision({ ...base, rows: [...base.rows].reverse() }); + assert.throws(() => mediaVectorRows(writeIndex(t, reordered)), /malformed or out of order/); + assert.throws( + () => mediaVectorRows(writeIndex(t, { ...base, names: base.names.slice(1) })), + /do not line up/, + ); +}); + +test("a missing media index has no rows", () => { + assert.deepEqual(mediaVectorRows(join(tmpdir(), "hf-no-such-index", "media-vectors.json")), []); }); diff --git a/packages/cli/src/media-use/resolve.mjs b/packages/cli/src/media-use/resolve.mjs index ed8e1d0e40..3cbe0f1675 100644 --- a/packages/cli/src/media-use/resolve.mjs +++ b/packages/cli/src/media-use/resolve.mjs @@ -59,6 +59,7 @@ import { import { BUNDLED_SFX_RECOVERY_COMMAND, BundledSfxAssetsError, + bundledSfxAssetPath, inspectBundledSfxAssets, } from "./lib/bundled-sfx-provider.mjs"; import { @@ -68,11 +69,7 @@ import { scoreMatch, } from "./lib/sfx-provider.mjs"; import { writeSfxCredit } from "./lib/openverse.mjs"; -import { - fetchMediaVectors, - mediaVectorRows, - rankMediaRowsWithVectors, -} from "./lib/local-media-search.mjs"; +import { mediaVectorRows, rankMediaRowsWithVectors } from "./lib/local-media-search.mjs"; const INGEST_TYPES = listTypes(); const DEFAULT_EXT = { @@ -468,21 +465,14 @@ async function run() { try { searchResult = await runCapability(type, "search", intent, ctx); if (!searchResult && type === "sfx" && !args.provider && !localOnly) { - const registry = - process.env.HYPERFRAMES_REGISTRY || - "https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry"; try { - await fetchMediaVectors(registry); const ranked = await rankMediaRowsWithVectors( intent, mediaVectorRows().filter((row) => row.kind === "sfx"), ); const row = ranked.rows[0]; - const candidates = row - ? [resolve(row.file), join(import.meta.dirname, "..", "..", "..", row.file)] - : []; - const localPath = candidates.find((candidate) => existsSync(candidate)); - if (row && localPath) { + const localPath = row ? bundledSfxAssetPath(row.file) : null; + if (localPath) { searchResult = { localPath, ext: extname(localPath), diff --git a/packages/cli/src/media-use/resolve.test.mjs b/packages/cli/src/media-use/resolve.test.mjs index f1427aa4e6..09a0ebb107 100644 --- a/packages/cli/src/media-use/resolve.test.mjs +++ b/packages/cli/src/media-use/resolve.test.mjs @@ -374,10 +374,38 @@ test("music without the HeyGen CLI fails naming it and the host app's own tool", test("a sound effect nothing matches is a plain miss that never names the HeyGen CLI", async () => { setup(); - // The local media index answers 404, so the miss never depends on the network. + // A registry and a cached index that both name files outside the bundled SFX + // folder: the resolve reads neither, copies nothing and reports the miss. + const tampered = JSON.stringify({ + dimensions: 384, + names: ["dog-barking", "dog-barking-2"], + rows: [ + { + id: "dog-barking", + kind: "sfx", + title: "dog barking", + description: "dog barking", + tags: ["sfx"], + file: "/etc/hosts", + }, + { + id: "dog-barking-2", + kind: "sfx", + title: "dog barking", + description: "dog barking", + tags: ["sfx"], + file: "../../.ssh/id_ed25519", + }, + ], + }); + const cacheDir = join(tmp, "cache"); + mkdirSync(join(cacheDir, "catalog"), { recursive: true }); + writeFileSync(join(cacheDir, "catalog", "media-vectors.json"), tampered); + let registryRequests = 0; const server = createServer((_req, res) => { - res.writeHead(404); - res.end(); + registryRequests++; + res.writeHead(200, { "content-type": "application/json" }); + res.end(tampered); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { @@ -387,6 +415,7 @@ test("a sound effect nothing matches is a plain miss that never names the HeyGen env: { HOME: tmp, PATH: tmp, + HYPERFRAMES_CACHE_DIR: cacheDir, HYPERFRAMES_REGISTRY: `http://127.0.0.1:${server.address().port}`, }, nodeArgs: openverseImport([]), @@ -399,6 +428,8 @@ test("a sound effect nothing matches is a plain miss that never names the HeyGen parsed.error, /^no sound effect matched "dog barking": HeyGen sound search \(min score 0\.4\)/, ); + assert.equal(registryRequests, 0); + assert.equal(existsSync(join(tmp, ".media/audio/sfx")), false); } finally { server.close(); cleanup();