mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): media-use copies only bundled SFX named by the bundled index (#7895)
* wip: paused before review * chore: keep only the meaningful hunks * chore(cli): drop formatting-only changes from the media index fix
This commit is contained in:
@@ -155,6 +155,12 @@ async function main() {
|
||||
join(mediaEngine, "resolve.mjs"),
|
||||
join(DIST, "skills", "media-use", "scripts", "resolve.mjs"),
|
||||
);
|
||||
// The SFX media index ships with the release whose bundled SFX files it names.
|
||||
mkdirSync(join(DIST, "catalog-artifact"), { recursive: true });
|
||||
cpSync(
|
||||
join(REPO_ROOT, "registry", "catalog-artifact", "media-vectors.json"),
|
||||
join(DIST, "catalog-artifact", "media-vectors.json"),
|
||||
);
|
||||
|
||||
const dockerfile = join(CLI_ROOT, "src", "docker", "Dockerfile.render");
|
||||
if (existsSync(dockerfile)) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { extname, join } from "node:path";
|
||||
import { existsSync, readFileSync, realpathSync, statSync } from "node:fs";
|
||||
import { extname, isAbsolute, join, relative } from "node:path";
|
||||
import { rankMediaRows } from "./media-search.mjs";
|
||||
|
||||
const LIB_DIR =
|
||||
@@ -80,6 +80,31 @@ export function extensionForBundledSfxFile(filename) {
|
||||
return extname(filename) || ".mp3";
|
||||
}
|
||||
|
||||
const INDEX_FILE_PREFIX = "skills/media-use/audio/assets/sfx/";
|
||||
const AUDIO_FILE_NAME =
|
||||
/^[A-Za-z0-9][A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*\.(?:mp3|wav|ogg|m4a|aac|flac)$/i;
|
||||
|
||||
/**
|
||||
* Map a media index row's `file` to an audio file inside the bundled SFX folder, or null.
|
||||
* The only accepted form is `skills/media-use/audio/assets/sfx/<name>` with a plain audio
|
||||
* file name; the result must still be a regular file inside the folder after realpath.
|
||||
*/
|
||||
export function bundledSfxAssetPath(file, libraryDir = LIB_DIR) {
|
||||
if (typeof file !== "string" || !file.startsWith(INDEX_FILE_PREFIX)) return null;
|
||||
const name = file.slice(INDEX_FILE_PREFIX.length);
|
||||
if (!AUDIO_FILE_NAME.test(name)) return null;
|
||||
try {
|
||||
const root = realpathSync(libraryDir);
|
||||
const asset = realpathSync(join(root, name));
|
||||
const inside = relative(root, asset);
|
||||
if (!inside || inside.startsWith("..") || isAbsolute(inside)) return null;
|
||||
if (!AUDIO_FILE_NAME.test(inside) || !statSync(asset).isFile()) return null;
|
||||
return asset;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const bundledSfxProvider = {
|
||||
async search(intent, ctx = {}) {
|
||||
const libraryDir = ctx.libraryDir || LIB_DIR;
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
BUNDLED_SFX_RECOVERY_COMMAND,
|
||||
BundledSfxAssetsError,
|
||||
bundledSfxAssetPath,
|
||||
bundledSfxProvider,
|
||||
extensionForBundledSfxFile,
|
||||
inspectBundledSfxAssets,
|
||||
@@ -137,3 +138,41 @@ test("literal ids outrank distinct ids with the same stem", () => {
|
||||
["cats", "cat"],
|
||||
);
|
||||
});
|
||||
|
||||
test("media index files map only to regular audio files inside the bundled SFX folder", (t) => {
|
||||
const root = mkdtempSync(join(tmpdir(), "media-use-sfx-paths-"));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const libraryDir = join(root, "sfx");
|
||||
mkdirSync(join(libraryDir, "folder.mp3"), { recursive: true });
|
||||
writeFileSync(join(libraryDir, "chime.mp3"), "chime");
|
||||
writeFileSync(join(libraryDir, "notes.txt"), "notes");
|
||||
writeFileSync(join(root, "outside.mp3"), "outside");
|
||||
symlinkSync(join(root, "outside.mp3"), join(libraryDir, "escape.mp3"));
|
||||
const prefix = "skills/media-use/audio/assets/sfx/";
|
||||
|
||||
assert.equal(
|
||||
bundledSfxAssetPath(`${prefix}chime.mp3`, libraryDir),
|
||||
realpathSync(join(libraryDir, "chime.mp3")),
|
||||
);
|
||||
for (const file of [
|
||||
"/etc/hosts",
|
||||
"../../.ssh/id_ed25519",
|
||||
"chime.mp3",
|
||||
"./chime.mp3",
|
||||
"skills/media-use/audio/assets/sfx",
|
||||
`${prefix}../../../../.ssh/id_ed25519`,
|
||||
`${prefix}../outside.mp3`,
|
||||
`${prefix}/chime.mp3`,
|
||||
`${prefix}.\\chime.mp3`,
|
||||
`${prefix}sub/chime.mp3`,
|
||||
`${prefix}notes.txt`,
|
||||
`${prefix}missing.mp3`,
|
||||
`${prefix}folder.mp3`,
|
||||
`${prefix}escape.mp3`,
|
||||
`${prefix}${join(libraryDir, "chime.mp3")}`,
|
||||
"skills\\media-use\\audio\\assets\\sfx\\chime.mp3",
|
||||
7,
|
||||
]) {
|
||||
assert.equal(bundledSfxAssetPath(file, libraryDir), null, String(file));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,36 +1,65 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const DEFAULT_DIRECTORY = join(homedir(), ".hyperframes", "catalog");
|
||||
const MEDIA_VECTOR_DIMENSIONS = 384;
|
||||
|
||||
function directory() {
|
||||
const cache = process.env.HYPERFRAMES_CACHE_DIR;
|
||||
// The index ships inside the CLI (dist/catalog-artifact, or registry/catalog-artifact in a
|
||||
// source checkout), so it always describes the SFX files of this same release.
|
||||
const BUNDLED_MEDIA_VECTORS = [
|
||||
join(import.meta.dirname, "..", "..", "..", "..", "catalog-artifact", "media-vectors.json"),
|
||||
join(
|
||||
import.meta.dirname,
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"registry",
|
||||
"catalog-artifact",
|
||||
"media-vectors.json",
|
||||
),
|
||||
].find((candidate) => existsSync(candidate));
|
||||
|
||||
function isMediaVectorRow(row) {
|
||||
return (
|
||||
process.env.HYPERFRAMES_CATALOG_ARTIFACT_DIR ||
|
||||
(cache ? join(resolve(cache), "catalog") : DEFAULT_DIRECTORY)
|
||||
!!row &&
|
||||
typeof row === "object" &&
|
||||
typeof row.id === "string" &&
|
||||
typeof row.kind === "string" &&
|
||||
typeof row.title === "string" &&
|
||||
typeof row.description === "string" &&
|
||||
Array.isArray(row.tags) &&
|
||||
row.tags.every((tag) => typeof tag === "string") &&
|
||||
typeof row.file === "string" &&
|
||||
(row.duration === undefined ||
|
||||
(typeof row.duration === "number" && Number.isFinite(row.duration) && row.duration >= 0))
|
||||
);
|
||||
}
|
||||
|
||||
export async function fetchMediaVectors(registryBaseUrl, options = {}) {
|
||||
const target = options.directory || directory();
|
||||
const base = registryBaseUrl.replace(/\/+$/, "");
|
||||
mkdirSync(target, { recursive: true, mode: 0o700 });
|
||||
const fetched = [];
|
||||
for (const file of ["media-vectors.json", "media-vectors.bin"]) {
|
||||
const response = await fetch(`${base}/catalog-artifact/${file}`);
|
||||
if (!response.ok) return false;
|
||||
fetched.push([file, Buffer.from(await response.arrayBuffer())]);
|
||||
function indexProblem(metadata) {
|
||||
if (!metadata || typeof metadata !== "object") return "not an object";
|
||||
if (metadata.dimensions !== MEDIA_VECTOR_DIMENSIONS) {
|
||||
return `expected ${MEDIA_VECTOR_DIMENSIONS} dimensions, got ${metadata.dimensions}`;
|
||||
}
|
||||
for (const [file, bytes] of fetched) writeFileSync(join(target, file), bytes, { mode: 0o600 });
|
||||
return true;
|
||||
const { names, rows } = metadata;
|
||||
if (!Array.isArray(names) || !Array.isArray(rows) || names.length !== rows.length) {
|
||||
return "names and rows do not line up";
|
||||
}
|
||||
if (rows.some((row, index) => !isMediaVectorRow(row) || row.id !== names[index])) {
|
||||
return "a row is malformed or out of order";
|
||||
}
|
||||
const revision = createHash("sha256").update(JSON.stringify(rows)).digest("hex");
|
||||
if (metadata.metadataRevision !== revision) return "rows do not match metadataRevision";
|
||||
return null;
|
||||
}
|
||||
|
||||
export function mediaVectorRows(target = directory()) {
|
||||
const metadataPath = join(target, "media-vectors.json");
|
||||
if (!existsSync(metadataPath)) return [];
|
||||
export function mediaVectorRows(metadataPath = BUNDLED_MEDIA_VECTORS) {
|
||||
if (!metadataPath || !existsSync(metadataPath)) return [];
|
||||
const metadata = JSON.parse(readFileSync(metadataPath, "utf8"));
|
||||
return Array.isArray(metadata.rows) ? metadata.rows : [];
|
||||
const problem = indexProblem(metadata);
|
||||
if (problem) throw new Error(`media index ${metadataPath} is invalid: ${problem}`);
|
||||
return metadata.rows;
|
||||
}
|
||||
|
||||
export { rankMediaRowsWithVectors } from "./media-search.mjs";
|
||||
|
||||
@@ -1,11 +1,49 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { bundledSfxAssetPath } from "./bundled-sfx-provider.mjs";
|
||||
import { mediaVectorRows } from "./local-media-search.mjs";
|
||||
|
||||
test("media vectors are read from the cache folder's catalog when HYPERFRAMES_CACHE_DIR is set", (t) => {
|
||||
const SHIPPED_INDEX = join(
|
||||
import.meta.dirname,
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"registry",
|
||||
"catalog-artifact",
|
||||
"media-vectors.json",
|
||||
);
|
||||
|
||||
function shippedIndex() {
|
||||
return JSON.parse(readFileSync(SHIPPED_INDEX, "utf8"));
|
||||
}
|
||||
|
||||
function writeIndex(t, metadata) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-media-index-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
const path = join(dir, "media-vectors.json");
|
||||
writeFileSync(path, JSON.stringify(metadata));
|
||||
return path;
|
||||
}
|
||||
|
||||
function withRevision(metadata) {
|
||||
const metadataRevision = createHash("sha256").update(JSON.stringify(metadata.rows)).digest("hex");
|
||||
return { ...metadata, metadataRevision };
|
||||
}
|
||||
|
||||
test("the bundled media index validates and every row names a bundled SFX file", () => {
|
||||
const rows = mediaVectorRows();
|
||||
assert.equal(rows.length, shippedIndex().rows.length);
|
||||
assert.ok(rows.length > 0);
|
||||
for (const row of rows) assert.ok(bundledSfxAssetPath(row.file), row.file);
|
||||
});
|
||||
|
||||
test("a media index in the user's cache folder is never read", (t) => {
|
||||
const cache = mkdtempSync(join(tmpdir(), "hf-cache-"));
|
||||
const prior = process.env.HYPERFRAMES_CACHE_DIR;
|
||||
process.env.HYPERFRAMES_CACHE_DIR = cache;
|
||||
@@ -17,7 +55,38 @@ test("media vectors are read from the cache folder's catalog when HYPERFRAMES_CA
|
||||
mkdirSync(join(cache, "catalog"));
|
||||
writeFileSync(
|
||||
join(cache, "catalog", "media-vectors.json"),
|
||||
JSON.stringify({ rows: [{ id: "a" }] }),
|
||||
JSON.stringify({
|
||||
rows: [{ id: "hosts", kind: "sfx", file: "/etc/hosts" }],
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(mediaVectorRows(), [{ id: "a" }]);
|
||||
assert.deepEqual(mediaVectorRows(), shippedIndex().rows);
|
||||
});
|
||||
|
||||
test("an index whose rows do not match its metadata revision is refused", (t) => {
|
||||
const metadata = shippedIndex();
|
||||
metadata.rows[0] = { ...metadata.rows[0], file: "/etc/hosts" };
|
||||
assert.throws(() => mediaVectorRows(writeIndex(t, metadata)), /metadataRevision/);
|
||||
});
|
||||
|
||||
test("an index with the wrong dimensions or malformed rows is refused", (t) => {
|
||||
const base = shippedIndex();
|
||||
assert.throws(
|
||||
() => mediaVectorRows(writeIndex(t, { ...base, dimensions: 3072 })),
|
||||
/384 dimensions/,
|
||||
);
|
||||
const badRow = withRevision({
|
||||
...base,
|
||||
rows: [{ ...base.rows[0], file: 7 }, ...base.rows.slice(1)],
|
||||
});
|
||||
assert.throws(() => mediaVectorRows(writeIndex(t, badRow)), /malformed or out of order/);
|
||||
const reordered = withRevision({ ...base, rows: [...base.rows].reverse() });
|
||||
assert.throws(() => mediaVectorRows(writeIndex(t, reordered)), /malformed or out of order/);
|
||||
assert.throws(
|
||||
() => mediaVectorRows(writeIndex(t, { ...base, names: base.names.slice(1) })),
|
||||
/do not line up/,
|
||||
);
|
||||
});
|
||||
|
||||
test("a missing media index has no rows", () => {
|
||||
assert.deepEqual(mediaVectorRows(join(tmpdir(), "hf-no-such-index", "media-vectors.json")), []);
|
||||
});
|
||||
|
||||
@@ -59,6 +59,7 @@ import {
|
||||
import {
|
||||
BUNDLED_SFX_RECOVERY_COMMAND,
|
||||
BundledSfxAssetsError,
|
||||
bundledSfxAssetPath,
|
||||
inspectBundledSfxAssets,
|
||||
} from "./lib/bundled-sfx-provider.mjs";
|
||||
import {
|
||||
@@ -68,11 +69,7 @@ import {
|
||||
scoreMatch,
|
||||
} from "./lib/sfx-provider.mjs";
|
||||
import { writeSfxCredit } from "./lib/openverse.mjs";
|
||||
import {
|
||||
fetchMediaVectors,
|
||||
mediaVectorRows,
|
||||
rankMediaRowsWithVectors,
|
||||
} from "./lib/local-media-search.mjs";
|
||||
import { mediaVectorRows, rankMediaRowsWithVectors } from "./lib/local-media-search.mjs";
|
||||
|
||||
const INGEST_TYPES = listTypes();
|
||||
const DEFAULT_EXT = {
|
||||
@@ -468,21 +465,14 @@ async function run() {
|
||||
try {
|
||||
searchResult = await runCapability(type, "search", intent, ctx);
|
||||
if (!searchResult && type === "sfx" && !args.provider && !localOnly) {
|
||||
const registry =
|
||||
process.env.HYPERFRAMES_REGISTRY ||
|
||||
"https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry";
|
||||
try {
|
||||
await fetchMediaVectors(registry);
|
||||
const ranked = await rankMediaRowsWithVectors(
|
||||
intent,
|
||||
mediaVectorRows().filter((row) => row.kind === "sfx"),
|
||||
);
|
||||
const row = ranked.rows[0];
|
||||
const candidates = row
|
||||
? [resolve(row.file), join(import.meta.dirname, "..", "..", "..", row.file)]
|
||||
: [];
|
||||
const localPath = candidates.find((candidate) => existsSync(candidate));
|
||||
if (row && localPath) {
|
||||
const localPath = row ? bundledSfxAssetPath(row.file) : null;
|
||||
if (localPath) {
|
||||
searchResult = {
|
||||
localPath,
|
||||
ext: extname(localPath),
|
||||
|
||||
@@ -374,10 +374,38 @@ test("music without the HeyGen CLI fails naming it and the host app's own tool",
|
||||
|
||||
test("a sound effect nothing matches is a plain miss that never names the HeyGen CLI", async () => {
|
||||
setup();
|
||||
// The local media index answers 404, so the miss never depends on the network.
|
||||
// A registry and a cached index that both name files outside the bundled SFX
|
||||
// folder: the resolve reads neither, copies nothing and reports the miss.
|
||||
const tampered = JSON.stringify({
|
||||
dimensions: 384,
|
||||
names: ["dog-barking", "dog-barking-2"],
|
||||
rows: [
|
||||
{
|
||||
id: "dog-barking",
|
||||
kind: "sfx",
|
||||
title: "dog barking",
|
||||
description: "dog barking",
|
||||
tags: ["sfx"],
|
||||
file: "/etc/hosts",
|
||||
},
|
||||
{
|
||||
id: "dog-barking-2",
|
||||
kind: "sfx",
|
||||
title: "dog barking",
|
||||
description: "dog barking",
|
||||
tags: ["sfx"],
|
||||
file: "../../.ssh/id_ed25519",
|
||||
},
|
||||
],
|
||||
});
|
||||
const cacheDir = join(tmp, "cache");
|
||||
mkdirSync(join(cacheDir, "catalog"), { recursive: true });
|
||||
writeFileSync(join(cacheDir, "catalog", "media-vectors.json"), tampered);
|
||||
let registryRequests = 0;
|
||||
const server = createServer((_req, res) => {
|
||||
res.writeHead(404);
|
||||
res.end();
|
||||
registryRequests++;
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(tampered);
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
@@ -387,6 +415,7 @@ test("a sound effect nothing matches is a plain miss that never names the HeyGen
|
||||
env: {
|
||||
HOME: tmp,
|
||||
PATH: tmp,
|
||||
HYPERFRAMES_CACHE_DIR: cacheDir,
|
||||
HYPERFRAMES_REGISTRY: `http://127.0.0.1:${server.address().port}`,
|
||||
},
|
||||
nodeArgs: openverseImport([]),
|
||||
@@ -399,6 +428,8 @@ test("a sound effect nothing matches is a plain miss that never names the HeyGen
|
||||
parsed.error,
|
||||
/^no sound effect matched "dog barking": HeyGen sound search \(min score 0\.4\)/,
|
||||
);
|
||||
assert.equal(registryRequests, 0);
|
||||
assert.equal(existsSync(join(tmp, ".media/audio/sfx")), false);
|
||||
} finally {
|
||||
server.close();
|
||||
cleanup();
|
||||
|
||||
Reference in New Issue
Block a user