mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
feat(media-use): source sound effects from Openverse with the bundled library offline (#5753)
* feat(media-use): source sound effects from Openverse with the bundled library offline * docs(media-use): name the resolve credits file without a dead path * style(skills): restore the slideshow harness formatting the pinned formatter expects * fix(media-use): keep Openverse credit text to one safe line and read credits without a race * fix(media-use): Openverse sound effects record as the engine's, so a rerun writes no copy * fix(media-use): two sound effect resolves at once each keep their credit line * fix(media-use): the standalone media-use copy takes the same credit lock * fix(media-use): a sound effect credit is always one safe line with a provider link writeSfxCredit cleans the credit and source link itself, since the CLI writes credits from a saved record. Backticks, brackets, angle brackets and backslashes are stripped, and a source link must be https on an Openverse audio provider host (Freesound, Jamendo, Wikimedia Commons) or it is left out.
This commit is contained in:
Miguel Ángel
@@ -102,19 +102,16 @@ const pendingFailureTracking = new Set();
|
||||
// API, move this state into a per-resolve context before reusing that path.
|
||||
let pendingRemediation = null;
|
||||
|
||||
const TOOL_WORDS = { bgm: "music", sfx: "sound-effect" };
|
||||
|
||||
/** A music or sound-effect resolve miss after the heygen CLI was missing or too old: what is missing, the host
|
||||
* app's own tool, the fix. Other types keep the generic miss. */
|
||||
/** A music resolve miss after the heygen CLI was missing or too old: what is missing, the host app's own
|
||||
* tool, the fix. Other types keep the generic miss. */
|
||||
export function heygenMiss(type, { code }) {
|
||||
const tool = TOOL_WORDS[type];
|
||||
if (!tool) return null;
|
||||
if (type !== "bgm") return null;
|
||||
const outdated = code === "outdated";
|
||||
const state = outdated ? `older than v${HEYGEN_MIN_VERSION}` : "not installed";
|
||||
return {
|
||||
code: outdated ? "heygen_cli_outdated" : "heygen_cli_missing",
|
||||
fix: outdated ? HEYGEN_UPDATE_COMMAND : HEYGEN_INSTALL_COMMAND,
|
||||
error: `${type} needs the heygen CLI, which is ${state}: use your host app's own ${tool} tool if it has one, or ${outdated ? "update" : "install"} the CLI.`,
|
||||
error: `${type} needs the heygen CLI, which is ${state}: use your host app's own music tool if it has one, or ${outdated ? "update" : "install"} the CLI.`,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -301,13 +301,13 @@ test("flushHeygenFailureTracking resolves immediately when nothing is pending",
|
||||
await flushHeygenFailureTracking();
|
||||
});
|
||||
|
||||
test("only a music or sound-effect miss names the heygen CLI and the host app's own tool", () => {
|
||||
for (const type of ["image", "icon", "voice", "video"])
|
||||
test("only a music miss names the heygen CLI and the host app's own tool", () => {
|
||||
for (const type of ["sfx", "image", "icon", "voice", "video"])
|
||||
assert.equal(heygenMiss(type, { code: "not_found" }), null, type);
|
||||
assert.equal(heygenMiss("bgm", { code: "not_found" }).code, "heygen_cli_missing");
|
||||
const sfx = heygenMiss("sfx", { code: "outdated" });
|
||||
assert.equal(sfx.code, "heygen_cli_outdated");
|
||||
assert.match(sfx.error, /host app's own sound-effect tool/);
|
||||
const bgm = heygenMiss("bgm", { code: "outdated" });
|
||||
assert.equal(bgm.code, "heygen_cli_outdated");
|
||||
assert.match(bgm.error, /host app's own music tool/);
|
||||
});
|
||||
|
||||
test("a CLI that rejects --headers without printing a version is outdated", () => {
|
||||
|
||||
@@ -11,6 +11,7 @@ export function allocateId(
|
||||
type: string,
|
||||
ext: string,
|
||||
): Omit<Reservation, "fullPath">;
|
||||
export function withMediaLock<T>(projectDir: string, fn: () => T): T;
|
||||
export function withReservedFileSync<T>(
|
||||
projectDir: string,
|
||||
type: string,
|
||||
|
||||
@@ -107,8 +107,7 @@ export function recordInPlace(
|
||||
description: description || basename(path),
|
||||
...(duration != null && { duration: Math.round(duration * 10) / 10 }),
|
||||
};
|
||||
mkdirSync(mediaDir(projectDir), { recursive: true });
|
||||
return withLock(mediaDir(projectDir), () => {
|
||||
return withMediaLock(projectDir, () => {
|
||||
const latest = latestRecordFor(projectDir, path);
|
||||
const same = ["source", "description", "duration"].every(
|
||||
(key) => latest?.[key] === fields[key],
|
||||
@@ -203,6 +202,12 @@ function withLock(dir, fn) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs `fn` holding the project's `.media/.lock`, so concurrent resolves take turns on what it guards. */
|
||||
export function withMediaLock(projectDir, fn) {
|
||||
mkdirSync(mediaDir(projectDir), { recursive: true });
|
||||
return withLock(mediaDir(projectDir), fn);
|
||||
}
|
||||
|
||||
// Atomically allocate the next free id for `type` AND reserve it on disk, so a
|
||||
// slow download/copy between allocation and appendRecord can't let a concurrent
|
||||
// caller grab the same id (the MU-23 clobber). Under the lock we take the max id
|
||||
@@ -210,10 +215,8 @@ function withLock(dir, fn) {
|
||||
// O_EXCL-create an empty marker under core's atomic temp name, which Studio's
|
||||
// project history skips. Returns { id, localPath, markerPath }.
|
||||
export function allocateId(projectDir, type, ext) {
|
||||
mkdirSync(mediaDir(projectDir), { recursive: true });
|
||||
const typeDir = typeDirPath(projectDir, type);
|
||||
mkdirSync(typeDir, { recursive: true });
|
||||
return withLock(mediaDir(projectDir), () => {
|
||||
mkdirSync(typeDirPath(projectDir, type), { recursive: true });
|
||||
return withMediaLock(projectDir, () => {
|
||||
const id = nextFreeId(projectDir, type);
|
||||
const localPath = `.media/${typeSubdir(type)}/${id}${ext}`;
|
||||
// Same shape as core atomicTempPath: an empty file at the final name would be committed.
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
// Sound effects from Openverse (https://api.openverse.org): free, no key.
|
||||
// Anonymous use is rate limited (20 requests a minute, 200 a day); callers fall
|
||||
// back to the bundled library when it is unavailable.
|
||||
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
const OPENVERSE_AUDIO_URL = "https://api.openverse.org/v1/audio/";
|
||||
// The only licences media-use accepts: no NonCommercial, ShareAlike or NoDerivatives.
|
||||
export const OPENVERSE_SFX_LICENSES = new Set(["cc0", "by"]);
|
||||
const PLAYABLE_FILETYPES = new Set(["mp3", "wav", "ogg", "flac"]);
|
||||
const MAX_SFX_MS = 15000;
|
||||
const IDEAL_SFX_MS = 4000;
|
||||
const BY_PENALTY = 2;
|
||||
const SEARCH_TIMEOUT_MS = 10000;
|
||||
|
||||
// Openverse text lands in markdown and JSON: one line, no control or markdown-structure characters, bounded.
|
||||
const oneLine = (text) =>
|
||||
String(text ?? "")
|
||||
.replace(/[\p{Cc}`[\]<>\\]+/gu, " ")
|
||||
.trim()
|
||||
.slice(0, 300);
|
||||
// Openverse's audio providers (api.openverse.org/v1/audio/stats/); a new provider's link is dropped until listed.
|
||||
const SOURCE_HOSTS = new Set(["freesound.org", "www.jamendo.com", "commons.wikimedia.org"]);
|
||||
function httpsUrl(url, hosts) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.protocol === "https:" && (!hosts || hosts.has(parsed.hostname))
|
||||
? oneLine(parsed.href)
|
||||
: "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/** Openverse could not answer (offline, rate limited, server error): stop asking it this run. */
|
||||
export class OpenverseUnavailableError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = "OpenverseUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
function usable(result) {
|
||||
return (
|
||||
OPENVERSE_SFX_LICENSES.has(result?.license) &&
|
||||
PLAYABLE_FILETYPES.has(result.filetype) &&
|
||||
typeof result.url === "string" &&
|
||||
result.url.startsWith("https://") &&
|
||||
Number.isFinite(result.duration) &&
|
||||
result.duration > 0 &&
|
||||
result.duration <= MAX_SFX_MS
|
||||
);
|
||||
}
|
||||
|
||||
// Lower is better: relevance position, then seconds past a typical cue length, then CC0 over CC BY.
|
||||
function penalty(result, position) {
|
||||
const longBy = Math.max(0, result.duration - IDEAL_SFX_MS) / 1000;
|
||||
return position + longBy + (result.license === "by" ? BY_PENALTY : 0);
|
||||
}
|
||||
|
||||
/** Best usable sound in Openverse's relevance-ordered results, or null. */
|
||||
export function pickOpenverseSfx(results) {
|
||||
let best = null;
|
||||
results.forEach((result, position) => {
|
||||
if (!usable(result)) return;
|
||||
const score = penalty(result, position);
|
||||
if (!best || score < best.score) best = { result, score };
|
||||
});
|
||||
if (!best) return null;
|
||||
const r = best.result;
|
||||
return {
|
||||
id: r.id,
|
||||
url: r.url,
|
||||
ext: `.${r.filetype}`,
|
||||
title: oneLine(r.title),
|
||||
creator: oneLine(r.creator),
|
||||
license: r.license,
|
||||
licenseUrl: httpsUrl(r.license_url),
|
||||
sourceUrl: httpsUrl(r.foreign_landing_url, SOURCE_HOSTS),
|
||||
credit: oneLine(r.attribution),
|
||||
durationS: Math.round(r.duration) / 1000,
|
||||
};
|
||||
}
|
||||
|
||||
/** Best licensed sound effect for `query`, null on no match; throws OpenverseUnavailableError. */
|
||||
export async function searchOpenverseSfx(query, { fetchImpl = fetch } = {}) {
|
||||
const params = new URLSearchParams({
|
||||
q: query,
|
||||
page_size: "20",
|
||||
license: [...OPENVERSE_SFX_LICENSES].join(","),
|
||||
});
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(`${OPENVERSE_AUDIO_URL}?${params}`, {
|
||||
headers: { Accept: "application/json" },
|
||||
signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS),
|
||||
});
|
||||
} catch (error) {
|
||||
throw new OpenverseUnavailableError(`Openverse unreachable (${error.message})`);
|
||||
}
|
||||
if (response.status === 429)
|
||||
throw new OpenverseUnavailableError("Openverse rate limit reached (HTTP 429)");
|
||||
if (!response.ok) throw new OpenverseUnavailableError(`Openverse HTTP ${response.status}`);
|
||||
let body;
|
||||
try {
|
||||
body = await response.json();
|
||||
} catch {
|
||||
throw new OpenverseUnavailableError("Openverse returned non-JSON");
|
||||
}
|
||||
return pickOpenverseSfx(Array.isArray(body?.results) ? body.results : []);
|
||||
}
|
||||
|
||||
/** Record a CC BY sound's credit for `file` in the project's credits file (one line per file). */
|
||||
export function writeSfxCredit(creditsPath, file, sound) {
|
||||
if (sound.license !== "by") return;
|
||||
// The sound may come from a saved record, not this run's pick: clean it again here.
|
||||
const sourceUrl = httpsUrl(sound.sourceUrl, SOURCE_HOSTS);
|
||||
const source = sourceUrl ? ` Source: ${sourceUrl}` : "";
|
||||
const line = `- \`${file}\`: ${oneLine(sound.credit)}${source}`;
|
||||
let prior = "";
|
||||
try {
|
||||
prior = readFileSync(creditsPath, "utf8");
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
const kept = prior
|
||||
.split("\n")
|
||||
.filter((l) => l.startsWith("- ") && !l.startsWith(`- \`${file}\`:`));
|
||||
const header =
|
||||
"# Sound effect credits\n\nThese sounds are licensed CC BY: credit them wherever the video is published.\n\n";
|
||||
mkdirSync(dirname(creditsPath), { recursive: true });
|
||||
writeFileSync(creditsPath, `${header}${[...kept, line].join("\n")}\n`);
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
OpenverseUnavailableError,
|
||||
pickOpenverseSfx,
|
||||
searchOpenverseSfx,
|
||||
writeSfxCredit,
|
||||
} from "./openverse.mjs";
|
||||
|
||||
const result = (overrides = {}) => ({
|
||||
id: "ov-1",
|
||||
title: "Deep Whoosh",
|
||||
creator: "Kinoton",
|
||||
license: "cc0",
|
||||
license_url: "https://creativecommons.org/publicdomain/zero/1.0/",
|
||||
foreign_landing_url: "https://freesound.org/people/Kinoton/sounds/351256",
|
||||
url: "https://cdn.freesound.org/previews/351/351256_2247456-hq.mp3",
|
||||
filetype: "mp3",
|
||||
duration: 1200,
|
||||
attribution: '"Deep Whoosh" by Kinoton is marked with CC0 1.0.',
|
||||
...overrides,
|
||||
});
|
||||
|
||||
test("only CC0 and CC BY sounds are ever picked", () => {
|
||||
for (const license of ["by-nc", "by-sa", "by-nd", "by-nc-sa", "by-nc-nd", "pdm", "sampling+"])
|
||||
assert.equal(pickOpenverseSfx([result({ license })]), null, license);
|
||||
assert.equal(pickOpenverseSfx([result({ license: "by" })]).license, "by");
|
||||
assert.equal(pickOpenverseSfx([result({ license: "cc0" })]).license, "cc0");
|
||||
});
|
||||
|
||||
test("sounds too long for a cue, or not playable, are skipped", () => {
|
||||
assert.equal(pickOpenverseSfx([result({ duration: 15001 })]), null);
|
||||
assert.equal(pickOpenverseSfx([result({ duration: null })]), null);
|
||||
assert.equal(pickOpenverseSfx([result({ filetype: "aiff" })]), null);
|
||||
assert.equal(pickOpenverseSfx([result({ url: "http://insecure.test/a.mp3" })]), null);
|
||||
});
|
||||
|
||||
test("ranking keeps relevance order, then prefers short clips and CC0 over CC BY", () => {
|
||||
const pick = (results) => pickOpenverseSfx(results).id;
|
||||
assert.equal(pick([result({ id: "first" }), result({ id: "second" })]), "first");
|
||||
assert.equal(pick([result({ id: "long", duration: 12000 }), result({ id: "short" })]), "short");
|
||||
assert.equal(pick([result({ id: "by", license: "by" }), result({ id: "cc0" })]), "cc0");
|
||||
assert.equal(
|
||||
pick([
|
||||
result({ id: "by", license: "by" }),
|
||||
...Array(3).fill(result({ license: "by-nc" })),
|
||||
result({ id: "cc0" }),
|
||||
]),
|
||||
"by",
|
||||
"a much more relevant CC BY sound still wins",
|
||||
);
|
||||
});
|
||||
|
||||
test("a pick carries what the project needs to play and credit it", () => {
|
||||
assert.deepEqual(pickOpenverseSfx([result({ license: "by", filetype: "wav", duration: 450 })]), {
|
||||
id: "ov-1",
|
||||
url: "https://cdn.freesound.org/previews/351/351256_2247456-hq.mp3",
|
||||
ext: ".wav",
|
||||
title: "Deep Whoosh",
|
||||
creator: "Kinoton",
|
||||
license: "by",
|
||||
licenseUrl: "https://creativecommons.org/publicdomain/zero/1.0/",
|
||||
sourceUrl: "https://freesound.org/people/Kinoton/sounds/351256",
|
||||
credit: '"Deep Whoosh" by Kinoton is marked with CC0 1.0.',
|
||||
durationS: 0.45,
|
||||
});
|
||||
});
|
||||
|
||||
test("Openverse text is kept to one bounded line, and only https links survive", () => {
|
||||
const pick = pickOpenverseSfx([
|
||||
result({
|
||||
license: "by",
|
||||
title: "Pop\n# Heading",
|
||||
attribution: `"Pop" by Ana.\n- \`assets/sfx/other.mp3\`: forged credit\r\u0007${"x".repeat(400)}`,
|
||||
foreign_landing_url: "javascript:alert(1)",
|
||||
license_url: "http://creativecommons.org/licenses/by/4.0/",
|
||||
}),
|
||||
]);
|
||||
assert.equal(pick.title, "Pop # Heading");
|
||||
assert.equal(pick.credit.length, 300);
|
||||
assert.ok(!/\p{Cc}/u.test(pick.credit), "no line breaks or control characters");
|
||||
assert.equal(pick.sourceUrl, "");
|
||||
assert.equal(pick.licenseUrl, "");
|
||||
});
|
||||
|
||||
test("search asks Openverse for JSON and only the allowed licences", async () => {
|
||||
const calls = [];
|
||||
const sound = await searchOpenverseSfx("ui click", {
|
||||
fetchImpl: async (url, init) => {
|
||||
calls.push({ url: new URL(url), init });
|
||||
return Response.json({ results: [result({ license: "by-nc" }), result({ id: "ok" })] });
|
||||
},
|
||||
});
|
||||
assert.equal(sound.id, "ok", "the licence field is checked even on a filtered response");
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].url.origin + calls[0].url.pathname, "https://api.openverse.org/v1/audio/");
|
||||
assert.equal(calls[0].url.searchParams.get("q"), "ui click");
|
||||
assert.equal(calls[0].url.searchParams.get("license"), "cc0,by");
|
||||
assert.equal(calls[0].init.headers.Accept, "application/json");
|
||||
});
|
||||
|
||||
test("no usable result is a miss, not an error", async () => {
|
||||
const fetchImpl = async () => Response.json({ results: [] });
|
||||
assert.equal(await searchOpenverseSfx("nothing", { fetchImpl }), null);
|
||||
});
|
||||
|
||||
test("rate limits, server errors, bad bodies and network failures make Openverse unavailable", async () => {
|
||||
const failures = [
|
||||
[async () => new Response("slow down", { status: 429 }), /rate limit/],
|
||||
[async () => new Response("down", { status: 503 }), /HTTP 503/],
|
||||
[async () => new Response("<html>"), /non-JSON/],
|
||||
[
|
||||
async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
},
|
||||
/unreachable \(fetch failed\)/,
|
||||
],
|
||||
];
|
||||
for (const [fetchImpl, message] of failures)
|
||||
await assert.rejects(searchOpenverseSfx("whoosh", { fetchImpl }), (error) => {
|
||||
assert.ok(error instanceof OpenverseUnavailableError);
|
||||
assert.match(error.message, message);
|
||||
return true;
|
||||
});
|
||||
});
|
||||
|
||||
test("credits list each CC BY file once and leave CC0 out", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "openverse-credits-"));
|
||||
try {
|
||||
const path = join(dir, "assets", "sfx", "CREDITS.md");
|
||||
const by = {
|
||||
license: "by",
|
||||
credit: '"Pop" by Ana is licensed under CC BY 4.0.',
|
||||
sourceUrl: "https://freesound.org/s/1",
|
||||
};
|
||||
writeSfxCredit(path, "assets/sfx/zap.mp3", { ...by, license: "cc0" });
|
||||
assert.throws(() => readFileSync(path), /ENOENT/);
|
||||
writeSfxCredit(path, "assets/sfx/pop.mp3", by);
|
||||
writeSfxCredit(path, "assets/sfx/pop.mp3", {
|
||||
...by,
|
||||
credit: '"Pop 2" by Ana is licensed under CC BY 4.0.',
|
||||
});
|
||||
writeSfxCredit(path, "assets/sfx/ding.mp3", by);
|
||||
const lines = readFileSync(path, "utf8")
|
||||
.split("\n")
|
||||
.filter((l) => l.startsWith("- "));
|
||||
assert.deepEqual(lines, [
|
||||
'- `assets/sfx/pop.mp3`: "Pop 2" by Ana is licensed under CC BY 4.0. Source: https://freesound.org/s/1',
|
||||
'- `assets/sfx/ding.mp3`: "Pop" by Ana is licensed under CC BY 4.0. Source: https://freesound.org/s/1',
|
||||
]);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a credit with no https source link is written without one", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "openverse-credits-"));
|
||||
try {
|
||||
const path = join(dir, "CREDITS.md");
|
||||
writeSfxCredit(path, "assets/sfx/pop.mp3", { license: "by", credit: "c.", sourceUrl: "" });
|
||||
assert.match(readFileSync(path, "utf8"), /\n- `assets\/sfx\/pop\.mp3`: c\.\n$/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a hostile credit is written as one safe line, with no link off Openverse's providers", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "openverse-credits-"));
|
||||
try {
|
||||
const path = join(dir, "CREDITS.md");
|
||||
const hostile = `"Pop" by Ana.\n- \`assets/sfx/x.mp3\`: forged [click](https://evil.test) <img src=x> \\${"x".repeat(5000)}`;
|
||||
for (const sourceUrl of [
|
||||
"javascript:alert(1)",
|
||||
"https://evil.test/a",
|
||||
"https://freesound.org/s/1\n- forged",
|
||||
]) {
|
||||
writeSfxCredit(path, "assets/sfx/pop.mp3", { license: "by", credit: hostile, sourceUrl });
|
||||
const lines = readFileSync(path, "utf8")
|
||||
.split("\n")
|
||||
.filter((l) => l.startsWith("- "));
|
||||
assert.equal(lines.length, 1, sourceUrl);
|
||||
assert.ok(
|
||||
lines[0].startsWith('- `assets/sfx/pop.mp3`: "Pop" by Ana. - '),
|
||||
lines[0].slice(0, 80),
|
||||
);
|
||||
assert.ok(
|
||||
!/[\p{Cc}`[\]<>\\]/u.test(lines[0].slice("- `assets/sfx/pop.mp3`:".length)),
|
||||
sourceUrl,
|
||||
);
|
||||
assert.ok(lines[0].length < 400, "bounded");
|
||||
assert.doesNotMatch(lines[0], /javascript:|evil\.test\/a/);
|
||||
}
|
||||
assert.match(readFileSync(path, "utf8"), /Source: https:\/\/freesound\.org\/s\/1-%20forged\n$/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("earlier credit lines survive a new credit", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "openverse-credits-"));
|
||||
try {
|
||||
const path = join(dir, "CREDITS.md");
|
||||
writeFileSync(path, "# Sound effect credits\n\n- `assets/sfx/old.mp3`: kept\n");
|
||||
writeSfxCredit(path, "assets/sfx/new.mp3", {
|
||||
license: "by",
|
||||
credit: "c.",
|
||||
sourceUrl: "https://freesound.org/s/2",
|
||||
});
|
||||
assert.match(
|
||||
readFileSync(path, "utf8"),
|
||||
/- `assets\/sfx\/old\.mp3`: kept\n- `assets\/sfx\/new\.mp3`: c\. Source: https:\/\/freesound\.org\/s\/2\n$/,
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -10,6 +10,8 @@
|
||||
// own auth. Providers, by type:
|
||||
// - heygen CLI: catalog + TTS, first for every type it serves (OAuth free
|
||||
// allowance first, then the user's HeyGen billing path)
|
||||
// - Openverse: free, keyless sound effects (CC0 / CC BY), with the bundled
|
||||
// library as the offline fallback
|
||||
// - mflux: local FLUX-class image gen, spec-selected to the machine's RAM
|
||||
// (free, private, offline once cached)
|
||||
// - codex CLI: image gen on the user's ChatGPT sub — the better-quality upsell
|
||||
@@ -21,7 +23,7 @@
|
||||
// (e.g. "make an image with codex").
|
||||
|
||||
import { bgmProvider } from "./bgm-provider.mjs";
|
||||
import { sfxProvider } from "./sfx-provider.mjs";
|
||||
import { OPENVERSE_SFX_PROVIDER, sfxProvider } from "./sfx-provider.mjs";
|
||||
import { bundledSfxProvider } from "./bundled-sfx-provider.mjs";
|
||||
import { imageProvider, iconProvider } from "./image-provider.mjs";
|
||||
import { brandProvider } from "./brand-provider.mjs";
|
||||
@@ -46,8 +48,8 @@ const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // rem
|
||||
const REGISTRY = {
|
||||
bgm: [N("heygen.audio.sounds", { search: bgmProvider.search })],
|
||||
sfx: [
|
||||
N(OPENVERSE_SFX_PROVIDER, { search: sfxProvider.search }),
|
||||
A("bundled.sfx", { search: bundledSfxProvider.search }),
|
||||
N("heygen.audio.sounds", { search: sfxProvider.search }),
|
||||
],
|
||||
image: [
|
||||
N("heygen.asset.search", { search: imageProvider.search }),
|
||||
|
||||
@@ -32,18 +32,18 @@ test("listTypes exposes the v2 media types", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("heygen provider is first for every type it serves except bundled sfx", () => {
|
||||
test("heygen provider is first for every type it serves", () => {
|
||||
for (const t of ["bgm", "image", "icon"]) {
|
||||
const first = getProviders(t)[0];
|
||||
assert.ok(first, `no enabled provider for ${t}`);
|
||||
assert.match(first.name, /^heygen/, `${t} first provider is ${first.name}`);
|
||||
}
|
||||
assert.equal(getProviders("sfx")[0].name, "bundled.sfx");
|
||||
assert.ok(!getProviders("sfx").some((p) => p.name.startsWith("heygen")), "sfx never asks HeyGen");
|
||||
});
|
||||
|
||||
test("sanctioned providers only: heygen, local mflux/kokoro/ltx, codex, design spec, logo tiers", () => {
|
||||
test("sanctioned providers only: heygen, Openverse, local mflux/kokoro/ltx, codex, design spec, logo tiers", () => {
|
||||
const allowed =
|
||||
/^heygen|^bundled\.sfx$|^mflux\.local$|^kokoro\.local$|^ltx\.local$|^codex\.image_gen$|^design_spec$|^thesvg$|^github\.avatar$|^favicon\.ddg$|^color_grade\.local$|^cube_lut\.local$/;
|
||||
/^heygen|^openverse\.audio$|^bundled\.sfx$|^mflux\.local$|^kokoro\.local$|^ltx\.local$|^codex\.image_gen$|^design_spec$|^thesvg$|^github\.avatar$|^favicon\.ddg$|^color_grade\.local$|^cube_lut\.local$/;
|
||||
for (const t of listTypes()) {
|
||||
for (const p of getProviders(t)) {
|
||||
assert.ok(allowed.test(p.name), `${t} lists unsanctioned provider: ${p.name}`);
|
||||
@@ -85,13 +85,14 @@ test("video cascade: HeyGen first, LTX local fallback, generate-only", async ()
|
||||
assert.equal(await runCapability("video", "search", "x", {}), null);
|
||||
});
|
||||
|
||||
test("sfx cascade: bundled library first, HeyGen catalog remains the network fallback", () => {
|
||||
test("sfx cascade: Openverse first, the bundled library is the offline fallback", () => {
|
||||
const ps = getProviders("sfx");
|
||||
assert.equal(ps[0].name, "bundled.sfx");
|
||||
assert.equal(typeof ps[0].search, "function");
|
||||
assert.ok(!ps[0].network, "bundled SFX remain available offline");
|
||||
assert.equal(ps[1].name, "heygen.audio.sounds");
|
||||
assert.ok(ps[1].network, "HeyGen SFX catalog is network-only");
|
||||
assert.deepEqual(
|
||||
ps.map((p) => p.name),
|
||||
["openverse.audio", "bundled.sfx"],
|
||||
);
|
||||
assert.ok(ps[0].network && !ps[0].paid, "Openverse is free and network-only");
|
||||
assert.ok(!ps[1].network, "bundled SFX remain available offline");
|
||||
});
|
||||
|
||||
test("ctx.provider forces one generator (e.g. 'make an image WITH codex')", async () => {
|
||||
@@ -201,9 +202,8 @@ test("providerTierFor reports the registry's own A/N/P declaration", () => {
|
||||
});
|
||||
|
||||
test("providerTierFor agrees across every type that declares the same name", () => {
|
||||
// heygen.audio.sounds serves both bgm and sfx; heygen.asset.search serves both
|
||||
// image and icon. A name whose tier depended on the media type would make the
|
||||
// telemetry property meaningless.
|
||||
// heygen.asset.search serves both image and icon. A name whose tier depended
|
||||
// on the media type would make the telemetry property meaningless.
|
||||
const byName = new Map();
|
||||
for (const type of listTypes()) {
|
||||
for (const name of providerNamesFor(type)) {
|
||||
|
||||
@@ -1,22 +1,32 @@
|
||||
import { heygenSearch } from "./heygen-search.mjs";
|
||||
import { OpenverseUnavailableError, searchOpenverseSfx } from "./openverse.mjs";
|
||||
|
||||
export const OPENVERSE_SFX_PROVIDER = "openverse.audio";
|
||||
|
||||
export const sfxProvider = {
|
||||
async search(intent) {
|
||||
const results = heygenSearch("audio sounds list", intent, {
|
||||
type: "sound_effects",
|
||||
minScore: 0.4,
|
||||
});
|
||||
if (!results) return null;
|
||||
const best = results[0];
|
||||
let sound;
|
||||
try {
|
||||
sound = await searchOpenverseSfx(intent);
|
||||
} catch (error) {
|
||||
if (!(error instanceof OpenverseUnavailableError)) throw error;
|
||||
console.error(`media-use: ${error.message}; using the bundled sound effects`);
|
||||
return null;
|
||||
}
|
||||
if (!sound) return null;
|
||||
return {
|
||||
url: best.audio_url,
|
||||
url: sound.url,
|
||||
ext: sound.ext,
|
||||
source: "search",
|
||||
// ext derived from audio_url by resolve.mjs — catalog SFX are .mp3 or .wav
|
||||
metadata: {
|
||||
description: best.description || best.name || intent,
|
||||
duration: best.duration || null,
|
||||
provider: "heygen.audio.sounds",
|
||||
provenance: { track_id: best.id, score: best.score, query: intent },
|
||||
description: sound.title || intent,
|
||||
duration: sound.durationS,
|
||||
provider: OPENVERSE_SFX_PROVIDER,
|
||||
provenance: {
|
||||
openverse_id: sound.id,
|
||||
license: sound.license,
|
||||
credit: sound.credit,
|
||||
source_url: sound.sourceUrl,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
@@ -12,6 +12,8 @@ import {
|
||||
findByPrompt,
|
||||
findByEntity,
|
||||
nextId,
|
||||
typeDirPath,
|
||||
withMediaLock,
|
||||
withReservedFile,
|
||||
withReservedFileSync,
|
||||
} from "./lib/manifest.mjs";
|
||||
@@ -55,6 +57,8 @@ import {
|
||||
versionLessThan,
|
||||
} from "./lib/heygen-cli.mjs";
|
||||
import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs";
|
||||
import { OPENVERSE_SFX_PROVIDER } from "./lib/sfx-provider.mjs";
|
||||
import { writeSfxCredit } from "./lib/openverse.mjs";
|
||||
import {
|
||||
fetchMediaVectors,
|
||||
mediaVectorRows,
|
||||
@@ -441,57 +445,43 @@ async function run() {
|
||||
return resolveColor(type, intent, { projectDir });
|
||||
}
|
||||
|
||||
// SFX search is bundled, local-index, then HeyGen.
|
||||
// SFX search is Openverse, the bundled library, then the local media index.
|
||||
let searchResult = null;
|
||||
let providerFailure = null;
|
||||
let localIndexFailure = null;
|
||||
try {
|
||||
if (type === "sfx" && !args.provider) {
|
||||
searchResult = await runCapability(type, "search", intent, {
|
||||
...ctx,
|
||||
provider: "bundled.sfx",
|
||||
});
|
||||
if (!searchResult && !localOnly) {
|
||||
const registry =
|
||||
process.env.HYPERFRAMES_REGISTRY ||
|
||||
"https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry";
|
||||
try {
|
||||
await fetchMediaVectors(registry);
|
||||
const ranked = await rankMediaRowsWithVectors(
|
||||
intent,
|
||||
mediaVectorRows().filter((row) => row.kind === "sfx"),
|
||||
);
|
||||
const row = ranked.rows[0];
|
||||
const candidates = row
|
||||
? [resolve(row.file), join(import.meta.dirname, "..", "..", "..", row.file)]
|
||||
: [];
|
||||
const localPath = candidates.find((candidate) => existsSync(candidate));
|
||||
if (row && localPath) {
|
||||
searchResult = {
|
||||
localPath,
|
||||
ext: extname(localPath),
|
||||
source: "local-index",
|
||||
metadata: {
|
||||
description: row.description,
|
||||
duration: row.duration ?? null,
|
||||
provider: "catalog.local",
|
||||
provenance: { library_key: row.id, tier: ranked.tier },
|
||||
},
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
localIndexFailure = error;
|
||||
searchResult = await runCapability(type, "search", intent, ctx);
|
||||
if (!searchResult && type === "sfx" && !args.provider && !localOnly) {
|
||||
const registry =
|
||||
process.env.HYPERFRAMES_REGISTRY ||
|
||||
"https://raw.githubusercontent.com/heygen-com/hyperframes/main/registry";
|
||||
try {
|
||||
await fetchMediaVectors(registry);
|
||||
const ranked = await rankMediaRowsWithVectors(
|
||||
intent,
|
||||
mediaVectorRows().filter((row) => row.kind === "sfx"),
|
||||
);
|
||||
const row = ranked.rows[0];
|
||||
const candidates = row
|
||||
? [resolve(row.file), join(import.meta.dirname, "..", "..", "..", row.file)]
|
||||
: [];
|
||||
const localPath = candidates.find((candidate) => existsSync(candidate));
|
||||
if (row && localPath) {
|
||||
searchResult = {
|
||||
localPath,
|
||||
ext: extname(localPath),
|
||||
source: "local-index",
|
||||
metadata: {
|
||||
description: row.description,
|
||||
duration: row.duration ?? null,
|
||||
provider: "catalog.local",
|
||||
provenance: { library_key: row.id, tier: ranked.tier },
|
||||
},
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
localIndexFailure = error;
|
||||
}
|
||||
if (!searchResult) searchResult = await runCapability(type, "search", intent, ctx);
|
||||
// Keep HeyGen remediation diagnostics while bundled remains authoritative.
|
||||
else if (!localOnly)
|
||||
await runCapability(type, "search", intent, {
|
||||
...ctx,
|
||||
provider: "heygen.audio.sounds",
|
||||
});
|
||||
} else {
|
||||
searchResult = await runCapability(type, "search", intent, ctx);
|
||||
}
|
||||
} catch (error) {
|
||||
providerFailure = error;
|
||||
@@ -602,16 +592,6 @@ async function run() {
|
||||
},
|
||||
};
|
||||
|
||||
const heygenRemediation = consumeHeygenRemediation();
|
||||
if (
|
||||
searchResult.metadata?.provider === "bundled.sfx" &&
|
||||
!localOnly &&
|
||||
!args.provider &&
|
||||
heygenRemediation
|
||||
) {
|
||||
record.advisory = heygenRemediation;
|
||||
}
|
||||
|
||||
appendRecord(projectDir, record);
|
||||
regenerateIndex(projectDir);
|
||||
// Promote fetched assets into the global cache for reuse.
|
||||
@@ -1304,6 +1284,16 @@ async function reuseGlobal(shaArg) {
|
||||
}
|
||||
|
||||
async function result(record, source) {
|
||||
const { provider, license, credit, source_url: sourceUrl } = record.provenance ?? {};
|
||||
// Under the media lock: two resolves at once each read, then rewrite, the whole credits file.
|
||||
if (provider === OPENVERSE_SFX_PROVIDER)
|
||||
withMediaLock(projectDir, () =>
|
||||
writeSfxCredit(join(typeDirPath(projectDir, "sfx"), "CREDITS.md"), record.path, {
|
||||
license,
|
||||
credit,
|
||||
sourceUrl,
|
||||
}),
|
||||
);
|
||||
// Non-PII usage event: which media type, how it resolved, which provider won.
|
||||
// Never the intent text or paths. Awaited so a short-lived run flushes it.
|
||||
await track("media_use_resolve", {
|
||||
|
||||
@@ -16,11 +16,7 @@ import { execFileSync, spawn, spawnSync } from "node:child_process";
|
||||
import { appendRecord, findByPrompt, readManifest } from "./lib/manifest.mjs";
|
||||
import { regenerateIndex } from "./lib/index-gen.mjs";
|
||||
import { getProvider } from "./lib/providers.mjs";
|
||||
import {
|
||||
HEYGEN_INSTALL_COMMAND,
|
||||
HEYGEN_NOT_FOUND_MESSAGE,
|
||||
HEYGEN_UPDATE_COMMAND,
|
||||
} from "./lib/heygen-cli.mjs";
|
||||
import { HEYGEN_INSTALL_COMMAND, HEYGEN_UPDATE_COMMAND } from "./lib/heygen-cli.mjs";
|
||||
import { freezeLocalFile } from "./lib/freeze.mjs";
|
||||
import { cachePut, cacheGet, importFromCache } from "./lib/cache.mjs";
|
||||
import { validateCubeFile } from "./lib/cube-validate.mjs";
|
||||
@@ -78,8 +74,8 @@ function runResolve(args, opts = {}) {
|
||||
}
|
||||
|
||||
function spawnResolve(args, opts = {}) {
|
||||
const { env, ...rest } = opts;
|
||||
return spawnSync(process.execPath, [RESOLVE_CLI, ...args], {
|
||||
const { env, nodeArgs = [], ...rest } = opts;
|
||||
return spawnSync(process.execPath, [...nodeArgs, RESOLVE_CLI, ...args], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, DO_NOT_TRACK: "1", ...env },
|
||||
@@ -88,9 +84,9 @@ function spawnResolve(args, opts = {}) {
|
||||
}
|
||||
|
||||
function spawnResolveAsync(args, opts = {}) {
|
||||
const { env, ...rest } = opts;
|
||||
const { env, nodeArgs = [], ...rest } = opts;
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, [RESOLVE_CLI, ...args], {
|
||||
const child = spawn(process.execPath, [...nodeArgs, RESOLVE_CLI, ...args], {
|
||||
cwd: REPO_ROOT,
|
||||
env: { ...process.env, DO_NOT_TRACK: "1", ...env },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
@@ -107,6 +103,31 @@ function spawnResolveAsync(args, opts = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
const OPENVERSE_BY_WHOOSH = {
|
||||
id: "ov-whoosh",
|
||||
title: "Swoosh",
|
||||
creator: "Ana",
|
||||
license: "by",
|
||||
foreign_landing_url: "https://freesound.org/s/1",
|
||||
url: "https://cdn.openverse.test/swoosh.mp3",
|
||||
filetype: "mp3",
|
||||
duration: 820,
|
||||
attribution: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
};
|
||||
|
||||
// Node args that stand Openverse in for the spawned resolve: `results` answers every search, 429 rate-limits it.
|
||||
function openverseImport(results) {
|
||||
const stub = `const real = globalThis.fetch;
|
||||
globalThis.fetch = async (url, init) => {
|
||||
const u = String(url);
|
||||
if (u.startsWith("https://api.openverse.org/"))
|
||||
return ${results === 429 ? 'new Response("", { status: 429 })' : `Response.json({ results: ${JSON.stringify(results)} })`};
|
||||
if (u.startsWith("https://cdn.openverse.test/")) return new Response("ID3 openverse bytes");
|
||||
return real(url, init);
|
||||
};`;
|
||||
return [`--import=data:text/javascript,${encodeURIComponent(stub)}`];
|
||||
}
|
||||
|
||||
function makeFrame(dir, name, color) {
|
||||
const out = join(dir, name);
|
||||
execFileSync(
|
||||
@@ -160,18 +181,82 @@ function test(name, fn) {
|
||||
|
||||
// --- manifest cache hit ---
|
||||
|
||||
test("bundled SFX resolve without HeyGen on PATH", () => {
|
||||
test("SFX resolve from Openverse with the licence on the record and a credit", () => {
|
||||
setup();
|
||||
const result = spawnResolve(["--type", "sfx", "--intent", "whoosh", "--project", tmp, "--json"], {
|
||||
env: { HOME: tmp, PATH: tmp },
|
||||
nodeArgs: openverseImport([OPENVERSE_BY_WHOOSH]),
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
assert.equal(parsed.ok, true);
|
||||
assert.equal(parsed.provenance.provider, "bundled.sfx");
|
||||
assert.equal(parsed.advisory?.message, HEYGEN_NOT_FOUND_MESSAGE);
|
||||
assert.equal(parsed.advisory.message.includes("| bash"), false);
|
||||
assert.ok(existsSync(join(tmp, parsed.path)));
|
||||
assert.equal(parsed.provenance.provider, "openverse.audio");
|
||||
assert.equal(parsed.provenance.license, "by");
|
||||
assert.equal(parsed.provenance.credit, OPENVERSE_BY_WHOOSH.attribution);
|
||||
assert.equal(parsed.advisory, undefined);
|
||||
assert.equal(readFileSync(join(tmp, parsed.path), "utf8"), "ID3 openverse bytes");
|
||||
assert.equal(
|
||||
readFileSync(join(tmp, ".media/audio/sfx/CREDITS.md"), "utf8").split("\n").at(-2),
|
||||
`- \`${parsed.path}\`: ${OPENVERSE_BY_WHOOSH.attribution} Source: ${OPENVERSE_BY_WHOOSH.foreign_landing_url}`,
|
||||
);
|
||||
cleanup();
|
||||
});
|
||||
|
||||
// Node args that hold each resolve just after its credits read until two have read (2 s at most), and give each
|
||||
// sound its own bytes.
|
||||
function creditRaceImport(barrier) {
|
||||
const stub = `import fs from "node:fs";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
const read = fs.readFileSync;
|
||||
fs.readFileSync = (path, ...rest) => {
|
||||
if (!String(path).endsWith("CREDITS.md")) return read(path, ...rest);
|
||||
let text, error;
|
||||
try {
|
||||
text = read(path, ...rest);
|
||||
} catch (e) {
|
||||
error = e;
|
||||
}
|
||||
fs.writeFileSync(${JSON.stringify(barrier)} + "/" + process.pid, "");
|
||||
const until = Date.now() + 2000;
|
||||
while (fs.readdirSync(${JSON.stringify(barrier)}).length < 2 && Date.now() < until)
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
|
||||
if (error) throw error;
|
||||
return text;
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
const prior = globalThis.fetch;
|
||||
globalThis.fetch = async (url, init) =>
|
||||
String(url).startsWith("https://cdn.openverse.test/") ? new Response("ID3 " + url) : prior(url, init);`;
|
||||
return [`--import=data:text/javascript,${encodeURIComponent(stub)}`];
|
||||
}
|
||||
|
||||
test("two SFX resolves at once each keep their Openverse credit line", async () => {
|
||||
setup();
|
||||
const barrier = join(tmp, "credit-barrier");
|
||||
mkdirSync(barrier);
|
||||
const runs = await Promise.all(
|
||||
["swoosh", "thud"].map((intent) =>
|
||||
spawnResolveAsync(["--type", "sfx", "--intent", intent, "--project", tmp, "--json"], {
|
||||
env: { HOME: tmp, PATH: tmp },
|
||||
nodeArgs: [
|
||||
...openverseImport([
|
||||
{
|
||||
...OPENVERSE_BY_WHOOSH,
|
||||
id: `ov-${intent}`,
|
||||
url: `https://cdn.openverse.test/${intent}.mp3`,
|
||||
},
|
||||
]),
|
||||
...creditRaceImport(barrier),
|
||||
],
|
||||
}),
|
||||
),
|
||||
);
|
||||
for (const run of runs) assert.equal(run.status, 0, run.stderr);
|
||||
const credited = readFileSync(join(tmp, ".media/audio/sfx/CREDITS.md"), "utf8")
|
||||
.split("\n")
|
||||
.filter((line) => line.startsWith("- "))
|
||||
.map((line) => line.slice(3, line.indexOf("`:")));
|
||||
assert.deepEqual(credited.sort(), runs.map((run) => JSON.parse(run.stdout).path).sort());
|
||||
cleanup();
|
||||
});
|
||||
|
||||
@@ -207,24 +292,28 @@ function writeFakeHeygen(body, exitCode = 0) {
|
||||
return binDir;
|
||||
}
|
||||
|
||||
test("bundled SFX advises update when the HeyGen CLI is outdated", () => {
|
||||
test("an Openverse rate limit resolves SFX from the bundled library", () => {
|
||||
setup();
|
||||
const binDir = writeFakeHeygen('echo "heygen v0.1.5 does not support --headers" >&2', 1);
|
||||
const result = spawnResolve(["--type", "sfx", "--intent", "whoosh", "--project", tmp, "--json"], {
|
||||
env: { HOME: tmp, PATH: binDir },
|
||||
env: { HOME: tmp, PATH: tmp },
|
||||
nodeArgs: openverseImport(429),
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
assert.equal(parsed.provenance.provider, "bundled.sfx");
|
||||
assert.match(parsed.advisory?.message ?? "", /heygen update/);
|
||||
assert.match(
|
||||
result.stderr,
|
||||
/Openverse rate limit reached \(HTTP 429\); using the bundled sound effects/,
|
||||
);
|
||||
assert.equal(existsSync(join(tmp, ".media/audio/sfx/CREDITS.md")), false);
|
||||
cleanup();
|
||||
});
|
||||
|
||||
test("bundled SFX does not advise installation after a healthy catalog miss", () => {
|
||||
test("a sound Openverse has no licensed match for comes from the bundled library", () => {
|
||||
setup();
|
||||
const binDir = writeFakeHeygen(`echo '{"data":[]}'`);
|
||||
const result = spawnResolve(["--type", "sfx", "--intent", "whoosh", "--project", tmp, "--json"], {
|
||||
env: { HOME: tmp, PATH: binDir },
|
||||
env: { HOME: tmp, PATH: tmp },
|
||||
nodeArgs: openverseImport([{ ...OPENVERSE_BY_WHOOSH, license: "by-nc" }]),
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
@@ -233,12 +322,12 @@ test("bundled SFX does not advise installation after a healthy catalog miss", ()
|
||||
cleanup();
|
||||
});
|
||||
|
||||
test("explicit local bundled SFX resolution does not advise installation", () => {
|
||||
test("explicit local bundled SFX resolution never asks Openverse", () => {
|
||||
for (const extraArgs of [["--local-only"], ["--provider", "bundled.sfx"]]) {
|
||||
setup();
|
||||
const result = spawnResolve(
|
||||
["--type", "sfx", "--intent", "whoosh", "--project", tmp, "--json", ...extraArgs],
|
||||
{ env: { HOME: tmp, PATH: tmp } },
|
||||
{ env: { HOME: tmp, PATH: tmp }, nodeArgs: openverseImport([OPENVERSE_BY_WHOOSH]) },
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
@@ -266,7 +355,7 @@ test("music without the HeyGen CLI fails naming it and the host app's own tool",
|
||||
cleanup();
|
||||
});
|
||||
|
||||
test("a sound effect nothing bundled matches fails the same way without the HeyGen CLI", async () => {
|
||||
test("a sound effect nothing matches is a plain miss that never names the HeyGen CLI", async () => {
|
||||
setup();
|
||||
// The local media index answers 404, so the miss never depends on the network.
|
||||
const server = createServer((_req, res) => {
|
||||
@@ -283,12 +372,13 @@ test("a sound effect nothing bundled matches fails the same way without the HeyG
|
||||
PATH: tmp,
|
||||
HYPERFRAMES_REGISTRY: `http://127.0.0.1:${server.address().port}`,
|
||||
},
|
||||
nodeArgs: openverseImport([]),
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 1, result.stderr);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
assert.equal(parsed.code, "heygen_cli_missing");
|
||||
assert.match(parsed.error, /host app's own sound-effect tool/);
|
||||
assert.equal(parsed.code, undefined);
|
||||
assert.equal(parsed.error, 'no provider could resolve sfx: "dog barking"');
|
||||
} finally {
|
||||
server.close();
|
||||
cleanup();
|
||||
@@ -311,20 +401,6 @@ test("music with an outdated HeyGen CLI fails naming the update", () => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
test("human bundled fallback prints the install hint once", () => {
|
||||
setup();
|
||||
const result = spawnResolve(["--type", "sfx", "--intent", "whoosh", "--project", tmp], {
|
||||
env: { HOME: tmp, PATH: tmp },
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.equal(
|
||||
result.stderr.match(/Install the CLI from https:\/\/developers\.heygen\.com\/cli/g)?.length,
|
||||
1,
|
||||
);
|
||||
assert.match(result.stdout, /resolved sfx_001/);
|
||||
cleanup();
|
||||
});
|
||||
|
||||
test("project manifest hit skips providers", () => {
|
||||
setup();
|
||||
const record = makeRecord({ provenance: { prompt: "cached query", provider: "test" } });
|
||||
@@ -1403,6 +1479,7 @@ test("track() posts to MEDIA_USE_TELEMETRY_HOST when set, proving real intercept
|
||||
for (const [provider, type, expected] of [
|
||||
["heygen.tts", "voice", "network_paid"],
|
||||
["heygen.audio.sounds", "bgm", "network_free"],
|
||||
["openverse.audio", "sfx", "network_free"],
|
||||
["bundled.sfx", "sfx", "local"],
|
||||
]) {
|
||||
test(`a resolve won by ${provider} sends provider_tier: ${expected}`, async () => {
|
||||
|
||||
@@ -9,6 +9,10 @@ export const skillModuleCopies = [
|
||||
"packages/cli/src/media-use/lib/media-fetch.mjs",
|
||||
["skills/media-use/scripts/lib/media-fetch.mjs"],
|
||||
],
|
||||
[
|
||||
"packages/cli/src/media-use/lib/openverse.mjs",
|
||||
["skills/media-use/audio/scripts/lib/openverse.mjs"],
|
||||
],
|
||||
[
|
||||
"skills/media-use/audio/scripts/lib/bgm-volume.mjs",
|
||||
workflows.map((skill) => `skills/${skill}/scripts/lib/bgm-volume.mjs`),
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"files": 145
|
||||
},
|
||||
"faceless-explainer": {
|
||||
"hash": "395653774d035d3a",
|
||||
"hash": "47a0783444abc45a",
|
||||
"files": 29
|
||||
},
|
||||
"figma": {
|
||||
@@ -54,8 +54,8 @@
|
||||
"files": 1
|
||||
},
|
||||
"media-use": {
|
||||
"hash": "c4c3c723aa04bb7d",
|
||||
"files": 112
|
||||
"hash": "2816313a2f72e636",
|
||||
"files": 113
|
||||
},
|
||||
"motion-graphics": {
|
||||
"hash": "7e4af0881390ac2e",
|
||||
@@ -66,11 +66,11 @@
|
||||
"files": 169
|
||||
},
|
||||
"pr-to-video": {
|
||||
"hash": "5e69368e66a27179",
|
||||
"hash": "c9a397a3d1ccbbfe",
|
||||
"files": 35
|
||||
},
|
||||
"product-launch-video": {
|
||||
"hash": "14d03406b4fc6904",
|
||||
"hash": "a705b9232e91f51f",
|
||||
"files": 37
|
||||
},
|
||||
"remotion-to-hyperframes": {
|
||||
@@ -78,7 +78,7 @@
|
||||
"files": 77
|
||||
},
|
||||
"slideshow": {
|
||||
"hash": "efc3c0be622fa9a1",
|
||||
"hash": "873bc459cc9c200b",
|
||||
"files": 2
|
||||
},
|
||||
"talking-head-recut": {
|
||||
|
||||
@@ -7,7 +7,7 @@ description: "Turn arbitrary text — an article, notes, a topic, a brief — in
|
||||
|
||||
> **First, keep this skill fresh — confirm with the user before running:** `npx hyperframes skills update faceless-explainer`. A fast no-op when everything is current; otherwise it refreshes this skill plus the core domain skills it depends on before you rely on them.
|
||||
|
||||
> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. When the host app provides its own music or sound-effect tools, use those for music and sound effects.
|
||||
> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/images from the HeyGen catalog, SFX from Openverse, and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. When the host app provides its own music or sound-effect tools, use those for music and sound effects.
|
||||
|
||||
# Faceless Explainer to HyperFrames
|
||||
|
||||
|
||||
@@ -117,6 +117,7 @@ function toProductLaunchMeta(neutral) {
|
||||
offset_s: s.offset_s ?? 0,
|
||||
duration_s: s.duration_s ?? 1,
|
||||
volume: s.volume ?? 0.35,
|
||||
...(s.license && { license: s.license, credit: s.credit }),
|
||||
}));
|
||||
return { bgm, bgm_pending: bgmPending, voices, sfx };
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: media-use
|
||||
description: Agent Media OS for a HyperFrames project. Resolve BGM, SFX, image, icon, brand logo, voice, color grade, or LUT into a frozen local file or paste-ready block + ledger record (one verb, `resolve`); generate via TTS / music / image models when the catalog misses; produce voiceover, transcription, captions, and background removal through one shared audio engine; operate on media (cut / reframe / transform); and reuse assets across projects. Also use for vague feedback that real footage looks dark, flat, boring, should feel retro/camcorder/print/ASCII, needs privacy, or needs a media reveal. When the host app provides its own music or sound-effect tools, use those for music and sound effects; `resolve --type bgm|sfx` needs the heygen CLI. When `HEYGEN_API_BASE` is set, HeyGen calls go through that host with no CLI sign-in.
|
||||
description: Agent Media OS for a HyperFrames project. Resolve BGM, SFX, image, icon, brand logo, voice, color grade, or LUT into a frozen local file or paste-ready block + ledger record (one verb, `resolve`); generate via TTS / music / image models when the catalog misses; produce voiceover, transcription, captions, and background removal through one shared audio engine; operate on media (cut / reframe / transform); and reuse assets across projects. Also use for vague feedback that real footage looks dark, flat, boring, should feel retro/camcorder/print/ASCII, needs privacy, or needs a media reveal. When the host app provides its own music or sound-effect tools, use those for music and sound effects; `resolve --type bgm` needs the heygen CLI, while `--type sfx` needs no account (Openverse, then a bundled library). When `HEYGEN_API_BASE` is set, HeyGen calls go through that host with no CLI sign-in.
|
||||
---
|
||||
|
||||
**Plugin installs:** Before setup or freshness commands, follow [plugin execution rules](../hyperframes/references/plugin-installation.md) when this skill is inside a HyperFrames plugin. Standalone installs keep the update instructions below.
|
||||
@@ -13,7 +13,7 @@ Only when `HEYGEN_API_BASE` is set in your environment (a host app set it and pa
|
||||
|
||||
First run otherwise (no `HEYGEN_API_BASE`), when you will use HeyGen media (catalog search, TTS, avatar video): install and sign in to the `heygen` CLI (the free-usage path), then verify with `npx hyperframes media-use resolve --doctor`. Setup and providers: `references/setup-providers.md`.
|
||||
|
||||
Music and sound effects inside a host app: when the app you run in gives you its own music or sound-effect tools, use those. Without `HEYGEN_API_BASE`, `resolve --type bgm` and `--type sfx` search the HeyGen catalog through the `heygen` CLI; without it they fail and say so (`sfx` still answers from its bundled library).
|
||||
Music and sound effects inside a host app: when the app you run in gives you its own music or sound-effect tools, use those. Without `HEYGEN_API_BASE`, `resolve --type bgm` searches the HeyGen catalog through the `heygen` CLI and fails without it, saying so. `resolve --type sfx` searches Openverse (CC0 and CC BY only, no account) and falls back to its bundled library offline; credit CC BY sounds as `audio/references/sfx.md` says.
|
||||
|
||||
Without `HEYGEN_API_BASE`, before generating a voiceover or an avatar video, tell the person: signing in to the heygen CLI with OAuth (`heygen auth login --oauth`) gives a free allowance for TTS voiceover and avatar videos, while an API key bills API credits.
|
||||
|
||||
@@ -28,7 +28,7 @@ Returns one line: `resolved <id> → <path> (<type>, <metadata>)`. All search no
|
||||
| Type | One-line intent |
|
||||
| ------- | -------------------------------------------------------------------------------- |
|
||||
| `bgm` | background music (HeyGen catalog via the `heygen` CLI, 10k+ tracks) |
|
||||
| `sfx` | sound effects (bundled 19-file library + catalog via the `heygen` CLI) |
|
||||
| `sfx` | sound effects (Openverse CC0 / CC BY, bundled 21-file library offline) |
|
||||
| `image` | photos, backgrounds (HeyGen asset search, 75k+ vectors) |
|
||||
| `icon` | icons, symbols (transparent) |
|
||||
| `logo` | official brand marks (theSVG → GitHub avatar → favicon; never redrawn) |
|
||||
|
||||
@@ -4,13 +4,13 @@
|
||||
|
||||
Run `npx hyperframes auth status` to see what's configured and which engines a workflow will use (see the skill's **Preflight** section). Keys resolve in this order — **first match wins**:
|
||||
|
||||
| Provider | Resolution order (first non-empty wins) | Local deps when used |
|
||||
| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
|
||||
| **HeyGen** (TTS + BGM/SFX retrieval) | `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; `$HEYGEN_CONFIG_DIR` overrides the dir; written by `hyperframes auth login`) | none (REST) |
|
||||
| **ElevenLabs** (TTS fallback) | `$ELEVENLABS_API_KEY` | `pip install elevenlabs` |
|
||||
| **Lyria** (BGM fallback) | `$GEMINI_API_KEY` → `$GOOGLE_API_KEY` | `pip install google-genai` |
|
||||
| **Kokoro** (TTS, no key) | always — final voice fallback | `pip install kokoro-onnx soundfile` |
|
||||
| **MusicGen** (BGM, no key) | always — final music fallback | `pip install transformers torch soundfile numpy` |
|
||||
| Provider | Resolution order (first non-empty wins) | Local deps when used |
|
||||
| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
|
||||
| **HeyGen** (TTS + BGM retrieval) | `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; `$HEYGEN_CONFIG_DIR` overrides the dir; written by `hyperframes auth login`) | none (REST) |
|
||||
| **ElevenLabs** (TTS fallback) | `$ELEVENLABS_API_KEY` | `pip install elevenlabs` |
|
||||
| **Lyria** (BGM fallback) | `$GEMINI_API_KEY` → `$GOOGLE_API_KEY` | `pip install google-genai` |
|
||||
| **Kokoro** (TTS, no key) | always — final voice fallback | `pip install kokoro-onnx soundfile` |
|
||||
| **MusicGen** (BGM, no key) | always — final music fallback | `pip install transformers torch soundfile numpy` |
|
||||
|
||||
`hyperframes auth login` (browser OAuth) is the recommended setup: one sign-in, every project, no per-repo `.env`. An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`; both are tagged with `X-HeyGen-Source: cli`. OAuth CLI users can consume the web-plan free allowance for HeyGen TTS (10 min/month); API keys follow the normal API billing path. With no HeyGen credential, voice/BGM run fully locally (Kokoro / MusicGen) — `hyperframes auth status` and `hyperframes doctor` both report whether those local deps are installed.
|
||||
|
||||
|
||||
@@ -1,35 +1,41 @@
|
||||
# Sound effects (SFX)
|
||||
|
||||
Named sound effects, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/sfx.mjs`). **Provider-gated** by the engine's one switch — whether a HeyGen credential is present, decided once (not per cue):
|
||||
Named sound effects, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/sfx.mjs`). No account or key is needed:
|
||||
|
||||
- **HeyGen credential present → retrieve every cue** from HeyGen's audio library (`/v3/audio/sounds`, `type=sound_effects`, `min_score=0.4`). Search-and-download, **not** generation. The bundled library is NOT consulted.
|
||||
- **No credential → the bundled 21-file library** (`assets/sfx/` + `manifest.json`): match each cue name, copy the matched file into the project. Offline, deterministic, free.
|
||||
- **Openverse first, per cue.** Each cue is searched on [Openverse](https://openverse.org)'s free audio API (most sound effects come from Freesound). Search-and-download, **not** generation.
|
||||
- **The bundled 21-file library is the fallback** (`assets/sfx/` + `manifest.json`): a cue Openverse has no usable match for, and every cue once Openverse is offline or rate limited, is matched against it and the file copied into the project.
|
||||
|
||||
There is no `npx hyperframes sfx` command. SFX is never generated — it is retrieved (online) or taken from the bundled library (offline).
|
||||
There is no `npx hyperframes sfx` command. SFX is never generated — it is retrieved from Openverse (online) or taken from the bundled library (offline). `npx hyperframes media-use resolve --type sfx` follows the same order (`--local-only` skips Openverse).
|
||||
|
||||
## Licences
|
||||
|
||||
Only **CC0** and **CC BY** sounds are used: never NonCommercial, ShareAlike or NoDerivatives. Each CC BY sound is credited in the project's `assets/sfx/CREDITS.md` (title, creator, licence, source link; for `resolve`, in a `CREDITS.md` beside the sound files it saves) and its `audio_meta.sfx[]` entry carries `license` and `credit`. Tell the person to include those credits wherever the video is published.
|
||||
|
||||
## Cues — request → meta
|
||||
|
||||
Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The engine flattens these into cues, resolves them per the switch, dedupes identical `(id, name)` pairs (the same effect named twice downloads/copies once), and writes `audio_meta.sfx[]`:
|
||||
Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The engine flattens these into cues, resolves each, dedupes identical `(id, name)` pairs (the same effect named twice downloads/copies once), and writes `audio_meta.sfx[]`:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"id": "3", // joins the cue to the caller's model (frame / scene / segment)
|
||||
"name": "whoosh",
|
||||
"file": "assets/sfx/whoosh.mp3", // downloaded or copied, relative to project root
|
||||
"source": "heygen" | "local", // which route resolved it
|
||||
"source": "openverse" | "local", // which route resolved it
|
||||
"offset_s": 0, // delay from the line's start
|
||||
"duration_s": 0.57,
|
||||
"volume": 0.35 // SFX sit UNDER voice + BGM
|
||||
"volume": 0.35, // SFX sit UNDER voice + BGM
|
||||
"license": "by", // Openverse only: "cc0" or "by"
|
||||
"credit": "\"Swoosh\" by Ana is licensed under CC BY 4.0. …" // Openverse only
|
||||
}
|
||||
```
|
||||
|
||||
A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never blocks a render. Neither route replaces a file of yours already at the output name: the cue gets the next free name (`whoosh-2.mp3`), reported as an anomaly, and `file` carries the real path.
|
||||
|
||||
## HeyGen retrieval (credentialed)
|
||||
## Openverse
|
||||
|
||||
`searchSounds(name, "sound_effects", { limit: 3, minScore: 0.4 })` → top hit → `assets/sfx/<slug>.mp3`. Results are ranked by `score` (each carries a presigned `audio_url`, `duration`, `description`). The floor is **0.4** because good SFX hits score ~0.5–0.67 — below the API's default `0.7`, which would silently drop most named cues (only whoosh/swoosh-family clears 0.7). `duration_s` comes from the result (else 1.0). Name effects concretely (`glass shatter`, not `dramatic sound`); a vague query returns a poor match.
|
||||
One search per distinct effect name (20 results, `license=cc0,by`; each result's licence is checked again). The pick keeps Openverse's relevance order, prefers clips that fit a cue (15 s at most; past ~4 s ranks lower) and CC0 over CC BY when they are close. The file lands at `assets/sfx/<slug>.<ext>` and `duration_s` comes from the result. Name effects concretely (`glass shatter`, not `dramatic sound`); a vague query returns a poor match. Anonymous use allows 20 searches a minute and 200 a day: past that, the rest of the run uses the bundled library.
|
||||
|
||||
## Bundled library (no credential)
|
||||
## Bundled library (fallback)
|
||||
|
||||
21 curated files in `assets/sfx/`, indexed by `manifest.json` — `{ file, duration, description }` per key (e.g. `whoosh`, `pop`, `click`, `chime`, `riser`, `impact-bass-1`, `glitch-1`, `typing`, …). A cue name resolves by **manifest key, file basename, or slug**, so `whoosh`, `whoosh.mp3`, or `"ui click"` (→ slug) all match. Matched files are copied into the project's `assets/sfx/`; `duration_s` comes from the manifest, so timing is known **offline** — e.g. `riser` is 10.03s, so trigger it at `climax − 10.03s`. The manifest's `description` field carries placement hints per effect; read `assets/sfx/manifest.json` for the full set and usage.
|
||||
|
||||
@@ -37,6 +43,6 @@ A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never bl
|
||||
|
||||
- **Volume ~0.35.** SFX must sit under narration and BGM, not fight them.
|
||||
- **No match → skip, don't fail.** A missing effect logs an anomaly and moves on; never a render blocker.
|
||||
- **Retrieval (credentialed) or bundled library (offline) — never generation.** You search HeyGen by text, or match a name against the 21-file manifest.
|
||||
- **One asset per distinct name.** Reuse across lines is deduped to a single download/copy, many cues.
|
||||
- **The switch is global, not per cue.** With a credential, retrieval handles even the long tail (effects not in the 21); without one, only the 21 bundled names resolve.
|
||||
- **Retrieval (Openverse) or bundled library (offline) — never generation.**
|
||||
- **One asset per distinct name.** Reuse across lines is deduped to a single search and download, many cues.
|
||||
- **Credit CC BY sounds.** `CREDITS.md` lists them; keep it with the project.
|
||||
|
||||
@@ -6,12 +6,12 @@
|
||||
//
|
||||
// node <MEDIA_DIR>/scripts/audio.mjs --request ./audio_request.json --hyperframes . --out ./audio_meta.json
|
||||
//
|
||||
// The three capabilities degrade on ONE switch — whether HeyGen is configured
|
||||
// (credential present, NOT the CLI). This mirrors the table in ../SKILL.md:
|
||||
// TTS and BGM degrade on ONE switch — whether HeyGen is configured (credential
|
||||
// present, NOT the CLI). SFX needs no credential. This mirrors ../SKILL.md:
|
||||
//
|
||||
// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
|
||||
// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
|
||||
// SFX : HeyGen retrieve → (no credential) bundled 19-file library
|
||||
// SFX : Openverse (CC0 / CC BY) → (offline, rate limited, no match) bundled library
|
||||
//
|
||||
// ── audio_request.json (input) ────────────────────────────────────────────────
|
||||
// {
|
||||
@@ -31,7 +31,7 @@
|
||||
// bgm: { path, volume, mode, query?, duration_s? } | null,
|
||||
// bgm_pending, bgm_provider, bgm_pid, bgm_log, bgm_mode, bgm_target_duration_s, …,
|
||||
// voices: [ { id, path, duration_s, words: [{id,text,start,end}] } ],
|
||||
// sfx: [ { id, name, file, source, offset_s, duration_s, volume } ],
|
||||
// sfx: [ { id, name, file, source, offset_s, duration_s, volume, license?, credit? } ],
|
||||
// total_duration_s }
|
||||
//
|
||||
// --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a
|
||||
@@ -52,7 +52,7 @@ import {
|
||||
withWordIds,
|
||||
} from "./lib/tts.mjs";
|
||||
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs";
|
||||
import { mapWithConcurrency } from "./lib/concurrency.mjs";
|
||||
import { openAudioMeta } from "./lib/audio-meta.mjs";
|
||||
import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
|
||||
@@ -83,7 +83,7 @@ const ttsConcurrency = Math.max(1, Number(process.env.HYPERFRAMES_TTS_CONCURRENC
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request.json")));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json")));
|
||||
const sfxLibDir = resolve(flag("sfx-lib", join(HERE, "..", "assets", "sfx")));
|
||||
const sfxLibDir = resolve(flag("sfx-lib", BUNDLED_SFX_DIR));
|
||||
const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
|
||||
const onlyArg = flag("only", "tts,bgm,sfx");
|
||||
const only = new Set(
|
||||
@@ -275,12 +275,12 @@ if (only.has("sfx")) {
|
||||
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
|
||||
.filter((c) => c.name),
|
||||
);
|
||||
const headers = heygenOK && cues.length ? heygenAuthHeaders() : null;
|
||||
const res = await resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir });
|
||||
const res = await resolveSfx({ cues, hyperframesDir, sfxLibDir });
|
||||
sfx = res.sfx;
|
||||
anomalies.push(...res.anomalies);
|
||||
const openverse = sfx.filter((s) => s.source === "openverse").length;
|
||||
console.error(
|
||||
`· sfx: ${sfx.length} cue(s) resolved (${heygenOK ? "heygen retrieval" : "bundled library"})`,
|
||||
`· sfx: ${sfx.length} cue(s) resolved (${openverse} Openverse, ${sfx.length - openverse} bundled library)`,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,11 @@ import { resolveSfx } from "./lib/sfx.mjs";
|
||||
|
||||
// Proves the relocated engine (skills/media-use/audio/) still resolves its
|
||||
// bundled SFX library from the moved location — the path most likely to break
|
||||
// on a subtree move. Offline (heygenOK:false), no network.
|
||||
// on a subtree move. Openverse is unreachable unless a test serves it.
|
||||
const offline = async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
};
|
||||
globalThis.fetch = offline;
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const sfxLibDir = join(HERE, "..", "assets", "sfx"); // same offset the engine uses
|
||||
@@ -21,7 +25,6 @@ test("bundled SFX library resolves from the relocated path", async () => {
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
@@ -39,7 +42,6 @@ test("an unknown cue is reported, not fatal", async () => {
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "definitely-not-a-real-sfx" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
@@ -59,7 +61,6 @@ test("a person's own file under a bundled name survives, and the cue gets the ne
|
||||
const resolve = () =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
@@ -89,18 +90,24 @@ test("two effects never share a file when one's name is taken by the person, run
|
||||
try {
|
||||
mkdirSync(join(dir, "assets", "sfx"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets", "sfx", "glitch.mp3"), "the person's own glitch");
|
||||
const realFetch = globalThis.fetch;
|
||||
globalThis.fetch = async (url) => {
|
||||
const query = new URL(url).searchParams.get("query");
|
||||
const query = new URL(url).searchParams.get("q");
|
||||
if (query)
|
||||
return Response.json({ data: [{ audio_url: `https://sound.test/${query}`, score: 0.6 }] });
|
||||
return Response.json({
|
||||
results: [
|
||||
{
|
||||
license: "cc0",
|
||||
filetype: "mp3",
|
||||
duration: 500,
|
||||
url: `https://sound.test/${encodeURIComponent(query)}`,
|
||||
},
|
||||
],
|
||||
});
|
||||
return new Response(`bytes of ${new URL(url).pathname}`);
|
||||
};
|
||||
const run = async (names) => {
|
||||
const { sfx } = await resolveSfx({
|
||||
cues: names.map((name, index) => ({ id: String(index), name })),
|
||||
heygenOK: true,
|
||||
headers: {},
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
@@ -119,7 +126,7 @@ test("two effects never share a file when one's name is taken by the person, run
|
||||
"assets/sfx/glitch-2.mp3",
|
||||
]);
|
||||
} finally {
|
||||
globalThis.fetch = realFetch;
|
||||
globalThis.fetch = offline;
|
||||
}
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
|
||||
@@ -43,7 +43,12 @@ export function voicePaths(hyperframesDir, lines, anomalies) {
|
||||
return paths;
|
||||
}
|
||||
|
||||
const SFX_SOURCES = { heygen: "search", local: "bundled" };
|
||||
// A cue's source as the manifest records it, and who provided it. A source missing here records as the person's
|
||||
// file, so every rerun would write a numbered copy beside it.
|
||||
const SFX_SOURCES = {
|
||||
openverse: { source: "search", provider: "openverse.audio" },
|
||||
local: { source: "bundled", provider: "bundled.sfx" },
|
||||
};
|
||||
|
||||
// The files one engine run wrote, each with how it was made, for the project's media manifest.
|
||||
export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx }) {
|
||||
@@ -73,8 +78,7 @@ export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields
|
||||
}
|
||||
if (only.has("sfx")) {
|
||||
for (const cue of new Map(sfx.map((entry) => [entry.file, entry])).values()) {
|
||||
const source = SFX_SOURCES[cue.source];
|
||||
const provider = source === "search" ? "heygen" : "bundled.sfx";
|
||||
const { source, provider } = SFX_SOURCES[cue.source];
|
||||
assets.push({
|
||||
path: cue.file,
|
||||
type: "sfx",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
|
||||
@@ -97,7 +97,7 @@ test("music and sound effects are marked by where they came from", () => {
|
||||
sfx: [
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/glass.mp3", name: "glass", source: "heygen" },
|
||||
{ file: "assets/sfx/glass.mp3", name: "glass", source: "openverse" },
|
||||
],
|
||||
});
|
||||
|
||||
@@ -106,11 +106,30 @@ test("music and sound effects are marked by where they came from", () => {
|
||||
[
|
||||
["assets/bgm/track.mp3", "search", undefined],
|
||||
["assets/sfx/whoosh.mp3", "bundled", "bundled.sfx"],
|
||||
["assets/sfx/glass.mp3", "search", "heygen"],
|
||||
["assets/sfx/glass.mp3", "search", "openverse.audio"],
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("a rerun of an Openverse sound effect writes over its own file, not a numbered copy", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/sfx"), { recursive: true });
|
||||
const anomalies = [];
|
||||
for (let run = 0; run < 3; run++) {
|
||||
const file = agentWritePath(dir, "assets/sfx/door-knock.mp3", { anomalies });
|
||||
writeFileSync(join(dir, file), "knock");
|
||||
const sfx = [{ file, name: "door knock", source: "openverse", duration_s: 1 }];
|
||||
recordInManifest(
|
||||
dir,
|
||||
writtenAssets({ only: new Set(["sfx"]), lines: [], voices: [], sfx, ...noBgm }),
|
||||
);
|
||||
}
|
||||
|
||||
assert.deepEqual(readdirSync(join(dir, "assets/sfx")), ["door-knock.mp3"]);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(readManifest(dir).length, 1);
|
||||
});
|
||||
|
||||
test("music made locally is marked generated once it is ready, not while pending", () => {
|
||||
const written = (bgm_pending) =>
|
||||
writtenAssets({
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
// Sound effects from Openverse (https://api.openverse.org): free, no key.
|
||||
// Anonymous use is rate limited (20 requests a minute, 200 a day); callers fall
|
||||
// back to the bundled library when it is unavailable.
|
||||
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
const OPENVERSE_AUDIO_URL = "https://api.openverse.org/v1/audio/";
|
||||
// The only licences media-use accepts: no NonCommercial, ShareAlike or NoDerivatives.
|
||||
export const OPENVERSE_SFX_LICENSES = new Set(["cc0", "by"]);
|
||||
const PLAYABLE_FILETYPES = new Set(["mp3", "wav", "ogg", "flac"]);
|
||||
const MAX_SFX_MS = 15000;
|
||||
const IDEAL_SFX_MS = 4000;
|
||||
const BY_PENALTY = 2;
|
||||
const SEARCH_TIMEOUT_MS = 10000;
|
||||
|
||||
// Openverse text lands in markdown and JSON: one line, no control or markdown-structure characters, bounded.
|
||||
const oneLine = (text) =>
|
||||
String(text ?? "")
|
||||
.replace(/[\p{Cc}`[\]<>\\]+/gu, " ")
|
||||
.trim()
|
||||
.slice(0, 300);
|
||||
// Openverse's audio providers (api.openverse.org/v1/audio/stats/); a new provider's link is dropped until listed.
|
||||
const SOURCE_HOSTS = new Set(["freesound.org", "www.jamendo.com", "commons.wikimedia.org"]);
|
||||
function httpsUrl(url, hosts) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.protocol === "https:" && (!hosts || hosts.has(parsed.hostname))
|
||||
? oneLine(parsed.href)
|
||||
: "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/** Openverse could not answer (offline, rate limited, server error): stop asking it this run. */
|
||||
export class OpenverseUnavailableError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = "OpenverseUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
function usable(result) {
|
||||
return (
|
||||
OPENVERSE_SFX_LICENSES.has(result?.license) &&
|
||||
PLAYABLE_FILETYPES.has(result.filetype) &&
|
||||
typeof result.url === "string" &&
|
||||
result.url.startsWith("https://") &&
|
||||
Number.isFinite(result.duration) &&
|
||||
result.duration > 0 &&
|
||||
result.duration <= MAX_SFX_MS
|
||||
);
|
||||
}
|
||||
|
||||
// Lower is better: relevance position, then seconds past a typical cue length, then CC0 over CC BY.
|
||||
function penalty(result, position) {
|
||||
const longBy = Math.max(0, result.duration - IDEAL_SFX_MS) / 1000;
|
||||
return position + longBy + (result.license === "by" ? BY_PENALTY : 0);
|
||||
}
|
||||
|
||||
/** Best usable sound in Openverse's relevance-ordered results, or null. */
|
||||
export function pickOpenverseSfx(results) {
|
||||
let best = null;
|
||||
results.forEach((result, position) => {
|
||||
if (!usable(result)) return;
|
||||
const score = penalty(result, position);
|
||||
if (!best || score < best.score) best = { result, score };
|
||||
});
|
||||
if (!best) return null;
|
||||
const r = best.result;
|
||||
return {
|
||||
id: r.id,
|
||||
url: r.url,
|
||||
ext: `.${r.filetype}`,
|
||||
title: oneLine(r.title),
|
||||
creator: oneLine(r.creator),
|
||||
license: r.license,
|
||||
licenseUrl: httpsUrl(r.license_url),
|
||||
sourceUrl: httpsUrl(r.foreign_landing_url, SOURCE_HOSTS),
|
||||
credit: oneLine(r.attribution),
|
||||
durationS: Math.round(r.duration) / 1000,
|
||||
};
|
||||
}
|
||||
|
||||
/** Best licensed sound effect for `query`, null on no match; throws OpenverseUnavailableError. */
|
||||
export async function searchOpenverseSfx(query, { fetchImpl = fetch } = {}) {
|
||||
const params = new URLSearchParams({
|
||||
q: query,
|
||||
page_size: "20",
|
||||
license: [...OPENVERSE_SFX_LICENSES].join(","),
|
||||
});
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(`${OPENVERSE_AUDIO_URL}?${params}`, {
|
||||
headers: { Accept: "application/json" },
|
||||
signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS),
|
||||
});
|
||||
} catch (error) {
|
||||
throw new OpenverseUnavailableError(`Openverse unreachable (${error.message})`);
|
||||
}
|
||||
if (response.status === 429)
|
||||
throw new OpenverseUnavailableError("Openverse rate limit reached (HTTP 429)");
|
||||
if (!response.ok) throw new OpenverseUnavailableError(`Openverse HTTP ${response.status}`);
|
||||
let body;
|
||||
try {
|
||||
body = await response.json();
|
||||
} catch {
|
||||
throw new OpenverseUnavailableError("Openverse returned non-JSON");
|
||||
}
|
||||
return pickOpenverseSfx(Array.isArray(body?.results) ? body.results : []);
|
||||
}
|
||||
|
||||
/** Record a CC BY sound's credit for `file` in the project's credits file (one line per file). */
|
||||
export function writeSfxCredit(creditsPath, file, sound) {
|
||||
if (sound.license !== "by") return;
|
||||
// The sound may come from a saved record, not this run's pick: clean it again here.
|
||||
const sourceUrl = httpsUrl(sound.sourceUrl, SOURCE_HOSTS);
|
||||
const source = sourceUrl ? ` Source: ${sourceUrl}` : "";
|
||||
const line = `- \`${file}\`: ${oneLine(sound.credit)}${source}`;
|
||||
let prior = "";
|
||||
try {
|
||||
prior = readFileSync(creditsPath, "utf8");
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
const kept = prior
|
||||
.split("\n")
|
||||
.filter((l) => l.startsWith("- ") && !l.startsWith(`- \`${file}\`:`));
|
||||
const header =
|
||||
"# Sound effect credits\n\nThese sounds are licensed CC BY: credit them wherever the video is published.\n\n";
|
||||
mkdirSync(dirname(creditsPath), { recursive: true });
|
||||
writeFileSync(creditsPath, `${header}${[...kept, line].join("\n")}\n`);
|
||||
}
|
||||
@@ -1,23 +1,26 @@
|
||||
// sfx.mjs — sound effects for the media audio engine. Provider-gated (NOT a
|
||||
// per-cue merge): the decision is made once, by whether HeyGen is configured —
|
||||
// mirroring how TTS and BGM degrade.
|
||||
//
|
||||
// HeyGen credential present → retrieve EVERY cue from HeyGen's audio library
|
||||
// (/v3/audio/sounds, type=sound_effects, min_score=0.4). The bundled
|
||||
// library is NOT consulted.
|
||||
// HeyGen credential absent → resolve cues against the bundled 21-file
|
||||
// library (assets/sfx/manifest.json), copying matched files into the
|
||||
// project. Offline, deterministic, free.
|
||||
// sfx.mjs — sound effects for the media audio engine. Each cue is searched on
|
||||
// Openverse (CC0 / CC BY only, see openverse.mjs). A cue Openverse cannot answer,
|
||||
// and every cue after Openverse is unreachable or rate limited, resolves against
|
||||
// the bundled 21-file library (assets/sfx/manifest.json) instead.
|
||||
//
|
||||
// A cue that matches nothing is skipped (recorded as an anomaly); SFX never
|
||||
// blocks a render. Every cue sits at volume ~0.35, under voice + BGM.
|
||||
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { downloadTo } from "./heygen.mjs";
|
||||
import { OpenverseUnavailableError, searchOpenverseSfx, writeSfxCredit } from "./openverse.mjs";
|
||||
import { latestRecordFor } from "../../../scripts/lib/manifest.mjs";
|
||||
import { agentWritePath } from "./media-record.mjs";
|
||||
|
||||
export const BUNDLED_SFX_DIR = join(
|
||||
dirname(fileURLToPath(import.meta.url)),
|
||||
"..",
|
||||
"..",
|
||||
"assets",
|
||||
"sfx",
|
||||
);
|
||||
const SFX_VOLUME = 0.35;
|
||||
const slug = (s) =>
|
||||
s
|
||||
@@ -27,14 +30,42 @@ const slug = (s) =>
|
||||
.slice(0, 40) || "x";
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// Lookups by manifest key, file basename, and slug of either, so a cue can name
|
||||
// "whoosh", "whoosh.mp3", or "ui click" (→ slug match). { error } when unusable.
|
||||
function loadBundledLibrary(sfxLibDir) {
|
||||
const manifestPath = join(sfxLibDir, "manifest.json");
|
||||
if (!existsSync(manifestPath)) return { error: `no bundled SFX library at ${sfxLibDir}` };
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (e) {
|
||||
return { error: `bundled SFX manifest parse failed (${e.message})` };
|
||||
}
|
||||
const byKey = new Map();
|
||||
for (const [key, entry] of Object.entries(manifest)) {
|
||||
if (!entry?.file || !isFinite(entry.duration)) continue;
|
||||
const rec = { key, file: entry.file, duration: entry.duration };
|
||||
byKey.set(key, rec);
|
||||
byKey.set(entry.file, rec);
|
||||
byKey.set(slug(key), rec);
|
||||
byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
|
||||
}
|
||||
return { byKey };
|
||||
}
|
||||
|
||||
// cues: [{ id, name }] (id = the line/frame/scene the cue fires in). Returns
|
||||
// { sfx: [{ id, name, file, source, offset_s, duration_s, volume }], anomalies }.
|
||||
export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir }) {
|
||||
// { sfx: [{ id, name, file, source, offset_s, duration_s, volume, license?, credit? }], anomalies }.
|
||||
export async function resolveSfx({ cues, hyperframesDir, sfxLibDir = BUNDLED_SFX_DIR }) {
|
||||
const sfx = [];
|
||||
const anomalies = [];
|
||||
const destDir = join(hyperframesDir, "assets", "sfx");
|
||||
// Each effect's file this run: one effect named twice gets one file, and two effects never share one.
|
||||
const fileFor = new Map();
|
||||
const taken = () => new Set(fileFor.values());
|
||||
// One Openverse sound per effect name this run, so a repeated cue costs no second request.
|
||||
const openverseFor = new Map();
|
||||
let openverseDown = false;
|
||||
let library;
|
||||
|
||||
// Dedupe identical (id,name) cues — the same effect named twice in one line
|
||||
// downloads/copies once.
|
||||
@@ -46,112 +77,82 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL
|
||||
return true;
|
||||
});
|
||||
|
||||
if (heygenOK) {
|
||||
for (const { id, name } of uniq) {
|
||||
async function fromOpenverse({ id, name }) {
|
||||
const key = `openverse:${slug(name)}`;
|
||||
let entry = openverseFor.get(key);
|
||||
if (!entry) {
|
||||
const sound = await searchOpenverseSfx(name);
|
||||
if (!sound) return null;
|
||||
const file = agentWritePath(hyperframesDir, `assets/sfx/${slug(name)}${sound.ext}`, {
|
||||
anomalies,
|
||||
taken: taken(),
|
||||
});
|
||||
try {
|
||||
// SFX hits score low (~0.5–0.67), below the API's default 0.7 which
|
||||
// silently drops most named cues — floor to 0.4. (BGM/music score high
|
||||
// and keep the default.)
|
||||
const results = await searchSounds(name, "sound_effects", headers, {
|
||||
limit: 3,
|
||||
minScore: 0.4,
|
||||
});
|
||||
if (!results.length) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): no HeyGen match — skipped`);
|
||||
continue;
|
||||
}
|
||||
const top = results[0];
|
||||
const file =
|
||||
fileFor.get(slug(name)) ??
|
||||
agentWritePath(hyperframesDir, `assets/sfx/${slug(name)}.mp3`, {
|
||||
anomalies,
|
||||
taken: new Set(fileFor.values()),
|
||||
});
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, file));
|
||||
fileFor.set(slug(name), file);
|
||||
sfx.push({
|
||||
id,
|
||||
name,
|
||||
file,
|
||||
source: "heygen",
|
||||
offset_s: 0,
|
||||
duration_s: typeof top.duration === "number" ? r3(top.duration) : 1.0,
|
||||
volume: SFX_VOLUME,
|
||||
});
|
||||
await downloadTo(sound.url, join(hyperframesDir, file));
|
||||
} catch (e) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): retrieval failed — ${e.message}`);
|
||||
anomalies.push(`sfx "${name}" (id ${id}): Openverse download failed — ${e.message}`);
|
||||
return null;
|
||||
}
|
||||
fileFor.set(key, file);
|
||||
writeSfxCredit(join(destDir, "CREDITS.md"), file, sound);
|
||||
entry = {
|
||||
file,
|
||||
source: "openverse",
|
||||
duration_s: r3(sound.durationS),
|
||||
license: sound.license,
|
||||
credit: sound.credit,
|
||||
};
|
||||
openverseFor.set(key, entry);
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
return { id, name, ...entry, offset_s: 0, volume: SFX_VOLUME };
|
||||
}
|
||||
|
||||
// ── offline: bundled library ──
|
||||
const manifestPath = join(sfxLibDir, "manifest.json");
|
||||
if (!existsSync(manifestPath)) {
|
||||
if (uniq.length)
|
||||
anomalies.push(`no HeyGen credential and no SFX library at ${sfxLibDir} — all cues dropped`);
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (e) {
|
||||
anomalies.push(`SFX manifest parse failed (${e.message}) — all cues dropped`);
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
// Build lookups: by manifest key, by file basename, and by slug of either, so
|
||||
// a cue can name "whoosh", "whoosh.mp3", or "ui click" (→ slug match).
|
||||
const byKey = new Map();
|
||||
for (const [key, entry] of Object.entries(manifest)) {
|
||||
if (!entry?.file || !isFinite(entry.duration)) continue;
|
||||
const rec = { key, file: entry.file, duration: entry.duration };
|
||||
byKey.set(key, rec);
|
||||
byKey.set(entry.file, rec);
|
||||
byKey.set(slug(key), rec);
|
||||
byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
for (const { id, name } of uniq) {
|
||||
function fromBundled({ id, name }) {
|
||||
library ??= loadBundledLibrary(sfxLibDir);
|
||||
if (library.error) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): ${library.error} — skipped`);
|
||||
return null;
|
||||
}
|
||||
const { byKey } = library;
|
||||
const hit = byKey.get(name) ?? byKey.get(slug(name));
|
||||
if (!hit) {
|
||||
const known = [...new Set([...byKey.values()].map((v) => v.key))].slice(0, 8).join(", ");
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): not in bundled library — skipped (have: ${known}…)`,
|
||||
);
|
||||
continue;
|
||||
return null;
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
const src = join(sfxLibDir, hit.file);
|
||||
const library = existsSync(src) ? readFileSync(src) : null;
|
||||
const bytes = existsSync(src) ? readFileSync(src) : null;
|
||||
const isLibraryCopy = (rel) =>
|
||||
library &&
|
||||
bytes &&
|
||||
existsSync(join(hyperframesDir, rel)) &&
|
||||
readFileSync(join(hyperframesDir, rel)).equals(library);
|
||||
readFileSync(join(hyperframesDir, rel)).equals(bytes);
|
||||
const destRel =
|
||||
fileFor.get(hit.file) ??
|
||||
agentWritePath(hyperframesDir, `assets/sfx/${hit.file}`, {
|
||||
anomalies,
|
||||
taken: new Set(fileFor.values()),
|
||||
taken: taken(),
|
||||
// An unrecorded copy of the library file is one an engine run made before the manifest had it.
|
||||
reusable: (rel) => !latestRecordFor(hyperframesDir, rel) && isLibraryCopy(rel),
|
||||
});
|
||||
const dest = join(hyperframesDir, destRel);
|
||||
// The bundled library may be incomplete: some installs of the skill ship
|
||||
// manifest.json without the actual mp3s. Pushing an sfx entry that points at
|
||||
// a file we never copied produces a dangling reference that silently drops
|
||||
// downstream ("not on disk"). Surface it as a loud anomaly and skip the cue
|
||||
// instead, so the audio_meta never references a missing file.
|
||||
if (library) {
|
||||
// Some installs of the skill ship manifest.json without the mp3s. An entry
|
||||
// pointing at a file we never copied would silently drop downstream, so
|
||||
// skip the cue with a loud anomaly instead.
|
||||
if (bytes) {
|
||||
if (!isLibraryCopy(destRel)) copyFileSync(src, dest);
|
||||
} else if (!existsSync(dest)) {
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
|
||||
`library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` +
|
||||
`restore assets/sfx/*.mp3, or configure a HeyGen credential for retrieval.`,
|
||||
`restore assets/sfx/*.mp3.`,
|
||||
);
|
||||
continue;
|
||||
return null;
|
||||
} // else the engine's earlier copy at dest stands in for the file this install lacks
|
||||
fileFor.set(hit.file, destRel);
|
||||
sfx.push({
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
file: destRel,
|
||||
@@ -159,7 +160,37 @@ export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxL
|
||||
offset_s: 0,
|
||||
duration_s: r3(hit.duration),
|
||||
volume: SFX_VOLUME,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
for (const cue of uniq) {
|
||||
let entry = null;
|
||||
if (!openverseDown) {
|
||||
try {
|
||||
entry = await fromOpenverse(cue);
|
||||
} catch (e) {
|
||||
if (!(e instanceof OpenverseUnavailableError)) throw e;
|
||||
openverseDown = true;
|
||||
anomalies.push(`${e.message}: the remaining SFX cues use the bundled library`);
|
||||
}
|
||||
}
|
||||
entry ??= fromBundled(cue);
|
||||
if (entry) sfx.push(entry);
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
|
||||
/**
|
||||
* One sound effect for `query`, saved under the project's assets/sfx/ (the entry point for hosts).
|
||||
* { sound: { file, source, duration_s, license?, credit? } | null, anomalies }.
|
||||
*/
|
||||
export async function findSoundEffect(query, { hyperframesDir, sfxLibDir = BUNDLED_SFX_DIR }) {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: query }],
|
||||
hyperframesDir,
|
||||
sfxLibDir,
|
||||
});
|
||||
if (!sfx.length) return { sound: null, anomalies };
|
||||
const { file, source, duration_s, license, credit } = sfx[0];
|
||||
return { sound: { file, source, duration_s, ...(license && { license, credit }) }, anomalies };
|
||||
}
|
||||
|
||||
@@ -4,9 +4,18 @@ import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readFileSync, rmSync
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { appendRecord } from "../../../scripts/lib/manifest.mjs";
|
||||
import { resolveSfx } from "./sfx.mjs";
|
||||
import { BUNDLED_SFX_DIR, findSoundEffect, resolveSfx } from "./sfx.mjs";
|
||||
|
||||
// Offline (no HeyGen) SFX resolution: the bundled library may ship manifest.json
|
||||
// No real network: each test serves Openverse's answers; by default it is unreachable.
|
||||
const offline = async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
};
|
||||
let serve = offline;
|
||||
const UNREACHABLE =
|
||||
"Openverse unreachable (fetch failed): the remaining SFX cues use the bundled library";
|
||||
globalThis.fetch = (url, init) => serve(String(url), init);
|
||||
|
||||
// Offline SFX resolution: the bundled library may ship manifest.json
|
||||
// without the actual mp3s. The old code copied only when the source existed but
|
||||
// pushed the sfx entry unconditionally — producing a dangling reference that
|
||||
// silently dropped downstream ("not on disk"). These tests lock in the loud
|
||||
@@ -37,7 +46,6 @@ test("offline: copies and references a present bundled file", async () => {
|
||||
writeFileSync(join(libDir, "whoosh.mp3"), "ID3-fake-bytes");
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
@@ -45,7 +53,7 @@ test("offline: copies and references a present bundled file", async () => {
|
||||
assert.equal(sfx[0].file, "assets/sfx/whoosh.mp3");
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.ok(existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "mp3 copied into project");
|
||||
assert.equal(anomalies.length, 0);
|
||||
assert.deepEqual(anomalies, [UNREACHABLE]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -58,13 +66,12 @@ test("offline: a matched-but-missing bundled file yields an anomaly and NO dangl
|
||||
);
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0, "no dangling entry for a file that was never copied");
|
||||
assert.equal(anomalies.length, 1);
|
||||
assert.match(anomalies[0], /missing from the offline library/);
|
||||
assert.equal(anomalies.length, 2);
|
||||
assert.match(anomalies[1], /missing from the offline library/);
|
||||
assert.ok(!existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "nothing copied");
|
||||
});
|
||||
});
|
||||
@@ -78,10 +85,9 @@ const whooshLibrary = (libDir, withFile) => {
|
||||
};
|
||||
const record = (projDir, source) =>
|
||||
appendRecord(projDir, { id: "sfx_001", type: "sfx", path: "assets/sfx/whoosh.mp3", source });
|
||||
const offline = (libDir, projDir) =>
|
||||
const bundledOnly = (libDir, projDir) =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
@@ -93,13 +99,13 @@ test("offline: the engine's earlier copy stands in for a library file this insta
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "engine copy");
|
||||
record(projDir, "bundled");
|
||||
|
||||
const { sfx, anomalies } = await offline(libDir, projDir);
|
||||
const { sfx, anomalies } = await bundledOnly(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
["assets/sfx/whoosh.mp3"],
|
||||
);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.deepEqual(anomalies, [UNREACHABLE]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -110,7 +116,7 @@ test("offline: a person's file with the library's bytes stays theirs once record
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "ID3-library-bytes");
|
||||
record(projDir, "existing");
|
||||
|
||||
const { sfx } = await offline(libDir, projDir);
|
||||
const { sfx } = await bundledOnly(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
@@ -119,3 +125,207 @@ test("offline: a person's file with the library's bytes stays theirs once record
|
||||
assert.equal(readFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "utf8"), "ID3-library-bytes");
|
||||
});
|
||||
});
|
||||
|
||||
const libraryWith = (libDir, names) => {
|
||||
const manifest = {};
|
||||
for (const name of names) {
|
||||
manifest[name] = { file: `${name}.mp3`, duration: 0.5 };
|
||||
writeFileSync(join(libDir, `${name}.mp3`), `library ${name}`);
|
||||
}
|
||||
writeFileSync(join(libDir, "manifest.json"), JSON.stringify(manifest));
|
||||
};
|
||||
const sound = (overrides = {}) => ({
|
||||
id: "ov-1",
|
||||
title: "Swoosh",
|
||||
creator: "Ana",
|
||||
license: "by",
|
||||
foreign_landing_url: "https://freesound.org/s/1",
|
||||
url: "https://cdn.freesound.test/swoosh.mp3",
|
||||
filetype: "mp3",
|
||||
duration: 820,
|
||||
attribution: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
...overrides,
|
||||
});
|
||||
// Openverse stand-in: `answer(query)` gives a search response; any other URL is the sound file.
|
||||
function openverse(answer) {
|
||||
const searches = [];
|
||||
serve = async (url) => {
|
||||
const query = new URL(url).searchParams.get("q");
|
||||
if (!url.startsWith("https://api.openverse.org/")) return new Response(`bytes of ${url}`);
|
||||
searches.push(query);
|
||||
return answer(query);
|
||||
};
|
||||
return searches;
|
||||
}
|
||||
const run = (libDir, projDir, names) =>
|
||||
resolveSfx({
|
||||
cues: names.map((name, index) => ({ id: String(index + 1), name })),
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
|
||||
test("openverse: each cue gets the best licensed sound, with its licence and credit", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["whoosh"]);
|
||||
openverse(() => Response.json({ results: [sound()] }));
|
||||
try {
|
||||
const { sfx, anomalies } = await run(libDir, projDir, ["whoosh"]);
|
||||
assert.deepEqual(sfx, [
|
||||
{
|
||||
id: "1",
|
||||
name: "whoosh",
|
||||
file: "assets/sfx/whoosh.mp3",
|
||||
source: "openverse",
|
||||
duration_s: 0.82,
|
||||
license: "by",
|
||||
credit: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
offset_s: 0,
|
||||
volume: 0.35,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(
|
||||
readFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "utf8"),
|
||||
"bytes of https://cdn.freesound.test/swoosh.mp3",
|
||||
);
|
||||
assert.match(
|
||||
readFileSync(join(projDir, "assets/sfx/CREDITS.md"), "utf8"),
|
||||
/- `assets\/sfx\/whoosh\.mp3`: "Swoosh" by Ana is licensed under CC BY 4\.0\. Source: https:\/\/freesound\.org\/s\/1/,
|
||||
);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: a rate limit sends this and every later cue to the bundled library, asking once", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["whoosh", "pop"]);
|
||||
const searches = openverse(() => new Response("slow down", { status: 429 }));
|
||||
try {
|
||||
const { sfx, anomalies } = await run(libDir, projDir, ["whoosh", "pop"]);
|
||||
assert.deepEqual(searches, ["whoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file, source }) => [file, source]),
|
||||
[
|
||||
["assets/sfx/whoosh.mp3", "local"],
|
||||
["assets/sfx/pop.mp3", "local"],
|
||||
],
|
||||
);
|
||||
assert.equal(readFileSync(join(projDir, "assets/sfx/pop.mp3"), "utf8"), "library pop");
|
||||
assert.deepEqual(anomalies, [
|
||||
"Openverse rate limit reached (HTTP 429): the remaining SFX cues use the bundled library",
|
||||
]);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: a cue with no licensed match falls back to the bundled library alone", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["pop"]);
|
||||
const searches = openverse((query) =>
|
||||
Response.json({ results: query === "pop" ? [sound({ license: "by-nc" })] : [sound()] }),
|
||||
);
|
||||
try {
|
||||
const { sfx } = await run(libDir, projDir, ["pop", "swoosh"]);
|
||||
assert.deepEqual(searches, ["pop", "swoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file, source }) => [file, source]),
|
||||
[
|
||||
["assets/sfx/pop.mp3", "local"],
|
||||
["assets/sfx/swoosh.mp3", "openverse"],
|
||||
],
|
||||
);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: a failed download falls back to the bundled file with an anomaly", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["whoosh"]);
|
||||
serve = async (url) =>
|
||||
url.startsWith("https://api.openverse.org/")
|
||||
? Response.json({ results: [sound()] })
|
||||
: new Response("gone", { status: 404 });
|
||||
try {
|
||||
const { sfx, anomalies } = await run(libDir, projDir, ["whoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file, source }) => [file, source]),
|
||||
[["assets/sfx/whoosh.mp3", "local"]],
|
||||
);
|
||||
assert.match(anomalies[0], /Openverse download failed — download HTTP 404/);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: one effect named in two lines is searched and downloaded once", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, []);
|
||||
const searches = openverse(() => Response.json({ results: [sound({ license: "cc0" })] }));
|
||||
try {
|
||||
const { sfx } = await run(libDir, projDir, ["whoosh", "whoosh"]);
|
||||
assert.deepEqual(searches, ["whoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ id, file }) => [id, file]),
|
||||
[
|
||||
["1", "assets/sfx/whoosh.mp3"],
|
||||
["2", "assets/sfx/whoosh.mp3"],
|
||||
],
|
||||
);
|
||||
assert.equal(
|
||||
existsSync(join(projDir, "assets/sfx/CREDITS.md")),
|
||||
false,
|
||||
"CC0 needs no credit",
|
||||
);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("findSoundEffect: a host gets the saved file, licence, credit and duration for one query", async () => {
|
||||
await withDirs(async ({ projDir }) => {
|
||||
const searches = openverse(() => Response.json({ results: [sound()] }));
|
||||
try {
|
||||
assert.deepEqual(await findSoundEffect("whoosh", { hyperframesDir: projDir }), {
|
||||
sound: {
|
||||
file: "assets/sfx/whoosh.mp3",
|
||||
source: "openverse",
|
||||
duration_s: 0.82,
|
||||
license: "by",
|
||||
credit: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
},
|
||||
anomalies: [],
|
||||
});
|
||||
assert.deepEqual(searches, ["whoosh"]);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("findSoundEffect: offline, the skill's own bundled library answers", async () => {
|
||||
await withDirs(async ({ projDir }) => {
|
||||
const { sound, anomalies } = await findSoundEffect("whoosh", { hyperframesDir: projDir });
|
||||
assert.deepEqual(sound, {
|
||||
file: "assets/sfx/whoosh.mp3",
|
||||
source: "local",
|
||||
duration_s: sound.duration_s,
|
||||
});
|
||||
assert.deepEqual(
|
||||
readFileSync(join(projDir, sound.file)),
|
||||
readFileSync(join(BUNDLED_SFX_DIR, "whoosh.mp3")),
|
||||
);
|
||||
assert.deepEqual(anomalies, [UNREACHABLE]);
|
||||
assert.equal(
|
||||
(await findSoundEffect("no such sound at all", { hyperframesDir: projDir })).sound,
|
||||
null,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,7 +12,7 @@ node <SKILL_DIR>/audio/scripts/audio.mjs --request ./audio_request.json --out ./
|
||||
- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
|
||||
- **Gemini narration** is opt-in with `provider: "gemini"` and an API key or service-account credentials. `tts_model` defaults to `gemini-3.8-flash-tts`; 3.8 Flash-Lite, 3.1 Flash Preview, and 2.5 Pro/Flash Preview TTS are also supported. `style` directs delivery; a line's style overrides the request's style. CLI overrides: `--tts-model`, `--style`. See `audio/references/tts.md` for a complete request. The existing automatic provider order is unchanged.
|
||||
- **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`.
|
||||
- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work.
|
||||
- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music retrieval; SFX come from Openverse with no account (`audio/references/sfx.md`). Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work.
|
||||
- If BGM took the generate path (`bgm_pending: true`), run `audio/scripts/wait-bgm.mjs` before final render.
|
||||
|
||||
Single-shot helpers: `audio/scripts/heygen-tts.mjs` (one voice file). Transcription / background removal / captions use the `hyperframes` CLI (`transcribe`, `remove-background`), see the per-topic guides in `audio/references/` (`tts.md`, `bgm.md`, `sfx.md`, `transcribe.md`, `remove-background.md`, `captions/`).
|
||||
|
||||
@@ -11,7 +11,7 @@ Returns one line: `resolved <id> → <path> (<type>, <metadata>)`
|
||||
| Type | What it finds | Provider / cascade |
|
||||
| ------- | -------------------------------- | ------------------------------------------------------------ |
|
||||
| `bgm` | Background music | HeyGen audio catalog (10k+ tracks) |
|
||||
| `sfx` | Sound effects | Bundled 19-file library + HeyGen catalog |
|
||||
| `sfx` | Sound effects | Openverse (CC0 / CC BY), then the bundled 21-file library |
|
||||
| `image` | Photos, backgrounds | HeyGen asset search (75k+ vectors) |
|
||||
| `icon` | Icons, symbols | HeyGen asset search (type=icon) |
|
||||
| `logo` | Official brand marks | theSVG → GitHub org avatar → domain favicon |
|
||||
|
||||
@@ -9,7 +9,7 @@ heygen update # free usage needs the OAuth-capable CLI (v0.3.0+)
|
||||
heygen auth login --oauth # OAuth = free subscription credits; --api-key bills API credits
|
||||
```
|
||||
|
||||
This unlocks the FREE path for bgm/sfx/image/icon catalog search, TTS (voice), and avatar videos. Sign in with `--oauth` — the free allowance rides on the OAuth session (an API key bills API credits instead). **media-use requires heygen >= v0.3.0 uniformly** (the OAuth free-usage path needs it), so `--doctor` nudges older CLIs to update even for API-key-only use. Before resolving anything, verify setup with:
|
||||
This unlocks the FREE path for bgm/image/icon catalog search, TTS (voice), and avatar videos. Sign in with `--oauth` — the free allowance rides on the OAuth session (an API key bills API credits instead). **media-use requires heygen >= v0.3.0 uniformly** (the OAuth free-usage path needs it), so `--doctor` nudges older CLIs to update even for API-key-only use. Before resolving anything, verify setup with:
|
||||
|
||||
```bash
|
||||
npx hyperframes media-use resolve --doctor
|
||||
@@ -41,7 +41,8 @@ in `references/operations.md`).
|
||||
|
||||
| Type | Provider / path |
|
||||
| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| bgm/sfx | heygen catalog free-usage path; a host app's own music and sound tools come first |
|
||||
| bgm | heygen catalog free-usage path; a host app's own music tools come first |
|
||||
| sfx | Openverse (CC0 / CC BY, no account), bundled library offline; a host app's own sound tools come first |
|
||||
| image | heygen search free-usage path; optional local mflux; codex `image_gen` upsell |
|
||||
| voice | heygen tts free-usage path; optional local **Kokoro** (free, on-device) |
|
||||
| icon | heygen asset search free-usage path |
|
||||
@@ -67,21 +68,21 @@ HeyGen-only types, that means no fresh resolve.
|
||||
## CLI tools used (what to run, and how to enable each)
|
||||
|
||||
`resolve` auto-cascades; each provider shells one CLI. HeyGen is the
|
||||
free-usage path for bgm/sfx/image/icon catalog search, TTS (voice), and avatar
|
||||
free-usage path for bgm/image/icon catalog search, TTS (voice), and avatar
|
||||
video, so those capabilities need `heygen` installed and authenticated. Local
|
||||
tools are OPT-IN alternatives where they exist; install one to unlock its free,
|
||||
private, on-device path instead of or ahead of HeyGen for that type. Only
|
||||
`ffmpeg`/`ffprobe` are strictly required for the tool to run at all.
|
||||
|
||||
| Tool | Serves | Install |
|
||||
| ------------------ | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `ffmpeg`/`ffprobe` | adopt probing, smart-grade signalstats, cut, duck bake, loudnorm | system package (`brew install ffmpeg`) |
|
||||
| `heygen` | catalog (bgm/sfx/image/icon) + TTS (voice) + avatar video — the free-usage path | install through [verified HeyGen release instructions](https://developers.heygen.com/cli), then `heygen auth login --oauth` (needs >= v0.3.0) |
|
||||
| `mflux-generate` | local image gen (FLUX), best-for-RAM | `uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux==0.9.6` |
|
||||
| `codex` | image gen upsell (ChatGPT sub) | Codex CLI, logged in via ChatGPT (owns its own auth) |
|
||||
| `parakeet-mlx` | local transcription (default ASR, best) on Apple Silicon | `uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx` |
|
||||
| `ltx-2-mlx` | local video gen | `git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras` |
|
||||
| `npx hyperframes` | Kokoro TTS (voice), Parakeet or whisper.cpp (transcribe), remove-background | via the hyperframes CLI; Parakeet on macOS, Windows and glibc 2.32+ Linux: `npx hyperframes models install parakeet`; whisper.cpp is built on first use (Homebrew on macOS, else git+cmake) |
|
||||
| Tool | Serves | Install |
|
||||
| ------------------ | --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `ffmpeg`/`ffprobe` | adopt probing, smart-grade signalstats, cut, duck bake, loudnorm | system package (`brew install ffmpeg`) |
|
||||
| `heygen` | catalog (bgm/image/icon) + TTS (voice) + avatar video — the free-usage path | install through [verified HeyGen release instructions](https://developers.heygen.com/cli), then `heygen auth login --oauth` (needs >= v0.3.0) |
|
||||
| `mflux-generate` | local image gen (FLUX), best-for-RAM | `uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux==0.9.6` |
|
||||
| `codex` | image gen upsell (ChatGPT sub) | Codex CLI, logged in via ChatGPT (owns its own auth) |
|
||||
| `parakeet-mlx` | local transcription (default ASR, best) on Apple Silicon | `uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx` |
|
||||
| `ltx-2-mlx` | local video gen | `git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras` |
|
||||
| `npx hyperframes` | Kokoro TTS (voice), Parakeet or whisper.cpp (transcribe), remove-background | via the hyperframes CLI; Parakeet on macOS, Windows and glibc 2.32+ Linux: `npx hyperframes models install parakeet`; whisper.cpp is built on first use (Homebrew on macOS, else git+cmake) |
|
||||
|
||||
The RAM-graded local-model shortlist + exact per-tier install/invoke lives in
|
||||
the CLI's bundled media-use engine, not in the standalone skill. Without a tool on PATH, its provider
|
||||
|
||||
@@ -107,8 +107,7 @@ export function recordInPlace(
|
||||
description: description || basename(path),
|
||||
...(duration != null && { duration: Math.round(duration * 10) / 10 }),
|
||||
};
|
||||
mkdirSync(mediaDir(projectDir), { recursive: true });
|
||||
return withLock(mediaDir(projectDir), () => {
|
||||
return withMediaLock(projectDir, () => {
|
||||
const latest = latestRecordFor(projectDir, path);
|
||||
const same = ["source", "description", "duration"].every(
|
||||
(key) => latest?.[key] === fields[key],
|
||||
@@ -203,6 +202,12 @@ function withLock(dir, fn) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs `fn` holding the project's `.media/.lock`, so concurrent resolves take turns on what it guards. */
|
||||
export function withMediaLock(projectDir, fn) {
|
||||
mkdirSync(mediaDir(projectDir), { recursive: true });
|
||||
return withLock(mediaDir(projectDir), fn);
|
||||
}
|
||||
|
||||
// Atomically allocate the next free id for `type` AND reserve it on disk, so a
|
||||
// slow download/copy between allocation and appendRecord can't let a concurrent
|
||||
// caller grab the same id (the MU-23 clobber). Under the lock we take the max id
|
||||
@@ -210,10 +215,8 @@ function withLock(dir, fn) {
|
||||
// O_EXCL-create an empty marker under core's atomic temp name, which Studio's
|
||||
// project history skips. Returns { id, localPath, markerPath }.
|
||||
export function allocateId(projectDir, type, ext) {
|
||||
mkdirSync(mediaDir(projectDir), { recursive: true });
|
||||
const typeDir = typeDirPath(projectDir, type);
|
||||
mkdirSync(typeDir, { recursive: true });
|
||||
return withLock(mediaDir(projectDir), () => {
|
||||
mkdirSync(typeDirPath(projectDir, type), { recursive: true });
|
||||
return withMediaLock(projectDir, () => {
|
||||
const id = nextFreeId(projectDir, type);
|
||||
const localPath = `.media/${typeSubdir(type)}/${id}${ext}`;
|
||||
// Same shape as core atomicTempPath: an empty file at the final name would be committed.
|
||||
|
||||
@@ -7,7 +7,7 @@ description: "Turn a GitHub pull request (a PR URL, owner/repo#N, or 'this PR' i
|
||||
|
||||
> **First, keep this skill fresh — confirm with the user before running:** `npx hyperframes skills update pr-to-video`. A fast no-op when everything is current; otherwise it refreshes this skill plus the core domain skills it depends on before you rely on them.
|
||||
|
||||
> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. When the host app provides its own music or sound-effect tools, use those for music and sound effects.
|
||||
> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/images from the HeyGen catalog, SFX from Openverse, and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. When the host app provides its own music or sound-effect tools, use those for music and sound effects.
|
||||
|
||||
# PR to HyperFrames
|
||||
|
||||
|
||||
@@ -117,6 +117,7 @@ function toProductLaunchMeta(neutral) {
|
||||
offset_s: s.offset_s ?? 0,
|
||||
duration_s: s.duration_s ?? 1,
|
||||
volume: s.volume ?? 0.35,
|
||||
...(s.license && { license: s.license, credit: s.credit }),
|
||||
}));
|
||||
return { bgm, bgm_pending: bgmPending, voices, sfx };
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ description: "Turn a product or marketing URL, pasted script, or brief into a pr
|
||||
|
||||
> **First, keep this skill fresh — confirm with the user before running:** `npx hyperframes skills update product-launch-video`. A fast no-op when everything is current; otherwise it refreshes this skill plus the core domain skills it depends on before you rely on them.
|
||||
|
||||
> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. When the host app provides its own music or sound-effect tools, use those for music and sound effects.
|
||||
> **media-use**: Before sourcing audio/images/logos, call `/media-use` to resolve BGM/images from the HeyGen catalog, SFX from Openverse, and brand logos from their official sources. Run `--adopt` first to register existing assets. See `/media-use` skill. When the host app provides its own music or sound-effect tools, use those for music and sound effects.
|
||||
|
||||
> **figma source**: If the source is a figma.com URL, run `/figma` first — asset export, brand tokens, and components/storyboard reconstruction if needed — then build this workflow from its output. Don't drive Figma via raw MCP tools directly: that skips SVG sanitization, `.media/manifest.jsonl` provenance, and brand-token `var()` binding, so a later brand change can't propagate without a full re-import.
|
||||
|
||||
|
||||
@@ -115,6 +115,7 @@ function toProductLaunchMeta(neutral) {
|
||||
offset_s: s.offset_s ?? 0,
|
||||
duration_s: s.duration_s ?? 1,
|
||||
volume: s.volume ?? 0.35,
|
||||
...(s.license && { license: s.license, credit: s.credit }),
|
||||
}));
|
||||
return { bgm, bgm_pending: bgmPending, voices, sfx };
|
||||
}
|
||||
|
||||
@@ -875,7 +875,7 @@ Do NOT add a mute button inside the composition. The `#sfx-mute` coral button pa
|
||||
</script>
|
||||
```
|
||||
|
||||
**Sourcing SFX files:** use the HeyGen MCP `search_audio_sounds` tool with `type=sound_effects` and keywords like "whoosh", "click", "transition". Download the results to a local `sfx/` directory next to `demo.html` and reference them by relative path. Do not fetch SFX at render time — the HyperFrames determinism rule forbids runtime network requests; pre-download and commit them.
|
||||
**Sourcing SFX files:** resolve each one with `npx hyperframes media-use resolve --type sfx --intent "<whoosh | click | transition>" --project .` (Openverse CC0 / CC BY, bundled library offline; CC BY credits land in a `CREDITS.md` beside the resolved files), copy the files to a local `sfx/` directory next to `demo.html` and reference them by relative path. Do not fetch SFX at render time — the HyperFrames determinism rule forbids runtime network requests; pre-download and commit them.
|
||||
|
||||
---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user