mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
Merge 7d97b12149 into 0e558d5916
This commit is contained in:
@@ -19,7 +19,7 @@ Returns one line: `resolved <id> → <path> (<type>, <metadata>)`. All search no
|
||||
|
||||
| Type | One-line intent |
|
||||
| ------- | ----------------------------------------------------------------------------------- |
|
||||
| `bgm` | background music (HeyGen catalog, 10k+ tracks) |
|
||||
| `bgm` | background music (HeyGen catalog or configured ACE-Step generation API) |
|
||||
| `sfx` | sound effects (bundled 19-file library + catalog) |
|
||||
| `image` | photos, backgrounds (HeyGen asset search, 75k+ vectors) |
|
||||
| `icon` | icons, symbols (transparent) |
|
||||
|
||||
@@ -3,15 +3,17 @@
|
||||
One music bed per composition, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/bgm.mjs`). Two routes, chosen by the engine's one switch — whether a HeyGen credential is present:
|
||||
|
||||
- **HeyGen retrieval — the default when credentialed.** Search HeyGen's music catalog by mood, download the top track. No generation; same `~/.heygen` / `$HEYGEN_API_KEY` credential as TTS.
|
||||
- **Local generation (Lyria → MusicGen) — the fallback when there is no credential** (or when asked for explicitly). Generate a WAV from a mood prompt. There is **no `npx hyperframes bgm` command**; the engine spawns `scripts/lyria-recipe.py` or an inline MusicGen script directly.
|
||||
- **ACE-Step remote generation — the preferred generation path when `bgm.provider` or the user provider configuration selects `acestep`.** It submits the native asynchronous API, waits in the existing detached BGM flow, and freezes the completed MP3.
|
||||
- **Local generation (Lyria → MusicGen) — the fallback when there is no credential or configured ACE-Step provider** (or when asked for explicitly). Generate a WAV from a mood prompt. There is **no `npx hyperframes bgm` command**; the engine spawns `scripts/lyria-recipe.py` or an inline MusicGen script directly.
|
||||
|
||||
> **Run the Preflight first — no credential is not a green light to silently generate locally.** Before generating, complete the sign-in **Preflight** (see `../SKILL.md` → Preflight): run `npx hyperframes auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen's music library, or continue offline with local generation). This applies to a one-off "generate a BGM" request just as much as inside a full workflow.
|
||||
|
||||
## Driving it from the request
|
||||
|
||||
`audio_request.json` → `bgm: { mode?, query?, prompt? }`:
|
||||
`audio_request.json` → `bgm: { mode?, provider?, query?, prompt? }`:
|
||||
|
||||
- **`mode`** — `retrieve | generate | none`. Omit for **auto** (retrieve when credentialed, else generate). An **explicit** `retrieve` is strict: no credential ⇒ skip, never a detached generate (so a caller with no `wait-bgm` step, e.g. product-launch, can't get a pending job it won't await).
|
||||
- **`provider`** — `acestep` forces the configured ACE-Step endpoint for generation. Omit to use the user provider default and then the existing HeyGen/Lyria/MusicGen cascade.
|
||||
- **`query`** — the mood, used for retrieval and as a fallback prompt seed (e.g. a storyboard's `music:` field, falling back to `message` → `arc` → `"calm cinematic underscore"`).
|
||||
- **`prompt`** — an explicit full prompt for generation; omit and the engine infers one (see Mood inference). Optional `blob` / `archetype` / `arc` feed that inference.
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ Run `npx hyperframes auth status` to see what's configured and which engines a w
|
||||
| Provider | Resolution order (first non-empty wins) | Local deps when used |
|
||||
| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
|
||||
| **HeyGen** (TTS + BGM/SFX retrieval) | `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; `$HEYGEN_CONFIG_DIR` overrides the dir; written by `hyperframes auth login`) | none (REST) |
|
||||
| **ACE-Step** (BGM generation) | `~/.media/providers.json` `bgm.acestep.base_url`; optional key named by `api_key_env` (default `$ACESTEP_API_KEY`) | none (native REST API) |
|
||||
| **ElevenLabs** (TTS fallback) | `$ELEVENLABS_API_KEY` | `pip install elevenlabs` |
|
||||
| **Lyria** (BGM fallback) | `$GEMINI_API_KEY` → `$GOOGLE_API_KEY` | `pip install google-genai` |
|
||||
| **Kokoro** (TTS, no key) | always — final voice fallback | `pip install kokoro-onnx soundfile` |
|
||||
@@ -22,6 +23,7 @@ Each command downloads its own model on first run and caches it under `~/.cache/
|
||||
- **TTS (ElevenLabs)** — same as HeyGen: API key + `ffmpeg`.
|
||||
- **TTS (Kokoro)** — Kokoro-82M (~311 MB) + voices (~27 MB) in `tts/`. Requires Python 3.8+ with `kokoro-onnx` and `soundfile` (`pip install kokoro-onnx soundfile`). Non-English text also needs `espeak-ng` system-wide.
|
||||
- **BGM (Lyria)** — needs `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `pip install google-genai`. No local model cache.
|
||||
- **BGM (ACE-Step)** — remote native API; no local dependency. Configure `bgm.default` as `acestep`, set `bgm.acestep.base_url`, and optionally set the named key environment variable.
|
||||
- **BGM (MusicGen)** — `pip install transformers torch soundfile`. `facebook/musicgen-small` (~300 MB) cached under `~/.cache/huggingface/` on first run.
|
||||
- **Transcribe** — Whisper model size depending on choice (75 MB – 3.1 GB) in `whisper/`, downloaded from HuggingFace on first use. `whisper.cpp` itself is NOT bundled: the CLI resolves it from PATH, installs via Homebrew (macOS), or builds it from source with git+cmake on first use (`$HYPERFRAMES_WHISPER_PATH` overrides).
|
||||
- **Remove-background** — `u2net_human_seg` (~168 MB ONNX) in `background-removal/models/`. Peak inference RAM ~1.5 GB.
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env node
|
||||
import { resolve } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { generateWithAceStep } from "../../scripts/lib/acestep-provider.mjs";
|
||||
import { freezeUrl } from "../../scripts/lib/freeze.mjs";
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
output: { type: "string" },
|
||||
duration: { type: "string", default: "30" },
|
||||
prompt: { type: "string" },
|
||||
},
|
||||
strict: true,
|
||||
});
|
||||
|
||||
if (!values.output || !values.prompt) {
|
||||
console.error(
|
||||
"usage: acestep-recipe.mjs --output <path> --duration <seconds> --prompt <description>",
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const result = await generateWithAceStep(values.prompt, { duration: Number(values.duration) });
|
||||
if (!result?.url) throw new Error("ACE-Step is not configured");
|
||||
await freezeUrl(result.url, resolve(values.output), { headers: result.downloadHeaders });
|
||||
console.log(`ACE-Step wrote ${values.output}`);
|
||||
@@ -10,7 +10,7 @@
|
||||
// (credential present, NOT the CLI). This mirrors the table in ../SKILL.md:
|
||||
//
|
||||
// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
|
||||
// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
|
||||
// BGM : HeyGen retrieve → configured ACE-Step → Lyria/MusicGen generate
|
||||
// SFX : HeyGen retrieve → (no credential) bundled 19-file library
|
||||
//
|
||||
// ── audio_request.json (input) ────────────────────────────────────────────────
|
||||
@@ -52,6 +52,7 @@ import {
|
||||
withWordIds,
|
||||
} from "./lib/tts.mjs";
|
||||
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
|
||||
import { configuredBgmProvider } from "../../scripts/lib/acestep-provider.mjs";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
import { mapWithConcurrency } from "./lib/concurrency.mjs";
|
||||
|
||||
@@ -83,6 +84,7 @@ const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request.
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json")));
|
||||
const sfxLibDir = resolve(flag("sfx-lib", join(HERE, "..", "assets", "sfx")));
|
||||
const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
|
||||
const acestepRecipe = resolve(flag("acestep-recipe", join(HERE, "acestep-recipe.mjs")));
|
||||
const onlyArg = flag("only", "tts,bgm,sfx");
|
||||
const only = new Set(
|
||||
onlyArg
|
||||
@@ -196,7 +198,11 @@ if (only.has("bgm")) {
|
||||
// a pending job it can't await). Only the UNSET/auto default picks generate
|
||||
// when HeyGen is absent.
|
||||
const explicitMode = bgmModeOverride || request.bgm?.mode || null;
|
||||
let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate");
|
||||
const requestedBgmProvider = request.bgm?.provider || configuredBgmProvider();
|
||||
let mode = noBgm
|
||||
? "none"
|
||||
: explicitMode ||
|
||||
(requestedBgmProvider === "acestep" ? "generate" : heygenOK ? "retrieve" : "generate");
|
||||
if (mode === "retrieve" && !heygenOK) {
|
||||
anomalies.push(
|
||||
"bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)",
|
||||
@@ -232,6 +238,8 @@ if (only.has("bgm")) {
|
||||
durationS: totalDuration || 30,
|
||||
hyperframesDir,
|
||||
lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
|
||||
acestepRecipe: existsSync(acestepRecipe) ? acestepRecipe : null,
|
||||
provider: requestedBgmProvider,
|
||||
seedSeconds,
|
||||
hasVoice,
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@ import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, openSync, closeSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { aceStepConfig, configuredBgmProvider } from "../../../scripts/lib/acestep-provider.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
@@ -115,10 +116,17 @@ export function generateBgmDetached({
|
||||
durationS,
|
||||
hyperframesDir,
|
||||
lyriaRecipe,
|
||||
acestepRecipe,
|
||||
provider,
|
||||
seedSeconds = 28,
|
||||
hasVoice,
|
||||
}) {
|
||||
const rel = "assets/bgm/track.wav";
|
||||
const useAceStep =
|
||||
(provider || configuredBgmProvider()) === "acestep" &&
|
||||
!!aceStepConfig().baseUrl &&
|
||||
!!acestepRecipe &&
|
||||
existsSync(acestepRecipe);
|
||||
const rel = useAceStep ? "assets/bgm/track.mp3" : "assets/bgm/track.wav";
|
||||
const abs = join(hyperframesDir, rel);
|
||||
mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true });
|
||||
const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`);
|
||||
@@ -126,6 +134,25 @@ export function generateBgmDetached({
|
||||
const baseMeta = { path: rel, mode: null, volume: bgmDefaultVolume(hasVoice), pending: true };
|
||||
|
||||
const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe);
|
||||
const fd = openSync(log, "w");
|
||||
|
||||
if (useAceStep) {
|
||||
const proc = spawn(
|
||||
process.execPath,
|
||||
[acestepRecipe, "--output", abs, "--duration", String(targetS), "--prompt", prompt],
|
||||
{ detached: true, stdio: ["ignore", fd, fd] },
|
||||
);
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: "detached-single",
|
||||
provider: "acestep",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
};
|
||||
}
|
||||
|
||||
// Make a backend runnable: prefer Lyria when configured (install google-genai
|
||||
// on demand), else ensure local MusicGen deps. Installs are synchronous here —
|
||||
@@ -134,7 +161,6 @@ export function generateBgmDetached({
|
||||
const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE);
|
||||
if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS);
|
||||
|
||||
const fd = openSync(log, "w");
|
||||
if (useLyria) {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
lyriaRecipe,
|
||||
|
||||
@@ -11,7 +11,7 @@ node <SKILL_DIR>/audio/scripts/audio.mjs --request ./audio_request.json --out ./
|
||||
|
||||
- **Request** `{ provider?, lang?, speed?, lines: [{ id, text, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
|
||||
- **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`.
|
||||
- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. Local/provider-specific generators are explicit alternatives where installed; run `node <SKILL_DIR>/scripts/resolve.mjs --doctor` before assuming retrieval or TTS will work.
|
||||
- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. A configured `acestep.remote` endpoint is the standard generated-BGM path; local/provider-specific generators remain explicit alternatives where installed. Run `node <SKILL_DIR>/scripts/resolve.mjs --doctor` before assuming retrieval or TTS will work.
|
||||
- If BGM took the generate path (`bgm_pending: true`), run `audio/scripts/wait-bgm.mjs` before final render.
|
||||
|
||||
Single-shot helpers: `audio/scripts/heygen-tts.mjs` (one voice file). Transcription / background removal / captions use the `hyperframes` CLI (`transcribe`, `remove-background`), see the per-topic guides in `audio/references/` (`tts.md`, `bgm.md`, `sfx.md`, `transcribe.md`, `remove-background.md`, `captions/`).
|
||||
|
||||
@@ -10,7 +10,7 @@ Returns one line: `resolved <id> → <path> (<type>, <metadata>)`
|
||||
|
||||
| Type | What it finds | Provider / cascade |
|
||||
| ------- | -------------------------------- | ------------------------------------------------------------ |
|
||||
| `bgm` | Background music | HeyGen audio catalog (10k+ tracks) |
|
||||
| `bgm` | Background music | HeyGen catalog; optional ACE-Step remote generation |
|
||||
| `sfx` | Sound effects | Bundled 19-file library + HeyGen catalog |
|
||||
| `image` | Photos, backgrounds | HeyGen asset search (75k+ vectors) |
|
||||
| `icon` | Icons, symbols | HeyGen asset search (type=icon) |
|
||||
@@ -65,6 +65,7 @@ node <SKILL_DIR>/scripts/resolve.mjs --type lut --intent "teal orange blockbuste
|
||||
| `--for` | Analyze a local image/video and add measured adjust suggestions (`grade` only) |
|
||||
| `--local-only` | Offline: skip every network provider (cache + local only) |
|
||||
| `--provider` | Force one generator (e.g. `codex`, `mflux`, `kokoro`, `heygen`) |
|
||||
| `--duration` | Generated BGM length in seconds, 10-600 (default 30) |
|
||||
| `--adopt` | Bulk-import existing assets/ into manifest |
|
||||
| `--doctor` | Check local CLI dependencies; no manifest changes |
|
||||
| `--stats` | Print local usage stats from `.media/` and `~/.media`; no manifest changes |
|
||||
@@ -103,6 +104,25 @@ The deterministic floor still runs automatically: an identical (case/whitespace-
|
||||
4. Search via provider (HeyGen audio catalog, HeyGen asset search), or resolve color locally
|
||||
5. Freeze file to `.media/<type>/`, register in manifest, regenerate `index.md`, auto-promote to `~/.media/`
|
||||
|
||||
### ACE-Step remote generation
|
||||
|
||||
`acestep.remote` is the supported self-hosted BGM generator. Configure it in `~/.media/providers.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"bgm": {
|
||||
"default": "acestep",
|
||||
"acestep": {
|
||||
"base_url": "https://music.example.ts.net",
|
||||
"api_key_env": "ACESTEP_API_KEY"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The key is optional and is read only from the named environment variable. The provider submits the native `/release_task` request, polls `/query_result`, downloads `/v1/audio`, and then follows the same freeze, ledger, and global-cache path as every other provider. Use `--provider acestep --duration 45` to force it for one resolve. `--local-only` always blocks it because a Tailnet service is still a network provider.
|
||||
|
||||
Steps 1 and 3 are the **deterministic floor**: they only auto-reuse an exact-normalized match, never a fuzzy one. Semantic reuse ("close enough") is the agent's explicit call via [Reuse before you resolve](#reuse-before-you-resolve) — it never happens automatically. The agent gets back **one line**; candidates, scores, provenance stay on disk.
|
||||
|
||||
## Adopt existing projects
|
||||
|
||||
@@ -27,7 +27,8 @@ see the ladder and override.
|
||||
|
||||
| Type | Provider / path |
|
||||
| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| bgm/sfx | heygen catalog free-usage path |
|
||||
| bgm | heygen catalog free-usage path; optional self-hosted `acestep.remote` generation configured in `~/.media/providers.json` |
|
||||
| sfx | heygen catalog free-usage path |
|
||||
| image | heygen search free-usage path; optional local mflux; codex `image_gen` upsell |
|
||||
| voice | heygen tts free-usage path; optional local **Kokoro** (free, on-device) |
|
||||
| icon | heygen asset search free-usage path |
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function readUserConfig() {
|
||||
const path = join(homedir(), ".media", "providers.json");
|
||||
if (!existsSync(path)) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8"));
|
||||
return parsed && typeof parsed === "object" ? parsed : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export function configuredBgmProvider() {
|
||||
return readUserConfig()?.bgm?.default || process.env.HYPERFRAMES_BGM_PROVIDER || null;
|
||||
}
|
||||
|
||||
export function aceStepConfig() {
|
||||
const configured = readUserConfig()?.bgm?.acestep || {};
|
||||
const baseUrl = String(process.env.ACESTEP_API_URL || configured.base_url || "").replace(
|
||||
/\/$/,
|
||||
"",
|
||||
);
|
||||
const keyName = configured.api_key_env || "ACESTEP_API_KEY";
|
||||
return { baseUrl, apiKey: process.env[keyName] || "" };
|
||||
}
|
||||
|
||||
function headers(apiKey) {
|
||||
return {
|
||||
"content-type": "application/json",
|
||||
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function requestJson(url, options, fetchImpl) {
|
||||
const response = await fetchImpl(url, options);
|
||||
let body = {};
|
||||
try {
|
||||
body = await response.json();
|
||||
} catch {
|
||||
// The status below is more useful than a JSON parser stack.
|
||||
}
|
||||
if (!response.ok) {
|
||||
const detail = body?.detail || body?.error || `HTTP ${response.status}`;
|
||||
throw new Error(`ACE-Step request failed: ${detail}`);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
function resultObject(value) {
|
||||
let parsed = value;
|
||||
if (typeof parsed === "string") {
|
||||
try {
|
||||
parsed = JSON.parse(parsed);
|
||||
} catch {
|
||||
throw new Error("ACE-Step returned an invalid result payload");
|
||||
}
|
||||
}
|
||||
if (Array.isArray(parsed)) return parsed[0] || null;
|
||||
return parsed && typeof parsed === "object" ? parsed : null;
|
||||
}
|
||||
|
||||
export async function generateWithAceStep(intent, ctx = {}, deps = {}) {
|
||||
const fetchImpl = deps.fetch || fetch;
|
||||
const sleepImpl = deps.sleep || sleep;
|
||||
const now = deps.now || Date.now;
|
||||
const { baseUrl, apiKey } = deps.config || aceStepConfig();
|
||||
if (!baseUrl) return null;
|
||||
|
||||
const duration = Math.max(10, Math.min(600, Number(ctx.duration) || 30));
|
||||
const submitted = await requestJson(
|
||||
`${baseUrl}/release_task`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: headers(apiKey),
|
||||
body: JSON.stringify({
|
||||
prompt: intent,
|
||||
lyrics: "[Instrumental]",
|
||||
thinking: true,
|
||||
use_format: false,
|
||||
audio_duration: duration,
|
||||
audio_format: "mp3",
|
||||
batch_size: 1,
|
||||
model: "acestep-v15-sft",
|
||||
lm_backend: "pt",
|
||||
inference_steps: 50,
|
||||
}),
|
||||
},
|
||||
fetchImpl,
|
||||
);
|
||||
const taskId = submitted?.data?.task_id || submitted?.task_id || submitted?.data?.id;
|
||||
if (!taskId) throw new Error("ACE-Step did not return a task ID");
|
||||
|
||||
const started = now();
|
||||
const timeoutMs = Number(process.env.ACESTEP_POLL_TIMEOUT_MS) || 60 * 60 * 1000;
|
||||
while (now() - started < timeoutMs) {
|
||||
const queried = await requestJson(
|
||||
`${baseUrl}/query_result`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: headers(apiKey),
|
||||
body: JSON.stringify({ task_id_list: [taskId] }),
|
||||
},
|
||||
fetchImpl,
|
||||
);
|
||||
const row = queried?.data?.[0];
|
||||
if (!row) throw new Error(`ACE-Step lost task ${taskId}`);
|
||||
if (Number(row.status) === 2)
|
||||
throw new Error(row.error || row.message || "ACE-Step generation failed");
|
||||
if (Number(row.status) === 1) {
|
||||
const result = resultObject(row.result);
|
||||
if (!result?.file) throw new Error("ACE-Step completed without an audio file");
|
||||
const audioUrl = new URL(String(result.file), `${baseUrl}/`).toString();
|
||||
const metadata = result.metas && typeof result.metas === "object" ? result.metas : {};
|
||||
return {
|
||||
url: audioUrl,
|
||||
downloadHeaders: apiKey ? { authorization: `Bearer ${apiKey}` } : undefined,
|
||||
ext: ".mp3",
|
||||
source: "generate",
|
||||
metadata: {
|
||||
description: intent,
|
||||
duration: Number(metadata.duration) || duration,
|
||||
provider: "acestep.remote",
|
||||
provenance: {
|
||||
endpoint: new URL(baseUrl).origin,
|
||||
task_id: String(taskId),
|
||||
dit_model: result.dit_model || "acestep-v15-sft",
|
||||
lm_model: result.lm_model || null,
|
||||
seed: result.seed_value || null,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
await sleepImpl(now() - started < 30_000 ? 2000 : 5000);
|
||||
}
|
||||
throw new Error(`ACE-Step task ${taskId} exceeded the generation timeout`);
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { generateWithAceStep } from "./acestep-provider.mjs";
|
||||
|
||||
function json(body, status = 200) {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
test("ACE-Step submits, polls, and resolves a generated audio URL", async () => {
|
||||
const calls = [];
|
||||
const responses = [
|
||||
json({ data: { task_id: "task-1" } }),
|
||||
json({ data: [{ task_id: "task-1", status: 0, result: "" }] }),
|
||||
json({
|
||||
data: [
|
||||
{
|
||||
task_id: "task-1",
|
||||
status: 1,
|
||||
result: JSON.stringify([
|
||||
{
|
||||
file: "/v1/audio?path=track.mp3",
|
||||
metas: { duration: 42 },
|
||||
dit_model: "xl",
|
||||
lm_model: "4b",
|
||||
},
|
||||
]),
|
||||
},
|
||||
],
|
||||
}),
|
||||
];
|
||||
let clock = 0;
|
||||
const result = await generateWithAceStep(
|
||||
"cinematic pulse",
|
||||
{ duration: 42 },
|
||||
{
|
||||
config: { baseUrl: "https://music.example", apiKey: "secret" },
|
||||
fetch: async (url, options) => {
|
||||
calls.push({ url, options });
|
||||
return responses.shift();
|
||||
},
|
||||
sleep: async (ms) => {
|
||||
clock += ms;
|
||||
},
|
||||
now: () => clock,
|
||||
},
|
||||
);
|
||||
assert.equal(result.url, "https://music.example/v1/audio?path=track.mp3");
|
||||
assert.equal(result.metadata.provider, "acestep.remote");
|
||||
assert.equal(result.metadata.duration, 42);
|
||||
assert.deepEqual(result.downloadHeaders, { authorization: "Bearer secret" });
|
||||
assert.equal(calls.length, 3);
|
||||
assert.match(calls[0].options.body, /acestep-v15-sft/);
|
||||
assert.match(calls[0].options.body, /"lm_backend":"pt"/);
|
||||
assert.equal(calls[0].options.headers.authorization, "Bearer secret");
|
||||
});
|
||||
|
||||
test("ACE-Step is unavailable when it has no configured endpoint", async () => {
|
||||
assert.equal(await generateWithAceStep("x", {}, { config: { baseUrl: "", apiKey: "" } }), null);
|
||||
});
|
||||
|
||||
test("ACE-Step surfaces failed jobs", async () => {
|
||||
const responses = [
|
||||
json({ data: { task_id: "bad" } }),
|
||||
json({ data: [{ status: 2, error: "out of memory" }] }),
|
||||
];
|
||||
await assert.rejects(
|
||||
generateWithAceStep(
|
||||
"x",
|
||||
{},
|
||||
{
|
||||
config: { baseUrl: "https://music.example", apiKey: "" },
|
||||
fetch: async () => responses.shift(),
|
||||
},
|
||||
),
|
||||
/out of memory/,
|
||||
);
|
||||
});
|
||||
@@ -5,9 +5,9 @@ import { dirname } from "node:path";
|
||||
// 256MB covers any real media asset; raise if 4K video sources ever exceed it.
|
||||
const MAX_FREEZE_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
export async function freezeUrl(url, destPath) {
|
||||
export async function freezeUrl(url, destPath, { headers } = {}) {
|
||||
const where = String(url).slice(0, 80);
|
||||
const res = await fetch(url);
|
||||
const res = await fetch(url, headers ? { headers } : undefined);
|
||||
if (!res.ok) throw new Error(`freeze failed: HTTP ${res.status} for ${where}`);
|
||||
|
||||
// Fail fast on an advertised oversize body before reading a single byte.
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
// (e.g. "make an image with codex").
|
||||
|
||||
import { bgmProvider } from "./bgm-provider.mjs";
|
||||
import { generateWithAceStep } from "./acestep-provider.mjs";
|
||||
import { sfxProvider } from "./sfx-provider.mjs";
|
||||
import { bundledSfxProvider } from "./bundled-sfx-provider.mjs";
|
||||
import { imageProvider, iconProvider } from "./image-provider.mjs";
|
||||
@@ -49,7 +50,10 @@ const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // rem
|
||||
|
||||
// heygen-CLI first. All remote providers are skipped by --local-only.
|
||||
const REGISTRY = {
|
||||
bgm: [N("heygen.audio.sounds", { search: bgmProvider.search })],
|
||||
bgm: [
|
||||
N("heygen.audio.sounds", { search: bgmProvider.search }),
|
||||
N("acestep.remote", { generate: generateWithAceStep }),
|
||||
],
|
||||
sfx: [
|
||||
N("heygen.audio.sounds", { search: sfxProvider.search }),
|
||||
A("bundled.sfx", { search: bundledSfxProvider.search }),
|
||||
|
||||
@@ -42,7 +42,7 @@ test("heygen provider is first for every type it serves", () => {
|
||||
|
||||
test("sanctioned providers only: heygen, local mflux/kokoro/ltx, codex, design spec, logo tiers", () => {
|
||||
const allowed =
|
||||
/^heygen|^bundled\.sfx$|^mflux\.local$|^kokoro\.local$|^ltx\.local$|^codex\.image_gen$|^design_spec$|^svgl$|^simple-icons$|^github\.avatar$|^favicon\.ddg$|^color_grade\.local$|^cube_lut\.local$/;
|
||||
/^heygen|^acestep\.remote$|^bundled\.sfx$|^mflux\.local$|^kokoro\.local$|^ltx\.local$|^codex\.image_gen$|^design_spec$|^svgl$|^simple-icons$|^github\.avatar$|^favicon\.ddg$|^color_grade\.local$|^cube_lut\.local$/;
|
||||
for (const t of listTypes()) {
|
||||
for (const p of getProviders(t)) {
|
||||
assert.ok(allowed.test(p.name), `${t} lists unsanctioned provider: ${p.name}`);
|
||||
@@ -63,6 +63,14 @@ test("image cascade: heygen catalog, then local mflux, then the codex upsell", (
|
||||
assert.ok(codex.network, "codex is network (skipped under --local-only)");
|
||||
});
|
||||
|
||||
test("bgm supports ACE-Step generation without replacing HeyGen retrieval", () => {
|
||||
const ps = getProviders("bgm");
|
||||
assert.equal(ps[0].name, "heygen.audio.sounds");
|
||||
assert.equal(ps[1].name, "acestep.remote");
|
||||
assert.equal(typeof ps[1].generate, "function");
|
||||
assert.ok(ps[1].network, "ACE-Step is skipped by --local-only");
|
||||
});
|
||||
|
||||
test("voice cascade: HeyGen TTS first, Kokoro remains the local fallback", () => {
|
||||
const ps = getProviders("voice");
|
||||
assert.equal(ps[0].name, "heygen.tts", "HeyGen TTS is first when credentials exist");
|
||||
|
||||
@@ -49,6 +49,7 @@ import {
|
||||
versionLessThan,
|
||||
} from "./lib/heygen-cli.mjs";
|
||||
import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs";
|
||||
import { configuredBgmProvider } from "./lib/acestep-provider.mjs";
|
||||
|
||||
const INGEST_TYPES = listTypes();
|
||||
const DEFAULT_EXT = {
|
||||
@@ -88,6 +89,7 @@ const { values: args } = parseArgs({
|
||||
analyze: { type: "boolean", default: false },
|
||||
"local-only": { type: "boolean", default: false },
|
||||
provider: { type: "string" },
|
||||
duration: { type: "string" },
|
||||
"avatar-id": { type: "string" },
|
||||
"voice-id": { type: "string" },
|
||||
json: { type: "boolean", default: false },
|
||||
@@ -125,6 +127,7 @@ Options:
|
||||
--analyze Return --for grade evidence without recording a candidate
|
||||
--local-only Offline: skip every network provider
|
||||
--provider Force one generator (e.g. codex, mflux, kokoro, heygen)
|
||||
--duration Generated BGM duration in seconds (10-600; default: 30)
|
||||
--avatar-id Override the default avatar for heygen.video generation
|
||||
--voice-id Override the default voice for voice/heygen.video generation
|
||||
--json Output JSON instead of one-line result
|
||||
@@ -136,6 +139,7 @@ const projectDir = resolve(args.project);
|
||||
const type = args.type;
|
||||
const intent = args.intent;
|
||||
const entity = args.entity || null;
|
||||
const configuredProvider = args.provider || (type === "bgm" ? configuredBgmProvider() : null);
|
||||
|
||||
if (args.adopt) {
|
||||
const { adoptExistingAssets } = await import("./lib/adopt.mjs");
|
||||
@@ -282,9 +286,9 @@ if (!listTypes().includes(args.type)) {
|
||||
// Forced-provider validation: reject an unknown/unavailable provider name up
|
||||
// front so a typo reads as a typo, not a catalog miss (`no provider could
|
||||
// resolve`). Match rule mirrors runProviders (full name or dotted prefix).
|
||||
if (args.provider && !providerMatches(args.type, args.provider)) {
|
||||
if (configuredProvider && !providerMatches(args.type, configuredProvider)) {
|
||||
console.error(
|
||||
`error: unknown provider "${args.provider}" for type ${args.type} (available: ${providerNamesFor(args.type).join(", ")})`,
|
||||
`error: unknown provider "${configuredProvider}" for type ${args.type} (available: ${providerNamesFor(args.type).join(", ")})`,
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
@@ -396,7 +400,8 @@ async function run() {
|
||||
entity,
|
||||
projectDir,
|
||||
localOnly,
|
||||
provider: args.provider,
|
||||
provider: configuredProvider,
|
||||
duration: args.duration ? Number(args.duration) : undefined,
|
||||
avatarId: args["avatar-id"],
|
||||
voiceId: args["voice-id"],
|
||||
};
|
||||
@@ -498,7 +503,9 @@ async function run() {
|
||||
if (searchResult.localPath) {
|
||||
freezeLocalFile(searchResult.localPath, reservation.fullPath);
|
||||
} else if (searchResult.url) {
|
||||
await freezeUrl(searchResult.url, reservation.fullPath);
|
||||
await freezeUrl(searchResult.url, reservation.fullPath, {
|
||||
headers: searchResult.downloadHeaders,
|
||||
});
|
||||
} else {
|
||||
throw new Error("provider returned no url or localPath");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user