// tts.mjs — multi-provider TTS for the media audio engine. The provider chain, // auto-detected from env, is the one documented in ../SKILL.md: // // 1. HeyGen (Starfish) — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY / ~/.heygen. // Direct v3 REST (NOT `hyperframes tts`, which in the published build is // Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps // in the same call, so no separate transcribe pass. // 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No // word timings → caller chains transcribeWav(). // 3. Kokoro-82M (local) — always available, via the published `hyperframes tts` // CLI. No word timings → caller chains transcribeWav(). // // "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never // by the CLI — see the note above. import { spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { heygenAuthHeaders, heygenCredential, heygenJSON } from "./heygen.mjs"; import { pythonInvocation } from "./python.mjs"; // ── provider detection ──────────────────────────────────────────────────────── export function heygenAvailable() { return heygenCredential() !== null; } export function elevenlabsAvailable() { if (!process.env.ELEVENLABS_API_KEY) return false; const { cmd, args } = pythonInvocation(["-c", "import elevenlabs"]); const r = spawnSync(cmd, args, { stdio: "ignore", }); return r.status === 0; } // First available provider wins; an explicit choice is honored (and validated). export function pickProvider(userProvider) { if (userProvider) { if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider)) throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`); if (userProvider === "heygen" && !heygenAvailable()) throw new Error( "provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx hyperframes auth login`)", ); if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY) throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set"); return userProvider; } return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro"; } // ── voice resolution ────────────────────────────────────────────────────────── // HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first // English public starfish voice when none is pinned. ElevenLabs/Kokoro have // their own defaults. export async function resolveVoiceId({ provider, userVoice, lang = "en" }) { if (userVoice) return userVoice; if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel if (provider === "kokoro") { if (lang === "en") return "am_michael"; throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)"); } // heygen — pin a fixed English default so the choice is deterministic. The old // "first English voice the API returns" drifts whenever HeyGen re-sorts the // public catalog. Marcia (mature, low female). Override with --voice / request.voice. if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female // Non-English: no fixed default — fall back to the first matching catalog voice. const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, { headers: heygenAuthHeaders(), }); const voices = payload.data ?? payload.voices ?? []; const pick = voices.find((v) => v.language === "English") ?? voices[0]; if (!pick) throw new Error("no public starfish voice to default to — pass --voice"); return pick.voice_id; } // ── helpers ───────────────────────────────────────────────────────────────── export function withWordIds(words) { return (words ?? []).map((w, i) => ({ id: `w${i}`, text: w.text, start: w.start, end: w.end, })); } // `ffmpeg -i ` prints a `Duration: HH:MM:SS.ms` line to stderr even // though it exits non-zero with no output requested. Parsing pulled out as // a pure function so the ENOENT fallback below can be tested without // depending on whether ffprobe/ffmpeg are actually installed on the // machine running the tests. export function parseFfmpegDurationBanner(stderrText) { const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderrText ?? ""); if (!match) return NaN; const [, hours, minutes, seconds] = match; return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds); } // Some "essentials"-style ffmpeg distributions (common on Windows) ship // ffmpeg.exe without ffprobe.exe. ffprobeDuration's caller (audio.mjs) // otherwise reads a spurious NaN as "the WAV file is corrupt" and drops an // already-successfully-synthesized TTS line, rather than "the tool for // measuring it is missing". function ffmpegDurationFallback(absPath) { const r = spawnSync("ffmpeg", ["-i", absPath], { encoding: "utf8" }); return parseFfmpegDurationBanner(r.stderr); } export function ffprobeDuration(absPath) { const r = spawnSync( "ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", absPath], { encoding: "utf8" }, ); if (r.error?.code === "ENOENT") return ffmpegDurationFallback(absPath); if (r.status !== 0) return NaN; return parseFloat(String(r.stdout).trim()); } export function resolveNpxCliFromNpmExecPath( npmExecPath = process.env.npm_execpath, pathExists = existsSync, ) { if (!npmExecPath) return null; const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase(); const npxCliPath = fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js"); return pathExists(npxCliPath) ? npxCliPath : null; } export function resolveSpawnCommand( cmd, args, opts = {}, platform = process.platform, env = process.env, pathExists = existsSync, ) { if (cmd !== "npx" || platform !== "win32") { return { cmd, args, opts: { stdio: "ignore", ...opts } }; } // On Windows, npx resolves to npx.cmd, which Node cannot execute directly. // Avoid `shell:true` and the .cmd shim entirely by invoking npm's JS CLI with // node, preserving request-provided values as argv data instead of shell text. const npxCliPath = resolveNpxCliFromNpmExecPath(env.npm_execpath, pathExists); if (!npxCliPath) return null; return { cmd: env.npm_node_execpath || process.execPath, args: [npxCliPath, ...args.map((arg) => String(arg))], opts: { stdio: "ignore", windowsHide: true, ...opts }, }; } // `platform`/`spawnFn` params (default process.platform / the real spawn) // exist so tests can exercise the win32 branch without mocking node:child_process // (its ESM exports are non-configurable, so mock.method can't patch it). export function spawnP( cmd, args, opts = {}, platform = process.platform, spawnFn = spawn, env = process.env, pathExists = existsSync, ) { const resolved = resolveSpawnCommand(cmd, args, opts, platform, env, pathExists); if (!resolved) return Promise.resolve({ status: -1 }); return new Promise((resolve) => { const p = spawnFn(resolved.cmd, resolved.args, resolved.opts); p.on("exit", (code) => resolve({ status: code ?? -1 })); p.on("error", () => resolve({ status: -1 })); }); } // mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format). function transcodeToWav(bytes, destWav) { const td = mkdtempSync(join(tmpdir(), "hf-tts-")); const tmp = join(td, "a.mp3"); writeFileSync(tmp, bytes); mkdirSync(dirname(destWav), { recursive: true }); const ff = spawnSync( "ffmpeg", ["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav], { stdio: "ignore" }, ); rmSync(td, { recursive: true, force: true }); return ff.status === 0 && existsSync(destWav); } const ELEVENLABS_PY = ` import os, sys from elevenlabs.client import ElevenLabs from elevenlabs import save client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"]) text = open(sys.argv[1]).read() audio = client.text_to_speech.convert( text=text, voice_id=sys.argv[2], model_id="eleven_multilingual_v2", output_format="mp3_44100_128", ) save(audio, sys.argv[3]) `; // ── synthesize one line ─────────────────────────────────────────────────────── // Writes wav at wavAbs. Returns { ok, words } — words is the raw // [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro // (caller must transcribeWav). Never throws; failures return { ok:false }. export async function synthesizeOne({ provider, text, voiceId, lang = "en", speed = 1.0, wavAbs, hyperframesDir, }) { if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }); if (provider === "elevenlabs") { const { cmd, args } = pythonInvocation([ "-c", ELEVENLABS_PY, writeTmpText(text), voiceId, wavAbs, ]); const r = await spawnP(cmd, args, {}); return { ok: r.status === 0 && existsSync(wavAbs), words: null }; } // kokoro — via the published CLI; --output is relative to the project dir. const wavRel = relTo(hyperframesDir, wavAbs); const args = ["hyperframes", "tts", writeTmpText(text), "--voice", voiceId, "--output", wavRel]; if (lang !== "en") args.push("--lang", lang); const r = await spawnP("npx", args, { cwd: hyperframesDir }); return { ok: r.status === 0 && existsSync(wavAbs), words: null }; } async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }) { try { const body = { text, voice_id: voiceId, speed }; if (lang !== "en") body.language = lang; const payload = await heygenJSON(`/voices/speech`, { method: "POST", headers: heygenAuthHeaders(), body, }); const inner = payload.data ?? payload; if (!inner.audio_url) return { ok: false, words: null }; const res = await fetch(inner.audio_url); if (!res.ok) return { ok: false, words: null }; const bytes = Buffer.from(await res.arrayBuffer()); // .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The // engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3. if (wavAbs.endsWith(".wav")) { if (!transcodeToWav(bytes, wavAbs)) return { ok: false, words: null }; } else { mkdirSync(dirname(wavAbs), { recursive: true }); writeFileSync(wavAbs, bytes); } const words = Array.isArray(inner.word_timestamps) ? inner.word_timestamps .filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end)) .filter((w) => !/^<.*>$/.test(w.word.trim())) // drop / sentinels .map((w) => ({ text: w.word, start: w.start, end: w.end })) : []; return { ok: true, words }; } catch { return { ok: false, words: null }; } } // ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the // flat [{id,text,start,end}] word array, or null. Each call uses a throwaway // --dir so parallel scenes don't collide on transcript.json. export async function transcribeWav({ wavRel, lang = "en", hyperframesDir }) { const model = lang === "en" ? "small.en" : "small"; const td = mkdtempSync(join(tmpdir(), "hf-trans-")); const args = ["hyperframes", "transcribe", wavRel, "--model", model, "--dir", td]; if (lang !== "en") args.push("--language", lang); const r = await spawnP("npx", args, { cwd: hyperframesDir }); let words = null; if (r.status === 0) { const src = join(td, "transcript.json"); if (existsSync(src)) { try { const arr = JSON.parse(readFileSync(src, "utf8")); if (Array.isArray(arr) && arr.length) words = arr; } catch {} } } rmSync(td, { recursive: true, force: true }); return words; } // ── tiny local utils ────────────────────────────────────────────────────────── function writeTmpText(text) { const td = mkdtempSync(join(tmpdir(), "hf-txt-")); const p = join(td, "line.txt"); writeFileSync(p, text); return p; } function relTo(base, abs) { return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs; }