feat(media-use): v2 media OS core (resolve cascade, providers, local generation, telemetry) + retire hyperframes-media

2026-09-12 15:20:13 +00:00 · 2026-07-06 23:41:05 -04:00
parent 676f461f6a
commit 5fe957363d
105 changed files with 2362 additions and 287 deletions
@@ -0,0 +1,326 @@
+// tts.mjs — multi-provider TTS for the media audio engine. The provider chain,
+// auto-detected from env, is the one documented in ../SKILL.md:
+//
+//   1. HeyGen (Starfish)  — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY / ~/.heygen.
+//        Direct v3 REST (NOT `hyperframes tts`, which in the published build is
+//        Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
+//        in the same call, so no separate transcribe pass.
+//   2. ElevenLabs         — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
+//        word timings → caller chains transcribeWav().
+//   3. Kokoro-82M (local) — always available, via the published `hyperframes tts`
+//        CLI. No word timings → caller chains transcribeWav().
+//
+// "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
+// by the CLI — see the note above.
+
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { heygenAuthHeaders, heygenCredential, heygenJSON } from "./heygen.mjs";
+import { pythonInvocation } from "./python.mjs";
+
+// ── provider detection ────────────────────────────────────────────────────────
+export function heygenAvailable() {
+  return heygenCredential() !== null;
+}
+export function elevenlabsAvailable() {
+  if (!process.env.ELEVENLABS_API_KEY) return false;
+  const { cmd, args } = pythonInvocation(["-c", "import elevenlabs"]);
+  const r = spawnSync(cmd, args, {
+    stdio: "ignore",
+  });
+  return r.status === 0;
+}
+
+// First available provider wins; an explicit choice is honored (and validated).
+export function pickProvider(userProvider) {
+  if (userProvider) {
+    if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider))
+      throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`);
+    if (userProvider === "heygen" && !heygenAvailable())
+      throw new Error(
+        "provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx hyperframes auth login`)",
+      );
+    if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
+      throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
+    return userProvider;
+  }
+  return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
+}
+
+// ── voice resolution ──────────────────────────────────────────────────────────
+// HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first
+// English public starfish voice when none is pinned. ElevenLabs/Kokoro have
+// their own defaults.
+export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
+  if (userVoice) return userVoice;
+  if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
+  if (provider === "kokoro") {
+    if (lang === "en") return "am_michael";
+    throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)");
+  }
+  // heygen — pin a fixed English default so the choice is deterministic. The old
+  // "first English voice the API returns" drifts whenever HeyGen re-sorts the
+  // public catalog. Marcia (mature, low female). Override with --voice / request.voice.
+  if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female
+  // Non-English: no fixed default — fall back to the first matching catalog voice.
+  const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
+    headers: heygenAuthHeaders(),
+  });
+  const voices = payload.data ?? payload.voices ?? [];
+  const pick = voices.find((v) => v.language === "English") ?? voices[0];
+  if (!pick) throw new Error("no public starfish voice to default to — pass --voice");
+  return pick.voice_id;
+}
+
+// ── helpers ─────────────────────────────────────────────────────────────────
+export function withWordIds(words) {
+  return (words ?? []).map((w, i) => ({
+    id: `w${i}`,
+    text: w.text,
+    start: w.start,
+    end: w.end,
+  }));
+}
+
+// `ffmpeg -i <file>` prints a `Duration: HH:MM:SS.ms` line to stderr even
+// though it exits non-zero with no output requested. Parsing pulled out as
+// a pure function so the ENOENT fallback below can be tested without
+// depending on whether ffprobe/ffmpeg are actually installed on the
+// machine running the tests.
+export function parseFfmpegDurationBanner(stderrText) {
+  const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderrText ?? "");
+  if (!match) return NaN;
+  const [, hours, minutes, seconds] = match;
+  return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds);
+}
+
+// Some "essentials"-style ffmpeg distributions (common on Windows) ship
+// ffmpeg.exe without ffprobe.exe. ffprobeDuration's caller (audio.mjs)
+// otherwise reads a spurious NaN as "the WAV file is corrupt" and drops an
+// already-successfully-synthesized TTS line, rather than "the tool for
+// measuring it is missing".
+function ffmpegDurationFallback(absPath) {
+  const r = spawnSync("ffmpeg", ["-i", absPath], { encoding: "utf8" });
+  return parseFfmpegDurationBanner(r.stderr);
+}
+
+export function ffprobeDuration(absPath) {
+  const r = spawnSync(
+    "ffprobe",
+    ["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", absPath],
+    { encoding: "utf8" },
+  );
+  if (r.error?.code === "ENOENT") return ffmpegDurationFallback(absPath);
+  if (r.status !== 0) return NaN;
+  return parseFloat(String(r.stdout).trim());
+}
+
+export function resolveNpxCliFromNpmExecPath(
+  npmExecPath = process.env.npm_execpath,
+  pathExists = existsSync,
+) {
+  if (!npmExecPath) return null;
+  const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase();
+  const npxCliPath =
+    fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js");
+  return pathExists(npxCliPath) ? npxCliPath : null;
+}
+
+export function resolveSpawnCommand(
+  cmd,
+  args,
+  opts = {},
+  platform = process.platform,
+  env = process.env,
+  pathExists = existsSync,
+) {
+  if (cmd !== "npx" || platform !== "win32") {
+    return { cmd, args, opts: { stdio: "ignore", ...opts } };
+  }
+
+  // On Windows, npx resolves to npx.cmd, which Node cannot execute directly.
+  // Avoid `shell:true` and the .cmd shim entirely by invoking npm's JS CLI with
+  // node, preserving request-provided values as argv data instead of shell text.
+  const npxCliPath = resolveNpxCliFromNpmExecPath(env.npm_execpath, pathExists);
+  if (!npxCliPath) return null;
+  return {
+    cmd: env.npm_node_execpath || process.execPath,
+    args: [npxCliPath, ...args.map((arg) => String(arg))],
+    opts: { stdio: "ignore", windowsHide: true, ...opts },
+  };
+}
+
+// `platform`/`spawnFn` params (default process.platform / the real spawn)
+// exist so tests can exercise the win32 branch without mocking node:child_process
+// (its ESM exports are non-configurable, so mock.method can't patch it).
+// One-shot so a whole batch of TTS lines doesn't repeat the same diagnostic.
+let _warnedNpxResolution = false;
+/** Test-only: reset the one-shot npx-resolution warning latch. */
+export function _resetNpxResolutionWarnForTests() {
+  _warnedNpxResolution = false;
+}
+
+export function spawnP(
+  cmd,
+  args,
+  opts = {},
+  platform = process.platform,
+  spawnFn = spawn,
+  env = process.env,
+  pathExists = existsSync,
+) {
+  const resolved = resolveSpawnCommand(cmd, args, opts, platform, env, pathExists);
+  if (!resolved) {
+    // resolveSpawnCommand only returns null for the npx-on-win32 case where
+    // npm_execpath isn't set (e.g. audio.mjs invoked directly with `node`, not
+    // through npm/npx). Without this, every call silently returns status:-1 and
+    // stdio:"ignore" hides why — callers just report "TTS failed - omitted" for
+    // every line. Surface the real reason once so it's diagnosable.
+    if (!_warnedNpxResolution) {
+      _warnedNpxResolution = true;
+      console.error(
+        `[media-use] Cannot run "${cmd}" on Windows: npm_execpath is not set, so the ` +
+          `npx JS CLI can't be located. This happens when this script is run directly with ` +
+          `\`node\` instead of through npm/npx. Every "${cmd}" call is being skipped. ` +
+          `Fix: run via \`npx\`/\`npm run\`, or export npm_execpath pointing at your npm-cli.js.`,
+      );
+    }
+    return Promise.resolve({ status: -1 });
+  }
+  return new Promise((resolve) => {
+    const p = spawnFn(resolved.cmd, resolved.args, resolved.opts);
+    p.on("exit", (code) => resolve({ status: code ?? -1 }));
+    p.on("error", () => resolve({ status: -1 }));
+  });
+}
+
+// mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format).
+function transcodeToWav(bytes, destWav) {
+  const td = mkdtempSync(join(tmpdir(), "hf-tts-"));
+  const tmp = join(td, "a.mp3");
+  writeFileSync(tmp, bytes);
+  mkdirSync(dirname(destWav), { recursive: true });
+  const ff = spawnSync(
+    "ffmpeg",
+    ["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav],
+    { stdio: "ignore" },
+  );
+  rmSync(td, { recursive: true, force: true });
+  return ff.status === 0 && existsSync(destWav);
+}
+
+const ELEVENLABS_PY = `
+import os, sys
+from elevenlabs.client import ElevenLabs
+from elevenlabs import save
+client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
+text = open(sys.argv[1]).read()
+audio = client.text_to_speech.convert(
+    text=text, voice_id=sys.argv[2],
+    model_id="eleven_multilingual_v2", output_format="mp3_44100_128",
+)
+save(audio, sys.argv[3])
+`;
+
+// ── synthesize one line ───────────────────────────────────────────────────────
+// Writes wav at wavAbs. Returns { ok, words } — words is the raw
+// [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro
+// (caller must transcribeWav). Never throws; failures return { ok:false }.
+export async function synthesizeOne({
+  provider,
+  text,
+  voiceId,
+  lang = "en",
+  speed = 1.0,
+  wavAbs,
+  hyperframesDir,
+}) {
+  if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
+  if (provider === "elevenlabs") {
+    const { cmd, args } = pythonInvocation([
+      "-c",
+      ELEVENLABS_PY,
+      writeTmpText(text),
+      voiceId,
+      wavAbs,
+    ]);
+    const r = await spawnP(cmd, args, {});
+    return { ok: r.status === 0 && existsSync(wavAbs), words: null };
+  }
+  // kokoro — via the published CLI; --output is relative to the project dir.
+  const wavRel = relTo(hyperframesDir, wavAbs);
+  const args = ["hyperframes", "tts", writeTmpText(text), "--voice", voiceId, "--output", wavRel];
+  if (lang !== "en") args.push("--lang", lang);
+  const r = await spawnP("npx", args, { cwd: hyperframesDir });
+  return { ok: r.status === 0 && existsSync(wavAbs), words: null };
+}
+
+async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }) {
+  try {
+    const body = { text, voice_id: voiceId, speed };
+    if (lang !== "en") body.language = lang;
+    const payload = await heygenJSON(`/voices/speech`, {
+      method: "POST",
+      headers: heygenAuthHeaders(),
+      body,
+    });
+    const inner = payload.data ?? payload;
+    if (!inner.audio_url) return { ok: false, words: null };
+    const res = await fetch(inner.audio_url);
+    if (!res.ok) return { ok: false, words: null };
+    const bytes = Buffer.from(await res.arrayBuffer());
+    // .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The
+    // engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3.
+    if (wavAbs.endsWith(".wav")) {
+      if (!transcodeToWav(bytes, wavAbs)) return { ok: false, words: null };
+    } else {
+      mkdirSync(dirname(wavAbs), { recursive: true });
+      writeFileSync(wavAbs, bytes);
+    }
+    const words = Array.isArray(inner.word_timestamps)
+      ? inner.word_timestamps
+          .filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end))
+          .filter((w) => !/^<.*>$/.test(w.word.trim())) // drop <start>/<end> sentinels
+          .map((w) => ({ text: w.word, start: w.start, end: w.end }))
+      : [];
+    return { ok: true, words };
+  } catch {
+    return { ok: false, words: null };
+  }
+}
+
+// ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
+// flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
+// --dir so parallel scenes don't collide on transcript.json.
+export async function transcribeWav({ wavRel, lang = "en", hyperframesDir }) {
+  const model = lang === "en" ? "small.en" : "small";
+  const td = mkdtempSync(join(tmpdir(), "hf-trans-"));
+  const args = ["hyperframes", "transcribe", wavRel, "--model", model, "--dir", td];
+  if (lang !== "en") args.push("--language", lang);
+  const r = await spawnP("npx", args, { cwd: hyperframesDir });
+  let words = null;
+  if (r.status === 0) {
+    const src = join(td, "transcript.json");
+    if (existsSync(src)) {
+      try {
+        const arr = JSON.parse(readFileSync(src, "utf8"));
+        if (Array.isArray(arr) && arr.length) words = arr;
+      } catch {}
+    }
+  }
+  rmSync(td, { recursive: true, force: true });
+  return words;
+}
+
+// ── tiny local utils ──────────────────────────────────────────────────────────
+function writeTmpText(text) {
+  const td = mkdtempSync(join(tmpdir(), "hf-txt-"));
+  const p = join(td, "line.txt");
+  writeFileSync(p, text);
+  return p;
+}
+function relTo(base, abs) {
+  return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs;
+}