feat(cli): parakeet ASR engine for transcribe (--engine) + HYPERFRAMES_PYTHON override

packages/cli/src/audio/providers.ts

@@ -1,7 +1,7 @@
 /**
  * Which voice / music engine a workflow will actually use, and whether
  * its local dependencies are present. Mirrors the resolution order the
- * hyperframes-media skill scripts use, so `auth status` and `doctor`
+ * media-use skill scripts use, so `auth status` and `doctor`
  * report the same engine the render pipeline would pick:
  *
  *   voice: HeyGen Starfish → ElevenLabs (key + `elevenlabs`) → Kokoro (local)

packages/cli/src/commands/auth/status-guidance.ts

@@ -63,7 +63,7 @@ function offlineEngineLines(engines?: OfflineEngineLine[]): string[] {
  * so it's left to the docs — not dangled here as a command a fresh machine
  * can't run. Names the local fallback so "no key" never reads as a failure,
  * and never steers users toward a per-repo `.env`. Mirrors the
- * hyperframes-media skill's Preflight section.
+ * media-use skill's Preflight section.
  */
 export function buildUnconfiguredLines(
   ctx: UnconfiguredContext,

packages/cli/src/commands/transcribe.ts

@@ -1,6 +1,8 @@
+// fallow-ignore-file code-duplication
 import { defineCommand } from "citty";
 import type { Example } from "./_examples.js";
 import { existsSync, writeFileSync } from "node:fs";
+import { findParakeet, transcribeWithParakeet } from "../whisper/parakeet.js";
 
 type CaptionExportFormat = "srt" | "vtt";
 
@@ -41,6 +43,12 @@ export default defineCommand({
       description: "Project directory (default: current directory)",
       alias: "d",
     },
+    engine: {
+      type: "string",
+      description:
+        "ASR engine: auto (Parakeet if installed, else whisper), parakeet, or whisper. Default: auto. Parakeet is more accurate and faster; enable with `uv pip install parakeet-mlx`.",
+      alias: "e",
+    },
     model: {
       type: "string",
       description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny.en, base.en, small.en, medium.en, large-v3`,
@@ -111,8 +119,9 @@ export default defineCommand({
       return importTranscript(inputPath, dir, args.json);
     }
 
-    // ── Transcribe mode: run whisper ─────────────────────────────────────
+    // ── Transcribe mode: run the ASR engine ──────────────────────────────
     return transcribeAudio(inputPath, dir, {
+      engine: args.engine,
       model: args.model,
       language: args.language,
       json: args.json,
@@ -213,25 +222,45 @@ async function exportTranscript(
 // Transcribe audio/video with whisper
 // ---------------------------------------------------------------------------
 
+// fallow-ignore-next-line complexity
 async function transcribeAudio(
   inputPath: string,
   dir: string,
-  opts: { model?: string; language?: string; json?: boolean; optional?: boolean },
+  opts: { engine?: string; model?: string; language?: string; json?: boolean; optional?: boolean },
 ): Promise<void> {
   const { transcribe } = await import("../whisper/transcribe.js");
   const { loadTranscript, patchCaptionHtml, stripBeforeOnset } =
     await import("../whisper/normalize.js");
 
+  // Engine: auto (Parakeet if installed, else whisper), or forced parakeet/whisper.
+  const engine = (opts.engine ?? "auto").toLowerCase();
+  if (engine !== "auto" && engine !== "parakeet" && engine !== "whisper") {
+    failWith(`Unknown --engine: ${opts.engine}. Use auto, parakeet, or whisper.`, !!opts.json);
+  }
+  const useParakeet = engine === "parakeet" || (engine === "auto" && !!findParakeet());
+
   const model = opts.model ?? DEFAULT_MODEL;
+  // --model selects the whisper model only; Parakeet uses its own fixed model.
+  if (useParakeet && opts.model && !opts.json) {
+    console.error(
+      c.dim(`  Note: --model applies to the whisper engine only; ignored under Parakeet.`),
+    );
+  }
+  const label = useParakeet ? "Parakeet" : model;
   const spin = opts.json ? null : clack.spinner();
-  spin?.start(`Transcribing with ${c.accent(model)}...`);
+  spin?.start(`Transcribing with ${c.accent(label)}...`);
 
   try {
-    const result = await transcribe(inputPath, dir, {
-      model,
-      language: opts.language,
-      onProgress: spin ? (msg) => spin.message(msg) : undefined,
-    });
+    const result = useParakeet
+      ? transcribeWithParakeet(inputPath, dir, {
+          language: opts.language,
+          onProgress: spin ? (msg) => spin.message(msg) : undefined,
+        })
+      : await transcribe(inputPath, dir, {
+          model,
+          language: opts.language,
+          onProgress: spin ? (msg) => spin.message(msg) : undefined,
+        });
 
     let { words } = loadTranscript(result.transcriptPath);
 
@@ -253,7 +282,8 @@ async function transcribeAudio(
       console.log(
         JSON.stringify({
           ok: true,
-          model,
+          engine: useParakeet ? "parakeet" : "whisper",
+          model: useParakeet ? "parakeet-tdt-0.6b-v3" : model,
           wordCount: words.length,
           durationSeconds: result.durationSeconds,
           speechOnsetSeconds: result.speechOnsetSeconds,
@@ -272,7 +302,15 @@ async function transcribeAudio(
       );
     }
   } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
+    // Surface the last few lines of the ASR subprocess's stderr, which
+    // execFileSync captures but otherwise drops on the floor — that's where
+    // parakeet-mlx / whisper report the actual failure cause.
+    const stderr =
+      err && typeof err === "object" && "stderr" in err && err.stderr
+        ? String(err.stderr).trim().split("\n").slice(-3).join("\n")
+        : "";
+    const base = err instanceof Error ? err.message : String(err);
+    const message = stderr ? `${base}\n${stderr}` : base;
 
     // whisper-cpp is an optional prerequisite, not part of the CLI. When it is
     // simply unavailable (no binary, no toolchain to build one), that is a setup

packages/cli/src/commands/tts.ts

@@ -1,3 +1,4 @@
+// fallow-ignore-file code-duplication
 import { defineCommand } from "citty";
 import type { Example } from "./_examples.js";
 import { existsSync, readFileSync } from "node:fs";
@@ -48,7 +49,7 @@ export default defineCommand({
     output: {
       type: "string",
       description: "Output file path (default: speech.wav in current directory)",
-      alias: "o",
+      alias: ["o", "out"],
     },
     voice: {
       type: "string",
@@ -76,6 +77,7 @@ export default defineCommand({
       default: false,
     },
   },
+  // fallow-ignore-next-line complexity
   async run({ args }) {
     // ── List voices mode ──────────────────────────────────────────────
     if (args.list) {

packages/cli/src/templates/_shared/AGENTS.md

@@ -17,7 +17,7 @@
 
 **Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above.
 
-The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/hyperframes-media`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent.
+The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent.
 
 > **Tailwind v4 projects** (`hyperframes init --tailwind`): see `/hyperframes-core` → `references/tailwind.md`.
 

packages/cli/src/templates/_shared/CLAUDE.md

@@ -17,7 +17,7 @@
 
 **Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above.
 
-The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/hyperframes-media`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent.
+The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent.
 
 > **Tailwind v4 projects** (`hyperframes init --tailwind`): see `/hyperframes-core` → `references/tailwind.md`.
 

packages/cli/src/tts/python.ts

@@ -8,8 +8,21 @@
 
 import { execFileSync } from "node:child_process";
 
-/** Locate a `python3` (or `python`) on PATH that reports as Python 3. */
+/** Locate a Python 3: `HYPERFRAMES_PYTHON` env override first, then PATH. */
 export function findPython(): string | undefined {
+  const override = process.env.HYPERFRAMES_PYTHON;
+  if (override) {
+    try {
+      const version = execFileSync(override, ["--version"], {
+        encoding: "utf-8",
+        stdio: ["pipe", "pipe", "pipe"],
+        timeout: 5000,
+      });
+      if (/Python 3/.test(version)) return override;
+    } catch {
+      // fall through to the PATH probe
+    }
+  }
   for (const name of ["python3", "python"]) {
     try {
       const cmd = process.platform === "win32" ? "where" : "which";

packages/cli/src/tts/synthesize.ts

@@ -110,6 +110,7 @@ export interface SynthesizeResult {
 /**
  * Synthesize text to speech using Kokoro-82M via kokoro-onnx.
  */
+// fallow-ignore-next-line complexity
 export async function synthesize(
   text: string,
   outputPath: string,
@@ -124,13 +125,13 @@ export async function synthesize(
   const python = findPython();
   if (!python) {
     throw new Error(
-      "Python 3 is required for text-to-speech. Install Python 3.8+ and run: pip install kokoro-onnx soundfile",
+      "Python 3 is required for text-to-speech. Install Python 3.10+ and run: pip install kokoro-onnx soundfile (or point HYPERFRAMES_PYTHON at a venv python that has them)",
     );
   }
 
   if (!hasPythonPackage(python, "kokoro_onnx")) {
     throw new Error(
-      "The kokoro-onnx package is not installed. Run: pip install kokoro-onnx soundfile",
+      "The kokoro-onnx package is not installed. Run: pip install kokoro-onnx soundfile (or point HYPERFRAMES_PYTHON at a venv python that has them)",
     );
   }
 

packages/cli/src/whisper/parakeet.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import { mergeTokensToWords } from "./parakeet.js";
+
+describe("mergeTokensToWords", () => {
+  it("joins Parakeet sub-word tokens into words on the space boundary", () => {
+    const words = mergeTokensToWords({
+      text: "Hello everyone. Um,",
+      sentences: [
+        {
+          tokens: [
+            { text: " H", start: 0.0, end: 0.24 },
+            { text: "ello", start: 0.24, end: 0.48 },
+            { text: " everyone.", start: 0.48, end: 1.28 },
+            { text: " Um,", start: 1.28, end: 1.92 },
+          ],
+        },
+      ],
+    });
+    expect(words).toEqual([
+      { text: "Hello", start: 0.0, end: 0.48 },
+      { text: "everyone.", start: 0.48, end: 1.28 },
+      { text: "Um,", start: 1.28, end: 1.92 },
+    ]);
+  });
+
+  it("spans sentences and tolerates missing tokens", () => {
+    expect(mergeTokensToWords({}).length).toBe(0);
+    const words = mergeTokensToWords({
+      sentences: [
+        { tokens: [{ text: "Hi", start: 0, end: 0.2 }] },
+        { tokens: [{ text: " there", start: 0.5, end: 0.9 }] },
+      ],
+    });
+    expect(words.map((w) => w.text)).toEqual(["Hi", "there"]);
+    expect(words[1]!.start).toBe(0.5);
+  });
+});

packages/cli/src/whisper/parakeet.ts

@@ -0,0 +1,151 @@
+/**
+ * Parakeet-TDT transcription engine (via parakeet-mlx on Apple Silicon).
+ *
+ * The higher-accuracy alternative to the whisper.cpp engine: NVIDIA Parakeet
+ * beats whisper-large-v3 on the Open ASR Leaderboard (~6.05% vs 7.44% avg WER,
+ * and 4.73% vs 5.96% on noisy audio where whisper-v3 hallucinates), while being
+ * 5-10x faster. Covers English + 25 European languages; whisper stays the
+ * multilingual fallback.
+ *
+ * Like the Kokoro TTS path, this is a user-installed local model: we DETECT it
+ * and, if absent, tell the user how to enable it (no auto-install). parakeet-mlx
+ * emits sub-word TOKENS; we merge them into the word timestamps the rest of the
+ * pipeline consumes.
+ */
+
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { basename, extname, join } from "node:path";
+import type { Word } from "./normalize.js";
+import type { TranscribeResult } from "./transcribe.js";
+
+const DEFAULT_MODEL = "mlx-community/parakeet-tdt-0.6b-v3";
+const PARAKEET_INSTALL =
+  "uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx";
+
+/** Verify a candidate binary actually runs (mirrors the --version gate on
+ *  HYPERFRAMES_PYTHON) so a stale $HYPERFRAMES_PARAKEET path can't shadow a
+ *  working install on PATH. */
+function isRunnable(bin: string): boolean {
+  try {
+    execFileSync(bin, ["--help"], { stdio: ["ignore", "ignore", "ignore"], timeout: 10000 });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Locate the `parakeet-mlx` runner: env override, the documented venv, then PATH. */
+export function findParakeet(): string | undefined {
+  const candidates = [
+    process.env.HYPERFRAMES_PARAKEET,
+    join(homedir(), ".venvs", "parakeet", "bin", "parakeet-mlx"),
+  ].filter((p): p is string => Boolean(p));
+
+  for (const path of candidates) {
+    if (existsSync(path) && isRunnable(path)) return path;
+  }
+  try {
+    const which = process.platform === "win32" ? "where" : "which";
+    const out = execFileSync(which, ["parakeet-mlx"], {
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "ignore"],
+      timeout: 5000,
+    });
+    const first = out
+      .split(/\r?\n/)
+      .map((s) => s.trim())
+      .find(Boolean);
+    if (first && existsSync(first)) return first;
+  } catch {
+    // not on PATH
+  }
+  return undefined;
+}
+
+interface ParakeetToken {
+  text?: string;
+  start?: number;
+  end?: number;
+}
+interface ParakeetJson {
+  text?: string;
+  sentences?: { tokens?: ParakeetToken[] }[];
+}
+
+/**
+ * Merge Parakeet's sub-word tokens (" H", "ello", ...) into words on the space
+ * boundary: a token starting with a space (or the first token) begins a word;
+ * the rest append. Produces the { text, start, end } words the pipeline uses.
+ */
+function tokenBounds(token: ParakeetToken): { text: string; start: number; end: number } {
+  const text = typeof token.text === "string" ? token.text : "";
+  const start = typeof token.start === "number" ? token.start : 0;
+  const end = typeof token.end === "number" ? token.end : start;
+  return { text, start, end };
+}
+
+export function mergeTokensToWords(parakeet: ParakeetJson): Word[] {
+  const words: Word[] = [];
+  for (const sentence of parakeet.sentences ?? []) {
+    for (const token of sentence.tokens ?? []) {
+      const { text, start, end } = tokenBounds(token);
+      if (text.startsWith(" ") || words.length === 0) {
+        words.push({ text: text.trim(), start, end });
+      } else {
+        const w = words[words.length - 1]!;
+        w.text += text;
+        w.end = end;
+      }
+    }
+  }
+  return words.filter((w) => w.text.length > 0);
+}
+
+interface ParakeetOptions {
+  language?: string;
+  model?: string;
+  onProgress?: (message: string) => void;
+}
+
+/** Transcribe with Parakeet and write `transcript.json` (Word[]) into `dir`. */
+export function transcribeWithParakeet(
+  inputPath: string,
+  dir: string,
+  options?: ParakeetOptions,
+): TranscribeResult {
+  const runner = findParakeet();
+  if (!runner) {
+    throw new Error(
+      `parakeet-mlx not found. Enable the Parakeet engine with:\n  ${PARAKEET_INSTALL}\n(or use --engine whisper)`,
+    );
+  }
+
+  const model = options?.model ?? DEFAULT_MODEL;
+  // First run pulls the model from HuggingFace (~600MB) — cue it so the wait
+  // doesn't read as a hang. HF caches at ~/.cache/huggingface/hub/models--<slug>.
+  const cached = existsSync(
+    join(homedir(), ".cache", "huggingface", "hub", `models--${model.replace(/\//g, "--")}`),
+  );
+  options?.onProgress?.(
+    cached ? "Transcribing with Parakeet..." : "Downloading Parakeet model (first run, ~600MB)...",
+  );
+  const workDir = mkdtempSync(join(tmpdir(), "hyperframes-parakeet-"));
+  try {
+    const argv = [inputPath, "--model", model, "--output-format", "json", "--output-dir", workDir];
+    if (options?.language) argv.push("--language", options.language);
+    execFileSync(runner, argv, { stdio: ["ignore", "pipe", "pipe"], timeout: 1_800_000 });
+
+    const produced = join(workDir, `${basename(inputPath, extname(inputPath))}.json`);
+    if (!existsSync(produced)) throw new Error("Parakeet did not produce output.");
+    const words = mergeTokensToWords(JSON.parse(readFileSync(produced, "utf-8")) as ParakeetJson);
+
+    const transcriptPath = join(dir, "transcript.json");
+    writeFileSync(transcriptPath, JSON.stringify(words, null, 2));
+    const durationSeconds = words.length > 0 ? words[words.length - 1]!.end : 0;
+    return { transcriptPath, wordCount: words.length, durationSeconds, speechOnsetSeconds: null };
+  } finally {
+    rmSync(workDir, { recursive: true, force: true });
+  }
+}