From 8c3590a90e4be76593ef7554a805004af789b373 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 5 Jul 2026 21:47:27 -0400 Subject: [PATCH] feat(cli): parakeet ASR engine for transcribe (--engine) + HYPERFRAMES_PYTHON override --- packages/cli/src/audio/providers.ts | 2 +- .../cli/src/commands/auth/status-guidance.ts | 2 +- packages/cli/src/commands/transcribe.ts | 58 +++++-- packages/cli/src/commands/tts.ts | 4 +- packages/cli/src/templates/_shared/AGENTS.md | 2 +- packages/cli/src/templates/_shared/CLAUDE.md | 2 +- packages/cli/src/tts/python.ts | 15 +- packages/cli/src/tts/synthesize.ts | 5 +- packages/cli/src/whisper/parakeet.test.ts | 37 +++++ packages/cli/src/whisper/parakeet.ts | 151 ++++++++++++++++++ 10 files changed, 260 insertions(+), 18 deletions(-) create mode 100644 packages/cli/src/whisper/parakeet.test.ts create mode 100644 packages/cli/src/whisper/parakeet.ts diff --git a/packages/cli/src/audio/providers.ts b/packages/cli/src/audio/providers.ts index a3eb3d489..d7981c24e 100644 --- a/packages/cli/src/audio/providers.ts +++ b/packages/cli/src/audio/providers.ts @@ -1,7 +1,7 @@ /** * Which voice / music engine a workflow will actually use, and whether * its local dependencies are present. Mirrors the resolution order the - * hyperframes-media skill scripts use, so `auth status` and `doctor` + * media-use skill scripts use, so `auth status` and `doctor` * report the same engine the render pipeline would pick: * * voice: HeyGen Starfish → ElevenLabs (key + `elevenlabs`) → Kokoro (local) diff --git a/packages/cli/src/commands/auth/status-guidance.ts b/packages/cli/src/commands/auth/status-guidance.ts index 08f524426..cb4269fe2 100644 --- a/packages/cli/src/commands/auth/status-guidance.ts +++ b/packages/cli/src/commands/auth/status-guidance.ts @@ -63,7 +63,7 @@ function offlineEngineLines(engines?: OfflineEngineLine[]): string[] { * so it's left to the docs — not dangled here as a command a fresh machine * can't run. Names the local fallback so "no key" never reads as a failure, * and never steers users toward a per-repo `.env`. Mirrors the - * hyperframes-media skill's Preflight section. + * media-use skill's Preflight section. */ export function buildUnconfiguredLines( ctx: UnconfiguredContext, diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.ts index d4add30ad..0c4ec4405 100644 --- a/packages/cli/src/commands/transcribe.ts +++ b/packages/cli/src/commands/transcribe.ts @@ -1,6 +1,8 @@ +// fallow-ignore-file code-duplication import { defineCommand } from "citty"; import type { Example } from "./_examples.js"; import { existsSync, writeFileSync } from "node:fs"; +import { findParakeet, transcribeWithParakeet } from "../whisper/parakeet.js"; type CaptionExportFormat = "srt" | "vtt"; @@ -41,6 +43,12 @@ export default defineCommand({ description: "Project directory (default: current directory)", alias: "d", }, + engine: { + type: "string", + description: + "ASR engine: auto (Parakeet if installed, else whisper), parakeet, or whisper. Default: auto. Parakeet is more accurate and faster; enable with `uv pip install parakeet-mlx`.", + alias: "e", + }, model: { type: "string", description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny.en, base.en, small.en, medium.en, large-v3`, @@ -111,8 +119,9 @@ export default defineCommand({ return importTranscript(inputPath, dir, args.json); } - // ── Transcribe mode: run whisper ───────────────────────────────────── + // ── Transcribe mode: run the ASR engine ────────────────────────────── return transcribeAudio(inputPath, dir, { + engine: args.engine, model: args.model, language: args.language, json: args.json, @@ -213,25 +222,45 @@ async function exportTranscript( // Transcribe audio/video with whisper // --------------------------------------------------------------------------- +// fallow-ignore-next-line complexity async function transcribeAudio( inputPath: string, dir: string, - opts: { model?: string; language?: string; json?: boolean; optional?: boolean }, + opts: { engine?: string; model?: string; language?: string; json?: boolean; optional?: boolean }, ): Promise { const { transcribe } = await import("../whisper/transcribe.js"); const { loadTranscript, patchCaptionHtml, stripBeforeOnset } = await import("../whisper/normalize.js"); + // Engine: auto (Parakeet if installed, else whisper), or forced parakeet/whisper. + const engine = (opts.engine ?? "auto").toLowerCase(); + if (engine !== "auto" && engine !== "parakeet" && engine !== "whisper") { + failWith(`Unknown --engine: ${opts.engine}. Use auto, parakeet, or whisper.`, !!opts.json); + } + const useParakeet = engine === "parakeet" || (engine === "auto" && !!findParakeet()); + const model = opts.model ?? DEFAULT_MODEL; + // --model selects the whisper model only; Parakeet uses its own fixed model. + if (useParakeet && opts.model && !opts.json) { + console.error( + c.dim(` Note: --model applies to the whisper engine only; ignored under Parakeet.`), + ); + } + const label = useParakeet ? "Parakeet" : model; const spin = opts.json ? null : clack.spinner(); - spin?.start(`Transcribing with ${c.accent(model)}...`); + spin?.start(`Transcribing with ${c.accent(label)}...`); try { - const result = await transcribe(inputPath, dir, { - model, - language: opts.language, - onProgress: spin ? (msg) => spin.message(msg) : undefined, - }); + const result = useParakeet + ? transcribeWithParakeet(inputPath, dir, { + language: opts.language, + onProgress: spin ? (msg) => spin.message(msg) : undefined, + }) + : await transcribe(inputPath, dir, { + model, + language: opts.language, + onProgress: spin ? (msg) => spin.message(msg) : undefined, + }); let { words } = loadTranscript(result.transcriptPath); @@ -253,7 +282,8 @@ async function transcribeAudio( console.log( JSON.stringify({ ok: true, - model, + engine: useParakeet ? "parakeet" : "whisper", + model: useParakeet ? "parakeet-tdt-0.6b-v3" : model, wordCount: words.length, durationSeconds: result.durationSeconds, speechOnsetSeconds: result.speechOnsetSeconds, @@ -272,7 +302,15 @@ async function transcribeAudio( ); } } catch (err) { - const message = err instanceof Error ? err.message : String(err); + // Surface the last few lines of the ASR subprocess's stderr, which + // execFileSync captures but otherwise drops on the floor — that's where + // parakeet-mlx / whisper report the actual failure cause. + const stderr = + err && typeof err === "object" && "stderr" in err && err.stderr + ? String(err.stderr).trim().split("\n").slice(-3).join("\n") + : ""; + const base = err instanceof Error ? err.message : String(err); + const message = stderr ? `${base}\n${stderr}` : base; // whisper-cpp is an optional prerequisite, not part of the CLI. When it is // simply unavailable (no binary, no toolchain to build one), that is a setup diff --git a/packages/cli/src/commands/tts.ts b/packages/cli/src/commands/tts.ts index bf0bc03c1..8ad0d3f41 100644 --- a/packages/cli/src/commands/tts.ts +++ b/packages/cli/src/commands/tts.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication import { defineCommand } from "citty"; import type { Example } from "./_examples.js"; import { existsSync, readFileSync } from "node:fs"; @@ -48,7 +49,7 @@ export default defineCommand({ output: { type: "string", description: "Output file path (default: speech.wav in current directory)", - alias: "o", + alias: ["o", "out"], }, voice: { type: "string", @@ -76,6 +77,7 @@ export default defineCommand({ default: false, }, }, + // fallow-ignore-next-line complexity async run({ args }) { // ── List voices mode ────────────────────────────────────────────── if (args.list) { diff --git a/packages/cli/src/templates/_shared/AGENTS.md b/packages/cli/src/templates/_shared/AGENTS.md index 72183329f..b72d7e5e6 100644 --- a/packages/cli/src/templates/_shared/AGENTS.md +++ b/packages/cli/src/templates/_shared/AGENTS.md @@ -17,7 +17,7 @@ **Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above. -The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/hyperframes-media`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. +The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. > **Tailwind v4 projects** (`hyperframes init --tailwind`): see `/hyperframes-core` → `references/tailwind.md`. diff --git a/packages/cli/src/templates/_shared/CLAUDE.md b/packages/cli/src/templates/_shared/CLAUDE.md index 72183329f..b72d7e5e6 100644 --- a/packages/cli/src/templates/_shared/CLAUDE.md +++ b/packages/cli/src/templates/_shared/CLAUDE.md @@ -17,7 +17,7 @@ **Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above. -The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/hyperframes-media`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. +The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. > **Tailwind v4 projects** (`hyperframes init --tailwind`): see `/hyperframes-core` → `references/tailwind.md`. diff --git a/packages/cli/src/tts/python.ts b/packages/cli/src/tts/python.ts index 13e6111d4..5c849b06a 100644 --- a/packages/cli/src/tts/python.ts +++ b/packages/cli/src/tts/python.ts @@ -8,8 +8,21 @@ import { execFileSync } from "node:child_process"; -/** Locate a `python3` (or `python`) on PATH that reports as Python 3. */ +/** Locate a Python 3: `HYPERFRAMES_PYTHON` env override first, then PATH. */ export function findPython(): string | undefined { + const override = process.env.HYPERFRAMES_PYTHON; + if (override) { + try { + const version = execFileSync(override, ["--version"], { + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + timeout: 5000, + }); + if (/Python 3/.test(version)) return override; + } catch { + // fall through to the PATH probe + } + } for (const name of ["python3", "python"]) { try { const cmd = process.platform === "win32" ? "where" : "which"; diff --git a/packages/cli/src/tts/synthesize.ts b/packages/cli/src/tts/synthesize.ts index cc887b81f..fefdc52c8 100644 --- a/packages/cli/src/tts/synthesize.ts +++ b/packages/cli/src/tts/synthesize.ts @@ -110,6 +110,7 @@ export interface SynthesizeResult { /** * Synthesize text to speech using Kokoro-82M via kokoro-onnx. */ +// fallow-ignore-next-line complexity export async function synthesize( text: string, outputPath: string, @@ -124,13 +125,13 @@ export async function synthesize( const python = findPython(); if (!python) { throw new Error( - "Python 3 is required for text-to-speech. Install Python 3.8+ and run: pip install kokoro-onnx soundfile", + "Python 3 is required for text-to-speech. Install Python 3.10+ and run: pip install kokoro-onnx soundfile (or point HYPERFRAMES_PYTHON at a venv python that has them)", ); } if (!hasPythonPackage(python, "kokoro_onnx")) { throw new Error( - "The kokoro-onnx package is not installed. Run: pip install kokoro-onnx soundfile", + "The kokoro-onnx package is not installed. Run: pip install kokoro-onnx soundfile (or point HYPERFRAMES_PYTHON at a venv python that has them)", ); } diff --git a/packages/cli/src/whisper/parakeet.test.ts b/packages/cli/src/whisper/parakeet.test.ts new file mode 100644 index 000000000..e707e4520 --- /dev/null +++ b/packages/cli/src/whisper/parakeet.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { mergeTokensToWords } from "./parakeet.js"; + +describe("mergeTokensToWords", () => { + it("joins Parakeet sub-word tokens into words on the space boundary", () => { + const words = mergeTokensToWords({ + text: "Hello everyone. Um,", + sentences: [ + { + tokens: [ + { text: " H", start: 0.0, end: 0.24 }, + { text: "ello", start: 0.24, end: 0.48 }, + { text: " everyone.", start: 0.48, end: 1.28 }, + { text: " Um,", start: 1.28, end: 1.92 }, + ], + }, + ], + }); + expect(words).toEqual([ + { text: "Hello", start: 0.0, end: 0.48 }, + { text: "everyone.", start: 0.48, end: 1.28 }, + { text: "Um,", start: 1.28, end: 1.92 }, + ]); + }); + + it("spans sentences and tolerates missing tokens", () => { + expect(mergeTokensToWords({}).length).toBe(0); + const words = mergeTokensToWords({ + sentences: [ + { tokens: [{ text: "Hi", start: 0, end: 0.2 }] }, + { tokens: [{ text: " there", start: 0.5, end: 0.9 }] }, + ], + }); + expect(words.map((w) => w.text)).toEqual(["Hi", "there"]); + expect(words[1]!.start).toBe(0.5); + }); +}); diff --git a/packages/cli/src/whisper/parakeet.ts b/packages/cli/src/whisper/parakeet.ts new file mode 100644 index 000000000..a53c18bfa --- /dev/null +++ b/packages/cli/src/whisper/parakeet.ts @@ -0,0 +1,151 @@ +/** + * Parakeet-TDT transcription engine (via parakeet-mlx on Apple Silicon). + * + * The higher-accuracy alternative to the whisper.cpp engine: NVIDIA Parakeet + * beats whisper-large-v3 on the Open ASR Leaderboard (~6.05% vs 7.44% avg WER, + * and 4.73% vs 5.96% on noisy audio where whisper-v3 hallucinates), while being + * 5-10x faster. Covers English + 25 European languages; whisper stays the + * multilingual fallback. + * + * Like the Kokoro TTS path, this is a user-installed local model: we DETECT it + * and, if absent, tell the user how to enable it (no auto-install). parakeet-mlx + * emits sub-word TOKENS; we merge them into the word timestamps the rest of the + * pipeline consumes. + */ + +import { execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { basename, extname, join } from "node:path"; +import type { Word } from "./normalize.js"; +import type { TranscribeResult } from "./transcribe.js"; + +const DEFAULT_MODEL = "mlx-community/parakeet-tdt-0.6b-v3"; +const PARAKEET_INSTALL = + "uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx"; + +/** Verify a candidate binary actually runs (mirrors the --version gate on + * HYPERFRAMES_PYTHON) so a stale $HYPERFRAMES_PARAKEET path can't shadow a + * working install on PATH. */ +function isRunnable(bin: string): boolean { + try { + execFileSync(bin, ["--help"], { stdio: ["ignore", "ignore", "ignore"], timeout: 10000 }); + return true; + } catch { + return false; + } +} + +/** Locate the `parakeet-mlx` runner: env override, the documented venv, then PATH. */ +export function findParakeet(): string | undefined { + const candidates = [ + process.env.HYPERFRAMES_PARAKEET, + join(homedir(), ".venvs", "parakeet", "bin", "parakeet-mlx"), + ].filter((p): p is string => Boolean(p)); + + for (const path of candidates) { + if (existsSync(path) && isRunnable(path)) return path; + } + try { + const which = process.platform === "win32" ? "where" : "which"; + const out = execFileSync(which, ["parakeet-mlx"], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + timeout: 5000, + }); + const first = out + .split(/\r?\n/) + .map((s) => s.trim()) + .find(Boolean); + if (first && existsSync(first)) return first; + } catch { + // not on PATH + } + return undefined; +} + +interface ParakeetToken { + text?: string; + start?: number; + end?: number; +} +interface ParakeetJson { + text?: string; + sentences?: { tokens?: ParakeetToken[] }[]; +} + +/** + * Merge Parakeet's sub-word tokens (" H", "ello", ...) into words on the space + * boundary: a token starting with a space (or the first token) begins a word; + * the rest append. Produces the { text, start, end } words the pipeline uses. + */ +function tokenBounds(token: ParakeetToken): { text: string; start: number; end: number } { + const text = typeof token.text === "string" ? token.text : ""; + const start = typeof token.start === "number" ? token.start : 0; + const end = typeof token.end === "number" ? token.end : start; + return { text, start, end }; +} + +export function mergeTokensToWords(parakeet: ParakeetJson): Word[] { + const words: Word[] = []; + for (const sentence of parakeet.sentences ?? []) { + for (const token of sentence.tokens ?? []) { + const { text, start, end } = tokenBounds(token); + if (text.startsWith(" ") || words.length === 0) { + words.push({ text: text.trim(), start, end }); + } else { + const w = words[words.length - 1]!; + w.text += text; + w.end = end; + } + } + } + return words.filter((w) => w.text.length > 0); +} + +interface ParakeetOptions { + language?: string; + model?: string; + onProgress?: (message: string) => void; +} + +/** Transcribe with Parakeet and write `transcript.json` (Word[]) into `dir`. */ +export function transcribeWithParakeet( + inputPath: string, + dir: string, + options?: ParakeetOptions, +): TranscribeResult { + const runner = findParakeet(); + if (!runner) { + throw new Error( + `parakeet-mlx not found. Enable the Parakeet engine with:\n ${PARAKEET_INSTALL}\n(or use --engine whisper)`, + ); + } + + const model = options?.model ?? DEFAULT_MODEL; + // First run pulls the model from HuggingFace (~600MB) — cue it so the wait + // doesn't read as a hang. HF caches at ~/.cache/huggingface/hub/models--. + const cached = existsSync( + join(homedir(), ".cache", "huggingface", "hub", `models--${model.replace(/\//g, "--")}`), + ); + options?.onProgress?.( + cached ? "Transcribing with Parakeet..." : "Downloading Parakeet model (first run, ~600MB)...", + ); + const workDir = mkdtempSync(join(tmpdir(), "hyperframes-parakeet-")); + try { + const argv = [inputPath, "--model", model, "--output-format", "json", "--output-dir", workDir]; + if (options?.language) argv.push("--language", options.language); + execFileSync(runner, argv, { stdio: ["ignore", "pipe", "pipe"], timeout: 1_800_000 }); + + const produced = join(workDir, `${basename(inputPath, extname(inputPath))}.json`); + if (!existsSync(produced)) throw new Error("Parakeet did not produce output."); + const words = mergeTokensToWords(JSON.parse(readFileSync(produced, "utf-8")) as ParakeetJson); + + const transcriptPath = join(dir, "transcript.json"); + writeFileSync(transcriptPath, JSON.stringify(words, null, 2)); + const durationSeconds = words.length > 0 ? words[words.length - 1]!.end : 0; + return { transcriptPath, wordCount: words.length, durationSeconds, speechOnsetSeconds: null }; + } finally { + rmSync(workDir, { recursive: true, force: true }); + } +}