diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index 2bcb54ac9e..a98fe708dd 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -46,6 +46,8 @@ "packages/producer/src/services/healthWorkerThread.ts", // Render-setup worker, spawned by path from cancellableProcess.ts. "packages/cli/src/renderSetupWorker.ts", + // Parakeet decode worker, spawned by path from whisper/sherpa.ts. + "packages/cli/src/whisper/sherpaWorker.ts", // Test fixture worker, spawned by path via the pools' workerEntryPath // option from the crash-recovery tests; has no import-graph referrer. "packages/producer/src/services/__fixtures__/crashOnMessageWorker.mjs", diff --git a/CREDITS.md b/CREDITS.md index 5f974018be..4d3c74e55a 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -28,6 +28,11 @@ broader Node.js ecosystem. - **[mediabunny](https://github.com/Vanilagy/mediabunny)** — media toolkit used in the studio for fast metadata extraction from file headers. Licensed under the [Mozilla Public License 2.0 (MPL-2.0)](https://mozilla.org/MPL/2.0/). +- **[Parakeet TDT 0.6B v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3)** by NVIDIA — the + speech recognition model `hyperframes models install parakeet` downloads, in the int8 ONNX export + from [csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8](https://huggingface.co/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8). + Licensed under [Creative Commons Attribution 4.0 (CC-BY-4.0)](https://creativecommons.org/licenses/by/4.0/). + It runs on **[sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx)**, Apache-2.0. - The seven 3D-motion catalog pieces (`canopy-part-title`, `glass-shard-title`, `code-slice-hero`, `frost-sequence-camera-orbit`, `cuboid-carousel`, `orbit-card`, `wireframe-portal-title`) are contributed with their author's diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index b3da43d55a..c9e3f62bc4 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -165,6 +165,7 @@ const commandLoaders = { browser: () => import("./commands/browser.js").then((m) => m.default), "remove-background": () => import("./commands/remove-background.js").then((m) => m.default), transcribe: () => import("./commands/transcribe.js").then((m) => m.default), + models: () => import("./commands/models.js").then((m) => m.default), tts: () => import("./commands/tts.js").then((m) => m.default), docs: () => import("./commands/docs.js").then((m) => m.default), doctor: () => import("./commands/doctor.js").then((m) => m.default), diff --git a/packages/cli/src/commands/models.ts b/packages/cli/src/commands/models.ts new file mode 100644 index 0000000000..f8ef4704ef --- /dev/null +++ b/packages/cli/src/commands/models.ts @@ -0,0 +1,83 @@ +import { defineCommand } from "citty"; +import * as clack from "@clack/prompts"; +import type { Example } from "./_examples.js"; +import { c } from "../ui/colors.js"; +import { formatBytes } from "../ui/format.js"; +import { failCommand } from "../utils/commandResult.js"; +import { PARAKEET_MODEL_LABEL } from "../whisper/parakeet.js"; + +export const examples: Example[] = [ + [ + "Download the Parakeet speech model that transcribe uses", + "hyperframes models install parakeet", + ], +]; + +function fail(message: string, json: boolean): never { + if (json) console.log(JSON.stringify({ ok: false, error: message })); + else console.error(c.error(message)); + failCommand(); +} + +async function installParakeet(json: boolean): Promise { + const sherpa = await import("../whisper/sherpa.js"); + const unsupported = sherpa.sherpaUnsupportedReason(); + if (unsupported) fail(unsupported, json); + + const spin = json ? null : clack.spinner({ output: process.stderr }); + spin?.start("Checking the sherpa-onnx runtime..."); + try { + const runtimeMissing = !sherpa.sherpaRuntimeInstalled(); + if (runtimeMissing) { + spin?.message("Installing the sherpa-onnx runtime from npm..."); + await sherpa.installSherpaRuntime(); + } + spin?.message("Verifying the Parakeet model..."); + let lastPct = -1; + const modelFetched = await sherpa.ensureParakeetModel({ + onBytes: (done, total) => { + const pct = Math.floor((done / total) * 100); + if (pct <= lastPct) return; + lastPct = pct; + spin?.message( + `Downloading Parakeet TDT 0.6B v3 — ${c.progress(pct + "%")} ${c.dim("(" + formatBytes(done) + " / " + formatBytes(total) + ")")}`, + ); + }, + }); + const changed = runtimeMissing || modelFetched; + if (json) { + console.log( + JSON.stringify({ + ok: true, + model: PARAKEET_MODEL_LABEL, + changed, + runtimeDir: sherpa.SHERPA_RUNTIME_DIR, + modelDir: sherpa.PARAKEET_MODEL_DIR, + }), + ); + } else { + spin?.stop(c.success(changed ? "Parakeet installed" : "Parakeet is already installed")); + } + } catch (err) { + spin?.stop(c.error("Parakeet install failed")); + fail(err instanceof Error ? err.message : String(err), json); + } +} + +export default defineCommand({ + meta: { name: "models", description: "Download on-device models (models install parakeet)" }, + args: { + action: { type: "positional", description: "install", required: true }, + name: { type: "positional", description: "Model to install: parakeet", required: true }, + json: { type: "boolean", description: "Print one JSON result, no progress", default: false }, + }, + async run({ args }) { + if (args.action !== "install" || args.name !== "parakeet") { + fail( + `Unknown: models ${args.action} ${args.name}. Try: hyperframes models install parakeet`, + args.json, + ); + } + return installParakeet(args.json); + }, +}); diff --git a/packages/cli/src/commands/transcribe.test.ts b/packages/cli/src/commands/transcribe.test.ts index 5c4bd521d5..c052fd7abd 100644 --- a/packages/cli/src/commands/transcribe.test.ts +++ b/packages/cli/src/commands/transcribe.test.ts @@ -7,7 +7,28 @@ import { CliRuntimeError, consumeCommandResult } from "../utils/commandResult.js // Make the whisper core report "unavailable" so we exercise the soft-skip path. const transcribeMock = vi.fn(); -vi.mock("../whisper/transcribe.js", () => ({ transcribe: transcribeMock })); +vi.mock("../whisper/transcribe.js", () => ({ + transcribe: transcribeMock, + prepareWav: (input: string) => input, +})); + +// Engine selection: which runners look installed, and the sherpa decode child it spawns. +const runners = { sherpa: false, mlx: false }; +vi.mock("../whisper/sherpa.js", async (importOriginal) => ({ + ...(await importOriginal()), + sherpaParakeetInstalled: () => runners.sherpa, +})); +const mlxMock = vi.fn(); +vi.mock("../whisper/parakeet.js", async (importOriginal) => ({ + ...(await importOriginal()), + findParakeet: () => (runners.mlx ? "/bin/parakeet-mlx" : undefined), + transcribeWithParakeet: (...a: unknown[]) => mlxMock(...a), +})); +const execFileMock = vi.fn(); +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + execFile: (...a: unknown[]) => execFileMock(...a), +})); const trackTranscribeUnavailable = vi.fn(); const trackCommandFailure = vi.fn(); @@ -18,6 +39,17 @@ vi.mock("../telemetry/events.js", () => ({ import transcribeCmd from "./transcribe.js"; +/** Makes each runner write a one-word transcript naming itself. */ +function fakeTranscript(dir: string, text: string) { + const transcriptPath = join(dir, "transcript.json"); + writeFileSync(transcriptPath, JSON.stringify([{ text, start: 0, end: 1 }])); + return { transcriptPath, wordCount: 1, durationSeconds: 1, speechOnsetSeconds: null }; +} + +function lastJson(): Record { + return JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])); +} + function dummyAudio(): { dir: string; input: string } { const dir = mkdtempSync(join(tmpdir(), "hf-transcribe-test-")); const input = join(dir, "narration.wav"); @@ -33,6 +65,8 @@ describe("transcribe command", () => { transcribeMock.mockReset(); trackTranscribeUnavailable.mockReset(); trackCommandFailure.mockReset(); + mlxMock.mockReset(); + Object.assign(runners, { sherpa: false, mlx: false }); transcribeMock.mockRejectedValue( new WhisperUnavailableError("whisper-cpp not found. Install: brew install whisper-cpp"), ); @@ -79,6 +113,93 @@ describe("transcribe command", () => { expect(trackCommandFailure).not.toHaveBeenCalled(); }); + describe("engine selection", () => { + beforeEach(() => { + transcribeMock.mockImplementation(async (_in: string, dir: string) => + fakeTranscript(dir, "whisper"), + ); + mlxMock.mockImplementation((_in: string, dir: string) => fakeTranscript(dir, "mlx")); + execFileMock.mockReset(); + execFileMock.mockImplementation((_cmd, _args, _opts, done) => { + const windows = [{ offset: 60, tokens: [" sherpa"], timestamps: [0.5], durations: [0.25] }]; + done(null, `HYPERFRAMES_PARAKEET_RESULT:${JSON.stringify(windows)}\n`, ""); + }); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.report, "getReport").mockReturnValue({ + header: { glibcVersionRuntime: "2.36" }, + } as never); + }); + + async function transcribeWith(engine: string, installed: typeof runners) { + Object.assign(runners, installed); + const { dir, input } = dummyAudio(); + dirs.push(dir); + await transcribeCmd.run!({ args: { input, json: true, engine } } as never); + const out = lastJson(); + const words = JSON.parse(readFileSync(String(out.transcriptPath), "utf-8")); + return { engine: out.engine, model: out.model, word: words[0]?.text, start: words[0]?.start }; + } + + it("auto and parakeet prefer sherpa-onnx, then parakeet-mlx, then whisper", async () => { + const sherpa = { + engine: "parakeet", + model: "parakeet-tdt-0.6b-v3", + word: "sherpa", + start: 60.5, + }; + expect(await transcribeWith("auto", { sherpa: true, mlx: true })).toEqual(sherpa); + expect(await transcribeWith("parakeet", { sherpa: true, mlx: true })).toEqual(sherpa); + expect(await transcribeWith("auto", { sherpa: false, mlx: true })).toMatchObject({ + engine: "parakeet", + word: "mlx", + }); + expect(await transcribeWith("auto", { sherpa: false, mlx: false })).toMatchObject({ + engine: "whisper", + word: "whisper", + }); + expect(await transcribeWith("whisper", { sherpa: true, mlx: true })).toMatchObject({ + engine: "whisper", + word: "whisper", + }); + }); + + it("--engine parakeet with nothing installed names the install command", async () => { + Object.assign(runners, { sherpa: false, mlx: false }); + const { dir, input } = dummyAudio(); + dirs.push(dir); + await expect( + transcribeCmd.run!({ args: { input, json: true, engine: "parakeet" } } as never), + ).rejects.toThrow(CliRuntimeError); + expect(lastJson().error).toContain("hyperframes models install parakeet"); + expect(transcribeMock).not.toHaveBeenCalled(); + }); + + it("falls back to whisper with one stderr line when the decode child crashes", async () => { + execFileMock.mockImplementation((_cmd, _args, _opts, done) => { + const crash = Object.assign(new Error("Command failed"), { code: null, signal: "SIGABRT" }); + done(crash, "", "terminate called after throwing an instance of 'Ort::Exception'\n"); + }); + expect(await transcribeWith("auto", { sherpa: true, mlx: false })).toMatchObject({ + engine: "whisper", + word: "whisper", + }); + const warnings = vi.mocked(console.error).mock.calls.map(([line]) => String(line)); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toMatch(/sherpa-onnx.*whisper.*SIGABRT.*Ort::Exception/); + }); + + it("falls back without spawning when glibc is too old for sherpa-onnx", async () => { + vi.mocked(process.report.getReport).mockReturnValue({ + header: { glibcVersionRuntime: "2.31" }, + } as never); + expect(await transcribeWith("auto", { sherpa: true, mlx: false })).toMatchObject({ + engine: "whisper", + }); + expect(execFileMock).not.toHaveBeenCalled(); + expect(String(vi.mocked(console.error).mock.calls[0]?.[0])).toMatch(/glibc 2\.32.*2\.31/); + }); + }); + it("imports an SRT and exports an SRT sidecar from transcript.json", async () => { const dir = mkdtempSync(join(tmpdir(), "hf-transcribe-test-")); dirs.push(dir); diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.ts index 501bde54bd..f66cb6a442 100644 --- a/packages/cli/src/commands/transcribe.ts +++ b/packages/cli/src/commands/transcribe.ts @@ -3,7 +3,7 @@ import { failCommand, setCommandExitCode } from "../utils/commandResult.js"; import { defineCommand } from "citty"; import type { Example } from "./_examples.js"; import { existsSync, writeFileSync } from "node:fs"; -import { findParakeet, transcribeWithParakeet } from "../whisper/parakeet.js"; +import { findParakeet, PARAKEET_MODEL_LABEL, transcribeWithParakeet } from "../whisper/parakeet.js"; type CaptionExportFormat = "srt" | "vtt"; @@ -55,7 +55,7 @@ export default defineCommand({ engine: { type: "string", description: - "ASR engine: auto (Parakeet if installed, else whisper), parakeet, or whisper. Default: auto. Parakeet is more accurate and faster; enable with `uv pip install parakeet-mlx`.", + "ASR engine: auto (Parakeet if installed, else whisper), parakeet, or whisper. Default: auto. Parakeet is more accurate and faster; install it with `hyperframes models install parakeet`.", alias: "e", }, model: { @@ -267,6 +267,15 @@ async function exportTranscript( // Transcribe audio/video with whisper // --------------------------------------------------------------------------- +type Runner = "sherpa" | "parakeet-mlx" | "whisper"; + +/** auto and parakeet prefer sherpa-onnx, then an installed parakeet-mlx runner, then whisper. */ +function pickRunner(engine: string, sherpaInstalled: () => boolean): Runner { + if (engine === "whisper") return "whisper"; + if (sherpaInstalled()) return "sherpa"; + return findParakeet() ? "parakeet-mlx" : "whisper"; +} + // fallow-ignore-next-line complexity async function transcribeAudio( inputPath: string, @@ -284,36 +293,56 @@ async function transcribeAudio( const { loadTranscript, patchCaptionHtml, stripBeforeOnset } = await import("../whisper/normalize.js"); - // Engine: auto (Parakeet if installed, else whisper), or forced parakeet/whisper. + const { sherpaParakeetInstalled, transcribeWithSherpa } = await import("../whisper/sherpa.js"); + const engine = (opts.engine ?? "auto").toLowerCase(); if (engine !== "auto" && engine !== "parakeet" && engine !== "whisper") { failWith(`Unknown --engine: ${opts.engine}. Use auto, parakeet, or whisper.`, !!opts.json); } - const useParakeet = engine === "parakeet" || (engine === "auto" && !!findParakeet()); + let runner = pickRunner(engine, sherpaParakeetInstalled); + if (engine === "parakeet" && runner === "whisper") { + failWith( + "Parakeet is not installed. Install it with: hyperframes models install parakeet (or use --engine whisper)", + !!opts.json, + ); + } const model = opts.model ?? DEFAULT_MODEL; // --model selects the whisper model only; Parakeet uses its own fixed model. - if (useParakeet && opts.model && !opts.json) { + if (runner !== "whisper" && opts.model && !opts.json) { console.error( c.dim(` Note: --model applies to the whisper engine only; ignored under Parakeet.`), ); } - const label = useParakeet ? "Parakeet" : model; + const label = runner === "whisper" ? model : "Parakeet"; const spin = opts.json ? null : clack.spinner(); spin?.start(`Transcribing with ${c.accent(label)}...`); + const onProgress = spin ? (msg: string) => spin.message(msg) : undefined; + const run = (r: Runner) => + r === "sherpa" + ? transcribeWithSherpa(inputPath, dir, { onProgress }) + : r === "parakeet-mlx" + ? transcribeWithParakeet(inputPath, dir, { language: opts.language, onProgress }) + : transcribe(inputPath, dir, { + model, + language: opts.language, + onProgress, + timeoutMs: opts.timeoutMs, + }); try { - const result = useParakeet - ? transcribeWithParakeet(inputPath, dir, { - language: opts.language, - onProgress: spin ? (msg) => spin.message(msg) : undefined, - }) - : await transcribe(inputPath, dir, { - model, - language: opts.language, - onProgress: spin ? (msg) => spin.message(msg) : undefined, - timeoutMs: opts.timeoutMs, - }); + let result: Awaited>; + try { + result = await run(runner); + } catch (err) { + if (runner !== "sherpa") throw err; + runner = pickRunner(engine, () => false); + const reason = err instanceof Error ? err.message : String(err); + spin?.clear(); + console.error(c.warn(`Parakeet (sherpa-onnx) failed, using ${runner} instead. ${reason}`)); + spin?.start(`Transcribing with ${c.accent(runner)}...`); + result = await run(runner); + } let { words } = loadTranscript(result.transcriptPath); @@ -335,8 +364,8 @@ async function transcribeAudio( console.log( JSON.stringify({ ok: true, - engine: useParakeet ? "parakeet" : "whisper", - model: useParakeet ? "parakeet-tdt-0.6b-v3" : model, + engine: runner === "whisper" ? "whisper" : "parakeet", + model: runner === "whisper" ? model : PARAKEET_MODEL_LABEL, wordCount: words.length, durationSeconds: result.durationSeconds, speechOnsetSeconds: result.speechOnsetSeconds, diff --git a/packages/cli/src/help.ts b/packages/cli/src/help.ts index 3210e3d46e..37173e9b98 100644 --- a/packages/cli/src/help.ts +++ b/packages/cli/src/help.ts @@ -90,6 +90,7 @@ const GROUPS: Group[] = [ "transcribe", "Transcribe audio/video to word-level timestamps, or import an existing transcript", ], + ["models", "Download on-device models (models install parakeet)"], ["tts", "Generate speech audio from text using a local AI model (Kokoro-82M)"], ["remove-background", "Remove background from a video or image to produce transparent media"], ], diff --git a/packages/cli/src/utils/optionalPackages.ts b/packages/cli/src/utils/optionalPackages.ts index d7e24612f8..d24b98ccb4 100644 --- a/packages/cli/src/utils/optionalPackages.ts +++ b/packages/cli/src/utils/optionalPackages.ts @@ -29,7 +29,7 @@ export const OPTIONAL_PACKAGES = { "@google/genai": "1.52.0", } as const satisfies Record; -const CACHE_DIR = join(homedir(), ".cache", "hyperframes", "optional"); +export const CACHE_DIR = join(homedir(), ".cache", "hyperframes", "optional"); export interface OptionalPackageDeps { cacheDir: string; @@ -108,11 +108,11 @@ function manifestPath(dir: string, name: string): string { return join(dir, "node_modules", name, "package.json"); } -function isInstalled(dir: string, name: string): boolean { +export function isInstalled(dir: string, name: string): boolean { return existsSync(manifestPath(dir, name)); } -function loadInstalled(dir: string, name: string): unknown | null { +export function loadInstalled(dir: string, name: string): unknown | null { if (!isInstalled(dir, name)) return null; return createRequire(join(dir, "package.json"))(name); } diff --git a/packages/cli/src/whisper/parakeet.test.ts b/packages/cli/src/whisper/parakeet.test.ts index e707e45208..d6bc5c94df 100644 --- a/packages/cli/src/whisper/parakeet.test.ts +++ b/packages/cli/src/whisper/parakeet.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { mergeTokensToWords } from "./parakeet.js"; +import { mergeTokensToWords, mergeWindowsToWords, silenceCuts } from "./parakeet.js"; describe("mergeTokensToWords", () => { it("joins Parakeet sub-word tokens into words on the space boundary", () => { @@ -35,3 +35,33 @@ describe("mergeTokensToWords", () => { expect(words[1]!.start).toBe(0.5); }); }); + +describe("mergeWindowsToWords", () => { + it("shifts each window's token times by the window's start", () => { + const words = mergeWindowsToWords([ + { offset: 0, tokens: [" Hel", "lo"], timestamps: [0.125, 0.25], durations: [0.125, 0.25] }, + { offset: 60.5, tokens: [" world"], timestamps: [0.25], durations: [0.5] }, + ]); + expect(words).toEqual([ + { text: "Hello", start: 0.125, end: 0.5 }, + { text: "world", start: 60.75, end: 61.25 }, + ]); + }); +}); + +describe("silenceCuts", () => { + // 100 samples/s keeps the fixture small: 150 s of steady sound with two 0.1 s silences. + const rate = 100; + const loud = (seconds: number) => new Float32Array(seconds * rate).fill(0.5); + + it("moves each cut to the quietest 100 ms within 5 s of the 60 s target", () => { + const samples = loud(150); + samples.fill(0, 5830, 5840); + samples.fill(0, 12195, 12205); + expect(silenceCuts(samples, rate)).toEqual([0, 5835, 12200, 15000]); + }); + + it("leaves audio shorter than one window whole", () => { + expect(silenceCuts(loud(42), rate)).toEqual([0, 4200]); + }); +}); diff --git a/packages/cli/src/whisper/parakeet.ts b/packages/cli/src/whisper/parakeet.ts index a53c18bfa7..71ba50cf0e 100644 --- a/packages/cli/src/whisper/parakeet.ts +++ b/packages/cli/src/whisper/parakeet.ts @@ -14,12 +14,18 @@ */ import { execFileSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { basename, extname, join } from "node:path"; import type { Word } from "./normalize.js"; import type { TranscribeResult } from "./transcribe.js"; +/** The model name `transcribe --json` reports for every Parakeet runner. */ +export const PARAKEET_MODEL_LABEL = "parakeet-tdt-0.6b-v3"; +/** Stdout/stderr line prefixes of the sherpa-onnx decode worker. */ +export const SHERPA_RESULT_PREFIX = "HYPERFRAMES_PARAKEET_RESULT:"; +export const SHERPA_ERROR_PREFIX = "HYPERFRAMES_PARAKEET_ERROR:"; + const DEFAULT_MODEL = "mlx-community/parakeet-tdt-0.6b-v3"; const PARAKEET_INSTALL = "uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx"; @@ -103,6 +109,63 @@ export function mergeTokensToWords(parakeet: ParakeetJson): Word[] { return words.filter((w) => w.text.length > 0); } +/** One decoded window from sherpa-onnx: token times are relative to the window's `offset` (s). */ +export interface SherpaWindow { + offset: number; + tokens: string[]; + timestamps: number[]; + durations?: number[]; +} + +const toMs = (seconds: number) => Math.round(seconds * 1000) / 1000; + +export function mergeWindowsToWords(windows: SherpaWindow[]): Word[] { + return mergeTokensToWords({ + sentences: windows.map((w) => ({ + tokens: w.tokens.map((text, i) => { + const start = w.offset + (w.timestamps[i] ?? 0); + return { text, start: toMs(start), end: toMs(start + (w.durations?.[i] ?? 0)) }; + }), + })), + }); +} + +function quietestBlockCenter(samples: Float32Array, lo: number, hi: number, block: number): number { + let best = hi; + let bestEnergy = Infinity; + for (let s = lo; s + block <= hi; s += block) { + let energy = 0; + for (let i = s; i < s + block; i++) energy += samples[i]! * samples[i]!; + if (energy < bestEnergy) { + bestEnergy = energy; + best = s + block / 2; + } + } + return best; +} + +/** + * Window boundaries (sample indices, first 0, last samples.length) about every `windowSeconds`, + * each moved to the quietest 100 ms within ±5 s. The encoder aborts past 400 s, and fixed cuts + * split words: measured 6.6% WER at fixed 60 s cuts, 1.5% snapped. + */ +export function silenceCuts( + samples: Float32Array, + sampleRate: number, + windowSeconds = 60, +): number[] { + const window = windowSeconds * sampleRate; + const slack = 5 * sampleRate; + const cuts = [0]; + while (cuts.at(-1)! + window < samples.length) { + const target = cuts.at(-1)! + window; + const hi = Math.min(target + slack, samples.length); + cuts.push(quietestBlockCenter(samples, target - slack, hi, sampleRate / 10)); + } + cuts.push(samples.length); + return cuts; +} + interface ParakeetOptions { language?: string; model?: string; @@ -139,13 +202,19 @@ export function transcribeWithParakeet( const produced = join(workDir, `${basename(inputPath, extname(inputPath))}.json`); if (!existsSync(produced)) throw new Error("Parakeet did not produce output."); - const words = mergeTokensToWords(JSON.parse(readFileSync(produced, "utf-8")) as ParakeetJson); - - const transcriptPath = join(dir, "transcript.json"); - writeFileSync(transcriptPath, JSON.stringify(words, null, 2)); - const durationSeconds = words.length > 0 ? words[words.length - 1]!.end : 0; - return { transcriptPath, wordCount: words.length, durationSeconds, speechOnsetSeconds: null }; + return writeParakeetTranscript( + dir, + mergeTokensToWords(JSON.parse(readFileSync(produced, "utf-8")) as ParakeetJson), + ); } finally { rmSync(workDir, { recursive: true, force: true }); } } + +export function writeParakeetTranscript(dir: string, words: Word[]): TranscribeResult { + mkdirSync(dir, { recursive: true }); + const transcriptPath = join(dir, "transcript.json"); + writeFileSync(transcriptPath, JSON.stringify(words, null, 2)); + const durationSeconds = words.length > 0 ? words[words.length - 1]!.end : 0; + return { transcriptPath, wordCount: words.length, durationSeconds, speechOnsetSeconds: null }; +} diff --git a/packages/cli/src/whisper/sherpa.test.ts b/packages/cli/src/whisper/sherpa.test.ts new file mode 100644 index 0000000000..b3c0fd3346 --- /dev/null +++ b/packages/cli/src/whisper/sherpa.test.ts @@ -0,0 +1,75 @@ +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DownloadOptions } from "../cloud/download.js"; +import { ensureParakeetModel, sherpaUnsupportedReason, type ModelFile } from "./sherpa.js"; + +const file = (name: string, content: string): ModelFile => ({ + name, + bytes: Buffer.byteLength(content), + sha256: createHash("sha256").update(content).digest("hex"), +}); + +/** A downloader that serves `served[name]` for any URL ending in that name. */ +function fakeDownload(served: Record) { + return vi.fn(async (url: string, dest: string, opts?: DownloadOptions) => { + const content = served[url.split("/").at(-1)!]!; + writeFileSync(dest, content); + opts?.onProgress?.(Buffer.byteLength(content), undefined); + return { path: dest, bytes: Buffer.byteLength(content) }; + }); +} + +describe("ensureParakeetModel", () => { + let dir: string; + afterEach(() => rmSync(dir, { recursive: true, force: true })); + const tempDir = () => (dir = mkdtempSync(join(tmpdir(), "hf-parakeet-model-"))); + + it("refuses a file whose sha256 does not match and leaves nothing under its name", async () => { + tempDir(); + const files = [file("encoder.onnx", "abc")]; + const download = fakeDownload({ "encoder.onnx": "abd" }); + + await expect(ensureParakeetModel({ dir, files, download })).rejects.toThrow(/sha256/); + expect(readdirSync(dir)).toEqual([]); + }); + + it("fetches only what is missing or corrupt, then is a no-op", async () => { + tempDir(); + mkdirSync(dir, { recursive: true }); + const files = [ + file("encoder.onnx", "enc"), + file("tokens.txt", "tok"), + file("joiner.onnx", "j"), + ]; + writeFileSync(join(dir, "encoder.onnx"), "enc"); + writeFileSync(join(dir, "tokens.txt"), "xxx"); + const download = fakeDownload({ "tokens.txt": "tok", "joiner.onnx": "j" }); + const onBytes = vi.fn(); + + expect(await ensureParakeetModel({ dir, files, download, onBytes })).toBe(true); + expect(download.mock.calls.map(([url]) => url.split("/").at(-1))).toEqual([ + "tokens.txt", + "joiner.onnx", + ]); + expect(readdirSync(dir).sort()).toEqual(["encoder.onnx", "joiner.onnx", "tokens.txt"]); + expect(onBytes.mock.calls.at(-1)).toEqual([4, 4]); + + download.mockClear(); + expect(await ensureParakeetModel({ dir, files, download })).toBe(false); + expect(download).not.toHaveBeenCalled(); + expect(existsSync(join(dir, "tokens.txt"))).toBe(true); + }); +}); + +describe("sherpaUnsupportedReason", () => { + it("refuses glibc older than 2.32 and accepts newer or non-glibc systems", () => { + const glibc = (version?: string) => ({ header: { glibcVersionRuntime: version } }); + expect(sherpaUnsupportedReason(glibc("2.31"))).toMatch(/glibc 2\.32/); + expect(sherpaUnsupportedReason(glibc("2.32"))).toBeNull(); + expect(sherpaUnsupportedReason(glibc("2.36"))).toBeNull(); + expect(sherpaUnsupportedReason(glibc())).toBeNull(); + }); +}); diff --git a/packages/cli/src/whisper/sherpa.ts b/packages/cli/src/whisper/sherpa.ts new file mode 100644 index 0000000000..4e0bb08981 --- /dev/null +++ b/packages/cli/src/whisper/sherpa.ts @@ -0,0 +1,223 @@ +import { execFile, type ExecFileException } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createReadStream, renameSync, rmSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { downloadToFile } from "../cloud/download.js"; +import { CACHE_DIR, install, isInstalled } from "../utils/optionalPackages.js"; +import { + mergeWindowsToWords, + SHERPA_ERROR_PREFIX, + SHERPA_RESULT_PREFIX, + writeParakeetTranscript, + type SherpaWindow, +} from "./parakeet.js"; +import { prepareWav, type TranscribeResult } from "./transcribe.js"; + +const RUNTIME = "sherpa-onnx-node"; +const RUNTIME_VERSION = "1.13.8"; +export const SHERPA_RUNTIME_DIR = join(CACHE_DIR, `${RUNTIME}@${RUNTIME_VERSION}`); + +const MODEL_REPO = "csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8"; +const MODEL_REVISION = "2bda32ec70b097a55adaa07d9a7173915b43cc78"; +export const PARAKEET_MODEL_DIR = join( + homedir(), + ".cache", + "hyperframes", + "parakeet", + "parakeet-tdt-0.6b-v3-int8", +); + +export interface ModelFile { + name: string; + bytes: number; + sha256: string; +} + +/** Order matters: encoder, decoder, joiner, tokens (see recognizerConfig). */ +const PARAKEET_MODEL_FILES: readonly ModelFile[] = [ + { + name: "encoder.int8.onnx", + bytes: 652_184_281, + sha256: "acfc2b4456377e15d04f0243af540b7fe7c992f8d898d751cf134c3a55fd2247", + }, + { + name: "decoder.int8.onnx", + bytes: 11_845_275, + sha256: "179e50c43d1a9de79c8a24149a2f9bac6eb5981823f2a2ed88d655b24248db4e", + }, + { + name: "joiner.int8.onnx", + bytes: 6_355_277, + sha256: "3164c13fc2821009440d20fcb5fdc78bff28b4db2f8d0f0b329101719c0948b3", + }, + { + name: "tokens.txt", + bytes: 93_939, + sha256: "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d", + }, +]; + +const DECODE_TIMEOUT_MS = 1_800_000; + +type ProcessReport = { header?: { glibcVersionRuntime?: string } }; + +/** Why sherpa-onnx cannot run here, or null. Its Linux prebuilt needs glibc 2.32. */ +export function sherpaUnsupportedReason( + report = process.report.getReport() as ProcessReport, +): string | null { + const glibc = report.header?.glibcVersionRuntime; + if (!glibc) return null; + const [major = 0, minor = 0] = glibc.split(".").map(Number); + if (major > 2 || (major === 2 && minor >= 32)) return null; + return `Parakeet needs glibc 2.32 or newer; this system has glibc ${glibc}.`; +} + +export function sherpaRuntimeInstalled(): boolean { + return isInstalled(SHERPA_RUNTIME_DIR, RUNTIME); +} + +export function installSherpaRuntime(): Promise { + return install(SHERPA_RUNTIME_DIR, RUNTIME, RUNTIME_VERSION); +} + +/** Sizes only: hashing 650 MB on every transcribe is too slow, and install already verified them. */ +export function sherpaParakeetInstalled(): boolean { + return ( + sherpaRuntimeInstalled() && + PARAKEET_MODEL_FILES.every( + (f) => + statSync(join(PARAKEET_MODEL_DIR, f.name), { throwIfNoEntry: false })?.size === f.bytes, + ) + ); +} + +async function sha256File(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer); + return hash.digest("hex"); +} + +async function verifies(path: string, file: ModelFile): Promise { + if (statSync(path, { throwIfNoEntry: false })?.size !== file.bytes) return false; + return (await sha256File(path)) === file.sha256; +} + +interface EnsureModelOptions { + dir?: string; + files?: readonly ModelFile[]; + download?: typeof downloadToFile; + onBytes?: (done: number, total: number) => void; +} + +/** + * Fetches every file that does not verify under a temp name, checks size and sha256, then renames + * it into place, so a file under its real name is always complete. False when all already verified. + */ +export async function ensureParakeetModel({ + dir = PARAKEET_MODEL_DIR, + files = PARAKEET_MODEL_FILES, + download = downloadToFile, + onBytes, +}: EnsureModelOptions = {}): Promise { + const missing: ModelFile[] = []; + for (const file of files) if (!(await verifies(join(dir, file.name), file))) missing.push(file); + const total = missing.reduce((sum, f) => sum + f.bytes, 0); + let done = 0; + for (const file of missing) { + const dest = join(dir, file.name); + const temp = `${dest}.${process.pid}.download`; + rmSync(dest, { force: true }); + try { + const url = `https://huggingface.co/${MODEL_REPO}/resolve/${MODEL_REVISION}/${file.name}`; + await download(url, temp, { onProgress: (bytes) => onBytes?.(done + bytes, total) }); + if (!(await verifies(temp, file))) { + throw new Error( + `${file.name} did not match its pinned size and sha256, so it was discarded. Re-run to retry.`, + ); + } + renameSync(temp, dest); + } finally { + rmSync(temp, { force: true }); + } + done += file.bytes; + } + return missing.length > 0; +} + +function recognizerConfig(): object { + const [encoder, decoder, joiner, tokens] = PARAKEET_MODEL_FILES.map((f) => + join(PARAKEET_MODEL_DIR, f.name), + ); + return { + featConfig: { sampleRate: 16000, featureDim: 80 }, + modelConfig: { + transducer: { encoder, decoder, joiner }, + tokens, + numThreads: 4, + provider: "cpu", + debug: 0, + modelType: "nemo_transducer", + }, + decodingMethod: "greedy_search", + }; +} + +function failureReason(err: ExecFileException | null, stderr: string): string { + const how = !err + ? "exited without a result" + : err.killed + ? "timed out" + : err.signal + ? `crashed (${err.signal})` + : `exited with code ${err.code}`; + const lines = stderr + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + const why = + lines.find((line) => line.startsWith(SHERPA_ERROR_PREFIX))?.slice(SHERPA_ERROR_PREFIX.length) ?? + lines.at(-1); + return why ? `${how}: ${why}` : how; +} + +/** Decodes in a child process: onnxruntime aborts the whole process on some inputs, uncatchably. */ +function decode(wavPath: string): Promise { + const sourceMode = import.meta.url.endsWith(".ts"); + const worker = new URL(sourceMode ? "./sherpaWorker.ts" : "./sherpaWorker.js", import.meta.url); + const args = [...(sourceMode ? ["--import", "tsx"] : []), fileURLToPath(worker)]; + const input = { wavPath, runtimeDir: SHERPA_RUNTIME_DIR, config: recognizerConfig() }; + return new Promise((resolve, reject) => { + execFile( + process.execPath, + args, + { + env: { ...process.env, HYPERFRAMES_PARAKEET_INPUT: JSON.stringify(input) }, + maxBuffer: 256 * 1024 * 1024, + timeout: DECODE_TIMEOUT_MS, + }, + (err, stdout, stderr) => { + const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX)); + if (!err && line) resolve(JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length))); + else reject(new Error(`Parakeet decoder ${failureReason(err, stderr)}`)); + }, + ); + }); +} + +export async function transcribeWithSherpa( + inputPath: string, + dir: string, + options?: { onProgress?: (message: string) => void }, +): Promise { + const unsupported = sherpaUnsupportedReason(); + if (unsupported) throw new Error(unsupported); + const wavPath = prepareWav(inputPath, options?.onProgress); + try { + options?.onProgress?.("Transcribing with Parakeet..."); + return writeParakeetTranscript(dir, mergeWindowsToWords(await decode(wavPath))); + } finally { + if (wavPath !== inputPath) rmSync(wavPath, { force: true }); + } +} diff --git a/packages/cli/src/whisper/sherpaWorker.ts b/packages/cli/src/whisper/sherpaWorker.ts new file mode 100644 index 0000000000..93c4881040 --- /dev/null +++ b/packages/cli/src/whisper/sherpaWorker.ts @@ -0,0 +1,47 @@ +import { loadInstalled } from "../utils/optionalPackages.js"; +import { + silenceCuts, + SHERPA_ERROR_PREFIX, + SHERPA_RESULT_PREFIX, + type SherpaWindow, +} from "./parakeet.js"; + +interface Wave { + samples: Float32Array; + sampleRate: number; +} + +interface SherpaOnnx { + readWave(path: string): Wave; + OfflineRecognizer: new (config: object) => { + createStream(): { acceptWaveform(wave: Wave): void }; + decode(stream: unknown): void; + getResult(stream: unknown): Omit; + }; +} + +const { wavPath, runtimeDir, config } = JSON.parse(process.env.HYPERFRAMES_PARAKEET_INPUT ?? "{}"); + +try { + const sherpa = loadInstalled(runtimeDir, "sherpa-onnx-node") as SherpaOnnx | null; + if (!sherpa) throw new Error(`sherpa-onnx-node is not installed in ${runtimeDir}`); + const recognizer = new sherpa.OfflineRecognizer(config); + const wave = sherpa.readWave(wavPath); + const cuts = silenceCuts(wave.samples, wave.sampleRate); + const windows: SherpaWindow[] = []; + for (let k = 0; k + 1 < cuts.length; k++) { + const stream = recognizer.createStream(); + const samples = wave.samples.subarray(cuts[k], cuts[k + 1]); + stream.acceptWaveform({ sampleRate: wave.sampleRate, samples }); + recognizer.decode(stream); + const { tokens, timestamps, durations } = recognizer.getResult(stream); + windows.push({ offset: cuts[k]! / wave.sampleRate, tokens, timestamps, durations }); + } + process.stdout.write(`${SHERPA_RESULT_PREFIX}${JSON.stringify(windows)}\n`); +} catch (err) { + const message = err instanceof Error ? err.message : String(err); + // Rethrow after the flush: a pipe write is asynchronous on macOS and a crash would drop it. + process.stderr.write(`${SHERPA_ERROR_PREFIX}${message}\n`, () => { + throw err; + }); +} diff --git a/packages/cli/src/whisper/transcribe.ts b/packages/cli/src/whisper/transcribe.ts index c511a2b535..fa8e422cc6 100644 --- a/packages/cli/src/whisper/transcribe.ts +++ b/packages/cli/src/whisper/transcribe.ts @@ -395,6 +395,23 @@ export function initialModelForLanguage(model: string, language?: string): strin return model; } +export function prepareWav(inputPath: string, onProgress?: (message: string) => void): string { + if (isAudioFile(inputPath)) { + onProgress?.("Preparing audio..."); + return prepareAudio(inputPath); + } + if (isVideoFile(inputPath)) { + if (!hasFFmpeg()) { + throw new Error( + `ffmpeg is required to extract audio from video. Install: ${getFFmpegInstallHint()}`, + ); + } + onProgress?.("Extracting audio from video..."); + return extractAudio(inputPath); + } + throw new Error(`Unsupported file type: ${extname(inputPath).toLowerCase()}`); +} + /** * Transcribe an audio or video file and save transcript.json to the output directory. */ @@ -417,23 +434,7 @@ export async function transcribe( }); // 3. Prepare audio - let wavPath: string; - const ext = extname(inputPath).toLowerCase(); - - if (isAudioFile(inputPath)) { - options?.onProgress?.("Preparing audio..."); - wavPath = prepareAudio(inputPath); - } else if (isVideoFile(inputPath)) { - if (!hasFFmpeg()) { - throw new Error( - `ffmpeg is required to extract audio from video. Install: ${getFFmpegInstallHint()}`, - ); - } - options?.onProgress?.("Extracting audio from video..."); - wavPath = extractAudio(inputPath); - } else { - throw new Error(`Unsupported file type: ${ext}`); - } + const wavPath = prepareWav(inputPath, options?.onProgress); // 4. Detect language and ensure correct model let effectiveModel = model; diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts index b9592f72d4..4e82ab1ddd 100644 --- a/packages/cli/tsup.config.ts +++ b/packages/cli/tsup.config.ts @@ -16,6 +16,7 @@ export default defineConfig({ fontLocalizeCli: "src/fontLocalizeCli.ts", runtimeVersion: "src/runtimeVersion.ts", renderSetupWorker: "src/renderSetupWorker.ts", + sherpaWorker: "src/whisper/sherpaWorker.ts", shaderTransitionWorker: "../producer/src/services/shaderTransitionWorker.ts", "registry/localSemantic": "src/registry/localSemantic.ts", },