feat(cli): download Parakeet with models install parakeet and transcribe with it

`hyperframes models install parakeet` installs the sherpa-onnx runtime from
npm and the Parakeet TDT 0.6B v3 int8 model from Hugging Face, pinned to a
revision. Every file is checked against its pinned size and sha256 under a
temp name before it is renamed into place. Re-running it is a no-op. On
Linux with glibc older than 2.32 it refuses up front.

`transcribe --engine auto|parakeet` now prefers that engine when it is
installed, then an installed parakeet-mlx runner, then whisper. Audio is cut
into about 60 s windows at the quietest 100 ms near each cut, and decoding
runs in a child process. If the child cannot load the runtime or crashes,
transcribe falls back with one stderr line saying why. The JSON output and
transcript.json shape are unchanged.
This commit is contained in:
Miguel Ángel
2026-09-26 22:48:17 -04:00
parent 31c00d2f0e
commit b7477c5729
15 changed files with 736 additions and 48 deletions
+1
View File
@@ -165,6 +165,7 @@ const commandLoaders = {
browser: () => import("./commands/browser.js").then((m) => m.default),
"remove-background": () => import("./commands/remove-background.js").then((m) => m.default),
transcribe: () => import("./commands/transcribe.js").then((m) => m.default),
models: () => import("./commands/models.js").then((m) => m.default),
tts: () => import("./commands/tts.js").then((m) => m.default),
docs: () => import("./commands/docs.js").then((m) => m.default),
doctor: () => import("./commands/doctor.js").then((m) => m.default),
+83
View File
@@ -0,0 +1,83 @@
import { defineCommand } from "citty";
import * as clack from "@clack/prompts";
import type { Example } from "./_examples.js";
import { c } from "../ui/colors.js";
import { formatBytes } from "../ui/format.js";
import { failCommand } from "../utils/commandResult.js";
import { PARAKEET_MODEL_LABEL } from "../whisper/parakeet.js";
export const examples: Example[] = [
[
"Download the Parakeet speech model that transcribe uses",
"hyperframes models install parakeet",
],
];
function fail(message: string, json: boolean): never {
if (json) console.log(JSON.stringify({ ok: false, error: message }));
else console.error(c.error(message));
failCommand();
}
async function installParakeet(json: boolean): Promise<void> {
const sherpa = await import("../whisper/sherpa.js");
const unsupported = sherpa.sherpaUnsupportedReason();
if (unsupported) fail(unsupported, json);
const spin = json ? null : clack.spinner({ output: process.stderr });
spin?.start("Checking the sherpa-onnx runtime...");
try {
const runtimeMissing = !sherpa.sherpaRuntimeInstalled();
if (runtimeMissing) {
spin?.message("Installing the sherpa-onnx runtime from npm...");
await sherpa.installSherpaRuntime();
}
spin?.message("Verifying the Parakeet model...");
let lastPct = -1;
const modelFetched = await sherpa.ensureParakeetModel({
onBytes: (done, total) => {
const pct = Math.floor((done / total) * 100);
if (pct <= lastPct) return;
lastPct = pct;
spin?.message(
`Downloading Parakeet TDT 0.6B v3 — ${c.progress(pct + "%")} ${c.dim("(" + formatBytes(done) + " / " + formatBytes(total) + ")")}`,
);
},
});
const changed = runtimeMissing || modelFetched;
if (json) {
console.log(
JSON.stringify({
ok: true,
model: PARAKEET_MODEL_LABEL,
changed,
runtimeDir: sherpa.SHERPA_RUNTIME_DIR,
modelDir: sherpa.PARAKEET_MODEL_DIR,
}),
);
} else {
spin?.stop(c.success(changed ? "Parakeet installed" : "Parakeet is already installed"));
}
} catch (err) {
spin?.stop(c.error("Parakeet install failed"));
fail(err instanceof Error ? err.message : String(err), json);
}
}
export default defineCommand({
meta: { name: "models", description: "Download on-device models (models install parakeet)" },
args: {
action: { type: "positional", description: "install", required: true },
name: { type: "positional", description: "Model to install: parakeet", required: true },
json: { type: "boolean", description: "Print one JSON result, no progress", default: false },
},
async run({ args }) {
if (args.action !== "install" || args.name !== "parakeet") {
fail(
`Unknown: models ${args.action} ${args.name}. Try: hyperframes models install parakeet`,
args.json,
);
}
return installParakeet(args.json);
},
});
+122 -1
View File
@@ -7,7 +7,28 @@ import { CliRuntimeError, consumeCommandResult } from "../utils/commandResult.js
// Make the whisper core report "unavailable" so we exercise the soft-skip path.
const transcribeMock = vi.fn();
vi.mock("../whisper/transcribe.js", () => ({ transcribe: transcribeMock }));
vi.mock("../whisper/transcribe.js", () => ({
transcribe: transcribeMock,
prepareWav: (input: string) => input,
}));
// Engine selection: which runners look installed, and the sherpa decode child it spawns.
const runners = { sherpa: false, mlx: false };
vi.mock("../whisper/sherpa.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../whisper/sherpa.js")>()),
sherpaParakeetInstalled: () => runners.sherpa,
}));
const mlxMock = vi.fn();
vi.mock("../whisper/parakeet.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../whisper/parakeet.js")>()),
findParakeet: () => (runners.mlx ? "/bin/parakeet-mlx" : undefined),
transcribeWithParakeet: (...a: unknown[]) => mlxMock(...a),
}));
const execFileMock = vi.fn();
vi.mock("node:child_process", async (importOriginal) => ({
...(await importOriginal<typeof import("node:child_process")>()),
execFile: (...a: unknown[]) => execFileMock(...a),
}));
const trackTranscribeUnavailable = vi.fn();
const trackCommandFailure = vi.fn();
@@ -18,6 +39,17 @@ vi.mock("../telemetry/events.js", () => ({
import transcribeCmd from "./transcribe.js";
/** Makes each runner write a one-word transcript naming itself. */
function fakeTranscript(dir: string, text: string) {
const transcriptPath = join(dir, "transcript.json");
writeFileSync(transcriptPath, JSON.stringify([{ text, start: 0, end: 1 }]));
return { transcriptPath, wordCount: 1, durationSeconds: 1, speechOnsetSeconds: null };
}
function lastJson(): Record<string, unknown> {
return JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0]));
}
function dummyAudio(): { dir: string; input: string } {
const dir = mkdtempSync(join(tmpdir(), "hf-transcribe-test-"));
const input = join(dir, "narration.wav");
@@ -33,6 +65,8 @@ describe("transcribe command", () => {
transcribeMock.mockReset();
trackTranscribeUnavailable.mockReset();
trackCommandFailure.mockReset();
mlxMock.mockReset();
Object.assign(runners, { sherpa: false, mlx: false });
transcribeMock.mockRejectedValue(
new WhisperUnavailableError("whisper-cpp not found. Install: brew install whisper-cpp"),
);
@@ -79,6 +113,93 @@ describe("transcribe command", () => {
expect(trackCommandFailure).not.toHaveBeenCalled();
});
describe("engine selection", () => {
beforeEach(() => {
transcribeMock.mockImplementation(async (_in: string, dir: string) =>
fakeTranscript(dir, "whisper"),
);
mlxMock.mockImplementation((_in: string, dir: string) => fakeTranscript(dir, "mlx"));
execFileMock.mockReset();
execFileMock.mockImplementation((_cmd, _args, _opts, done) => {
const windows = [{ offset: 60, tokens: [" sherpa"], timestamps: [0.5], durations: [0.25] }];
done(null, `HYPERFRAMES_PARAKEET_RESULT:${JSON.stringify(windows)}\n`, "");
});
vi.spyOn(console, "error").mockImplementation(() => {});
vi.spyOn(process.report, "getReport").mockReturnValue({
header: { glibcVersionRuntime: "2.36" },
} as never);
});
async function transcribeWith(engine: string, installed: typeof runners) {
Object.assign(runners, installed);
const { dir, input } = dummyAudio();
dirs.push(dir);
await transcribeCmd.run!({ args: { input, json: true, engine } } as never);
const out = lastJson();
const words = JSON.parse(readFileSync(String(out.transcriptPath), "utf-8"));
return { engine: out.engine, model: out.model, word: words[0]?.text, start: words[0]?.start };
}
it("auto and parakeet prefer sherpa-onnx, then parakeet-mlx, then whisper", async () => {
const sherpa = {
engine: "parakeet",
model: "parakeet-tdt-0.6b-v3",
word: "sherpa",
start: 60.5,
};
expect(await transcribeWith("auto", { sherpa: true, mlx: true })).toEqual(sherpa);
expect(await transcribeWith("parakeet", { sherpa: true, mlx: true })).toEqual(sherpa);
expect(await transcribeWith("auto", { sherpa: false, mlx: true })).toMatchObject({
engine: "parakeet",
word: "mlx",
});
expect(await transcribeWith("auto", { sherpa: false, mlx: false })).toMatchObject({
engine: "whisper",
word: "whisper",
});
expect(await transcribeWith("whisper", { sherpa: true, mlx: true })).toMatchObject({
engine: "whisper",
word: "whisper",
});
});
it("--engine parakeet with nothing installed names the install command", async () => {
Object.assign(runners, { sherpa: false, mlx: false });
const { dir, input } = dummyAudio();
dirs.push(dir);
await expect(
transcribeCmd.run!({ args: { input, json: true, engine: "parakeet" } } as never),
).rejects.toThrow(CliRuntimeError);
expect(lastJson().error).toContain("hyperframes models install parakeet");
expect(transcribeMock).not.toHaveBeenCalled();
});
it("falls back to whisper with one stderr line when the decode child crashes", async () => {
execFileMock.mockImplementation((_cmd, _args, _opts, done) => {
const crash = Object.assign(new Error("Command failed"), { code: null, signal: "SIGABRT" });
done(crash, "", "terminate called after throwing an instance of 'Ort::Exception'\n");
});
expect(await transcribeWith("auto", { sherpa: true, mlx: false })).toMatchObject({
engine: "whisper",
word: "whisper",
});
const warnings = vi.mocked(console.error).mock.calls.map(([line]) => String(line));
expect(warnings).toHaveLength(1);
expect(warnings[0]).toMatch(/sherpa-onnx.*whisper.*SIGABRT.*Ort::Exception/);
});
it("falls back without spawning when glibc is too old for sherpa-onnx", async () => {
vi.mocked(process.report.getReport).mockReturnValue({
header: { glibcVersionRuntime: "2.31" },
} as never);
expect(await transcribeWith("auto", { sherpa: true, mlx: false })).toMatchObject({
engine: "whisper",
});
expect(execFileMock).not.toHaveBeenCalled();
expect(String(vi.mocked(console.error).mock.calls[0]?.[0])).toMatch(/glibc 2\.32.*2\.31/);
});
});
it("imports an SRT and exports an SRT sidecar from transcript.json", async () => {
const dir = mkdtempSync(join(tmpdir(), "hf-transcribe-test-"));
dirs.push(dir);
+48 -19
View File
@@ -3,7 +3,7 @@ import { failCommand, setCommandExitCode } from "../utils/commandResult.js";
import { defineCommand } from "citty";
import type { Example } from "./_examples.js";
import { existsSync, writeFileSync } from "node:fs";
import { findParakeet, transcribeWithParakeet } from "../whisper/parakeet.js";
import { findParakeet, PARAKEET_MODEL_LABEL, transcribeWithParakeet } from "../whisper/parakeet.js";
type CaptionExportFormat = "srt" | "vtt";
@@ -55,7 +55,7 @@ export default defineCommand({
engine: {
type: "string",
description:
"ASR engine: auto (Parakeet if installed, else whisper), parakeet, or whisper. Default: auto. Parakeet is more accurate and faster; enable with `uv pip install parakeet-mlx`.",
"ASR engine: auto (Parakeet if installed, else whisper), parakeet, or whisper. Default: auto. Parakeet is more accurate and faster; install it with `hyperframes models install parakeet`.",
alias: "e",
},
model: {
@@ -267,6 +267,15 @@ async function exportTranscript(
// Transcribe audio/video with whisper
// ---------------------------------------------------------------------------
type Runner = "sherpa" | "parakeet-mlx" | "whisper";
/** auto and parakeet prefer sherpa-onnx, then an installed parakeet-mlx runner, then whisper. */
function pickRunner(engine: string, sherpaInstalled: () => boolean): Runner {
if (engine === "whisper") return "whisper";
if (sherpaInstalled()) return "sherpa";
return findParakeet() ? "parakeet-mlx" : "whisper";
}
// fallow-ignore-next-line complexity
async function transcribeAudio(
inputPath: string,
@@ -284,36 +293,56 @@ async function transcribeAudio(
const { loadTranscript, patchCaptionHtml, stripBeforeOnset } =
await import("../whisper/normalize.js");
// Engine: auto (Parakeet if installed, else whisper), or forced parakeet/whisper.
const { sherpaParakeetInstalled, transcribeWithSherpa } = await import("../whisper/sherpa.js");
const engine = (opts.engine ?? "auto").toLowerCase();
if (engine !== "auto" && engine !== "parakeet" && engine !== "whisper") {
failWith(`Unknown --engine: ${opts.engine}. Use auto, parakeet, or whisper.`, !!opts.json);
}
const useParakeet = engine === "parakeet" || (engine === "auto" && !!findParakeet());
let runner = pickRunner(engine, sherpaParakeetInstalled);
if (engine === "parakeet" && runner === "whisper") {
failWith(
"Parakeet is not installed. Install it with: hyperframes models install parakeet (or use --engine whisper)",
!!opts.json,
);
}
const model = opts.model ?? DEFAULT_MODEL;
// --model selects the whisper model only; Parakeet uses its own fixed model.
if (useParakeet && opts.model && !opts.json) {
if (runner !== "whisper" && opts.model && !opts.json) {
console.error(
c.dim(` Note: --model applies to the whisper engine only; ignored under Parakeet.`),
);
}
const label = useParakeet ? "Parakeet" : model;
const label = runner === "whisper" ? model : "Parakeet";
const spin = opts.json ? null : clack.spinner();
spin?.start(`Transcribing with ${c.accent(label)}...`);
const onProgress = spin ? (msg: string) => spin.message(msg) : undefined;
const run = (r: Runner) =>
r === "sherpa"
? transcribeWithSherpa(inputPath, dir, { onProgress })
: r === "parakeet-mlx"
? transcribeWithParakeet(inputPath, dir, { language: opts.language, onProgress })
: transcribe(inputPath, dir, {
model,
language: opts.language,
onProgress,
timeoutMs: opts.timeoutMs,
});
try {
const result = useParakeet
? transcribeWithParakeet(inputPath, dir, {
language: opts.language,
onProgress: spin ? (msg) => spin.message(msg) : undefined,
})
: await transcribe(inputPath, dir, {
model,
language: opts.language,
onProgress: spin ? (msg) => spin.message(msg) : undefined,
timeoutMs: opts.timeoutMs,
});
let result: Awaited<ReturnType<typeof run>>;
try {
result = await run(runner);
} catch (err) {
if (runner !== "sherpa") throw err;
runner = pickRunner(engine, () => false);
const reason = err instanceof Error ? err.message : String(err);
spin?.clear();
console.error(c.warn(`Parakeet (sherpa-onnx) failed, using ${runner} instead. ${reason}`));
spin?.start(`Transcribing with ${c.accent(runner)}...`);
result = await run(runner);
}
let { words } = loadTranscript(result.transcriptPath);
@@ -335,8 +364,8 @@ async function transcribeAudio(
console.log(
JSON.stringify({
ok: true,
engine: useParakeet ? "parakeet" : "whisper",
model: useParakeet ? "parakeet-tdt-0.6b-v3" : model,
engine: runner === "whisper" ? "whisper" : "parakeet",
model: runner === "whisper" ? model : PARAKEET_MODEL_LABEL,
wordCount: words.length,
durationSeconds: result.durationSeconds,
speechOnsetSeconds: result.speechOnsetSeconds,
+1
View File
@@ -90,6 +90,7 @@ const GROUPS: Group[] = [
"transcribe",
"Transcribe audio/video to word-level timestamps, or import an existing transcript",
],
["models", "Download on-device models (models install parakeet)"],
["tts", "Generate speech audio from text using a local AI model (Kokoro-82M)"],
["remove-background", "Remove background from a video or image to produce transparent media"],
],
+3 -3
View File
@@ -29,7 +29,7 @@ export const OPTIONAL_PACKAGES = {
"@google/genai": "1.52.0",
} as const satisfies Record<OptionalPackage, string>;
const CACHE_DIR = join(homedir(), ".cache", "hyperframes", "optional");
export const CACHE_DIR = join(homedir(), ".cache", "hyperframes", "optional");
export interface OptionalPackageDeps {
cacheDir: string;
@@ -108,11 +108,11 @@ function manifestPath(dir: string, name: string): string {
return join(dir, "node_modules", name, "package.json");
}
function isInstalled(dir: string, name: string): boolean {
export function isInstalled(dir: string, name: string): boolean {
return existsSync(manifestPath(dir, name));
}
function loadInstalled(dir: string, name: string): unknown | null {
export function loadInstalled(dir: string, name: string): unknown | null {
if (!isInstalled(dir, name)) return null;
return createRequire(join(dir, "package.json"))(name);
}
+31 -1
View File
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { mergeTokensToWords } from "./parakeet.js";
import { mergeTokensToWords, mergeWindowsToWords, silenceCuts } from "./parakeet.js";
describe("mergeTokensToWords", () => {
it("joins Parakeet sub-word tokens into words on the space boundary", () => {
@@ -35,3 +35,33 @@ describe("mergeTokensToWords", () => {
expect(words[1]!.start).toBe(0.5);
});
});
describe("mergeWindowsToWords", () => {
it("shifts each window's token times by the window's start", () => {
const words = mergeWindowsToWords([
{ offset: 0, tokens: [" Hel", "lo"], timestamps: [0.125, 0.25], durations: [0.125, 0.25] },
{ offset: 60.5, tokens: [" world"], timestamps: [0.25], durations: [0.5] },
]);
expect(words).toEqual([
{ text: "Hello", start: 0.125, end: 0.5 },
{ text: "world", start: 60.75, end: 61.25 },
]);
});
});
describe("silenceCuts", () => {
// 100 samples/s keeps the fixture small: 150 s of steady sound with two 0.1 s silences.
const rate = 100;
const loud = (seconds: number) => new Float32Array(seconds * rate).fill(0.5);
it("moves each cut to the quietest 100 ms within 5 s of the 60 s target", () => {
const samples = loud(150);
samples.fill(0, 5830, 5840);
samples.fill(0, 12195, 12205);
expect(silenceCuts(samples, rate)).toEqual([0, 5835, 12200, 15000]);
});
it("leaves audio shorter than one window whole", () => {
expect(silenceCuts(loud(42), rate)).toEqual([0, 4200]);
});
});
+76 -7
View File
@@ -14,12 +14,18 @@
*/
import { execFileSync } from "node:child_process";
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { homedir, tmpdir } from "node:os";
import { basename, extname, join } from "node:path";
import type { Word } from "./normalize.js";
import type { TranscribeResult } from "./transcribe.js";
/** The model name `transcribe --json` reports for every Parakeet runner. */
export const PARAKEET_MODEL_LABEL = "parakeet-tdt-0.6b-v3";
/** Stdout/stderr line prefixes of the sherpa-onnx decode worker. */
export const SHERPA_RESULT_PREFIX = "HYPERFRAMES_PARAKEET_RESULT:";
export const SHERPA_ERROR_PREFIX = "HYPERFRAMES_PARAKEET_ERROR:";
const DEFAULT_MODEL = "mlx-community/parakeet-tdt-0.6b-v3";
const PARAKEET_INSTALL =
"uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx";
@@ -103,6 +109,63 @@ export function mergeTokensToWords(parakeet: ParakeetJson): Word[] {
return words.filter((w) => w.text.length > 0);
}
/** One decoded window from sherpa-onnx: token times are relative to the window's `offset` (s). */
export interface SherpaWindow {
offset: number;
tokens: string[];
timestamps: number[];
durations?: number[];
}
const toMs = (seconds: number) => Math.round(seconds * 1000) / 1000;
export function mergeWindowsToWords(windows: SherpaWindow[]): Word[] {
return mergeTokensToWords({
sentences: windows.map((w) => ({
tokens: w.tokens.map((text, i) => {
const start = w.offset + (w.timestamps[i] ?? 0);
return { text, start: toMs(start), end: toMs(start + (w.durations?.[i] ?? 0)) };
}),
})),
});
}
function quietestBlockCenter(samples: Float32Array, lo: number, hi: number, block: number): number {
let best = hi;
let bestEnergy = Infinity;
for (let s = lo; s + block <= hi; s += block) {
let energy = 0;
for (let i = s; i < s + block; i++) energy += samples[i]! * samples[i]!;
if (energy < bestEnergy) {
bestEnergy = energy;
best = s + block / 2;
}
}
return best;
}
/**
* Window boundaries (sample indices, first 0, last samples.length) about every `windowSeconds`,
* each moved to the quietest 100 ms within ±5 s. The encoder aborts past 400 s, and fixed cuts
* split words: measured 6.6% WER at fixed 60 s cuts, 1.5% snapped.
*/
export function silenceCuts(
samples: Float32Array,
sampleRate: number,
windowSeconds = 60,
): number[] {
const window = windowSeconds * sampleRate;
const slack = 5 * sampleRate;
const cuts = [0];
while (cuts.at(-1)! + window < samples.length) {
const target = cuts.at(-1)! + window;
const hi = Math.min(target + slack, samples.length);
cuts.push(quietestBlockCenter(samples, target - slack, hi, sampleRate / 10));
}
cuts.push(samples.length);
return cuts;
}
interface ParakeetOptions {
language?: string;
model?: string;
@@ -139,13 +202,19 @@ export function transcribeWithParakeet(
const produced = join(workDir, `${basename(inputPath, extname(inputPath))}.json`);
if (!existsSync(produced)) throw new Error("Parakeet did not produce output.");
const words = mergeTokensToWords(JSON.parse(readFileSync(produced, "utf-8")) as ParakeetJson);
const transcriptPath = join(dir, "transcript.json");
writeFileSync(transcriptPath, JSON.stringify(words, null, 2));
const durationSeconds = words.length > 0 ? words[words.length - 1]!.end : 0;
return { transcriptPath, wordCount: words.length, durationSeconds, speechOnsetSeconds: null };
return writeParakeetTranscript(
dir,
mergeTokensToWords(JSON.parse(readFileSync(produced, "utf-8")) as ParakeetJson),
);
} finally {
rmSync(workDir, { recursive: true, force: true });
}
}
export function writeParakeetTranscript(dir: string, words: Word[]): TranscribeResult {
mkdirSync(dir, { recursive: true });
const transcriptPath = join(dir, "transcript.json");
writeFileSync(transcriptPath, JSON.stringify(words, null, 2));
const durationSeconds = words.length > 0 ? words[words.length - 1]!.end : 0;
return { transcriptPath, wordCount: words.length, durationSeconds, speechOnsetSeconds: null };
}
+75
View File
@@ -0,0 +1,75 @@
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { DownloadOptions } from "../cloud/download.js";
import { ensureParakeetModel, sherpaUnsupportedReason, type ModelFile } from "./sherpa.js";
const file = (name: string, content: string): ModelFile => ({
name,
bytes: Buffer.byteLength(content),
sha256: createHash("sha256").update(content).digest("hex"),
});
/** A downloader that serves `served[name]` for any URL ending in that name. */
function fakeDownload(served: Record<string, string>) {
return vi.fn(async (url: string, dest: string, opts?: DownloadOptions) => {
const content = served[url.split("/").at(-1)!]!;
writeFileSync(dest, content);
opts?.onProgress?.(Buffer.byteLength(content), undefined);
return { path: dest, bytes: Buffer.byteLength(content) };
});
}
describe("ensureParakeetModel", () => {
let dir: string;
afterEach(() => rmSync(dir, { recursive: true, force: true }));
const tempDir = () => (dir = mkdtempSync(join(tmpdir(), "hf-parakeet-model-")));
it("refuses a file whose sha256 does not match and leaves nothing under its name", async () => {
tempDir();
const files = [file("encoder.onnx", "abc")];
const download = fakeDownload({ "encoder.onnx": "abd" });
await expect(ensureParakeetModel({ dir, files, download })).rejects.toThrow(/sha256/);
expect(readdirSync(dir)).toEqual([]);
});
it("fetches only what is missing or corrupt, then is a no-op", async () => {
tempDir();
mkdirSync(dir, { recursive: true });
const files = [
file("encoder.onnx", "enc"),
file("tokens.txt", "tok"),
file("joiner.onnx", "j"),
];
writeFileSync(join(dir, "encoder.onnx"), "enc");
writeFileSync(join(dir, "tokens.txt"), "xxx");
const download = fakeDownload({ "tokens.txt": "tok", "joiner.onnx": "j" });
const onBytes = vi.fn();
expect(await ensureParakeetModel({ dir, files, download, onBytes })).toBe(true);
expect(download.mock.calls.map(([url]) => url.split("/").at(-1))).toEqual([
"tokens.txt",
"joiner.onnx",
]);
expect(readdirSync(dir).sort()).toEqual(["encoder.onnx", "joiner.onnx", "tokens.txt"]);
expect(onBytes.mock.calls.at(-1)).toEqual([4, 4]);
download.mockClear();
expect(await ensureParakeetModel({ dir, files, download })).toBe(false);
expect(download).not.toHaveBeenCalled();
expect(existsSync(join(dir, "tokens.txt"))).toBe(true);
});
});
describe("sherpaUnsupportedReason", () => {
it("refuses glibc older than 2.32 and accepts newer or non-glibc systems", () => {
const glibc = (version?: string) => ({ header: { glibcVersionRuntime: version } });
expect(sherpaUnsupportedReason(glibc("2.31"))).toMatch(/glibc 2\.32/);
expect(sherpaUnsupportedReason(glibc("2.32"))).toBeNull();
expect(sherpaUnsupportedReason(glibc("2.36"))).toBeNull();
expect(sherpaUnsupportedReason(glibc())).toBeNull();
});
});
+223
View File
@@ -0,0 +1,223 @@
import { execFile, type ExecFileException } from "node:child_process";
import { createHash } from "node:crypto";
import { createReadStream, renameSync, rmSync, statSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { downloadToFile } from "../cloud/download.js";
import { CACHE_DIR, install, isInstalled } from "../utils/optionalPackages.js";
import {
mergeWindowsToWords,
SHERPA_ERROR_PREFIX,
SHERPA_RESULT_PREFIX,
writeParakeetTranscript,
type SherpaWindow,
} from "./parakeet.js";
import { prepareWav, type TranscribeResult } from "./transcribe.js";
const RUNTIME = "sherpa-onnx-node";
const RUNTIME_VERSION = "1.13.8";
export const SHERPA_RUNTIME_DIR = join(CACHE_DIR, `${RUNTIME}@${RUNTIME_VERSION}`);
const MODEL_REPO = "csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8";
const MODEL_REVISION = "2bda32ec70b097a55adaa07d9a7173915b43cc78";
export const PARAKEET_MODEL_DIR = join(
homedir(),
".cache",
"hyperframes",
"parakeet",
"parakeet-tdt-0.6b-v3-int8",
);
export interface ModelFile {
name: string;
bytes: number;
sha256: string;
}
/** Order matters: encoder, decoder, joiner, tokens (see recognizerConfig). */
const PARAKEET_MODEL_FILES: readonly ModelFile[] = [
{
name: "encoder.int8.onnx",
bytes: 652_184_281,
sha256: "acfc2b4456377e15d04f0243af540b7fe7c992f8d898d751cf134c3a55fd2247",
},
{
name: "decoder.int8.onnx",
bytes: 11_845_275,
sha256: "179e50c43d1a9de79c8a24149a2f9bac6eb5981823f2a2ed88d655b24248db4e",
},
{
name: "joiner.int8.onnx",
bytes: 6_355_277,
sha256: "3164c13fc2821009440d20fcb5fdc78bff28b4db2f8d0f0b329101719c0948b3",
},
{
name: "tokens.txt",
bytes: 93_939,
sha256: "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d",
},
];
const DECODE_TIMEOUT_MS = 1_800_000;
type ProcessReport = { header?: { glibcVersionRuntime?: string } };
/** Why sherpa-onnx cannot run here, or null. Its Linux prebuilt needs glibc 2.32. */
export function sherpaUnsupportedReason(
report = process.report.getReport() as ProcessReport,
): string | null {
const glibc = report.header?.glibcVersionRuntime;
if (!glibc) return null;
const [major = 0, minor = 0] = glibc.split(".").map(Number);
if (major > 2 || (major === 2 && minor >= 32)) return null;
return `Parakeet needs glibc 2.32 or newer; this system has glibc ${glibc}.`;
}
export function sherpaRuntimeInstalled(): boolean {
return isInstalled(SHERPA_RUNTIME_DIR, RUNTIME);
}
export function installSherpaRuntime(): Promise<void> {
return install(SHERPA_RUNTIME_DIR, RUNTIME, RUNTIME_VERSION);
}
/** Sizes only: hashing 650 MB on every transcribe is too slow, and install already verified them. */
export function sherpaParakeetInstalled(): boolean {
return (
sherpaRuntimeInstalled() &&
PARAKEET_MODEL_FILES.every(
(f) =>
statSync(join(PARAKEET_MODEL_DIR, f.name), { throwIfNoEntry: false })?.size === f.bytes,
)
);
}
async function sha256File(path: string): Promise<string> {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer);
return hash.digest("hex");
}
async function verifies(path: string, file: ModelFile): Promise<boolean> {
if (statSync(path, { throwIfNoEntry: false })?.size !== file.bytes) return false;
return (await sha256File(path)) === file.sha256;
}
interface EnsureModelOptions {
dir?: string;
files?: readonly ModelFile[];
download?: typeof downloadToFile;
onBytes?: (done: number, total: number) => void;
}
/**
* Fetches every file that does not verify under a temp name, checks size and sha256, then renames
* it into place, so a file under its real name is always complete. False when all already verified.
*/
export async function ensureParakeetModel({
dir = PARAKEET_MODEL_DIR,
files = PARAKEET_MODEL_FILES,
download = downloadToFile,
onBytes,
}: EnsureModelOptions = {}): Promise<boolean> {
const missing: ModelFile[] = [];
for (const file of files) if (!(await verifies(join(dir, file.name), file))) missing.push(file);
const total = missing.reduce((sum, f) => sum + f.bytes, 0);
let done = 0;
for (const file of missing) {
const dest = join(dir, file.name);
const temp = `${dest}.${process.pid}.download`;
rmSync(dest, { force: true });
try {
const url = `https://huggingface.co/${MODEL_REPO}/resolve/${MODEL_REVISION}/${file.name}`;
await download(url, temp, { onProgress: (bytes) => onBytes?.(done + bytes, total) });
if (!(await verifies(temp, file))) {
throw new Error(
`${file.name} did not match its pinned size and sha256, so it was discarded. Re-run to retry.`,
);
}
renameSync(temp, dest);
} finally {
rmSync(temp, { force: true });
}
done += file.bytes;
}
return missing.length > 0;
}
function recognizerConfig(): object {
const [encoder, decoder, joiner, tokens] = PARAKEET_MODEL_FILES.map((f) =>
join(PARAKEET_MODEL_DIR, f.name),
);
return {
featConfig: { sampleRate: 16000, featureDim: 80 },
modelConfig: {
transducer: { encoder, decoder, joiner },
tokens,
numThreads: 4,
provider: "cpu",
debug: 0,
modelType: "nemo_transducer",
},
decodingMethod: "greedy_search",
};
}
function failureReason(err: ExecFileException | null, stderr: string): string {
const how = !err
? "exited without a result"
: err.killed
? "timed out"
: err.signal
? `crashed (${err.signal})`
: `exited with code ${err.code}`;
const lines = stderr
.split("\n")
.map((line) => line.trim())
.filter(Boolean);
const why =
lines.find((line) => line.startsWith(SHERPA_ERROR_PREFIX))?.slice(SHERPA_ERROR_PREFIX.length) ??
lines.at(-1);
return why ? `${how}: ${why}` : how;
}
/** Decodes in a child process: onnxruntime aborts the whole process on some inputs, uncatchably. */
function decode(wavPath: string): Promise<SherpaWindow[]> {
const sourceMode = import.meta.url.endsWith(".ts");
const worker = new URL(sourceMode ? "./sherpaWorker.ts" : "./sherpaWorker.js", import.meta.url);
const args = [...(sourceMode ? ["--import", "tsx"] : []), fileURLToPath(worker)];
const input = { wavPath, runtimeDir: SHERPA_RUNTIME_DIR, config: recognizerConfig() };
return new Promise((resolve, reject) => {
execFile(
process.execPath,
args,
{
env: { ...process.env, HYPERFRAMES_PARAKEET_INPUT: JSON.stringify(input) },
maxBuffer: 256 * 1024 * 1024,
timeout: DECODE_TIMEOUT_MS,
},
(err, stdout, stderr) => {
const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX));
if (!err && line) resolve(JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length)));
else reject(new Error(`Parakeet decoder ${failureReason(err, stderr)}`));
},
);
});
}
export async function transcribeWithSherpa(
inputPath: string,
dir: string,
options?: { onProgress?: (message: string) => void },
): Promise<TranscribeResult> {
const unsupported = sherpaUnsupportedReason();
if (unsupported) throw new Error(unsupported);
const wavPath = prepareWav(inputPath, options?.onProgress);
try {
options?.onProgress?.("Transcribing with Parakeet...");
return writeParakeetTranscript(dir, mergeWindowsToWords(await decode(wavPath)));
} finally {
if (wavPath !== inputPath) rmSync(wavPath, { force: true });
}
}
+47
View File
@@ -0,0 +1,47 @@
import { loadInstalled } from "../utils/optionalPackages.js";
import {
silenceCuts,
SHERPA_ERROR_PREFIX,
SHERPA_RESULT_PREFIX,
type SherpaWindow,
} from "./parakeet.js";
interface Wave {
samples: Float32Array;
sampleRate: number;
}
interface SherpaOnnx {
readWave(path: string): Wave;
OfflineRecognizer: new (config: object) => {
createStream(): { acceptWaveform(wave: Wave): void };
decode(stream: unknown): void;
getResult(stream: unknown): Omit<SherpaWindow, "offset">;
};
}
const { wavPath, runtimeDir, config } = JSON.parse(process.env.HYPERFRAMES_PARAKEET_INPUT ?? "{}");
try {
const sherpa = loadInstalled(runtimeDir, "sherpa-onnx-node") as SherpaOnnx | null;
if (!sherpa) throw new Error(`sherpa-onnx-node is not installed in ${runtimeDir}`);
const recognizer = new sherpa.OfflineRecognizer(config);
const wave = sherpa.readWave(wavPath);
const cuts = silenceCuts(wave.samples, wave.sampleRate);
const windows: SherpaWindow[] = [];
for (let k = 0; k + 1 < cuts.length; k++) {
const stream = recognizer.createStream();
const samples = wave.samples.subarray(cuts[k], cuts[k + 1]);
stream.acceptWaveform({ sampleRate: wave.sampleRate, samples });
recognizer.decode(stream);
const { tokens, timestamps, durations } = recognizer.getResult(stream);
windows.push({ offset: cuts[k]! / wave.sampleRate, tokens, timestamps, durations });
}
process.stdout.write(`${SHERPA_RESULT_PREFIX}${JSON.stringify(windows)}\n`);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
// Rethrow after the flush: a pipe write is asynchronous on macOS and a crash would drop it.
process.stderr.write(`${SHERPA_ERROR_PREFIX}${message}\n`, () => {
throw err;
});
}
+18 -17
View File
@@ -395,6 +395,23 @@ export function initialModelForLanguage(model: string, language?: string): strin
return model;
}
export function prepareWav(inputPath: string, onProgress?: (message: string) => void): string {
if (isAudioFile(inputPath)) {
onProgress?.("Preparing audio...");
return prepareAudio(inputPath);
}
if (isVideoFile(inputPath)) {
if (!hasFFmpeg()) {
throw new Error(
`ffmpeg is required to extract audio from video. Install: ${getFFmpegInstallHint()}`,
);
}
onProgress?.("Extracting audio from video...");
return extractAudio(inputPath);
}
throw new Error(`Unsupported file type: ${extname(inputPath).toLowerCase()}`);
}
/**
* Transcribe an audio or video file and save transcript.json to the output directory.
*/
@@ -417,23 +434,7 @@ export async function transcribe(
});
// 3. Prepare audio
let wavPath: string;
const ext = extname(inputPath).toLowerCase();
if (isAudioFile(inputPath)) {
options?.onProgress?.("Preparing audio...");
wavPath = prepareAudio(inputPath);
} else if (isVideoFile(inputPath)) {
if (!hasFFmpeg()) {
throw new Error(
`ffmpeg is required to extract audio from video. Install: ${getFFmpegInstallHint()}`,
);
}
options?.onProgress?.("Extracting audio from video...");
wavPath = extractAudio(inputPath);
} else {
throw new Error(`Unsupported file type: ${ext}`);
}
const wavPath = prepareWav(inputPath, options?.onProgress);
// 4. Detect language and ensure correct model
let effectiveModel = model;
+1
View File
@@ -16,6 +16,7 @@ export default defineConfig({
fontLocalizeCli: "src/fontLocalizeCli.ts",
runtimeVersion: "src/runtimeVersion.ts",
renderSetupWorker: "src/renderSetupWorker.ts",
sherpaWorker: "src/whisper/sherpaWorker.ts",
shaderTransitionWorker: "../producer/src/services/shaderTransitionWorker.ts",
"registry/localSemantic": "src/registry/localSemantic.ts",
},