diff --git a/packages/cli/src/commands/transcribe.test.ts b/packages/cli/src/commands/transcribe.test.ts index abb3e5db45..65a4b52788 100644 --- a/packages/cli/src/commands/transcribe.test.ts +++ b/packages/cli/src/commands/transcribe.test.ts @@ -136,11 +136,11 @@ describe("transcribe command", () => { } as never); }); - async function transcribeWith(engine: string, installed: typeof runners) { + async function transcribeWith(engine: string, installed: typeof runners, language?: string) { Object.assign(runners, installed); const { dir, input } = dummyAudio(); dirs.push(dir); - await transcribeCmd.run!({ args: { input, json: true, engine } } as never); + await transcribeCmd.run!({ args: { input, json: true, engine, language } } as never); const out = lastJson(); const words = JSON.parse(readFileSync(String(out.transcriptPath), "utf-8")); return { engine: out.engine, model: out.model, word: words[0]?.text, start: words[0]?.start }; @@ -169,6 +169,51 @@ describe("transcribe command", () => { }); }); + it("auto uses Parakeet only for a language it transcribes", async () => { + const both = { sherpa: true, mlx: true }; + expect(await transcribeWith("auto", both, "ja")).toMatchObject({ engine: "whisper" }); + expect(await transcribeWith("auto", both, "pt-BR")).toMatchObject({ engine: "parakeet" }); + expect(await transcribeWith("auto", both, "UK")).toMatchObject({ engine: "parakeet" }); + expect(transcribeMock).toHaveBeenCalledTimes(1); + }); + + it("--engine parakeet with a language it does not transcribe fails without falling back", async () => { + Object.assign(runners, { sherpa: true, mlx: true }); + const { exitCode, out } = await transcribeFails("parakeet", { language: "ja" }); + expect(exitCode).toBe(1); + expect(out.error).toMatch( + /^Parakeet does not transcribe --language ja; it covers en, es, .*uk\. Use --engine whisper\.$/, + ); + expect(execFileMock).not.toHaveBeenCalled(); + expect(transcribeMock).not.toHaveBeenCalled(); + }); + + it("listens for Ctrl-C from before audio prep until the run ends", async () => { + const before = process.listenerCount("SIGINT"); + let duringPrep = 0; + prepareWavMock.mockImplementation((input: string) => { + duringPrep = process.listenerCount("SIGINT"); + return input; + }); + expect(await transcribeWith("auto", { sherpa: true, mlx: false })).toMatchObject({ + engine: "parakeet", + }); + expect(duringPrep).toBe(before + 1); + expect(process.listenerCount("SIGINT")).toBe(before); + }); + + it("exits 130 when Ctrl-C stops the whisper fallback too", async () => { + crashChild("SIGABRT"); + transcribeMock.mockImplementation(async () => { + process.emit("SIGINT"); + throw new Error("whisper-cli was killed by SIGINT"); + }); + Object.assign(runners, { sherpa: true, mlx: false }); + const { exitCode, out } = await transcribeFails("auto"); + expect(exitCode).toBe(130); + expect(out).toEqual({ ok: false, error: "Transcription cancelled" }); + }); + it("--engine parakeet with nothing installed names the install command", async () => { Object.assign(runners, { sherpa: false, mlx: false }); const { dir, input } = dummyAudio(); @@ -333,7 +378,7 @@ describe("transcribe command", () => { }); it.runIf(process.platform === "linux" && process.arch === "x64")( - "falls back without spawning when glibc is too old for sherpa-onnx", + "skips Parakeet when glibc is too old for sherpa-onnx, never naming the install command", async () => { vi.mocked(process.report.getReport).mockReturnValue({ header: { glibcVersionRuntime: "2.31" }, @@ -342,7 +387,12 @@ describe("transcribe command", () => { engine: "whisper", }); expect(execFileMock).not.toHaveBeenCalled(); - expect(String(vi.mocked(console.error).mock.calls[0]?.[0])).toMatch(/glibc 2\.32.*2\.31/); + expect(console.error).not.toHaveBeenCalled(); + const { exitCode, out } = await transcribeFails("parakeet"); + expect(exitCode).toBe(1); + expect(out.error).toMatch( + /^Parakeet needs glibc 2\.32 or newer; this system has glibc 2\.31\.$/, + ); }, ); }); diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.ts index f7c67dcebf..34f7d5e126 100644 --- a/packages/cli/src/commands/transcribe.ts +++ b/packages/cli/src/commands/transcribe.ts @@ -4,7 +4,13 @@ import { normalizeErrorMessage } from "../utils/errorMessage.js"; import { defineCommand } from "citty"; import type { Example } from "./_examples.js"; import { existsSync, rmSync, writeFileSync } from "node:fs"; -import { findParakeet, PARAKEET_MODEL_LABEL, transcribeWithParakeet } from "../whisper/parakeet.js"; +import { + findParakeet, + PARAKEET_LANGUAGES, + PARAKEET_MODEL_LABEL, + parakeetSpeaks, + transcribeWithParakeet, +} from "../whisper/parakeet.js"; type CaptionExportFormat = "srt" | "vtt"; @@ -270,10 +276,10 @@ async function exportTranscript( type Runner = "sherpa" | "parakeet-mlx" | "whisper"; -/** auto and parakeet prefer sherpa-onnx, then an installed parakeet-mlx runner, then whisper. */ -function pickRunner(engine: string, sherpaInstalled: () => boolean): Runner { - if (engine === "whisper") return "whisper"; - if (sherpaInstalled()) return "sherpa"; +/** auto and parakeet prefer sherpa-onnx, then parakeet-mlx, then whisper, in Parakeet's languages. */ +function pickRunner(engine: string, sherpaUsable: () => boolean, language?: string): Runner { + if (engine === "whisper" || !parakeetSpeaks(language)) return "whisper"; + if (sherpaUsable()) return "sherpa"; return findParakeet() ? "parakeet-mlx" : "whisper"; } @@ -297,17 +303,28 @@ async function transcribeAudio( const { loadTranscript, patchCaptionHtml, stripBeforeOnset } = await import("../whisper/normalize.js"); - const { DecodeCancelled, prepareSherpaWav, sherpaParakeetInstalled, transcribeWithSherpa } = - await import("../whisper/sherpa.js"); + const { + DecodeCancelled, + prepareSherpaWav, + sherpaParakeetInstalled, + sherpaUnsupportedReason, + transcribeWithSherpa, + } = await import("../whisper/sherpa.js"); + const { createRenderCancellationScope } = await import("../utils/renderCancellation.js"); const engine = (opts.engine ?? "auto").toLowerCase(); if (engine !== "auto" && engine !== "parakeet" && engine !== "whisper") { failWith(`Unknown --engine: ${opts.engine}. Use auto, parakeet, or whisper.`, !!opts.json); } - let runner = pickRunner(engine, sherpaParakeetInstalled); + const unsupported = sherpaUnsupportedReason(); + const sherpaUsable = () => !unsupported && sherpaParakeetInstalled(); + let runner = pickRunner(engine, sherpaUsable, opts.language); if (engine === "parakeet" && runner === "whisper") { failWith( - "Parakeet is not installed. Install it with: hyperframes models install parakeet (or use --engine whisper)", + !parakeetSpeaks(opts.language) + ? `Parakeet does not transcribe --language ${opts.language}; it covers ${PARAKEET_LANGUAGES.split(" ").join(", ")}. Use --engine whisper.` + : (unsupported ?? + "Parakeet is not installed. Install it with: hyperframes models install parakeet (or use --engine whisper)"), !!opts.json, ); } @@ -324,9 +341,11 @@ async function transcribeAudio( spin?.start(`Transcribing with ${label(runner)}...`); const onProgress = spin ? (msg: string) => spin.message(msg) : undefined; let wavPath = inputPath; + // Before audio prep: under --json no spinner listens for SIGINT, so Ctrl-C would kill Node. + const cancellation = runner === "sherpa" ? createRenderCancellationScope() : null; const run = (r: Runner) => r === "sherpa" - ? transcribeWithSherpa(wavPath, dir, { onProgress }) + ? transcribeWithSherpa(wavPath, dir, { onProgress, signal: cancellation!.signal }) : r === "parakeet-mlx" ? transcribeWithParakeet(wavPath, dir, { language: opts.language, onProgress }) : transcribe(wavPath, dir, { @@ -347,7 +366,7 @@ async function transcribeAudio( const reason = normalizeErrorMessage(err).replace(/\.+$/, ""); const parakeetError = `Parakeet failed: ${reason}. To repair it, run: hyperframes models install parakeet`; if (!parakeetFallsBack(engine)) throw new Error(parakeetError); - runner = pickRunner(engine, () => false); + runner = pickRunner(engine, () => false, opts.language); spin?.clear(); console.error(c.warn(`${parakeetError}. Using ${runner} for this run.`)); spin?.start(`Transcribing with ${label(runner)}...`); @@ -399,9 +418,10 @@ async function transcribeAudio( ); } } catch (err) { - if (err instanceof DecodeCancelled) { - if (opts.json) console.log(JSON.stringify({ ok: false, error: err.message })); - else spin?.stop(c.warn(err.message)); + if (err instanceof DecodeCancelled || cancellation?.signal.aborted) { + const message = "Transcription cancelled"; + if (opts.json) console.log(JSON.stringify({ ok: false, error: message })); + else spin?.stop(c.warn(message)); failCommand(130); } // Surface the last few lines of the ASR subprocess's stderr, which @@ -440,6 +460,7 @@ async function transcribeAudio( } failCommand(); } finally { + cancellation?.dispose(); if (wavPath !== inputPath) rmSync(wavPath, { force: true }); } } diff --git a/packages/cli/src/whisper/parakeet.ts b/packages/cli/src/whisper/parakeet.ts index 90dcdba365..ea805210aa 100644 --- a/packages/cli/src/whisper/parakeet.ts +++ b/packages/cli/src/whisper/parakeet.ts @@ -26,6 +26,16 @@ export const PARAKEET_MODEL_LABEL = "parakeet-tdt-0.6b-v3"; export const SHERPA_RESULT_PREFIX = "HYPERFRAMES_PARAKEET_RESULT:"; export const SHERPA_ERROR_PREFIX = "HYPERFRAMES_PARAKEET_ERROR:"; +/** Parakeet TDT 0.6B v3's languages, as listed on its model card. */ +export const PARAKEET_LANGUAGES = + "en es fr de bg hr cs da nl et fi el hu it lv lt mt pl pt ro sk sl sv ru uk"; + +/** True when no language was asked for, or Parakeet transcribes it ("pt-BR" counts as pt). */ +export function parakeetSpeaks(language?: string): boolean { + const base = language?.trim().toLowerCase().split(/[-_]/, 1)[0]; + return !base || PARAKEET_LANGUAGES.split(" ").includes(base); +} + const DEFAULT_MODEL = "mlx-community/parakeet-tdt-0.6b-v3"; const PARAKEET_INSTALL = "uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx"; @@ -117,14 +127,14 @@ export interface SherpaWindow { durations?: number[]; } -const toMs = (seconds: number) => Math.round(seconds * 1000) / 1000; +const roundToMs = (seconds: number) => Math.round(seconds * 1000) / 1000; export function mergeWindowsToWords(windows: SherpaWindow[]): Word[] { return mergeTokensToWords({ sentences: windows.map((w) => ({ tokens: w.tokens.map((text, i) => { const start = w.offset + (w.timestamps[i] ?? 0); - return { text, start: toMs(start), end: toMs(start + (w.durations?.[i] ?? 0)) }; + return { text, start: roundToMs(start), end: roundToMs(start + (w.durations?.[i] ?? 0)) }; }), })), }); diff --git a/packages/cli/src/whisper/sherpa.test.ts b/packages/cli/src/whisper/sherpa.test.ts index 789934b02a..c3df458aea 100644 --- a/packages/cli/src/whisper/sherpa.test.ts +++ b/packages/cli/src/whisper/sherpa.test.ts @@ -7,7 +7,9 @@ import type { DownloadOptions } from "../cloud/download.js"; import { ensureParakeetModel, installSherpaRuntime, + SHERPA_RUNTIME_DIR, sherpaPlatformPackage, + sherpaRuntimeInstalled, sherpaUnsupportedReason, type ModelFile, } from "./sherpa.js"; @@ -52,6 +54,39 @@ describe("installSherpaRuntime", () => { expect(args).toContain("sherpa-onnx-node@1.13.8"); expect(args).toContain(sherpaPlatformPackage()); }); + + it("repairs a runtime whose native binary is missing instead of calling it installed", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-sherpa-runtime-")); + const native = sherpaPlatformPackage().replace(/@[^@]+$/, ""); + const manifest = (root: string, name: string) => { + mkdirSync(join(root, "node_modules", name), { recursive: true }); + writeFileSync(join(root, "node_modules", name, "package.json"), `{"name":"${name}"}`); + }; + try { + manifest(dir, "sherpa-onnx-node"); + manifest(dir, native); + expect(sherpaRuntimeInstalled(dir)).toBe(false); + const run = vi.fn(async (args: string[]) => { + const staging = args[args.indexOf("--prefix") + 1]!; + manifest(staging, "sherpa-onnx-node"); + manifest(staging, native); + writeFileSync(join(staging, "node_modules", native, "sherpa-onnx.node"), "binary"); + }); + + await installSherpaRuntime({ run, dir }); + expect(run).toHaveBeenCalledTimes(1); + expect(existsSync(join(dir, "node_modules", native, "sherpa-onnx.node"))).toBe(true); + expect(sherpaRuntimeInstalled(dir)).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("keeps each platform and arch in its own runtime dir", () => { + expect(SHERPA_RUNTIME_DIR).toMatch( + new RegExp(`sherpa-onnx-node@1\\.13\\.8-${process.platform}-${process.arch}$`), + ); + }); }); describe("ensureParakeetModel", () => { diff --git a/packages/cli/src/whisper/sherpa.ts b/packages/cli/src/whisper/sherpa.ts index 4e238865ce..9fca8ea8d7 100644 --- a/packages/cli/src/whisper/sherpa.ts +++ b/packages/cli/src/whisper/sherpa.ts @@ -1,6 +1,6 @@ import { execFile, type ExecFileException } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; -import { createReadStream, mkdirSync, renameSync, rmSync, statSync } from "node:fs"; +import { createReadStream, existsSync, mkdirSync, renameSync, rmSync, statSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -12,7 +12,6 @@ import { runNpm, sweepStaleStaging, } from "../utils/optionalPackages.js"; -import { createRenderCancellationScope } from "../utils/renderCancellation.js"; import { mergeWindowsToWords, SHERPA_ERROR_PREFIX, @@ -24,7 +23,11 @@ import { getPreparedWavDurationSeconds, prepareWav, type TranscribeResult } from const RUNTIME = "sherpa-onnx-node"; const RUNTIME_VERSION = "1.13.8"; -export const SHERPA_RUNTIME_DIR = join(CACHE_DIR, `${RUNTIME}@${RUNTIME_VERSION}`); +/** Per platform and arch, like the native package inside, so Rosetta never shadows arm64. */ +export const SHERPA_RUNTIME_DIR = join( + CACHE_DIR, + `${RUNTIME}@${RUNTIME_VERSION}-${process.platform}-${process.arch}`, +); const MODEL_REPO = "csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8"; const MODEL_REVISION = "2bda32ec70b097a55adaa07d9a7173915b43cc78"; @@ -107,23 +110,29 @@ export function sherpaUnsupportedReason(host: Host = currentHost()): string | nu return `Parakeet needs glibc 2.32 or newer; this system has glibc ${host.glibc}.`; } -export function sherpaRuntimeInstalled(): boolean { - return isInstalled(SHERPA_RUNTIME_DIR, RUNTIME); +const nativePackageName = (platform: string, arch: string) => + `sherpa-onnx-${platform === "win32" ? "win" : platform}-${arch}`; + +/** The runtime and the native binary its loader requires; either can be missing on its own. */ +export function sherpaRuntimeInstalled(dir = SHERPA_RUNTIME_DIR): boolean { + const binary = join(dir, "node_modules", nativePackageName(process.platform, process.arch)); + return isInstalled(dir, RUNTIME) && existsSync(join(binary, "sherpa-onnx.node")); } /** The native package, pinned too: the runtime's own optionalDependencies accept any 1.13.x. */ export function sherpaPlatformPackage(platform = process.platform, arch = process.arch): string { - return `sherpa-onnx-${platform === "win32" ? "win" : platform}-${arch}@${RUNTIME_VERSION}`; + return `${nativePackageName(platform, arch)}@${RUNTIME_VERSION}`; } export function installSherpaRuntime({ run = runNpm, signal, -}: { run?: typeof runNpm; signal?: AbortSignal } = {}): Promise { + dir = SHERPA_RUNTIME_DIR, +}: { run?: typeof runNpm; signal?: AbortSignal; dir?: string } = {}): Promise { + // install() keeps any dir holding the runtime manifest, so a half-installed one goes first. + if (!sherpaRuntimeInstalled(dir)) rmSync(dir, { recursive: true, force: true }); const native = sherpaPlatformPackage(); - return install(SHERPA_RUNTIME_DIR, RUNTIME, RUNTIME_VERSION, (args) => - run([...args, native], signal), - ); + return install(dir, RUNTIME, RUNTIME_VERSION, (args) => run([...args, native], signal)); } /** Sizes only: hashing 650 MB on every transcribe is too slow, and install already verified them. */ @@ -217,6 +226,8 @@ function recognizerConfig(): object { /** Ctrl-C or a SIGTERM stopped the decode: the user's cancel, never a reason to fall back. */ export class DecodeCancelled extends Error {} +const MAX_REASON_CHARS = 600; + function failureReason(err: ExecFileException | null, stderr: string): string { const how = !err ? "exited without a result" @@ -232,7 +243,8 @@ function failureReason(err: ExecFileException | null, stderr: string): string { const why = lines.find((line) => line.startsWith(SHERPA_ERROR_PREFIX))?.slice(SHERPA_ERROR_PREFIX.length) ?? lines.at(-1); - return why ? `${how}: ${why}` : how; + if (!why) return how; + return `${how}: ${why.length > MAX_REASON_CHARS ? `${why.slice(0, MAX_REASON_CHARS)}…` : why}`; } /** Decodes in a child process: onnxruntime aborts the whole process on some inputs, uncatchably. */ @@ -281,20 +293,13 @@ export function prepareSherpaWav( } } +/** The caller owns the cancellation scope: it must already cover audio preparation. */ export async function transcribeWithSherpa( wavPath: string, dir: string, - options?: { onProgress?: (message: string) => void }, + options: { signal: AbortSignal; onProgress?: (message: string) => void }, ): Promise { - const unsupported = sherpaUnsupportedReason(); - if (unsupported) throw new Error(unsupported); - // Aborts on SIGINT/SIGTERM/SIGHUP, which also kills the child if only this process was signalled. - const cancellation = createRenderCancellationScope(); - try { - options?.onProgress?.("Transcribing with Parakeet..."); - const windows = await decode(wavPath, cancellation.signal); - return writeParakeetTranscript(dir, mergeWindowsToWords(windows)); - } finally { - cancellation.dispose(); - } + options.onProgress?.("Transcribing with Parakeet..."); + const windows = await decode(wavPath, options.signal); + return writeParakeetTranscript(dir, mergeWindowsToWords(windows)); } diff --git a/packages/cli/src/whisper/sherpaWorker.test.ts b/packages/cli/src/whisper/sherpaWorker.test.ts index 62bfa76d5e..2a582b0371 100644 --- a/packages/cli/src/whisper/sherpaWorker.test.ts +++ b/packages/cli/src/whisper/sherpaWorker.test.ts @@ -16,11 +16,18 @@ module.exports = { readWave(p) { path = p; if (path.endsWith("broken.wav")) throw new Error("Failed to read " + path); + if (path.endsWith("lines.wav")) throw new Error("Could not find it. Tried\\n\\n ../a.node\\n ./b.node\\n"); + if (path.endsWith("long.wav")) return { sampleRate: 100, samples: new Float32Array(60000) }; return { sampleRate: 100, samples: new Float32Array(300).fill(0.5) }; }, OfflineRecognizer: class { + constructor() { + if (process.env.STARTED) require("fs").writeFileSync(process.env.STARTED, ""); + } createStream() { return { acceptWaveform(w) { this.w = w; } }; } - decode() {} + decode() { + if (path.endsWith("long.wav")) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1000); + } getResult(stream) { const n = stream.w.samples.length; if (n === 300) return { tokens: [" not"], timestamps: [2.08], durations: [0.4] }; @@ -85,6 +92,46 @@ describe("sherpaWorker", () => { expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}Failed to read broken.wav`); }); + it("keeps every line of a multi-line error on its one prefixed line", async () => { + const { code, stderr } = await runWorker(fakeRuntime(), "lines.wav"); + expect(code).toBe(1); + expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}Could not find it. Tried ../a.node ./b.node\n`); + }); + + it.skipIf(process.platform === "win32")( + "stops between windows once the CLI that spawned it is gone", + async () => { + const started = join(fakeRuntime(), "started"); + const input = JSON.stringify({ wavPath: "long.wav", runtimeDir: root, config: {} }); + // The shell waits for the first window, then exits, orphaning a 10-window decode. + const shell = + `"${process.execPath}" --import tsx "${WORKER}" >/dev/null 2>&1 & echo $!; ` + + `while [ ! -e "${started}" ]; do sleep 0.1; done`; + const pid = await new Promise((resolve) => + execFile( + "sh", + ["-c", shell], + { env: { ...process.env, HYPERFRAMES_PARAKEET_INPUT: input, STARTED: started } }, + (_e, out) => resolve(Number(out.trim())), + ), + ); + const alive = () => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }; + const deadline = Date.now() + 4000; + while (alive() && Date.now() < deadline) await new Promise((r) => setTimeout(r, 100)); + const orphanSurvived = alive(); + if (orphanSurvived) process.kill(pid, "SIGKILL"); + expect(orphanSurvived).toBe(false); + }, + 15_000, + ); + it("names the missing runtime when sherpa-onnx-node is not installed", async () => { root = mkdtempSync(join(tmpdir(), "hf-sherpa-worker-")); const { code, stderr } = await runWorker(root, "speech.wav"); diff --git a/packages/cli/src/whisper/sherpaWorker.ts b/packages/cli/src/whisper/sherpaWorker.ts index 1d69e70b34..45e91dce53 100644 --- a/packages/cli/src/whisper/sherpaWorker.ts +++ b/packages/cli/src/whisper/sherpaWorker.ts @@ -23,6 +23,7 @@ interface SherpaOnnx { } const { wavPath, runtimeDir, config } = JSON.parse(process.env.HYPERFRAMES_PARAKEET_INPUT ?? "{}"); +const parentPid = process.ppid; /** Leading silence moves the frame grid; 0.5 s recovered the dropped clause at every length tried. */ const RETRY_PAD_SECONDS = 0.5; @@ -52,6 +53,8 @@ try { const cuts = silenceCuts(wave.samples, wave.sampleRate); const windows: SherpaWindow[] = []; for (let k = 0; k + 1 < cuts.length; k++) { + // A SIGKILLed CLI cannot stop us, and nobody would read the result. + if (process.ppid !== parentPid) throw new Error("the CLI exited, so the decode stopped"); const slice = { sampleRate: wave.sampleRate, samples: wave.samples.subarray(cuts[k], cuts[k + 1]), @@ -79,7 +82,8 @@ try { } process.stdout.write(`${SHERPA_RESULT_PREFIX}${JSON.stringify(windows)}\n`); } catch (err) { - const message = err instanceof Error ? err.message : String(err); + // One line: the reader takes the prefixed line, and the loader's error spans several. + const message = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").trim(); // Rethrow after the flush: a pipe write is asynchronous on macOS and a crash would drop it. process.stderr.write(`${SHERPA_ERROR_PREFIX}${message}\n`, () => { throw err;