diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index 2bcb54ac9e..a98fe708dd 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -46,6 +46,8 @@ "packages/producer/src/services/healthWorkerThread.ts", // Render-setup worker, spawned by path from cancellableProcess.ts. "packages/cli/src/renderSetupWorker.ts", + // Parakeet decode worker, spawned by path from whisper/sherpa.ts. + "packages/cli/src/whisper/sherpaWorker.ts", // Test fixture worker, spawned by path via the pools' workerEntryPath // option from the crash-recovery tests; has no import-graph referrer. "packages/producer/src/services/__fixtures__/crashOnMessageWorker.mjs", diff --git a/CREDITS.md b/CREDITS.md index 5f974018be..4d3c74e55a 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -28,6 +28,11 @@ broader Node.js ecosystem. - **[mediabunny](https://github.com/Vanilagy/mediabunny)** — media toolkit used in the studio for fast metadata extraction from file headers. Licensed under the [Mozilla Public License 2.0 (MPL-2.0)](https://mozilla.org/MPL/2.0/). +- **[Parakeet TDT 0.6B v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3)** by NVIDIA — the + speech recognition model `hyperframes models install parakeet` downloads, in the int8 ONNX export + from [csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8](https://huggingface.co/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8). + Licensed under [Creative Commons Attribution 4.0 (CC-BY-4.0)](https://creativecommons.org/licenses/by/4.0/). + It runs on **[sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx)**, Apache-2.0. - The seven 3D-motion catalog pieces (`canopy-part-title`, `glass-shard-title`, `code-slice-hero`, `frost-sequence-camera-orbit`, `cuboid-carousel`, `orbit-card`, `wireframe-portal-title`) are contributed with their author's diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 8b41b45e5d..db81489abe 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -274,19 +274,22 @@ npx hyperframes transcribe transcript.json --to srt | Flag | Description | | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `--dir, -d` | Project directory (default: current directory) | -| `--engine, -e` | ASR engine: `auto` (default — Parakeet if installed, else whisper), `parakeet`, or `whisper` | +| `--engine, -e` | ASR engine: `auto` (default — Parakeet if installed and the language is one of its 25, else whisper), `parakeet` (fails if Parakeet is missing, broken, or lacks the `--language`), or `whisper` | | `--model, -m` | Whisper model (default `small.en`): `tiny.en`, `base.en`, `small.en`, `medium.en`, `large-v3` | | `--language, -l` | Language code (`en`, `es`, `ja`, …). Filters out speech in other languages. | | `--to` | Export a sidecar instead: `srt` or `vtt` | | `--output, -o` | Where the exported SRT/VTT goes | | `--preserve-cues` | Keep each transcript entry as its own caption cue, skipping word-level grouping | -| `--optional` | Skip and exit 0 when whisper-cpp is unavailable, instead of failing. For pipelines that can carry on without captions. | +| `--optional` | Skip and exit 0 when whisper-cpp is unavailable, instead of failing. For pipelines that can carry on without captions. A broken Parakeet install is a real failure: it exits 1 even when whisper-cpp is missing. | | `--timeout` | Whisper spawn timeout in ms, overriding the auto-scaled default. Raise it on slow or emulated CPUs where large models take many seconds per audio second. Whisper only — Parakeet has its own. Minimum 5000. Env: `HYPERFRAMES_TRANSCRIBE_TIMEOUT_MS`. | | `--json` | Output the result as JSON | The command works out what you gave it. Audio and video are transcribed -locally — Parakeet when it is installed (`uv pip install parakeet-mlx`; faster -and more accurate), whisper.cpp otherwise. Transcript files are imported and +locally — Parakeet when it is installed (`hyperframes models install parakeet`; +faster and more accurate), whisper.cpp otherwise. Parakeet covers en, es, fr, +de, bg, hr, cs, da, nl, et, fi, el, hu, it, lv, lt, mt, pl, pt, ro, sk, sl, sv, +ru and uk; `auto` uses whisper for any other `--language`. If Parakeet fails under +`auto`, the run says why, names the repair command, and uses whisper. Transcript files are imported and normalized instead: whisper.cpp JSON, OpenAI Whisper API JSON with word timestamps, SRT, and VTT all land as the same `[{text, start, end}]` word array in `transcript.json`. If the project has caption HTML, it is patched @@ -303,6 +306,26 @@ cues with no internal spaces — single-word or CJK captions. the JSON. +### `models` + +Download an on-device model. Only asked-for installs happen; nothing downloads +on first use. + +```bash +npx hyperframes models install parakeet # progress on stderr +npx hyperframes models install parakeet --json # one JSON result +``` + +`parakeet` installs the sherpa-onnx runtime from npm and NVIDIA's Parakeet TDT +0.6B v3 speech model (int8, about 640 MB, CC-BY-4.0) into +`~/.cache/hyperframes`. Every file is checked against a pinned size and +sha256. Re-running loads the runtime in a separate process and re-checks each +model file's sha256, reinstalls whichever fails, and changes nothing on a good +install. It runs on macOS +(arm64, x64), Windows (x64, ia32) and Linux (x64, arm64, glibc 2.32 or newer), +and refuses up front anywhere else. Ctrl-C stops the download and keeps no +partial file. + ### `tts` Generate speech with a local model (Kokoro-82M). No API key, nothing leaves the diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index b3da43d55a..c9e3f62bc4 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -165,6 +165,7 @@ const commandLoaders = { browser: () => import("./commands/browser.js").then((m) => m.default), "remove-background": () => import("./commands/remove-background.js").then((m) => m.default), transcribe: () => import("./commands/transcribe.js").then((m) => m.default), + models: () => import("./commands/models.js").then((m) => m.default), tts: () => import("./commands/tts.js").then((m) => m.default), docs: () => import("./commands/docs.js").then((m) => m.default), doctor: () => import("./commands/doctor.js").then((m) => m.default), diff --git a/packages/cli/src/commands/models.test.ts b/packages/cli/src/commands/models.test.ts new file mode 100644 index 0000000000..57a1ef49c1 --- /dev/null +++ b/packages/cli/src/commands/models.test.ts @@ -0,0 +1,111 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CliRuntimeError, consumeCommandResult } from "../utils/commandResult.js"; + +const sherpa = { + SHERPA_RUNTIME_DIR: "/cache/optional/sherpa-onnx-node@1.13.8", + PARAKEET_MODEL_DIR: "/cache/parakeet/parakeet-tdt-0.6b-v3-int8", + sherpaUnsupportedReason: vi.fn(), + installSherpaRuntime: vi.fn(), + ensureParakeetModel: vi.fn(), + DecodeCancelled: class extends Error {}, +}; +vi.mock("../whisper/sherpa.js", () => sherpa); + +// The command's Ctrl-C scope, driven by the test instead of a real signal. +let cancel = new AbortController(); +const dispose = vi.fn(); +vi.mock("../utils/renderCancellation.js", () => ({ + createRenderCancellationScope: () => ({ signal: cancel.signal, dispose, checkAncestors() {} }), +})); + +import modelsCmd from "./models.js"; + +async function install() { + let exitCode = 0; + let threw = false; + try { + await modelsCmd.run!({ args: { action: "install", name: "parakeet", json: true } } as never); + } catch (err) { + if (!(err instanceof CliRuntimeError)) throw err; + exitCode = err.result.exitCode; + threw = true; + } + exitCode ||= consumeCommandResult().exitCode; + const out: Record = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]), + ); + return { exitCode, threw, out }; +} + +describe("models install parakeet --json", () => { + beforeEach(() => { + vi.clearAllMocks(); + cancel = new AbortController(); + consumeCommandResult(); + vi.spyOn(console, "log").mockImplementation(() => {}); + sherpa.sherpaUnsupportedReason.mockReturnValue(null); + sherpa.installSherpaRuntime.mockResolvedValue(true); + sherpa.ensureParakeetModel.mockResolvedValue(false); + }); + afterEach(() => vi.restoreAllMocks()); + + it("prints one result naming what it installed and where", async () => { + expect(await install()).toEqual({ + exitCode: 0, + threw: false, + out: { + ok: true, + model: "parakeet-tdt-0.6b-v3", + changed: true, + runtimeDir: sherpa.SHERPA_RUNTIME_DIR, + modelDir: sherpa.PARAKEET_MODEL_DIR, + }, + }); + expect(dispose).toHaveBeenCalled(); + }); + + it("reports changed: false when the runtime loads and the model verifies", async () => { + sherpa.installSherpaRuntime.mockResolvedValue(false); + expect((await install()).out).toMatchObject({ ok: true, changed: false }); + }); + + it("reports a failed install as ok:false with exit 1", async () => { + sherpa.ensureParakeetModel.mockRejectedValue(new Error("tokens.txt did not match")); + expect(await install()).toEqual({ + exitCode: 1, + threw: true, + out: { ok: false, error: "tokens.txt did not match" }, + }); + }); + + it("refuses an unsupported system before downloading anything", async () => { + sherpa.sherpaUnsupportedReason.mockReturnValue("Parakeet needs glibc 2.32 or newer"); + expect(await install()).toEqual({ + exitCode: 1, + threw: true, + out: { ok: false, error: "Parakeet needs glibc 2.32 or newer" }, + }); + expect(sherpa.installSherpaRuntime).not.toHaveBeenCalled(); + expect(sherpa.ensureParakeetModel).not.toHaveBeenCalled(); + }); + + it("stops with exit 130 when Ctrl-C stops the runtime check before any listener runs", async () => { + sherpa.installSherpaRuntime.mockRejectedValue(new sherpa.DecodeCancelled("cancelled")); + const { exitCode, threw, out } = await install(); + expect(exitCode).toBe(130); + expect(threw).toBe(false); + expect(out).toMatchObject({ ok: false, error: expect.stringMatching(/cancelled/) }); + }); + + it("stops with exit 130 when cancelled mid-download", async () => { + sherpa.ensureParakeetModel.mockImplementation(async ({ signal }: { signal: AbortSignal }) => { + cancel.abort(); + signal.throwIfAborted(); + }); + const { exitCode, threw, out } = await install(); + expect(exitCode).toBe(130); + expect(threw).toBe(false); + expect(out).toMatchObject({ ok: false, error: expect.stringMatching(/cancelled/) }); + expect(sherpa.installSherpaRuntime).toHaveBeenCalledWith({ signal: cancel.signal }); + }); +}); diff --git a/packages/cli/src/commands/models.ts b/packages/cli/src/commands/models.ts new file mode 100644 index 0000000000..b405daa91b --- /dev/null +++ b/packages/cli/src/commands/models.ts @@ -0,0 +1,107 @@ +import { defineCommand } from "citty"; +import * as clack from "@clack/prompts"; +import type { Example } from "./_examples.js"; +import { c } from "../ui/colors.js"; +import { formatBytes } from "../ui/format.js"; +import { failCommand, setCommandExitCode } from "../utils/commandResult.js"; +import { createRenderCancellationScope } from "../utils/renderCancellation.js"; +import { PARAKEET_MODEL_LABEL } from "../whisper/parakeet.js"; + +export const examples: Example[] = [ + [ + "Download the Parakeet speech model that transcribe uses", + "hyperframes models install parakeet", + ], +]; + +function fail(message: string, json: boolean): never { + if (json) console.log(JSON.stringify({ ok: false, error: message })); + else console.error(c.error(message)); + failCommand(); +} + +/** A cancel is the user's choice, not a command failure: exit 130 without a cli_error. */ +function reportCancel(json: boolean): void { + const message = "Parakeet install cancelled; nothing partial was kept."; + if (json) console.log(JSON.stringify({ ok: false, error: message })); + else console.error(c.warn(message)); + setCommandExitCode(130); +} + +function downloadProgress(spin: Spinner) { + let lastPct = -1; + return (done: number, total: number) => { + const pct = Math.floor((done / total) * 100); + if (pct <= lastPct) return; + lastPct = pct; + spin?.message( + `Downloading Parakeet TDT 0.6B v3 — ${c.progress(pct + "%")} ${c.dim("(" + formatBytes(done) + " / " + formatBytes(total) + ")")}`, + ); + }; +} + +type Sherpa = typeof import("../whisper/sherpa.js"); +type Spinner = ReturnType | null; + +/** Installs what is missing; true when anything changed. */ +async function installMissing(sherpa: Sherpa, spin: Spinner, signal: AbortSignal) { + const runtimeInstalled = await sherpa.installSherpaRuntime({ signal }); + spin?.message("Verifying the Parakeet model..."); + const modelFetched = await sherpa.ensureParakeetModel({ + signal, + onBytes: downloadProgress(spin), + }); + return runtimeInstalled || modelFetched; +} + +/** A synchronous child's Ctrl-C shows as its error before the scope's listener runs. */ +const wasCancelled = (err: unknown, signal: AbortSignal, sherpa: Sherpa) => + signal.aborted || err instanceof sherpa.DecodeCancelled; + +async function installParakeet(json: boolean): Promise { + const sherpa = await import("../whisper/sherpa.js"); + const unsupported = sherpa.sherpaUnsupportedReason(); + if (unsupported) fail(unsupported, json); + + const spin = json ? null : clack.spinner({ output: process.stderr }); + // Ctrl-C must stop a 650 MB download, not just print "Canceled" over it. + const cancellation = createRenderCancellationScope(); + spin?.start("Checking the sherpa-onnx runtime (installing it from npm if it does not load)..."); + try { + const changed = await installMissing(sherpa, spin, cancellation.signal); + spin?.stop(c.success(changed ? "Parakeet installed" : "Parakeet is already installed")); + if (json) { + const { SHERPA_RUNTIME_DIR: runtimeDir, PARAKEET_MODEL_DIR: modelDir } = sherpa; + console.log( + JSON.stringify({ ok: true, model: PARAKEET_MODEL_LABEL, changed, runtimeDir, modelDir }), + ); + } + } catch (err) { + const cancelled = wasCancelled(err, cancellation.signal, sherpa); + spin?.stop( + cancelled ? c.warn("Parakeet install cancelled") : c.error("Parakeet install failed"), + ); + if (cancelled) return reportCancel(json); + fail(err instanceof Error ? err.message : String(err), json); + } finally { + cancellation.dispose(); + } +} + +export default defineCommand({ + meta: { name: "models", description: "Download on-device models (models install parakeet)" }, + args: { + action: { type: "positional", description: "install", required: true }, + name: { type: "positional", description: "Model to install: parakeet", required: true }, + json: { type: "boolean", description: "Print one JSON result, no progress", default: false }, + }, + async run({ args }) { + if (args.action !== "install" || args.name !== "parakeet") { + fail( + `Unknown: models ${args.action} ${args.name}. Try: hyperframes models install parakeet`, + args.json, + ); + } + return installParakeet(args.json); + }, +}); diff --git a/packages/cli/src/commands/transcribe.test.ts b/packages/cli/src/commands/transcribe.test.ts index 329d62339d..307765cf00 100644 --- a/packages/cli/src/commands/transcribe.test.ts +++ b/packages/cli/src/commands/transcribe.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { writeFileSync, readFileSync, mkdtempSync, rmSync } from "node:fs"; +import { chmodSync, existsSync, writeFileSync, readFileSync, mkdtempSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { WhisperUnavailableError } from "../whisper/manager.js"; @@ -7,7 +7,31 @@ import { CliRuntimeError, consumeCommandResult } from "../utils/commandResult.js // Make the whisper core report "unavailable" so we exercise the soft-skip path. const transcribeMock = vi.fn(); -vi.mock("../whisper/transcribe.js", () => ({ transcribe: transcribeMock })); +const prepareWavMock = vi.fn((input: string) => input); +let audioSeconds = 1; +vi.mock("../whisper/transcribe.js", () => ({ + transcribe: transcribeMock, + prepareWav: (input: string) => prepareWavMock(input), + getPreparedWavDurationSeconds: () => audioSeconds, +})); + +// Engine selection: which runners look installed, and the sherpa decode child it spawns. +const runners = { sherpa: false, mlx: false }; +vi.mock("../whisper/sherpa.js", async (importOriginal) => ({ + ...(await importOriginal()), + sherpaParakeetInstalled: () => runners.sherpa, +})); +const mlxMock = vi.fn(); +vi.mock("../whisper/parakeet.js", async (importOriginal) => ({ + ...(await importOriginal()), + findParakeet: () => (runners.mlx ? "/bin/parakeet-mlx" : undefined), + transcribeWithParakeet: (...a: unknown[]) => mlxMock(...a), +})); +const execFileMock = vi.fn(); +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + execFile: (...a: unknown[]) => execFileMock(...a), +})); const trackTranscribeUnavailable = vi.fn(); vi.mock("../telemetry/events.js", () => ({ @@ -16,6 +40,17 @@ vi.mock("../telemetry/events.js", () => ({ import transcribeCmd from "./transcribe.js"; +/** Makes each runner write a one-word transcript naming itself. */ +function fakeTranscript(dir: string, text: string) { + const transcriptPath = join(dir, "transcript.json"); + writeFileSync(transcriptPath, JSON.stringify([{ text, start: 0, end: 1 }])); + return { transcriptPath, wordCount: 1, durationSeconds: 1, speechOnsetSeconds: null }; +} + +function lastJson(): Record { + return JSON.parse(String(vi.mocked(console.log).mock.calls.at(-1)?.[0])); +} + function dummyAudio(): { dir: string; input: string } { const dir = mkdtempSync(join(tmpdir(), "hf-transcribe-test-")); const input = join(dir, "narration.wav"); @@ -29,7 +64,10 @@ describe("transcribe command", () => { dirs = []; consumeCommandResult(); transcribeMock.mockReset(); + prepareWavMock.mockReset().mockImplementation((input: string) => input); trackTranscribeUnavailable.mockReset(); + mlxMock.mockReset(); + Object.assign(runners, { sherpa: false, mlx: false }); transcribeMock.mockRejectedValue( new WhisperUnavailableError("whisper-cpp not found. Install: brew install whisper-cpp"), ); @@ -40,6 +78,7 @@ describe("transcribe command", () => { consumeCommandResult(); for (const d of dirs) rmSync(d, { recursive: true, force: true }); vi.restoreAllMocks(); + vi.unstubAllEnvs(); }); it("explicit run exits non-zero and is NOT reported as a command failure", async () => { @@ -70,10 +109,320 @@ describe("transcribe command", () => { // passes on a machine with no Parakeet and silently tests nothing on one // that has it. expect(transcribeMock).toHaveBeenCalled(); + expect(prepareWavMock).not.toHaveBeenCalled(); expect(consumeCommandResult().exitCode).toBe(0); expect(trackTranscribeUnavailable).toHaveBeenCalledWith({ optional: true }); }); + describe("engine selection", () => { + beforeEach(() => { + transcribeMock.mockImplementation(async (_in: string, dir: string) => + fakeTranscript(dir, "whisper"), + ); + mlxMock.mockImplementation((_in: string, dir: string) => fakeTranscript(dir, "mlx")); + execFileMock.mockReset(); + execFileMock.mockImplementation((_cmd, _args, _opts, done) => { + const windows = [{ offset: 60, tokens: [" sherpa"], timestamps: [0.5], durations: [0.25] }]; + done(null, `HYPERFRAMES_PARAKEET_RESULT:${JSON.stringify(windows)}\n`, ""); + }); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.report, "getReport").mockReturnValue({ + header: { glibcVersionRuntime: "2.36" }, + } as never); + }); + + async function transcribeWith(engine: string, installed: typeof runners, language?: string) { + Object.assign(runners, installed); + const { dir, input } = dummyAudio(); + dirs.push(dir); + await transcribeCmd.run!({ args: { input, json: true, engine, language } } as never); + const out = lastJson(); + const words = JSON.parse(readFileSync(String(out.transcriptPath), "utf-8")); + return { engine: out.engine, model: out.model, word: words[0]?.text, start: words[0]?.start }; + } + + it("auto and parakeet prefer sherpa-onnx, then parakeet-mlx, then whisper", async () => { + const sherpa = { + engine: "parakeet", + model: "parakeet-tdt-0.6b-v3", + word: "sherpa", + start: 60.5, + }; + expect(await transcribeWith("auto", { sherpa: true, mlx: true })).toEqual(sherpa); + expect(await transcribeWith("parakeet", { sherpa: true, mlx: true })).toEqual(sherpa); + expect(await transcribeWith("auto", { sherpa: false, mlx: true })).toMatchObject({ + engine: "parakeet", + word: "mlx", + }); + expect(await transcribeWith("auto", { sherpa: false, mlx: false })).toMatchObject({ + engine: "whisper", + word: "whisper", + }); + expect(await transcribeWith("whisper", { sherpa: true, mlx: true })).toMatchObject({ + engine: "whisper", + word: "whisper", + }); + }); + + it("auto uses Parakeet only for a language it transcribes", async () => { + const both = { sherpa: true, mlx: true }; + expect(await transcribeWith("auto", both, "ja")).toMatchObject({ engine: "whisper" }); + expect(await transcribeWith("auto", both, "pt-BR")).toMatchObject({ engine: "parakeet" }); + expect(await transcribeWith("auto", both, "UK")).toMatchObject({ engine: "parakeet" }); + expect(transcribeMock).toHaveBeenCalledTimes(1); + }); + + it("--engine parakeet with a language it does not transcribe fails without falling back", async () => { + Object.assign(runners, { sherpa: true, mlx: true }); + const { exitCode, out } = await transcribeFails("parakeet", { language: "ja" }); + expect(exitCode).toBe(1); + expect(out.error).toMatch( + /^Parakeet does not transcribe --language ja; it covers en, es, .*uk\. Use --engine whisper\.$/, + ); + expect(execFileMock).not.toHaveBeenCalled(); + expect(transcribeMock).not.toHaveBeenCalled(); + }); + + it("listens for Ctrl-C from before audio prep until the run ends", async () => { + const before = process.listenerCount("SIGINT"); + let duringPrep = 0; + prepareWavMock.mockImplementation((input: string) => { + duringPrep = process.listenerCount("SIGINT"); + return input; + }); + expect(await transcribeWith("auto", { sherpa: true, mlx: false })).toMatchObject({ + engine: "parakeet", + }); + expect(duringPrep).toBe(before + 1); + expect(process.listenerCount("SIGINT")).toBe(before); + }); + + it("exits 130 when Ctrl-C stops the whisper fallback too", async () => { + crashChild("SIGABRT"); + // No signal reaches the listener in time: the synchronous whisper child's own signal decides. + transcribeMock.mockImplementation(async () => { + throw Object.assign(new Error("Command failed: whisper-cli"), { signal: "SIGINT" }); + }); + Object.assign(runners, { sherpa: true, mlx: false }); + const { exitCode, out } = await transcribeFails("auto"); + expect(exitCode).toBe(130); + expect(out).toEqual({ ok: false, error: "Transcription cancelled" }); + }); + + it("reports the fallback's own timeout as a failure, not a cancel", async () => { + crashChild("SIGABRT"); + transcribeMock.mockImplementation(async () => { + throw Object.assign(new Error("spawnSync whisper-cli ETIMEDOUT"), { + code: "ETIMEDOUT", + signal: "SIGTERM", + }); + }); + Object.assign(runners, { sherpa: true, mlx: false }); + const { exitCode, out } = await transcribeFails("auto"); + expect(exitCode).toBe(1); + expect(out.error).toMatch(/The whisper fallback failed too: spawnSync whisper-cli ETIMEDOUT/); + }); + + it("--engine parakeet with nothing installed names the install command", async () => { + Object.assign(runners, { sherpa: false, mlx: false }); + const { dir, input } = dummyAudio(); + dirs.push(dir); + await expect( + transcribeCmd.run!({ args: { input, json: true, engine: "parakeet" } } as never), + ).rejects.toThrow(CliRuntimeError); + expect(lastJson().error).toContain("hyperframes models install parakeet"); + expect(transcribeMock).not.toHaveBeenCalled(); + }); + + const crashChild = (signal: string, stderr = "") => + execFileMock.mockImplementation((_cmd, _args, _opts, done) => { + done(Object.assign(new Error("Command failed"), { code: null, signal }), "", stderr); + }); + + async function transcribeFails(engine: string, extra: Record = {}) { + const { dir, input } = dummyAudio(); + dirs.push(dir); + let exitCode = 0; + try { + await transcribeCmd.run!({ args: { input, json: true, engine, ...extra } } as never); + } catch (err) { + if (!(err instanceof CliRuntimeError)) throw err; + exitCode = err.result.exitCode; + } + return { exitCode: exitCode || consumeCommandResult().exitCode, out: lastJson() }; + } + + it("auto falls back to whisper with one line naming the Parakeet error and the repair", async () => { + crashChild("SIGABRT", "terminate called after throwing an instance of 'Ort::Exception'\n"); + expect(await transcribeWith("auto", { sherpa: true, mlx: false })).toMatchObject({ + engine: "whisper", + word: "whisper", + }); + const warnings = vi.mocked(console.error).mock.calls.map(([line]) => String(line)); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toMatch( + /^.*Parakeet failed: .*SIGABRT.*Ort::Exception.*hyperframes models install parakeet.*Using whisper/, + ); + }); + + it("an explicit --engine parakeet fails with the Parakeet error instead of falling back", async () => { + crashChild("SIGABRT", "Protobuf parsing failed.\n"); + Object.assign(runners, { sherpa: true, mlx: false }); + const { exitCode, out } = await transcribeFails("parakeet"); + expect(exitCode).toBe(1); + expect(out.error).toMatch( + /^Parakeet failed: .*Protobuf parsing failed\. To repair it, run: hyperframes models install parakeet$/, + ); + expect(transcribeMock).not.toHaveBeenCalled(); + }); + + it("keeps a long decoder error whole up to a bound", async () => { + crashChild( + "SIGABRT", + `HYPERFRAMES_PARAKEET_ERROR:${"a".repeat(590)} tail ${"b".repeat(2000)}\n`, + ); + Object.assign(runners, { sherpa: true, mlx: false }); + const { out } = await transcribeFails("parakeet"); + expect(out.error).toContain(`${"a".repeat(590)} tail`); + expect(out.error).toContain("b…. To repair it"); + expect(String(out.error).length).toBeLessThan(800); + }); + + it("reports the Parakeet error, not whisper_unavailable, when the fallback is missing too", async () => { + crashChild("SIGABRT"); + transcribeMock.mockRejectedValue(new WhisperUnavailableError("whisper-cpp not found")); + Object.assign(runners, { sherpa: true, mlx: false }); + const { exitCode, out } = await transcribeFails("auto", { optional: true }); + expect(exitCode).toBe(1); + expect(out.skipped).toBeUndefined(); + expect(out.error).toMatch(/^Parakeet failed: .*install parakeet.*whisper-cpp not found/); + }); + + it("stops with exit 130 on Ctrl-C instead of falling back", async () => { + crashChild("SIGINT"); + Object.assign(runners, { sherpa: true, mlx: false }); + const { exitCode, out } = await transcribeFails("auto"); + expect(exitCode).toBe(130); + expect(out).toEqual({ ok: false, error: "Transcription cancelled" }); + expect(transcribeMock).not.toHaveBeenCalled(); + }); + + describe.skipIf(process.platform === "win32")("when ffmpeg exits 255 while preparing", () => { + it.each([ + ["Exiting normally, received signal 2.", 130, "Transcription cancelled"], + [ + "Error opening input files: Operation not permitted", + 1, + "ffmpeg failed: Error opening input files: Operation not permitted", + ], + [ + "Output file does not contain any stream\nError opening output files: Invalid argument", + 1, + "ffmpeg failed: Error opening output files: Invalid argument\nOutput file does not contain any stream", + ], + ])("after saying %j, the run exits %i", async (said, code, error) => { + const actual = await vi.importActual( + "../whisper/transcribe.js", + ); + prepareWavMock.mockImplementation((input: string) => actual.prepareWav(input)); + const dir = mkdtempSync(join(tmpdir(), "hf-transcribe-ffmpeg-")); + dirs.push(dir); + const ffmpeg = join(dir, "ffmpeg"); + const script = `process.stderr.write(${JSON.stringify(`${said}\n`)}); process.exit(255);`; + writeFileSync(ffmpeg, `#!${process.execPath}\n${script}\n`); + chmodSync(ffmpeg, 0o755); + vi.stubEnv("HYPERFRAMES_FFMPEG_PATH", ffmpeg); + Object.assign(runners, { sherpa: true, mlx: false }); + const { exitCode, out } = await transcribeFails("auto"); + expect(exitCode).toBe(code); + expect(out).toEqual({ ok: false, error }); + expect(transcribeMock).not.toHaveBeenCalled(); + }); + }); + + it("fails an unreadable input as an input error, with no Parakeet repair and no fallback", async () => { + prepareWavMock.mockImplementation(() => { + throw new Error("Command failed: ffmpeg -i silent.mp4 -vn -ar 16000"); + }); + Object.assign(runners, { sherpa: true, mlx: false }); + for (const optional of [false, true]) { + const { exitCode, out } = await transcribeFails("auto", { optional }); + expect(exitCode).toBe(1); + expect(out).toEqual({ + ok: false, + error: "Command failed: ffmpeg -i silent.mp4 -vn -ar 16000", + }); + } + expect(execFileMock).not.toHaveBeenCalled(); + expect(transcribeMock).not.toHaveBeenCalled(); + }); + + it("prepares the audio once, hands the same WAV to the fallback, and removes it", async () => { + crashChild("SIGABRT"); + const prepared: string[] = []; + prepareWavMock.mockImplementation((input: string) => { + prepared.push(`${input}.16k.wav`); + writeFileSync(prepared[0]!, "wav"); + return prepared[0]!; + }); + expect(await transcribeWith("auto", { sherpa: true, mlx: false })).toMatchObject({ + engine: "whisper", + }); + expect(prepared).toHaveLength(1); + const sherpaInput = JSON.parse( + execFileMock.mock.calls[0]?.[2].env.HYPERFRAMES_PARAKEET_INPUT, + ); + expect(sherpaInput.wavPath).toBe(prepared[0]); + expect(transcribeMock.mock.calls[0]?.[0]).toBe(prepared[0]); + expect(existsSync(prepared[0]!)).toBe(false); + }); + + it("gives the decode at least 30 minutes, and twice the audio length", async () => { + Object.assign(runners, { sherpa: true, mlx: false }); + for (const [seconds, timeout] of [ + [60, 1_800_000], + [3600, 7_200_000], + ]) { + audioSeconds = seconds!; + const { dir, input } = dummyAudio(); + dirs.push(dir); + await transcribeCmd.run!({ args: { input, json: true, engine: "auto" } } as never); + expect(execFileMock.mock.calls.at(-1)?.[2]).toMatchObject({ timeout }); + } + audioSeconds = 1; + }); + + it("succeeds with 0 words when the audio has no speech", async () => { + execFileMock.mockImplementation((_cmd, _args, _opts, done) => { + done(null, `HYPERFRAMES_PARAKEET_RESULT:${JSON.stringify([])}\n`, ""); + }); + Object.assign(runners, { sherpa: true, mlx: false }); + const { dir, input } = dummyAudio(); + dirs.push(dir); + await transcribeCmd.run!({ args: { input, json: true, engine: "auto" } } as never); + expect(lastJson()).toMatchObject({ ok: true, engine: "parakeet", wordCount: 0 }); + }); + + it.runIf(process.platform === "linux" && process.arch === "x64")( + "skips Parakeet when glibc is too old for sherpa-onnx, never naming the install command", + async () => { + vi.mocked(process.report.getReport).mockReturnValue({ + header: { glibcVersionRuntime: "2.31" }, + } as never); + expect(await transcribeWith("auto", { sherpa: true, mlx: false })).toMatchObject({ + engine: "whisper", + }); + expect(execFileMock).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + const { exitCode, out } = await transcribeFails("parakeet"); + expect(exitCode).toBe(1); + expect(out.error).toMatch( + /^Parakeet needs glibc 2\.32 or newer; this system has glibc 2\.31\.$/, + ); + }, + ); + }); + it("imports an SRT and exports an SRT sidecar from transcript.json", async () => { const dir = mkdtempSync(join(tmpdir(), "hf-transcribe-test-")); dirs.push(dir); diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.ts index eca73b1b1b..2ef336a48a 100644 --- a/packages/cli/src/commands/transcribe.ts +++ b/packages/cli/src/commands/transcribe.ts @@ -1,9 +1,16 @@ import { failCommand, setCommandExitCode } from "../utils/commandResult.js"; +import { normalizeErrorMessage } from "../utils/errorMessage.js"; // fallow-ignore-file code-duplication import { defineCommand } from "citty"; import type { Example } from "./_examples.js"; -import { existsSync, writeFileSync } from "node:fs"; -import { findParakeet, transcribeWithParakeet } from "../whisper/parakeet.js"; +import { existsSync, rmSync, writeFileSync } from "node:fs"; +import { + findParakeet, + PARAKEET_LANGUAGES, + PARAKEET_MODEL_LABEL, + parakeetSpeaks, + transcribeWithParakeet, +} from "../whisper/parakeet.js"; type CaptionExportFormat = "srt" | "vtt"; @@ -55,7 +62,7 @@ export default defineCommand({ engine: { type: "string", description: - "ASR engine: auto (Parakeet if installed, else whisper), parakeet, or whisper. Default: auto. Parakeet is more accurate and faster; enable with `uv pip install parakeet-mlx`.", + "ASR engine: auto (Parakeet if installed and it covers --language, else whisper), parakeet, or whisper. Default: auto. Parakeet is more accurate and faster; install it with `hyperframes models install parakeet`.", alias: "e", }, model: { @@ -265,6 +272,18 @@ async function exportTranscript( // Transcribe audio/video with whisper // --------------------------------------------------------------------------- +type Runner = "sherpa" | "parakeet-mlx" | "whisper"; + +/** auto and parakeet prefer sherpa-onnx, then parakeet-mlx, then whisper, in Parakeet's languages. */ +function pickRunner(engine: string, sherpaUsable: () => boolean, language?: string): Runner { + if (engine === "whisper" || !parakeetSpeaks(language)) return "whisper"; + if (sherpaUsable()) return "sherpa"; + return findParakeet() ? "parakeet-mlx" : "whisper"; +} + +/** When Parakeet fails, only auto falls back; an explicit --engine parakeet fails with the error. */ +const parakeetFallsBack = (engine: string) => engine === "auto"; + // fallow-ignore-next-line complexity async function transcribeAudio( inputPath: string, @@ -282,36 +301,85 @@ async function transcribeAudio( const { loadTranscript, patchCaptionHtml, stripBeforeOnset } = await import("../whisper/normalize.js"); - // Engine: auto (Parakeet if installed, else whisper), or forced parakeet/whisper. + const { + DecodeCancelled, + prepareSherpaWav, + sherpaParakeetInstalled, + sherpaUnsupportedReason, + transcribeWithSherpa, + } = await import("../whisper/sherpa.js"); + const { createRenderCancellationScope, stoppedByCancelSignal } = + await import("../utils/renderCancellation.js"); + const engine = (opts.engine ?? "auto").toLowerCase(); if (engine !== "auto" && engine !== "parakeet" && engine !== "whisper") { failWith(`Unknown --engine: ${opts.engine}. Use auto, parakeet, or whisper.`, !!opts.json); } - const useParakeet = engine === "parakeet" || (engine === "auto" && !!findParakeet()); + const unsupported = sherpaUnsupportedReason(); + const sherpaUsable = () => !unsupported && sherpaParakeetInstalled(); + let runner = pickRunner(engine, sherpaUsable, opts.language); + if (engine === "parakeet" && runner === "whisper") { + failWith( + !parakeetSpeaks(opts.language) + ? `Parakeet does not transcribe --language ${opts.language}; it covers ${PARAKEET_LANGUAGES.split(" ").join(", ")}. Use --engine whisper.` + : (unsupported ?? + "Parakeet is not installed. Install it with: hyperframes models install parakeet (or use --engine whisper)"), + !!opts.json, + ); + } const model = opts.model ?? DEFAULT_MODEL; // --model selects the whisper model only; Parakeet uses its own fixed model. - if (useParakeet && opts.model && !opts.json) { + if (runner !== "whisper" && opts.model && !opts.json) { console.error( c.dim(` Note: --model applies to the whisper engine only; ignored under Parakeet.`), ); } - const label = useParakeet ? "Parakeet" : model; + const label = (r: Runner) => c.accent(r === "whisper" ? model : "Parakeet"); const spin = opts.json ? null : clack.spinner(); - spin?.start(`Transcribing with ${c.accent(label)}...`); + spin?.start(`Transcribing with ${label(runner)}...`); + const onProgress = spin ? (msg: string) => spin.message(msg) : undefined; + let wavPath = inputPath; + // Before audio prep: under --json no spinner listens for SIGINT, so Ctrl-C would kill Node. + const cancellation = runner === "sherpa" ? createRenderCancellationScope() : null; + const run = (r: Runner) => + r === "sherpa" + ? transcribeWithSherpa(wavPath, dir, { onProgress, signal: cancellation!.signal }) + : r === "parakeet-mlx" + ? transcribeWithParakeet(wavPath, dir, { language: opts.language, onProgress }) + : transcribe(wavPath, dir, { + model, + language: opts.language, + onProgress, + timeoutMs: opts.timeoutMs, + }); try { - const result = useParakeet - ? transcribeWithParakeet(inputPath, dir, { - language: opts.language, - onProgress: spin ? (msg) => spin.message(msg) : undefined, - }) - : await transcribe(inputPath, dir, { - model, - language: opts.language, - onProgress: spin ? (msg) => spin.message(msg) : undefined, - timeoutMs: opts.timeoutMs, - }); + // Outside the fallback: an unreadable input is not a Parakeet failure. The fallback reuses it. + if (runner === "sherpa") wavPath = prepareSherpaWav(inputPath, onProgress); + let result: Awaited>; + try { + result = await run(runner); + } catch (err) { + if (runner !== "sherpa" || err instanceof DecodeCancelled) throw err; + const reason = normalizeErrorMessage(err).replace(/\.+$/, ""); + const parakeetError = `Parakeet failed: ${reason}. To repair it, run: hyperframes models install parakeet`; + if (!parakeetFallsBack(engine)) throw new Error(parakeetError); + runner = pickRunner(engine, () => false, opts.language); + spin?.clear(); + console.error(c.warn(`${parakeetError}. Using ${runner} for this run.`)); + spin?.start(`Transcribing with ${label(runner)}...`); + try { + result = await run(runner); + } catch (fallbackErr) { + // Whisper runs synchronously, so Ctrl-C shows as its child's signal before any listener runs. + if (stoppedByCancelSignal(fallbackErr as { signal?: string })) { + throw new DecodeCancelled("Transcription cancelled"); + } + const why = normalizeErrorMessage(fallbackErr); + throw new Error(`${parakeetError}. The ${runner} fallback failed too: ${why}`); + } + } let { words } = loadTranscript(result.transcriptPath); @@ -333,8 +401,8 @@ async function transcribeAudio( console.log( JSON.stringify({ ok: true, - engine: useParakeet ? "parakeet" : "whisper", - model: useParakeet ? "parakeet-tdt-0.6b-v3" : model, + engine: runner === "whisper" ? "whisper" : "parakeet", + model: runner === "whisper" ? model : PARAKEET_MODEL_LABEL, wordCount: words.length, durationSeconds: result.durationSeconds, speechOnsetSeconds: result.speechOnsetSeconds, @@ -353,14 +421,26 @@ async function transcribeAudio( ); } } catch (err) { + if (err instanceof DecodeCancelled || cancellation?.signal.aborted) { + const message = "Transcription cancelled"; + if (opts.json) console.log(JSON.stringify({ ok: false, error: message })); + else spin?.stop(c.warn(message)); + setCommandExitCode(130); + return; + } // Surface the last few lines of the ASR subprocess's stderr, which // execFileSync captures but otherwise drops on the floor — that's where // parakeet-mlx / whisper report the actual failure cause. + const base = err instanceof Error ? err.message : String(err); const stderr = err && typeof err === "object" && "stderr" in err && err.stderr - ? String(err.stderr).trim().split("\n").slice(-3).join("\n") + ? String(err.stderr) + .trim() + .split("\n") + .slice(-3) + .filter((line) => !base.includes(line)) + .join("\n") : ""; - const base = err instanceof Error ? err.message : String(err); const message = stderr ? `${base}\n${stderr}` : base; // whisper-cpp is an optional prerequisite, not part of the CLI. When it is @@ -387,5 +467,8 @@ async function transcribeAudio( spin?.stop(c.error(`Transcription failed: ${message}`)); } failCommand(1, err); + } finally { + cancellation?.dispose(); + if (wavPath !== inputPath) rmSync(wavPath, { force: true }); } } diff --git a/packages/cli/src/help.ts b/packages/cli/src/help.ts index 3210e3d46e..37173e9b98 100644 --- a/packages/cli/src/help.ts +++ b/packages/cli/src/help.ts @@ -90,6 +90,7 @@ const GROUPS: Group[] = [ "transcribe", "Transcribe audio/video to word-level timestamps, or import an existing transcript", ], + ["models", "Download on-device models (models install parakeet)"], ["tts", "Generate speech audio from text using a local AI model (Kokoro-82M)"], ["remove-background", "Remove background from a video or image to produce transparent media"], ], diff --git a/packages/cli/src/utils/optionalPackages.test.ts b/packages/cli/src/utils/optionalPackages.test.ts index fe07f2851b..e1c0a6e983 100644 --- a/packages/cli/src/utils/optionalPackages.test.ts +++ b/packages/cli/src/utils/optionalPackages.test.ts @@ -7,6 +7,7 @@ import { renameSync, rmSync, symlinkSync, + utimesSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -321,4 +322,18 @@ describe("install", () => { rmSync(cache, { recursive: true, force: true }); } }); + + it("sweeps a staging dir hours old even when its pid is alive, since pids get reused", async () => { + const { cache, dir, stubNpm } = setup(); + const reused = `${dir}.tmp-${process.ppid}-abcd1234`; + mkdirSync(reused, { recursive: true }); + const sevenHoursAgo = new Date(Date.now() - 7 * 60 * 60 * 1000); + utimesSync(reused, sevenHoursAgo, sevenHoursAgo); + try { + await install(dir, name, version, stubNpm); + expect(existsSync(reused)).toBe(false); + } finally { + rmSync(cache, { recursive: true, force: true }); + } + }); }); diff --git a/packages/cli/src/utils/optionalPackages.ts b/packages/cli/src/utils/optionalPackages.ts index d7e24612f8..1eabc22bcc 100644 --- a/packages/cli/src/utils/optionalPackages.ts +++ b/packages/cli/src/utils/optionalPackages.ts @@ -8,6 +8,7 @@ import { realpathSync, renameSync, rmSync, + statSync, writeFileSync, } from "node:fs"; import { createRequire } from "node:module"; @@ -29,7 +30,7 @@ export const OPTIONAL_PACKAGES = { "@google/genai": "1.52.0", } as const satisfies Record; -const CACHE_DIR = join(homedir(), ".cache", "hyperframes", "optional"); +export const CACHE_DIR = join(homedir(), ".cache", "hyperframes", "optional"); export interface OptionalPackageDeps { cacheDir: string; @@ -108,11 +109,11 @@ function manifestPath(dir: string, name: string): string { return join(dir, "node_modules", name, "package.json"); } -function isInstalled(dir: string, name: string): boolean { +export function isInstalled(dir: string, name: string): boolean { return existsSync(manifestPath(dir, name)); } -function loadInstalled(dir: string, name: string): unknown | null { +export function loadInstalled(dir: string, name: string): unknown | null { if (!isInstalled(dir, name)) return null; return createRequire(join(dir, "package.json"))(name); } @@ -136,9 +137,9 @@ export function loadBesideCli(name: OptionalPackage, cliUrl = import.meta.url): return pinnedCopyBesideCli(name, cliUrl) ? createRequire(cliUrl)(name) : null; } -function runNpm(args: string[]): Promise { +export function runNpm(args: string[], signal?: AbortSignal): Promise { const npm = buildNpmCommand(args); - const child = spawn(npm.command, npm.args, { stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn(npm.command, npm.args, { stdio: ["ignore", "pipe", "pipe"], signal }); return new Promise((resolve, reject) => { let output = ""; child.stdout.on("data", (chunk) => (output += chunk)); @@ -159,16 +160,20 @@ function isProcessAlive(pid: number): boolean { } } -/** Removes staging dirs whose owning pid is dead (crash or kill mid-install); never a live one. */ -function sweepStaleStaging(dir: string): void { +const STAGING_PID_TRUSTED_FOR_MS = 6 * 60 * 60 * 1000; + +/** Removes staging dirs whose pid is dead (killed install), or too old to trust a live pid (reuse). */ +export function sweepStaleStaging(dir: string): void { const prefix = `${basename(dir)}.tmp-`; const parent = dirname(dir); if (!existsSync(parent)) return; for (const entry of readdirSync(parent)) { if (!entry.startsWith(prefix)) continue; const pid = /^(\d+)(?:-|$)/.exec(entry.slice(prefix.length))?.[1]; - if (pid === undefined || isProcessAlive(Number(pid))) continue; + if (pid === undefined) continue; const stale = join(parent, entry); + const age = Date.now() - (statSync(stale, { throwIfNoEntry: false })?.mtimeMs ?? Date.now()); + if (isProcessAlive(Number(pid)) && age < STAGING_PID_TRUSTED_FOR_MS) continue; try { rmSync(stale, { recursive: true, force: true }); } catch (err) { diff --git a/packages/cli/src/utils/renderCancellation.test.ts b/packages/cli/src/utils/renderCancellation.test.ts index d6cba15de6..f732274fcd 100644 --- a/packages/cli/src/utils/renderCancellation.test.ts +++ b/packages/cli/src/utils/renderCancellation.test.ts @@ -1,6 +1,6 @@ // fallow-ignore-file code-duplication import { afterEach, describe, expect, it, vi } from "vitest"; -import { createRenderCancellationScope } from "./renderCancellation.js"; +import { createRenderCancellationScope, stoppedByCancelSignal } from "./renderCancellation.js"; afterEach(() => { vi.useRealTimers(); @@ -159,3 +159,20 @@ describe("createRenderCancellationScope ancestor monitoring", () => { } }); }); + +describe("stoppedByCancelSignal", () => { + it("is true for a child killed by any signal the scope cancels on, SIGHUP included", () => { + for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) { + expect(stoppedByCancelSignal({ signal })).toBe(true); + } + }); + + it("is false when Node stopped the child or it stopped another way", () => { + expect(stoppedByCancelSignal({ code: "ETIMEDOUT", signal: "SIGTERM" })).toBe(false); + expect(stoppedByCancelSignal({ code: "ENOBUFS", signal: "SIGTERM" })).toBe(false); + expect(stoppedByCancelSignal({ killed: true, signal: "SIGTERM" })).toBe(false); + expect(stoppedByCancelSignal({ error: new Error("ETIMEDOUT"), signal: "SIGTERM" })).toBe(false); + expect(stoppedByCancelSignal({ signal: "SIGKILL" })).toBe(false); + expect(stoppedByCancelSignal({ code: 1, signal: null })).toBe(false); + }); +}); diff --git a/packages/cli/src/utils/renderCancellation.ts b/packages/cli/src/utils/renderCancellation.ts index 0d441048d0..ef60cd3d45 100644 --- a/packages/cli/src/utils/renderCancellation.ts +++ b/packages/cli/src/utils/renderCancellation.ts @@ -39,6 +39,16 @@ const RENDER_CANCELLATION_SIGNALS: readonly RenderCancellationSignal[] = [ "SIGHUP", ]; +export function stoppedByCancelSignal(stop: { + code?: unknown; + signal?: string | null; + killed?: boolean; + error?: unknown; +}): boolean { + if (stop.code || stop.killed || stop.error) return false; + return RENDER_CANCELLATION_SIGNALS.some((signal) => signal === stop.signal); +} + function captureAncestors( pid: number, lookupParentPid: (pid: number) => number | null, diff --git a/packages/cli/src/whisper/normalize.test.ts b/packages/cli/src/whisper/normalize.test.ts index 08a1bb88a5..4439d00a6e 100644 --- a/packages/cli/src/whisper/normalize.test.ts +++ b/packages/cli/src/whisper/normalize.test.ts @@ -84,6 +84,19 @@ describe("detectFormat", () => { }); describe("loadTranscript", () => { + it("reads an empty word list as a transcript with no words", () => { + expect(loadTranscript(tmpFile("transcript.json", "[]"))).toEqual({ + words: [], + format: "words-json", + }); + }); + + it("still rejects a JSON array that is not a word list", () => { + expect(() => loadTranscript(tmpFile("transcript.json", '[{"foo":1}]'))).toThrow( + /Unrecognized JSON transcript format/, + ); + }); + it("parses whisper-cpp JSON with punctuation merging", () => { const path = tmpFile( "transcript.json", diff --git a/packages/cli/src/whisper/normalize.ts b/packages/cli/src/whisper/normalize.ts index 86160fead6..5c7bb29a2c 100644 --- a/packages/cli/src/whisper/normalize.ts +++ b/packages/cli/src/whisper/normalize.ts @@ -48,7 +48,10 @@ function detectJsonFormat(raw: unknown): TranscriptFormat { if (obj.transcription && Array.isArray(obj.transcription)) return "whisper-cpp"; if (obj.words && Array.isArray(obj.words)) return "openai"; } - if (Array.isArray(raw) && raw[0]?.text !== undefined && raw[0]?.start !== undefined) { + if ( + Array.isArray(raw) && + raw.slice(0, 1).every((w) => w?.text !== undefined && w?.start !== undefined) + ) { return "words-json"; } throw new Error( diff --git a/packages/cli/src/whisper/parakeet.test.ts b/packages/cli/src/whisper/parakeet.test.ts index e707e45208..cea12e842a 100644 --- a/packages/cli/src/whisper/parakeet.test.ts +++ b/packages/cli/src/whisper/parakeet.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from "vitest"; -import { mergeTokensToWords } from "./parakeet.js"; +import { + droppedSpeechGaps, + mergeTokensToWords, + mergeWindowsToWords, + silenceCuts, + spliceGap, +} from "./parakeet.js"; describe("mergeTokensToWords", () => { it("joins Parakeet sub-word tokens into words on the space boundary", () => { @@ -35,3 +41,132 @@ describe("mergeTokensToWords", () => { expect(words[1]!.start).toBe(0.5); }); }); + +describe("mergeWindowsToWords", () => { + it("shifts each window's token times by the window's start", () => { + const words = mergeWindowsToWords([ + { offset: 0, tokens: [" Hel", "lo"], timestamps: [0.125, 0.25], durations: [0.125, 0.25] }, + { offset: 60.5, tokens: [" world"], timestamps: [0.25], durations: [0.5] }, + ]); + expect(words).toEqual([ + { text: "Hello", start: 0.125, end: 0.5 }, + { text: "world", start: 60.75, end: 61.25 }, + ]); + }); +}); + +describe("silenceCuts", () => { + // 100 samples/s keeps the fixture small: 150 s of steady sound with two 0.1 s silences. + const rate = 100; + const loud = (seconds: number) => new Float32Array(seconds * rate).fill(0.5); + + it("moves each cut to the quietest 100 ms within 5 s of the 60 s target", () => { + const samples = loud(150); + samples.fill(0, 5830, 5840); + samples.fill(0, 12195, 12205); + expect(silenceCuts(samples, rate)).toEqual([0, 5835, 12200, 15000]); + }); + + it("leaves audio shorter than one window whole", () => { + expect(silenceCuts(loud(42), rate)).toEqual([0, 4200]); + }); + + it("makes no window for empty audio", () => { + expect(silenceCuts(new Float32Array(0), rate)).toEqual([0]); + }); +}); + +describe("droppedSpeechGaps", () => { + const rate = 100; + const tokens = (timestamps: number[]) => ({ + tokens: timestamps.map(() => " w"), + timestamps, + durations: timestamps.map(() => 0.2), + }); + + it("finds a second of loud audio before the first token and between tokens", () => { + const speech = new Float32Array(6 * rate).fill(0.5); + expect(droppedSpeechGaps(speech, rate, tokens([2.08, 2.5, 3, 3.5, 4, 4.5, 5, 5.5]))).toEqual([ + [0, 2.08], + ]); + expect(droppedSpeechGaps(speech, rate, tokens([0, 0.5, 1, 3.5, 4, 4.5, 5, 5.5]))).toEqual([ + [1.2, 3.5], + ]); + expect( + droppedSpeechGaps(speech, rate, tokens([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5])), + ).toEqual([]); + }); + + it("ignores a quiet pause, silence, and a loud window with no tokens at all", () => { + const pause = new Float32Array(6 * rate).fill(0.5); + pause.fill(0, 2 * rate, 4 * rate); + expect( + droppedSpeechGaps(pause, rate, tokens([0.1, 0.6, 1.1, 1.6, 4.1, 4.6, 5.1, 5.6])), + ).toEqual([]); + expect(droppedSpeechGaps(new Float32Array(3 * rate), rate, tokens([]))).toEqual([]); + expect(droppedSpeechGaps(new Float32Array(3 * rate).fill(0.5), rate, tokens([]))).toEqual([]); + }); +}); + +describe("spliceGap", () => { + const w = (...pairs: [string, number][]) => ({ + tokens: pairs.map(([text]) => text), + timestamps: pairs.map(([, start]) => start), + durations: pairs.map(() => 0.1), + }); + const text = (r: ReturnType) => + mergeWindowsToWords([{ offset: 0, ...r }]) + .map((word) => word.text) + .join(" "); + + it("adds the gap's words in order and drops neighbours re-heard off time, in another case", () => { + const decoded = w([" so", 9.5], [" A", 13], ["sk", 13.25]); + const patch = w( + [" so", 0.5], + [" ask", 0.75], + [" not", 2.75], + [",", 3.25], + [" a", 3.5], + ["sk", 3.75], + ); + expect(text(spliceGap(decoded, patch, 9.25, [9.75, 13]))).toBe("so ask not, Ask"); + }); + + it("drops a right neighbour the patch splits into other tokens", () => { + const decoded = w([" so", 9], [" ask", 10]); + const patch = w([" so", 8.95], [" not", 9.4], [" a", 9.8], ["sk", 9.95]); + expect(text(spliceGap(decoded, patch, 0, [9.2, 10]))).toBe("so not ask"); + }); + + it("never glues the rest of a re-split left neighbour onto it", () => { + const decoded = w([" thing", 5], [" right", 7]); + const patch = w([" th", 5], ["ing", 5.35], [" we", 5.8], [" said", 6.3]); + expect(text(spliceGap(decoded, patch, 0, [5.3, 7]))).toBe("thing we said right"); + }); + + it("compares whole words, so a gap word is not eaten by a neighbour's last token", () => { + const decoded = w([" it", 3], ["'s", 3.2], [" now", 5]); + const patch = w([" s", 3.4], ["o", 3.5], [" what", 4]); + expect(text(spliceGap(decoded, patch, 0, [3.3, 5]))).toBe("it's so what now"); + }); + + it("keeps a patch's first word even when it has no leading space", () => { + const decoded = w([" so", 9], [" right", 12]); + expect(text(spliceGap(decoded, w(["ask", 9.4], [" not", 10.1]), 0, [9.1, 12]))).toBe( + "so ask not right", + ); + }); + + it("keeps a real repeat that starts well inside the gap", () => { + const decoded = w([" that", 2], [" right", 4]); + const patch = w([" that", 3], [" is", 3.4]); + expect(text(spliceGap(decoded, patch, 0, [2.2, 4]))).toBe("that that is right"); + }); + + it("adds a word shared by two gaps once, however each patch splits it", () => { + const decoded = w([" a", 1], [" mid", 3], [" z", 5]); + const first = spliceGap(decoded, w([" one", 1.8], [" m", 2.8], ["id", 2.95]), 0, [1.2, 3]); + const both = spliceGap(first, w([" mi", 3.35], ["d", 3.45], [" two", 4]), 0, [3.3, 5]); + expect(text(both)).toBe("a one mid two z"); + }); +}); diff --git a/packages/cli/src/whisper/parakeet.ts b/packages/cli/src/whisper/parakeet.ts index a53c18bfa7..ea805210aa 100644 --- a/packages/cli/src/whisper/parakeet.ts +++ b/packages/cli/src/whisper/parakeet.ts @@ -14,12 +14,28 @@ */ import { execFileSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { basename, extname, join } from "node:path"; import type { Word } from "./normalize.js"; import type { TranscribeResult } from "./transcribe.js"; +/** The model name `transcribe --json` reports for every Parakeet runner. */ +export const PARAKEET_MODEL_LABEL = "parakeet-tdt-0.6b-v3"; +/** Stdout/stderr line prefixes of the sherpa-onnx decode worker. */ +export const SHERPA_RESULT_PREFIX = "HYPERFRAMES_PARAKEET_RESULT:"; +export const SHERPA_ERROR_PREFIX = "HYPERFRAMES_PARAKEET_ERROR:"; + +/** Parakeet TDT 0.6B v3's languages, as listed on its model card. */ +export const PARAKEET_LANGUAGES = + "en es fr de bg hr cs da nl et fi el hu it lv lt mt pl pt ro sk sl sv ru uk"; + +/** True when no language was asked for, or Parakeet transcribes it ("pt-BR" counts as pt). */ +export function parakeetSpeaks(language?: string): boolean { + const base = language?.trim().toLowerCase().split(/[-_]/, 1)[0]; + return !base || PARAKEET_LANGUAGES.split(" ").includes(base); +} + const DEFAULT_MODEL = "mlx-community/parakeet-tdt-0.6b-v3"; const PARAKEET_INSTALL = "uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx"; @@ -103,6 +119,156 @@ export function mergeTokensToWords(parakeet: ParakeetJson): Word[] { return words.filter((w) => w.text.length > 0); } +/** One decoded window from sherpa-onnx: token times are relative to the window's `offset` (s). */ +export interface SherpaWindow { + offset: number; + tokens: string[]; + timestamps: number[]; + durations?: number[]; +} + +const roundToMs = (seconds: number) => Math.round(seconds * 1000) / 1000; + +export function mergeWindowsToWords(windows: SherpaWindow[]): Word[] { + return mergeTokensToWords({ + sentences: windows.map((w) => ({ + tokens: w.tokens.map((text, i) => { + const start = w.offset + (w.timestamps[i] ?? 0); + return { text, start: roundToMs(start), end: roundToMs(start + (w.durations?.[i] ?? 0)) }; + }), + })), + }); +} + +function meanEnergy(samples: Float32Array, from: number, to: number): number { + let energy = 0; + for (let i = from; i < to; i++) energy += samples[i]! * samples[i]!; + return energy / Math.max(1, to - from); +} + +/** + * Stretches (s) of at least 1 s without tokens that are about as loud as the window: skipped speech. + * A window with no tokens at all is music or noise, not a skip. + */ +export function droppedSpeechGaps( + samples: Float32Array, + sampleRate: number, + decoded: Omit, +): [number, number][] { + if (decoded.tokens.length === 0) return []; + const loud = 0.3 * meanEnergy(samples, 0, samples.length); + const { timestamps, durations } = decoded; + const gapStarts = [0, ...timestamps.map((t, i) => t + (durations?.[i] ?? 0))]; + const gapEnds = [...timestamps, samples.length / sampleRate]; + const at = (seconds: number) => Math.round(seconds * sampleRate); + return gapStarts + .map((from, i): [number, number] => [from, gapEnds[i]!]) + .filter(([from, to]) => to - from >= 1 && meanEnergy(samples, at(from), at(to)) > loud); +} + +/** Number of trailing `a` entries that equal the leading `b` entries. */ +function overlap(a: string[], b: string[]): number { + for (let n = Math.min(a.length, b.length); n > 0; n--) { + if (a.slice(-n).every((text, i) => text === b[i])) return n; + } + return 0; +} + +interface Token { + text: string; + start: number; + duration: number; +} + +/** Tokens grouped into words: a token with a leading space starts one. */ +function wordsOf(tokens: Token[]): Token[][] { + const words: Token[][] = []; + for (const t of tokens) { + if (t.text.startsWith(" ") || words.length === 0) words.push([t]); + else words.at(-1)!.push(t); + } + return words; +} + +const letters = (word: Token[]) => + word + .map((t) => t.text) + .join("") + .toLowerCase() + .replace(/[^\p{L}\p{N}]/gu, ""); + +/** How far from a gap edge the patch may re-hear the neighbouring word (measured: 0.26 s). */ +const EDGE_SECONDS = 0.3; + +/** + * Adds the `patch` words, decoded from `offset` s into the window, that start inside the gap, minus + * the gap's neighbours re-heard near its edges (compared as whole words, by letters). + */ +export function spliceGap( + decoded: Omit, + patch: Omit, + offset: number, + [from, to]: [number, number], +): Omit { + const tokens = (w: Omit, shift: number): Token[] => + w.tokens.map((text, i) => ({ + text, + start: shift + w.timestamps[i]!, + duration: w.durations?.[i] ?? 0, + })); + const kept = wordsOf(tokens(decoded, 0)); + const added = wordsOf(tokens(patch, offset)).filter( + ([first]) => first!.start >= from && first!.start < to, + ); + const before = kept.filter(([first]) => first!.start < from).map(letters); + const after = kept.filter(([first]) => first!.start >= to).map(letters); + const head = added.filter(([first]) => first!.start < from + EDGE_SECONDS).map(letters); + const tail = added.filter(([first]) => first!.start >= to - EDGE_SECONDS).map(letters); + const fresh = added + .slice(overlap(before, head), added.length - overlap(tail, after)) + .map(([first, ...rest]) => [{ ...first!, text: ` ${first!.text.trimStart()}` }, ...rest]); + const merged = [...kept, ...fresh].sort((a, b) => a[0]!.start - b[0]!.start).flat(); + return { + tokens: merged.map((t) => t.text), + timestamps: merged.map((t) => t.start), + durations: merged.map((t) => t.duration), + }; +} + +function quietestBlockCenter(samples: Float32Array, lo: number, hi: number, block: number): number { + let best = hi; + let bestEnergy = Infinity; + for (let s = lo; s + block <= hi; s += block) { + const energy = meanEnergy(samples, s, s + block); + if (energy < bestEnergy) { + bestEnergy = energy; + best = s + block / 2; + } + } + return best; +} + +/** + * Window boundaries (sample indices, first 0, last samples.length) about every `windowSeconds`, + * each moved to the quietest 100 ms within ±5 s, so no cut lands inside a word. + */ +export function silenceCuts( + samples: Float32Array, + sampleRate: number, + windowSeconds = 60, +): number[] { + const window = windowSeconds * sampleRate; + const slack = 5 * sampleRate; + const cuts = [0]; + while (cuts.at(-1)! + window < samples.length) { + const target = cuts.at(-1)! + window; + const hi = Math.min(target + slack, samples.length); + cuts.push(quietestBlockCenter(samples, target - slack, hi, sampleRate / 10)); + } + if (samples.length > 0) cuts.push(samples.length); + return cuts; +} + interface ParakeetOptions { language?: string; model?: string; @@ -139,13 +305,19 @@ export function transcribeWithParakeet( const produced = join(workDir, `${basename(inputPath, extname(inputPath))}.json`); if (!existsSync(produced)) throw new Error("Parakeet did not produce output."); - const words = mergeTokensToWords(JSON.parse(readFileSync(produced, "utf-8")) as ParakeetJson); - - const transcriptPath = join(dir, "transcript.json"); - writeFileSync(transcriptPath, JSON.stringify(words, null, 2)); - const durationSeconds = words.length > 0 ? words[words.length - 1]!.end : 0; - return { transcriptPath, wordCount: words.length, durationSeconds, speechOnsetSeconds: null }; + return writeParakeetTranscript( + dir, + mergeTokensToWords(JSON.parse(readFileSync(produced, "utf-8")) as ParakeetJson), + ); } finally { rmSync(workDir, { recursive: true, force: true }); } } + +export function writeParakeetTranscript(dir: string, words: Word[]): TranscribeResult { + mkdirSync(dir, { recursive: true }); + const transcriptPath = join(dir, "transcript.json"); + writeFileSync(transcriptPath, JSON.stringify(words, null, 2)); + const durationSeconds = words.length > 0 ? words[words.length - 1]!.end : 0; + return { transcriptPath, wordCount: words.length, durationSeconds, speechOnsetSeconds: null }; +} diff --git a/packages/cli/src/whisper/sherpa.test.ts b/packages/cli/src/whisper/sherpa.test.ts new file mode 100644 index 0000000000..1250f69589 --- /dev/null +++ b/packages/cli/src/whisper/sherpa.test.ts @@ -0,0 +1,294 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + rmSync, + truncateSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DownloadOptions } from "../cloud/download.js"; +import { + ensureParakeetModel, + installSherpaRuntime, + SHERPA_RUNTIME_DIR, + sherpaPlatformPackage, + DecodeCancelled, + sherpaRuntimeLoadError, + sherpaUnsupportedReason, + type ModelFile, +} from "./sherpa.js"; + +const file = (name: string, content: string): ModelFile => ({ + name, + bytes: Buffer.byteLength(content), + sha256: createHash("sha256").update(content).digest("hex"), +}); + +/** A downloader that serves `served[name]` for any URL ending in that name. */ +function fakeDownload(served: Record) { + return vi.fn(async (url: string, dest: string, opts?: DownloadOptions) => { + const content = served[url.split("/").at(-1)!]!; + mkdirSync(dirname(dest), { recursive: true }); + writeFileSync(dest, content); + opts?.signal?.throwIfAborted(); + opts?.onProgress?.(Buffer.byteLength(content), undefined); + return { path: dest, bytes: Buffer.byteLength(content) }; + }); +} + +describe("installSherpaRuntime", () => { + it.each([ + ["darwin", "arm64", "sherpa-onnx-darwin-arm64@1.13.8"], + ["linux", "x64", "sherpa-onnx-linux-x64@1.13.8"], + ["win32", "ia32", "sherpa-onnx-win-ia32@1.13.8"], + ])("pins the %s-%s native package", (platform, arch, spec) => { + expect(sherpaPlatformPackage(platform as NodeJS.Platform, arch as NodeJS.Architecture)).toBe( + spec, + ); + }); + + it("asks npm for the runtime and its native package at the same exact version", async () => { + const run = vi.fn(async (_args: string[]) => { + throw new Error("stop before touching the cache"); + }); + const cancel = new AbortController(); + const dir = mkdtempSync(join(tmpdir(), "hf-sherpa-runtime-")); + await installSherpaRuntime({ run, signal: cancel.signal, dir }).catch(() => undefined); + rmSync(dir, { recursive: true, force: true }); + expect((run.mock.calls[0] as unknown[] | undefined)?.[1]).toBe(cancel.signal); + const args = run.mock.calls[0]?.[0] as string[] | undefined; + expect(args).toContain("sherpa-onnx-node@1.13.8"); + expect(args).toContain(sherpaPlatformPackage()); + }); + + /** A stand-in runtime in `root` whose entry point loads, or throws as a missing library does. */ + const LOADS = "module.exports = {};"; + const BROKEN = 'throw new Error("libonnxruntime.so: cannot open\\n shared object");'; + function fakeRuntime(root: string, body: string) { + const pkg = join(root, "node_modules", "sherpa-onnx-node"); + mkdirSync(pkg, { recursive: true }); + writeFileSync(join(pkg, "package.json"), '{"name":"sherpa-onnx-node","main":"index.js"}'); + writeFileSync(join(pkg, "index.js"), body); + } + + it("reinstalls a runtime that is present but does not load", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-sherpa-runtime-")); + try { + fakeRuntime(dir, BROKEN); + expect(sherpaRuntimeLoadError(dir)).toBe("libonnxruntime.so: cannot open shared object"); + const run = vi.fn(async (args: string[]) => + fakeRuntime(args[args.indexOf("--prefix") + 1]!, LOADS), + ); + + expect(await installSherpaRuntime({ run, dir })).toBe(true); + expect(run).toHaveBeenCalledTimes(1); + expect(sherpaRuntimeLoadError(dir)).toBeNull(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("leaves a runtime that loads alone", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-sherpa-runtime-")); + try { + fakeRuntime(dir, LOADS); + const run = vi.fn(); + expect(await installSherpaRuntime({ run, dir })).toBe(false); + expect(run).not.toHaveBeenCalled(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("fails with the loader's error when the reinstalled runtime still does not load", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-sherpa-runtime-")); + try { + const run = vi.fn(async (args: string[]) => + fakeRuntime(args[args.indexOf("--prefix") + 1]!, BROKEN), + ); + await expect(installSherpaRuntime({ run, dir })).rejects.toThrow( + "The sherpa-onnx runtime was reinstalled but still does not load (libonnxruntime.so: cannot open shared object). Use --engine whisper for now.", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it.skipIf(process.platform === "win32").each(["SIGINT", "SIGHUP"])( + "treats a probe stopped by %s as a cancel and keeps the runtime", + async (signal) => { + const dir = mkdtempSync(join(tmpdir(), "hf-sherpa-runtime-")); + try { + fakeRuntime(dir, `process.kill(process.pid, "${signal}");`); + const run = vi.fn(); + await expect(installSherpaRuntime({ run, dir })).rejects.toBeInstanceOf(DecodeCancelled); + expect(run).not.toHaveBeenCalled(); + expect(existsSync(join(dir, "node_modules", "sherpa-onnx-node", "index.js"))).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); + + it("reads its own probe timeout as a broken runtime, not a cancel, and says so", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-sherpa-runtime-")); + try { + fakeRuntime(dir, "Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5000);"); + expect(sherpaRuntimeLoadError(dir, 300)).toBe("loading it timed out after 0.3 s"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("keeps each platform and arch in its own runtime dir", () => { + expect(SHERPA_RUNTIME_DIR).toMatch( + new RegExp(`sherpa-onnx-node@1\\.13\\.8-${process.platform}-${process.arch}$`), + ); + }); +}); + +describe("sherpaParakeetInstalled", () => { + it("counts a runtime that does not load, so transcribe reports it instead of skipping it", async () => { + const home = mkdtempSync(join(tmpdir(), "hf-sherpa-home-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const sherpa = await import("./sherpa.js"); + const manifest = join(sherpa.SHERPA_RUNTIME_DIR, "node_modules", "sherpa-onnx-node"); + mkdirSync(manifest, { recursive: true }); + writeFileSync(join(manifest, "package.json"), "{}"); + mkdirSync(sherpa.PARAKEET_MODEL_DIR, { recursive: true }); + for (const [name, bytes] of [ + ["encoder.int8.onnx", 652_184_281], + ["decoder.int8.onnx", 11_845_275], + ["joiner.int8.onnx", 6_355_277], + ["tokens.txt", 93_939], + ] as const) { + writeFileSync(join(sherpa.PARAKEET_MODEL_DIR, name), ""); + truncateSync(join(sherpa.PARAKEET_MODEL_DIR, name), bytes); + } + expect(sherpa.sherpaRuntimeLoadError()).not.toBeNull(); + expect(sherpa.sherpaParakeetInstalled()).toBe(true); + } finally { + vi.unstubAllEnvs(); + rmSync(home, { recursive: true, force: true }); + } + }); +}); + +describe("ensureParakeetModel", () => { + let dir: string; + afterEach(() => rmSync(dir, { recursive: true, force: true })); + const tempDir = () => (dir = mkdtempSync(join(tmpdir(), "hf-parakeet-model-"))); + const stagingLeft = () => + readdirSync(dirname(dir)).filter((name) => name.startsWith(`${basename(dir)}.tmp-`)); + + it("refuses a file whose sha256 does not match and leaves nothing under its name", async () => { + tempDir(); + const files = [file("encoder.onnx", "abc")]; + const download = fakeDownload({ "encoder.onnx": "abd" }); + + await expect(ensureParakeetModel({ dir, files, download })).rejects.toThrow(/sha256/); + expect(readdirSync(dir)).toEqual([]); + expect(stagingLeft()).toEqual([]); + }); + + it("names the file, the host and a retry when a download fails", async () => { + tempDir(); + const download = vi.fn(async () => { + throw new TypeError("fetch failed"); + }); + await expect( + ensureParakeetModel({ dir, files: [file("encoder.onnx", "enc")], download }), + ).rejects.toThrow( + "Could not download encoder.onnx from huggingface.co (fetch failed). Check your network and re-run.", + ); + expect(stagingLeft()).toEqual([]); + }); + + it("stops at the next file on cancel, keeping only files that verified", async () => { + tempDir(); + const files = [file("encoder.onnx", "enc"), file("tokens.txt", "tok")]; + const cancel = new AbortController(); + const download = fakeDownload({ "encoder.onnx": "enc", "tokens.txt": "tok" }); + download.mockImplementationOnce(async (_url, dest) => { + mkdirSync(dirname(dest), { recursive: true }); + writeFileSync(dest, "enc"); + cancel.abort(); + return { path: dest, bytes: 3 }; + }); + + await expect( + ensureParakeetModel({ dir, files, download, signal: cancel.signal }), + ).rejects.toThrow(/abort/i); + expect(download).toHaveBeenCalledTimes(1); + expect(download.mock.calls[0]?.[2]?.signal).toBe(cancel.signal); + expect(readdirSync(dir)).toEqual(["encoder.onnx"]); + expect(stagingLeft()).toEqual([]); + }); + + it("sweeps the staging dir a killed install left behind", async () => { + tempDir(); + const stale = `${dir}.tmp-999999999-dead`; + mkdirSync(stale); + writeFileSync(join(stale, "encoder.onnx"), "half"); + const files = [file("tokens.txt", "tok")]; + + await ensureParakeetModel({ dir, files, download: fakeDownload({ "tokens.txt": "tok" }) }); + expect(existsSync(stale)).toBe(false); + }); + + it("fetches only what is missing or corrupt, then is a no-op", async () => { + tempDir(); + mkdirSync(dir, { recursive: true }); + const files = [ + file("encoder.onnx", "enc"), + file("tokens.txt", "tok"), + file("joiner.onnx", "j"), + ]; + writeFileSync(join(dir, "encoder.onnx"), "enc"); + writeFileSync(join(dir, "tokens.txt"), "xxx"); + const download = fakeDownload({ "tokens.txt": "tok", "joiner.onnx": "j" }); + const onBytes = vi.fn(); + + expect(await ensureParakeetModel({ dir, files, download, onBytes })).toBe(true); + expect(download.mock.calls.map(([url]) => url.split("/").at(-1))).toEqual([ + "tokens.txt", + "joiner.onnx", + ]); + expect(readdirSync(dir).sort()).toEqual(["encoder.onnx", "joiner.onnx", "tokens.txt"]); + expect(onBytes.mock.calls.at(-1)).toEqual([4, 4]); + + download.mockClear(); + expect(await ensureParakeetModel({ dir, files, download })).toBe(false); + expect(download).not.toHaveBeenCalled(); + expect(existsSync(join(dir, "tokens.txt"))).toBe(true); + }); +}); + +describe("sherpaUnsupportedReason", () => { + const linux = (glibc?: string) => ({ platform: "linux", arch: "x64", glibc }); + + it("refuses Linux without glibc 2.32 and accepts newer glibc", () => { + expect(sherpaUnsupportedReason(linux("2.31"))).toMatch(/glibc 2\.32.*2\.31/); + expect(sherpaUnsupportedReason(linux())).toMatch(/musl/); + expect(sherpaUnsupportedReason(linux("2.32"))).toBeNull(); + expect(sherpaUnsupportedReason(linux("2.36"))).toBeNull(); + }); + + it("refuses targets with no native package, naming the ones that have one", () => { + expect(sherpaUnsupportedReason({ platform: "darwin", arch: "arm64" })).toBeNull(); + expect(sherpaUnsupportedReason({ platform: "win32", arch: "x64" })).toBeNull(); + expect(sherpaUnsupportedReason({ platform: "win32", arch: "arm64" })).toMatch( + /darwin-arm64.*win32-ia32; this system is win32-arm64/, + ); + expect(sherpaUnsupportedReason({ platform: "linux", arch: "arm", glibc: "2.36" })).toMatch( + /this system is linux-arm/, + ); + }); +}); diff --git a/packages/cli/src/whisper/sherpa.ts b/packages/cli/src/whisper/sherpa.ts new file mode 100644 index 0000000000..35dfab44ee --- /dev/null +++ b/packages/cli/src/whisper/sherpa.ts @@ -0,0 +1,355 @@ +import { execFile, spawnSync, type ExecFileException } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { createReadStream, mkdirSync, renameSync, rmSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { downloadToFile } from "../cloud/download.js"; +import { stoppedByCancelSignal } from "../utils/renderCancellation.js"; +import { + CACHE_DIR, + install, + isInstalled, + runNpm, + sweepStaleStaging, +} from "../utils/optionalPackages.js"; +import { + mergeWindowsToWords, + SHERPA_ERROR_PREFIX, + SHERPA_RESULT_PREFIX, + writeParakeetTranscript, + type SherpaWindow, +} from "./parakeet.js"; +import { getPreparedWavDurationSeconds, prepareWav, type TranscribeResult } from "./transcribe.js"; + +const RUNTIME = "sherpa-onnx-node"; +const RUNTIME_VERSION = "1.13.8"; +/** Per platform and arch, like the native package inside, so Rosetta never shadows arm64. */ +export const SHERPA_RUNTIME_DIR = join( + CACHE_DIR, + `${RUNTIME}@${RUNTIME_VERSION}-${process.platform}-${process.arch}`, +); + +const MODEL_REPO = "csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8"; +const MODEL_REVISION = "2bda32ec70b097a55adaa07d9a7173915b43cc78"; +export const PARAKEET_MODEL_DIR = join( + homedir(), + ".cache", + "hyperframes", + "parakeet", + "parakeet-tdt-0.6b-v3-int8", +); + +export interface ModelFile { + name: string; + bytes: number; + sha256: string; +} + +/** Order matters: encoder, decoder, joiner, tokens (see recognizerConfig). */ +const PARAKEET_MODEL_FILES: readonly ModelFile[] = [ + { + name: "encoder.int8.onnx", + bytes: 652_184_281, + sha256: "acfc2b4456377e15d04f0243af540b7fe7c992f8d898d751cf134c3a55fd2247", + }, + { + name: "decoder.int8.onnx", + bytes: 11_845_275, + sha256: "179e50c43d1a9de79c8a24149a2f9bac6eb5981823f2a2ed88d655b24248db4e", + }, + { + name: "joiner.int8.onnx", + bytes: 6_355_277, + sha256: "3164c13fc2821009440d20fcb5fdc78bff28b4db2f8d0f0b329101719c0948b3", + }, + { + name: "tokens.txt", + bytes: 93_939, + sha256: "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d", + }, +]; + +/** At least 30 minutes, and twice the audio length for slow CPUs. */ +const decodeTimeoutMs = (audioSeconds: number) => Math.max(1_800_000, audioSeconds * 2000); + +/** The platforms sherpa-onnx-node publishes a native package for. */ +const SUPPORTED_TARGETS = [ + "darwin-arm64", + "darwin-x64", + "linux-x64", + "linux-arm64", + "win32-x64", + "win32-ia32", +]; + +interface Host { + platform: string; + arch: string; + glibc?: string; +} + +function currentHost(): Host { + const report = process.report.getReport() as { header?: { glibcVersionRuntime?: string } }; + return { + platform: process.platform, + arch: process.arch, + glibc: report.header?.glibcVersionRuntime, + }; +} + +/** Why sherpa-onnx cannot run here, or null. Its Linux prebuilt needs glibc 2.32 (so no musl). */ +export function sherpaUnsupportedReason(host: Host = currentHost()): string | null { + const target = `${host.platform}-${host.arch}`; + if (!SUPPORTED_TARGETS.includes(target)) { + return `Parakeet runs on ${SUPPORTED_TARGETS.join(", ")}; this system is ${target}.`; + } + if (host.platform !== "linux") return null; + if (!host.glibc) return "Parakeet needs glibc 2.32 or newer; this Linux has no glibc (musl?)."; + const [major = 0, minor = 0] = host.glibc.split(".").map(Number); + if (major > 2 || (major === 2 && minor >= 32)) return null; + return `Parakeet needs glibc 2.32 or newer; this system has glibc ${host.glibc}.`; +} + +const nativePackageName = (platform: string, arch: string) => + `sherpa-onnx-${platform === "win32" ? "win" : platform}-${arch}`; + +const LOAD_RUNTIME = `try { + require("node:module").createRequire(process.env.HF_SHERPA_MANIFEST)("${RUNTIME}"); +} catch (e) { + process.stderr.write(String(e && e.message).replace(/\\s+/g, " ")); + process.exitCode = 1; +}`; + +/** Loads it in a child as the decode worker does: null when it loads, else the loader's error. */ +export function sherpaRuntimeLoadError( + dir = SHERPA_RUNTIME_DIR, + timeoutMs = 60_000, +): string | null { + if (!isInstalled(dir, RUNTIME)) return `${RUNTIME} is not installed in ${dir}`; + const env = { ...process.env, HF_SHERPA_MANIFEST: join(dir, "package.json") }; + const probe = spawnSync(process.execPath, ["-e", LOAD_RUNTIME], { + env, + encoding: "utf8", + stdio: ["ignore", "ignore", "pipe"], + timeout: timeoutMs, + }); + if (probe.status === 0) return null; + // Ctrl-C reaches the probe too: a cancel, never proof that a working runtime is broken. + if (stoppedByCancelSignal(probe)) throw new DecodeCancelled("Parakeet install cancelled"); + const timedOut = (probe.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT"; + return bounded( + probe.stderr?.trim() || + (timedOut ? `loading it timed out after ${timeoutMs / 1000} s` : probe.error?.message) || + (probe.signal + ? `loading it crashed (${probe.signal})` + : `it exited ${probe.status} with no output`), + ); +} + +/** The native package, pinned too: the runtime's own optionalDependencies accept any 1.13.x. */ +export function sherpaPlatformPackage(platform = process.platform, arch = process.arch): string { + return `${nativePackageName(platform, arch)}@${RUNTIME_VERSION}`; +} + +/** Installs the runtime unless it already loads; true when it installed. */ +export async function installSherpaRuntime({ + run = runNpm, + signal, + dir = SHERPA_RUNTIME_DIR, +}: { run?: typeof runNpm; signal?: AbortSignal; dir?: string } = {}): Promise { + if (sherpaRuntimeLoadError(dir) === null) return false; + // install() keeps any dir holding the runtime manifest, so a broken one goes first. + rmSync(dir, { recursive: true, force: true }); + const native = sherpaPlatformPackage(); + await install(dir, RUNTIME, RUNTIME_VERSION, (args) => run([...args, native], signal)); + const stillBroken = sherpaRuntimeLoadError(dir); + if (stillBroken) { + throw new Error( + `The sherpa-onnx runtime was reinstalled but still does not load (${stillBroken}). Use --engine whisper for now.`, + ); + } + return true; +} + +/** + * Model sizes only (install verified the hashes). The runtime counts once its manifest is there, even + * if broken: transcribe must then fail with the repair, not quietly pick whisper. + */ +export function sherpaParakeetInstalled(): boolean { + return ( + isInstalled(SHERPA_RUNTIME_DIR, RUNTIME) && + PARAKEET_MODEL_FILES.every( + (f) => + statSync(join(PARAKEET_MODEL_DIR, f.name), { throwIfNoEntry: false })?.size === f.bytes, + ) + ); +} + +async function sha256File(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer); + return hash.digest("hex"); +} + +async function verifies(path: string, file: ModelFile): Promise { + if (statSync(path, { throwIfNoEntry: false })?.size !== file.bytes) return false; + return (await sha256File(path)) === file.sha256; +} + +interface EnsureModelOptions { + dir?: string; + files?: readonly ModelFile[]; + download?: typeof downloadToFile; + onBytes?: (done: number, total: number) => void; + signal?: AbortSignal; +} + +/** + * Fetches each file that does not verify into a pid-named staging dir, checks size and sha256, then + * renames it into place. False when all already verified. + */ +export async function ensureParakeetModel({ + dir = PARAKEET_MODEL_DIR, + files = PARAKEET_MODEL_FILES, + download = downloadToFile, + onBytes, + signal, +}: EnsureModelOptions = {}): Promise { + sweepStaleStaging(dir); + const missing: ModelFile[] = []; + for (const file of files) if (!(await verifies(join(dir, file.name), file))) missing.push(file); + const total = missing.reduce((sum, f) => sum + f.bytes, 0); + const staging = `${dir}.tmp-${process.pid}-${randomUUID().slice(0, 8)}`; + let done = 0; + try { + for (const file of missing) { + signal?.throwIfAborted(); + const dest = join(dir, file.name); + const temp = join(staging, file.name); + rmSync(dest, { force: true }); + const url = `https://huggingface.co/${MODEL_REPO}/resolve/${MODEL_REVISION}/${file.name}`; + await download(url, temp, { + signal, + onProgress: (bytes) => onBytes?.(done + bytes, total), + }).catch((err: unknown) => { + if (signal?.aborted) throw err; + const why = err instanceof Error ? err.message : String(err); + throw new Error( + `Could not download ${file.name} from huggingface.co (${why}). Check your network and re-run.`, + { cause: err }, + ); + }); + if (!(await verifies(temp, file))) { + throw new Error( + `${file.name} did not match its pinned size and sha256, so it was discarded. Re-run to retry.`, + ); + } + mkdirSync(dir, { recursive: true }); + renameSync(temp, dest); + done += file.bytes; + } + } finally { + rmSync(staging, { recursive: true, force: true }); + } + return missing.length > 0; +} + +function recognizerConfig(): object { + const [encoder, decoder, joiner, tokens] = PARAKEET_MODEL_FILES.map((f) => + join(PARAKEET_MODEL_DIR, f.name), + ); + return { + featConfig: { sampleRate: 16000, featureDim: 80 }, + modelConfig: { + transducer: { encoder, decoder, joiner }, + tokens, + numThreads: 4, + provider: "cpu", + debug: 0, + modelType: "nemo_transducer", + }, + decodingMethod: "greedy_search", + }; +} + +/** Ctrl-C or a SIGTERM stopped a Parakeet child: the user's cancel, never a reason to fall back. */ +export class DecodeCancelled extends Error {} + +const MAX_REASON_CHARS = 600; +const bounded = (why: string) => + why.length > MAX_REASON_CHARS ? `${why.slice(0, MAX_REASON_CHARS)}…` : why; + +function failureReason(err: ExecFileException | null, stderr: string): string { + const how = !err + ? "exited without a result" + : err.killed + ? "was stopped (timeout or output limit)" + : err.signal + ? `crashed (${err.signal})` + : `exited with code ${err.code}`; + const lines = stderr + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + const why = + lines.find((line) => line.startsWith(SHERPA_ERROR_PREFIX))?.slice(SHERPA_ERROR_PREFIX.length) ?? + lines.at(-1); + if (!why) return how; + return `${how}: ${bounded(why)}`; +} + +/** Decodes in a child process: onnxruntime aborts the whole process on some inputs, uncatchably. */ +function decode(wavPath: string, signal: AbortSignal): Promise { + const sourceMode = import.meta.url.endsWith(".ts"); + const worker = new URL(sourceMode ? "./sherpaWorker.ts" : "./sherpaWorker.js", import.meta.url); + const args = [...(sourceMode ? ["--import", "tsx"] : []), fileURLToPath(worker)]; + const input = { wavPath, runtimeDir: SHERPA_RUNTIME_DIR, config: recognizerConfig() }; + return new Promise((resolve, reject) => { + execFile( + process.execPath, + args, + { + env: { ...process.env, HYPERFRAMES_PARAKEET_INPUT: JSON.stringify(input) }, + maxBuffer: 256 * 1024 * 1024, + timeout: decodeTimeoutMs(getPreparedWavDurationSeconds(wavPath) ?? 0), + signal, + }, + (err, stdout, stderr) => { + if (signal.aborted || (err && stoppedByCancelSignal(err))) { + reject(new DecodeCancelled("Transcription cancelled")); + return; + } + const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX)); + if (!err && line) resolve(JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length))); + else reject(new Error(`Parakeet decoder ${failureReason(err, stderr)}`)); + }, + ); + }); +} + +/** Prepares the audio for Parakeet and its fallback alike; ffmpeg stopped by Ctrl-C is a cancel. */ +export function prepareSherpaWav( + inputPath: string, + onProgress?: (message: string) => void, +): string { + try { + return prepareWav(inputPath, onProgress); + } catch (err) { + if ((err as { cancelled?: boolean }).cancelled) { + throw new DecodeCancelled("Transcription cancelled"); + } + throw err; + } +} + +/** The caller owns the cancellation scope: it must already cover audio preparation. */ +export async function transcribeWithSherpa( + wavPath: string, + dir: string, + options: { signal: AbortSignal; onProgress?: (message: string) => void }, +): Promise { + options.onProgress?.("Transcribing with Parakeet..."); + const windows = await decode(wavPath, options.signal); + return writeParakeetTranscript(dir, mergeWindowsToWords(windows)); +} diff --git a/packages/cli/src/whisper/sherpaWorker.test.ts b/packages/cli/src/whisper/sherpaWorker.test.ts new file mode 100644 index 0000000000..2a582b0371 --- /dev/null +++ b/packages/cli/src/whisper/sherpaWorker.test.ts @@ -0,0 +1,141 @@ +import { execFile } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX } from "./parakeet.js"; + +const WORKER = fileURLToPath(new URL("./sherpaWorker.ts", import.meta.url)); + +// Stand-in sherpa-onnx-node, 3 s of sound: the whole window (300 samples) drops the first 2 s; the +// padded window (350) hears it, and for speech.wav so does the padded gap +-0.5 s (308) alone. +const FAKE_SHERPA = ` +let path; +module.exports = { + readWave(p) { + path = p; + if (path.endsWith("broken.wav")) throw new Error("Failed to read " + path); + if (path.endsWith("lines.wav")) throw new Error("Could not find it. Tried\\n\\n ../a.node\\n ./b.node\\n"); + if (path.endsWith("long.wav")) return { sampleRate: 100, samples: new Float32Array(60000) }; + return { sampleRate: 100, samples: new Float32Array(300).fill(0.5) }; + }, + OfflineRecognizer: class { + constructor() { + if (process.env.STARTED) require("fs").writeFileSync(process.env.STARTED, ""); + } + createStream() { return { acceptWaveform(w) { this.w = w; } }; } + decode() { + if (path.endsWith("long.wav")) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1000); + } + getResult(stream) { + const n = stream.w.samples.length; + if (n === 300) return { tokens: [" not"], timestamps: [2.08], durations: [0.4] }; + if (n === 350) return { tokens: [" ask", " not"], timestamps: [0.75, 1.5], durations: [0.4, 0.4] }; + return n === 308 && path.endsWith("speech.wav") + ? { tokens: [" ask", " not"], timestamps: [1, 2.58], durations: [0.4, 0.4] } + : { tokens: [], timestamps: [], durations: [] }; + } + }, +}; +`; + +function runWorker(runtimeDir: string, wavPath: string) { + const input = JSON.stringify({ wavPath, runtimeDir, config: {} }); + return new Promise<{ code: number | null; stdout: string; stderr: string }>((resolve) => { + execFile( + process.execPath, + ["--import", "tsx", WORKER], + { env: { ...process.env, HYPERFRAMES_PARAKEET_INPUT: input } }, + (err, stdout, stderr) => resolve({ code: err ? (err.code as number) : 0, stdout, stderr }), + ); + }); +} + +describe("sherpaWorker", () => { + let root: string; + afterEach(() => rmSync(root, { recursive: true, force: true })); + + function fakeRuntime(): string { + root = mkdtempSync(join(tmpdir(), "hf-sherpa-worker-")); + const pkg = join(root, "node_modules", "sherpa-onnx-node"); + mkdirSync(pkg, { recursive: true }); + writeFileSync(join(pkg, "package.json"), '{"name":"sherpa-onnx-node","main":"index.js"}'); + writeFileSync(join(pkg, "index.js"), FAKE_SHERPA); + return root; + } + + function windowsOf(stdout: string) { + const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX))!; + return JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length)); + } + + it("decodes a gap that skipped loud audio on its own and splices in its new tokens", async () => { + const { code, stdout } = await runWorker(fakeRuntime(), "speech.wav"); + expect(code).toBe(0); + expect(windowsOf(stdout)).toEqual([ + { offset: 0, tokens: [" ask", " not"], timestamps: [0.5, 2.08], durations: [0.4, 0.4] }, + ]); + }); + + it("re-decodes the window with leading silence when the gap alone gives nothing", async () => { + const { code, stdout } = await runWorker(fakeRuntime(), "hum.wav"); + expect(code).toBe(0); + expect(windowsOf(stdout)).toEqual([ + { offset: 0, tokens: [" ask", " not"], timestamps: [0.25, 1], durations: [0.4, 0.4] }, + ]); + }); + + it("exits non-zero with one prefixed error line when it cannot read the audio", async () => { + const { code, stderr } = await runWorker(fakeRuntime(), "broken.wav"); + expect(code).toBe(1); + expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}Failed to read broken.wav`); + }); + + it("keeps every line of a multi-line error on its one prefixed line", async () => { + const { code, stderr } = await runWorker(fakeRuntime(), "lines.wav"); + expect(code).toBe(1); + expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}Could not find it. Tried ../a.node ./b.node\n`); + }); + + it.skipIf(process.platform === "win32")( + "stops between windows once the CLI that spawned it is gone", + async () => { + const started = join(fakeRuntime(), "started"); + const input = JSON.stringify({ wavPath: "long.wav", runtimeDir: root, config: {} }); + // The shell waits for the first window, then exits, orphaning a 10-window decode. + const shell = + `"${process.execPath}" --import tsx "${WORKER}" >/dev/null 2>&1 & echo $!; ` + + `while [ ! -e "${started}" ]; do sleep 0.1; done`; + const pid = await new Promise((resolve) => + execFile( + "sh", + ["-c", shell], + { env: { ...process.env, HYPERFRAMES_PARAKEET_INPUT: input, STARTED: started } }, + (_e, out) => resolve(Number(out.trim())), + ), + ); + const alive = () => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }; + const deadline = Date.now() + 4000; + while (alive() && Date.now() < deadline) await new Promise((r) => setTimeout(r, 100)); + const orphanSurvived = alive(); + if (orphanSurvived) process.kill(pid, "SIGKILL"); + expect(orphanSurvived).toBe(false); + }, + 15_000, + ); + + it("names the missing runtime when sherpa-onnx-node is not installed", async () => { + root = mkdtempSync(join(tmpdir(), "hf-sherpa-worker-")); + const { code, stderr } = await runWorker(root, "speech.wav"); + expect(code).toBe(1); + expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}sherpa-onnx-node is not installed in ${root}`); + }); +}); diff --git a/packages/cli/src/whisper/sherpaWorker.ts b/packages/cli/src/whisper/sherpaWorker.ts new file mode 100644 index 0000000000..45e91dce53 --- /dev/null +++ b/packages/cli/src/whisper/sherpaWorker.ts @@ -0,0 +1,91 @@ +import { loadInstalled } from "../utils/optionalPackages.js"; +import { + droppedSpeechGaps, + silenceCuts, + spliceGap, + SHERPA_ERROR_PREFIX, + SHERPA_RESULT_PREFIX, + type SherpaWindow, +} from "./parakeet.js"; + +interface Wave { + samples: Float32Array; + sampleRate: number; +} + +interface SherpaOnnx { + readWave(path: string): Wave; + OfflineRecognizer: new (config: object) => { + createStream(): { acceptWaveform(wave: Wave): void }; + decode(stream: unknown): void; + getResult(stream: unknown): Omit; + }; +} + +const { wavPath, runtimeDir, config } = JSON.parse(process.env.HYPERFRAMES_PARAKEET_INPUT ?? "{}"); +const parentPid = process.ppid; + +/** Leading silence moves the frame grid; 0.5 s recovered the dropped clause at every length tried. */ +const RETRY_PAD_SECONDS = 0.5; +/** Audio kept on each side of a skipped gap when it is decoded on its own. */ +const GAP_MARGIN_SECONDS = 0.5; + +function decodeWindow( + recognizer: InstanceType, + wave: Wave, + padSeconds = 0, +): Omit { + const pad = Math.round(padSeconds * wave.sampleRate); + const samples = new Float32Array(pad + wave.samples.length); + samples.set(wave.samples, pad); + const stream = recognizer.createStream(); + stream.acceptWaveform({ sampleRate: wave.sampleRate, samples }); + recognizer.decode(stream); + const { tokens, timestamps, durations } = recognizer.getResult(stream); + return { tokens, timestamps: timestamps.map((t) => Math.max(0, t - padSeconds)), durations }; +} + +try { + const sherpa = loadInstalled(runtimeDir, "sherpa-onnx-node") as SherpaOnnx | null; + if (!sherpa) throw new Error(`sherpa-onnx-node is not installed in ${runtimeDir}`); + const recognizer = new sherpa.OfflineRecognizer(config); + const wave = sherpa.readWave(wavPath); + const cuts = silenceCuts(wave.samples, wave.sampleRate); + const windows: SherpaWindow[] = []; + for (let k = 0; k + 1 < cuts.length; k++) { + // A SIGKILLed CLI cannot stop us, and nobody would read the result. + if (process.ppid !== parentPid) throw new Error("the CLI exited, so the decode stopped"); + const slice = { + sampleRate: wave.sampleRate, + samples: wave.samples.subarray(cuts[k], cuts[k + 1]), + }; + let decoded = decodeWindow(recognizer, slice); + const gaps = droppedSpeechGaps(slice.samples, slice.sampleRate, decoded); + let spliced = decoded; + for (const [from, to] of gaps) { + const lo = Math.max(0, from - GAP_MARGIN_SECONDS); + const at = (seconds: number) => Math.round(seconds * slice.sampleRate); + const samples = slice.samples.subarray(at(lo), at(to + GAP_MARGIN_SECONDS)); + const patch = decodeWindow( + recognizer, + { sampleRate: slice.sampleRate, samples }, + RETRY_PAD_SECONDS, + ); + spliced = spliceGap(spliced, patch, lo, [from, to]); + } + if (spliced.tokens.length > decoded.tokens.length) decoded = spliced; + else if (gaps.length > 0) { + const retry = decodeWindow(recognizer, slice, RETRY_PAD_SECONDS); + if (retry.tokens.length > decoded.tokens.length) decoded = retry; + } + windows.push({ offset: cuts[k]! / wave.sampleRate, ...decoded }); + } + process.stdout.write(`${SHERPA_RESULT_PREFIX}${JSON.stringify(windows)}\n`); +} catch (err) { + // One line: the reader takes the prefixed line, and the loader's error spans several. + const message = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").trim(); + // Rethrow after the flush: a pipe write is asynchronous on macOS and a crash would drop it. + process.stderr.write(`${SHERPA_ERROR_PREFIX}${message}\n`, () => { + throw err; + }); +} diff --git a/packages/cli/src/whisper/transcribe.test.ts b/packages/cli/src/whisper/transcribe.test.ts index b6ad787a91..a8fa01c7df 100644 --- a/packages/cli/src/whisper/transcribe.test.ts +++ b/packages/cli/src/whisper/transcribe.test.ts @@ -1,8 +1,14 @@ +import { chmodSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it, test } from "vitest"; import { dtwPresetForModel, + getPreparedWavDurationSeconds, initialModelForLanguage, + isPcm16kMono, isWhisperTimeoutError, + prepareWav, resolveAudioPreparationTimeoutMs, resolveWhisperTimeoutMs, whisperModelSlowdownFactor, @@ -159,14 +165,17 @@ describe("resolveWhisperTimeoutMs with overrideMs", () => { }); describe("isWhisperTimeoutError", () => { - it("returns true for a Node SIGTERM child-timeout error", () => { - const err = Object.assign(new Error("Command failed"), { signal: "SIGTERM" }); + it("returns true for execFileSync's own timeout (ETIMEDOUT with SIGTERM)", () => { + const err = Object.assign(new Error("Command failed"), { + code: "ETIMEDOUT", + signal: "SIGTERM", + }); expect(isWhisperTimeoutError(err)).toBe(true); }); - it("returns true for an ETIMEDOUT-coded error", () => { - const err = Object.assign(new Error("Command failed"), { code: "ETIMEDOUT" }); - expect(isWhisperTimeoutError(err)).toBe(true); + it("returns false for a SIGTERM someone else sent, so it reaches the cancel rule", () => { + const err = Object.assign(new Error("Command failed"), { signal: "SIGTERM" }); + expect(isWhisperTimeoutError(err)).toBe(false); }); it("returns false for a non-timeout child error", () => { @@ -191,7 +200,10 @@ describe("wrapWhisperTimeoutError", () => { }); it("wraps SIGTERM timeouts with a discoverable hint naming --timeout and the env var", () => { - const original = Object.assign(new Error("Command failed"), { signal: "SIGTERM" }); + const original = Object.assign(new Error("Command failed"), { + code: "ETIMEDOUT", + signal: "SIGTERM", + }); const wrapped = wrapWhisperTimeoutError(original, { effectiveTimeoutMs: 1_260_000, model: "medium.en", @@ -264,3 +276,89 @@ describe("initialModelForLanguage", () => { expect(initialModelForLanguage("large-v3", "de")).toBe("large-v3"); }); }); + +describe("isPcm16kMono", () => { + it("passes only 16-bit PCM through; 24-bit and float WAV go through ffmpeg", () => { + const wav = (codec_name: string) => ({ codec_name, sample_rate: "16000", channels: 1 }); + expect(isPcm16kMono(wav("pcm_s16le"))).toBe(true); + expect(isPcm16kMono(wav("pcm_s24le"))).toBe(false); + expect(isPcm16kMono(wav("pcm_f32le"))).toBe(false); + expect(isPcm16kMono({ ...wav("pcm_s16le"), sample_rate: "24000" })).toBe(false); + }); +}); + +describe("getPreparedWavDurationSeconds", () => { + it("reads the length from the size alone, whatever comes before the samples", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-wav-duration-")); + const wav = join(dir, "prepared.wav"); + writeFileSync(wav, Buffer.concat([Buffer.alloc(6000), Buffer.alloc(1.5 * 32_000)])); + expect(getPreparedWavDurationSeconds(wav)).toBeCloseTo(1.69, 2); + expect(getPreparedWavDurationSeconds(join(dir, "missing.wav"))).toBeNull(); + rmSync(dir, { recursive: true, force: true }); + }); +}); + +describe.skipIf(process.platform === "win32")("prepareWav", () => { + it("removes the partial WAV when ffmpeg fails or is stopped", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-prepare-wav-")); + const ffmpeg = join(dir, "ffmpeg"); + writeFileSync( + ffmpeg, + `#!${process.execPath}\nrequire("fs").writeFileSync(process.argv.at(-1), "partial");\nprocess.exit(255);\n`, + ); + chmodSync(ffmpeg, 0o755); + const leftovers = () => + readdirSync(tmpdir()).filter((f) => f.startsWith(`hyperframes-audio-${process.pid}-`)); + const saved = process.env.HYPERFRAMES_FFMPEG_PATH; + process.env.HYPERFRAMES_FFMPEG_PATH = ffmpeg; + try { + for (const input of ["talk.mp3", "talk.mp4"]) { + expect(() => prepareWav(join(dir, input))).toThrow(/Command failed/); + expect(leftovers()).toEqual([]); + } + } finally { + if (saved === undefined) delete process.env.HYPERFRAMES_FFMPEG_PATH; + else process.env.HYPERFRAMES_FFMPEG_PATH = saved; + rmSync(dir, { recursive: true, force: true }); + } + }); + + function failWith(script: string): unknown { + const dir = mkdtempSync(join(tmpdir(), "hf-prepare-wav-")); + const ffmpeg = join(dir, "ffmpeg"); + writeFileSync(ffmpeg, `#!${process.execPath}\n${script}\n`); + chmodSync(ffmpeg, 0o755); + const saved = process.env.HYPERFRAMES_FFMPEG_PATH; + process.env.HYPERFRAMES_FFMPEG_PATH = ffmpeg; + try { + prepareWav(join(dir, "talk.mp3")); + } catch (err) { + return err; + } finally { + if (saved === undefined) delete process.env.HYPERFRAMES_FFMPEG_PATH; + else process.env.HYPERFRAMES_FFMPEG_PATH = saved; + rmSync(dir, { recursive: true, force: true }); + } + throw new Error("prepareWav did not fail"); + } + + it("keeps ffmpeg's stderr on the error, so the cause above its summary line is shown", () => { + const said = + "Output file does not contain any stream\nError opening output files: Invalid argument"; + const err = failWith(`process.stderr.write(${JSON.stringify(said)}); process.exit(234);`); + expect(err).toMatchObject({ cancelled: false, stderr: said }); + }); + + it("reads a cancel only from ffmpeg's own signal line, not a path that mentions one", () => { + const said = "/in/received signal.mp3: Invalid data found when processing input"; + const err = failWith(`process.stderr.write(${JSON.stringify(said)}); process.exit(255);`); + expect(err).toMatchObject({ cancelled: false }); + }); + + it("rethrows Node's own failure (output overflow) as is, never as a cancel", () => { + const flood = `require("fs").writeSync(2, "Exiting normally, received signal 2.\\n" + "x".repeat(65 << 20));`; + const err = failWith(`${flood} process.exit(255);`) as { code?: string; cancelled?: boolean }; + expect(err.code).toBe("ENOBUFS"); + expect(err.cancelled).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/whisper/transcribe.ts b/packages/cli/src/whisper/transcribe.ts index c511a2b535..f22df4e7e5 100644 --- a/packages/cli/src/whisper/transcribe.ts +++ b/packages/cli/src/whisper/transcribe.ts @@ -1,10 +1,11 @@ // fallow-ignore-file complexity import { execFileSync } from "node:child_process"; -import { existsSync, readFileSync, mkdirSync, unlinkSync } from "node:fs"; +import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } from "node:fs"; import { join, extname } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js"; +import { stoppedByCancelSignal } from "../utils/renderCancellation.js"; import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js"; /** @@ -183,11 +184,10 @@ function getMediaDurationSeconds(filePath: string): number | null { } } -function getPreparedWavDurationSeconds(wavPath: string): number | null { +/** A prepared WAV is 16 kHz mono s16, so its size gives the length (the header adds a few ms). */ +export function getPreparedWavDurationSeconds(wavPath: string): number | null { try { - const dataChunk = findWavDataChunk(readFileSync(wavPath)); - if (!dataChunk) return null; - return dataChunk.size / (16_000 * 2); + return statSync(wavPath).size / (16_000 * 2); } catch { return null; } @@ -299,6 +299,27 @@ function tempWavPath(): string { return join(tmpdir(), `hyperframes-audio-${process.pid}-${randomUUID()}.wav`); } +function runFfmpeg(ffmpegPath: string, args: string[], output: string, timeout: number): void { + try { + execFileSync(ffmpegPath, ["-nostats", "-hide_banner", ...args, "-y", output], { + stdio: ["ignore", "ignore", "pipe"], + maxBuffer: 64 * 1024 * 1024, + timeout, + }); + } catch (err) { + rmSync(output, { force: true }); + const stop = err as { code?: string; signal?: string; stderr?: Buffer }; + // A code means Node stopped it; ffmpeg traps Ctrl-C and says so (exit 255 is EPERM too on 7+). + if (stop.code) throw err; + const said = String(stop.stderr ?? "").trim(); + const cancelled = /Exiting normally, received signal/.test(said) || stoppedByCancelSignal(stop); + const reason = said.split("\n").at(-1) || (err as Error).message; + // stderr: the command shows its last lines, where ffmpeg names the cause above its summary line. + const failure = new Error(`ffmpeg failed: ${reason}`, { cause: err }); + throw Object.assign(failure, { cancelled, stderr: said }); + } +} + /** * Extract audio from a video file as 16kHz mono WAV (whisper requirement). */ @@ -310,17 +331,29 @@ function extractAudio(videoPath: string): string { ); } const wavPath = tempWavPath(); - execFileSync( + runFfmpeg( ffmpegPath, - ["-i", videoPath, "-vn", "-ar", "16000", "-ac", "1", "-f", "wav", "-y", wavPath], - { - stdio: "ignore", - timeout: resolveAudioPreparationTimeoutMs(getMediaDurationSeconds(videoPath)), - }, + ["-i", videoPath, "-vn", "-ar", "16000", "-ac", "1", "-f", "wav"], + wavPath, + resolveAudioPreparationTimeoutMs(getMediaDurationSeconds(videoPath)), ); return wavPath; } +interface AudioStream { + codec_type?: string; + codec_name?: string; + sample_rate?: string; + channels?: number; +} + +/** 16-bit PCM only: sherpa-onnx cannot read 24-bit WAV, so anything else goes through ffmpeg. */ +export function isPcm16kMono(stream: AudioStream | undefined): boolean { + return ( + stream?.codec_name === "pcm_s16le" && stream.sample_rate === "16000" && stream.channels === 1 + ); +} + /** * Check if a WAV file is already 16kHz mono via ffprobe. */ @@ -333,15 +366,8 @@ function isWav16kMono(filePath: string): boolean { ["-v", "quiet", "-print_format", "json", "-show_streams", "--", filePath], { encoding: "utf-8", timeout: 10_000 }, ); - const parsed: { - streams?: { - codec_type?: string; - sample_rate?: string; - channels?: number; - }[]; - } = JSON.parse(raw); - const audio = parsed.streams?.find((s) => s.codec_type === "audio"); - return audio?.sample_rate === "16000" && audio?.channels === 1; + const parsed: { streams?: AudioStream[] } = JSON.parse(raw); + return isPcm16kMono(parsed.streams?.find((s) => s.codec_type === "audio")); } catch { return false; } @@ -361,13 +387,11 @@ function prepareAudio(audioPath: string): string { throw new Error(`ffmpeg is required to prepare audio. Install: ${getFFmpegInstallHint()}`); } const wavPath = tempWavPath(); - execFileSync( + runFfmpeg( ffmpegPath, - ["-i", audioPath, "-ar", "16000", "-ac", "1", "-f", "wav", "-y", wavPath], - { - stdio: "ignore", - timeout: resolveAudioPreparationTimeoutMs(getMediaDurationSeconds(audioPath)), - }, + ["-i", audioPath, "-ar", "16000", "-ac", "1", "-f", "wav"], + wavPath, + resolveAudioPreparationTimeoutMs(getMediaDurationSeconds(audioPath)), ); return wavPath; } @@ -395,6 +419,23 @@ export function initialModelForLanguage(model: string, language?: string): strin return model; } +export function prepareWav(inputPath: string, onProgress?: (message: string) => void): string { + if (isAudioFile(inputPath)) { + onProgress?.("Preparing audio..."); + return prepareAudio(inputPath); + } + if (isVideoFile(inputPath)) { + if (!hasFFmpeg()) { + throw new Error( + `ffmpeg is required to extract audio from video. Install: ${getFFmpegInstallHint()}`, + ); + } + onProgress?.("Extracting audio from video..."); + return extractAudio(inputPath); + } + throw new Error(`Unsupported file type: ${extname(inputPath).toLowerCase()}`); +} + /** * Transcribe an audio or video file and save transcript.json to the output directory. */ @@ -417,23 +458,7 @@ export async function transcribe( }); // 3. Prepare audio - let wavPath: string; - const ext = extname(inputPath).toLowerCase(); - - if (isAudioFile(inputPath)) { - options?.onProgress?.("Preparing audio..."); - wavPath = prepareAudio(inputPath); - } else if (isVideoFile(inputPath)) { - if (!hasFFmpeg()) { - throw new Error( - `ffmpeg is required to extract audio from video. Install: ${getFFmpegInstallHint()}`, - ); - } - options?.onProgress?.("Extracting audio from video..."); - wavPath = extractAudio(inputPath); - } else { - throw new Error(`Unsupported file type: ${ext}`); - } + const wavPath = prepareWav(inputPath, options?.onProgress); // 4. Detect language and ensure correct model let effectiveModel = model; @@ -544,14 +569,9 @@ export async function transcribe( // Timeout error discoverability // --------------------------------------------------------------------------- -// Node's `execFileSync` kills the child with SIGTERM when its `timeout` option -// fires, so the resulting Error carries `signal === "SIGTERM"`. On some platforms -// / Node versions `code === "ETIMEDOUT"` is also set. Match either signal so we -// don't miss a timeout on a platform we haven't validated. +// execFileSync's own timeout sets code ETIMEDOUT (and SIGTERM); a bare SIGTERM is someone stopping it. export function isWhisperTimeoutError(err: unknown): boolean { - if (!(err instanceof Error)) return false; - const record = err as { signal?: unknown; code?: unknown }; - return record.signal === "SIGTERM" || record.code === "ETIMEDOUT"; + return err instanceof Error && (err as { code?: unknown }).code === "ETIMEDOUT"; } export interface WrapWhisperTimeoutOptions { diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts index b9592f72d4..4e82ab1ddd 100644 --- a/packages/cli/tsup.config.ts +++ b/packages/cli/tsup.config.ts @@ -16,6 +16,7 @@ export default defineConfig({ fontLocalizeCli: "src/fontLocalizeCli.ts", runtimeVersion: "src/runtimeVersion.ts", renderSetupWorker: "src/renderSetupWorker.ts", + sherpaWorker: "src/whisper/sherpaWorker.ts", shaderTransitionWorker: "../producer/src/services/shaderTransitionWorker.ts", "registry/localSemantic": "src/registry/localSemantic.ts", }, diff --git a/skills-manifest.json b/skills-manifest.json index 27d7b2180f..fd62a8d3ae 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -2,8 +2,8 @@ "source": "heygen-com/hyperframes", "skills": { "embedded-captions": { - "hash": "6a9850a8d250780e", - "files": 143 + "hash": "7c6f147f722cc4c4", + "files": 144 }, "faceless-explainer": { "hash": "7fda7a8bb75acabf", @@ -54,8 +54,8 @@ "files": 1 }, "media-use": { - "hash": "ebef14be0621a731", - "files": 101 + "hash": "004d60f9660a5f16", + "files": 102 }, "motion-graphics": { "hash": "3d3a35ffbe8b7e3e", diff --git a/skills/embedded-captions/scripts/transcribe.cjs b/skills/embedded-captions/scripts/transcribe.cjs index 07c4c2bb0d..5be62262ee 100644 --- a/skills/embedded-captions/scripts/transcribe.cjs +++ b/skills/embedded-captions/scripts/transcribe.cjs @@ -260,7 +260,8 @@ function main() { end: w.end ?? w.t1, type: "word", })); - engine = `whisper.cpp(${model})`; + // Under the default --engine auto the CLI may have run Parakeet; it reports which. + engine = info.engine === "parakeet" ? `parakeet(${info.model})` : `whisper.cpp(${model})`; } // Tail-hallucination guard: drop words whisper placed entirely inside a terminal diff --git a/skills/embedded-captions/scripts/transcribe.test.mjs b/skills/embedded-captions/scripts/transcribe.test.mjs new file mode 100644 index 0000000000..5882a52854 --- /dev/null +++ b/skills/embedded-captions/scripts/transcribe.test.mjs @@ -0,0 +1,41 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +// A stub CLI answers as `transcribe --json` does when --engine auto ran Parakeet. +const STUB_CLI = ` +const fs = require("fs"), path = require("path"); +const project = process.argv[process.argv.indexOf("-d") + 1]; +const transcriptPath = path.join(project, "cli-words.json"); +fs.writeFileSync(transcriptPath, JSON.stringify([{ text: "hello", start: 0.1, end: 0.5 }])); +console.log(JSON.stringify({ ok: true, engine: "parakeet", model: "parakeet-tdt-0.6b-v3", transcriptPath })); +`; + +test("the CLI fallback labels the transcript with the engine the CLI ran", () => { + const root = mkdtempSync(join(tmpdir(), "embedded-captions-transcribe-")); + try { + const cliDir = join(root, "hf", "packages", "cli", "dist"); + mkdirSync(cliDir, { recursive: true }); + writeFileSync(join(cliDir, "cli.js"), STUB_CLI); + const project = join(root, "project"); + mkdirSync(project); + writeFileSync(join(project, "source.mp4"), ""); + writeFileSync(join(project, "audio.mp3"), ""); + execFileSync( + process.execPath, + [fileURLToPath(new URL("./transcribe.cjs", import.meta.url)), project], + { + stdio: "ignore", + env: { ...process.env, HYPERFRAMES_ROOT: join(root, "hf"), TRANSCRIBE_ENGINE: "whisper" }, + }, + ); + const transcript = JSON.parse(readFileSync(join(project, "transcript.json"), "utf8")); + assert.equal(transcript.engine, "parakeet(parakeet-tdt-0.6b-v3)"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/skills/media-use/references/operations.md b/skills/media-use/references/operations.md index e2909f6c01..491abebf40 100644 --- a/skills/media-use/references/operations.md +++ b/skills/media-use/references/operations.md @@ -112,14 +112,16 @@ engine directly. # install once: uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx node /scripts/transcribe.mjs --input talk.mp4 --out talk.transcribe.json -# equivalently, the hyperframes CLI has Parakeet built in (auto-detects it, whisper fallback): +# the hyperframes CLI runs Parakeet on macOS, Linux and Windows once it is installed: +npx hyperframes models install parakeet # once, ~640 MB, ask the user first npx hyperframes transcribe talk.mp4 --engine parakeet # or --engine auto (default) ``` -VERIFIED on 24GB: accurate, ~3s (cached) for 8s audio. Parakeet covers English + -25 European languages. For other languages, or when parakeet-mlx is not -installed, transcribe.mjs auto-falls-back to whisper.cpp (99 languages) via -`hyperframes transcribe`. `--engine parakeet|whisper` forces one. (Cohere +VERIFIED on 24GB: accurate, ~3s (cached) for 8s audio. Parakeet covers 25 +European languages, English included. For other languages, or when parakeet-mlx is not +installed, transcribe.mjs falls back to `hyperframes transcribe`, which uses the +CLI's Parakeet when `models install parakeet` has run and whisper.cpp (99 +languages) otherwise. `--engine parakeet|whisper` forces one. (Cohere Transcribe tops the leaderboard on paper but its mlx-audio quants produced garbage and ran 40-70x slower on a Mac in testing, so it is not wired in.) diff --git a/skills/media-use/references/setup-providers.md b/skills/media-use/references/setup-providers.md index 79b1a56e6a..42ce0d5ae6 100644 --- a/skills/media-use/references/setup-providers.md +++ b/skills/media-use/references/setup-providers.md @@ -59,15 +59,15 @@ tools are OPT-IN alternatives where they exist; install one to unlock its free, private, on-device path instead of or ahead of HeyGen for that type. Only `ffmpeg`/`ffprobe` are strictly required for the tool to run at all. -| Tool | Serves | Install | -| ------------------ | ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| `ffmpeg`/`ffprobe` | adopt probing, smart-grade signalstats, cut, duck bake, loudnorm | system package (`brew install ffmpeg`) | -| `heygen` | catalog (bgm/sfx/image/icon) + TTS (voice) + avatar video — the free-usage path | install through [verified HeyGen release instructions](https://developers.heygen.com/cli), then `heygen auth login --oauth` (needs >= v0.3.0) | -| `mflux-generate` | local image gen (FLUX), best-for-RAM | `uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux==0.9.6` | -| `codex` | image gen upsell (ChatGPT sub) | Codex CLI, logged in via ChatGPT (owns its own auth) | -| `parakeet-mlx` | local transcription (default ASR, best) | `uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx` | -| `ltx-2-mlx` | local video gen | `git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras` | -| `npx hyperframes` | Kokoro TTS (voice), whisper.cpp (transcribe fallback), remove-background | via the hyperframes CLI; whisper.cpp is built on first use (Homebrew on macOS, else git+cmake), models download from HuggingFace | +| Tool | Serves | Install | +| ------------------ | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ffmpeg`/`ffprobe` | adopt probing, smart-grade signalstats, cut, duck bake, loudnorm | system package (`brew install ffmpeg`) | +| `heygen` | catalog (bgm/sfx/image/icon) + TTS (voice) + avatar video — the free-usage path | install through [verified HeyGen release instructions](https://developers.heygen.com/cli), then `heygen auth login --oauth` (needs >= v0.3.0) | +| `mflux-generate` | local image gen (FLUX), best-for-RAM | `uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux==0.9.6` | +| `codex` | image gen upsell (ChatGPT sub) | Codex CLI, logged in via ChatGPT (owns its own auth) | +| `parakeet-mlx` | local transcription (default ASR, best) on Apple Silicon | `uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx` | +| `ltx-2-mlx` | local video gen | `git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras` | +| `npx hyperframes` | Kokoro TTS (voice), Parakeet or whisper.cpp (transcribe), remove-background | via the hyperframes CLI; Parakeet on macOS, Windows and glibc 2.32+ Linux: `npx hyperframes models install parakeet`; whisper.cpp is built on first use (Homebrew on macOS, else git+cmake) | The RAM-graded local-model shortlist + exact per-tier install/invoke lives in `scripts/lib/local-models.mjs` (the agent can read `describeModelLadder(cap, specs)` diff --git a/skills/media-use/scripts/transcribe.mjs b/skills/media-use/scripts/transcribe.mjs index e702606966..ccf9b244dc 100644 --- a/skills/media-use/scripts/transcribe.mjs +++ b/skills/media-use/scripts/transcribe.mjs @@ -12,7 +12,7 @@ import { } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { basename, extname, join, resolve } from "node:path"; -import { parseArgs } from "node:util"; +import { parseArgs, stripVTControlCharacters } from "node:util"; import { mergeTokensToWords } from "./lib/parakeet-words.mjs"; import { track } from "./lib/telemetry.mjs"; import { resolveNpxInvocation } from "./lib/npx-sync.mjs"; @@ -49,7 +49,8 @@ Usage: Parakeet (default) beats whisper.cpp on accuracy + speed for English/European languages; whisper.cpp (99 languages) is the fallback. Install Parakeet once: - uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx`); + npx hyperframes models install parakeet (macOS, Windows, Linux with glibc 2.32+) + uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx (Apple Silicon)`); process.exit(0); } @@ -122,33 +123,57 @@ function runParakeet(runner) { } } -// whisper.cpp via the hyperframes CLI (fetched/built on first use — see +// The hyperframes CLI (its Parakeet, else whisper.cpp built on first use — see // SKILL.md): writes transcript.json into --dir; relocate to --out. -function runWhisper() { +function runCli(cliEngine) { const workDir = mkdtempSync(join(tmpdir(), "media-use-whisper-")); try { // On Windows a bare "npx" is npx.cmd, which execFileSync cannot exec // (spawnSync npx ENOENT) — resolveNpxInvocation reroutes it through // node + npx-cli.js (and throws actionably when it can't), same // mechanism as the audio engine's TTS spawns. + // Under auto or parakeet the CLI may run its own Parakeet; --json says which engine ran. const resolved = resolveNpxInvocation( - ["hyperframes", "transcribe", inputPath, "--dir", workDir], + ["hyperframes", "transcribe", inputPath, "--dir", workDir, "--engine", cliEngine, "--json"], { stdio: ["ignore", "pipe", "pipe"], timeout: 1_800_000 }, ); - execFileSync(resolved.cmd, resolved.args, resolved.opts); + let stdout; + try { + stdout = String(execFileSync(resolved.cmd, resolved.args, resolved.opts)); + } catch (e) { + // stderr holds spinner redraws and --json prints the reason on stdout, so keep the last lines. + const tail = stripVTControlCharacters(`${e.stderr ?? ""}\n${e.stdout ?? ""}`) + .split(/[\r\n]+/) + .map((l) => l.trim()) + .filter((l) => l && l !== "│") + .slice(-3) + .join("\n"); + throw new Error( + `hyperframes transcribe failed (exit ${e.status ?? e.code})${tail ? `:\n${tail}` : ""}`, + ); + } const produced = join(workDir, "transcript.json"); - if (!existsSync(produced)) throw new Error("whisper produced no transcript.json"); + if (!existsSync(produced)) + throw new Error("hyperframes transcribe produced no transcript.json"); const tmp = `${outPath}.tmp-${process.pid}`; copyFileSync(produced, tmp); renameSync(tmp, outPath); // atomic publish let words; try { const t = JSON.parse(readFileSync(outPath, "utf8")); - words = Array.isArray(t?.words) ? t.words.length : undefined; + // The CLI writes a flat word array, not { words }. + words = Array.isArray(t) ? t.length : undefined; } catch { /* leave undefined */ } - report("whisper", words); + let ran = "whisper"; + try { + ran = JSON.parse(stdout.trim().split("\n").at(-1)).engine === "parakeet" ? "parakeet" : ran; + } catch { + /* no JSON line: keep whisper */ + } + report(ran, words); + return ran; } finally { rmSync(workDir, { recursive: true, force: true }); } @@ -156,22 +181,11 @@ function runWhisper() { try { const parakeetBin = resolveParakeet(); - const engine = - args.engine === "parakeet" || args.engine === "whisper" - ? args.engine - : parakeetBin - ? "parakeet" - : "whisper"; - if (engine === "parakeet") { - if (!parakeetBin) { - throw new Error( - "parakeet-mlx not found (checked $HYPERFRAMES_PARAKEET, ~/.venvs/parakeet, and PATH). Install: uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx (or use --engine whisper)", - ); - } - runParakeet(parakeetBin); - } else { - runWhisper(); - } + const asked = args.engine === "parakeet" || args.engine === "whisper" ? args.engine : "auto"; + // Without parakeet-mlx, the CLI's own Parakeet (any OS, `hyperframes models install parakeet`). + let engine = "parakeet"; + if (asked !== "whisper" && parakeetBin) runParakeet(parakeetBin); + else engine = runCli(asked); await track("media_use_transcribe", { engine }); } catch (err) { if (args.json) console.log(JSON.stringify({ ok: false, error: err.message })); diff --git a/skills/media-use/scripts/transcribe.test.mjs b/skills/media-use/scripts/transcribe.test.mjs new file mode 100644 index 0000000000..3f0d2a53b1 --- /dev/null +++ b/skills/media-use/scripts/transcribe.test.mjs @@ -0,0 +1,112 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +// The stub stands in for `npx hyperframes transcribe --dir ...`: it records its args, +// writes the CLI's flat array and prints the CLI's --json line naming `cliEngine`. +function runScript(engine, cliEngine) { + const root = mkdtempSync(join(tmpdir(), "media-use-transcribe-")); + try { + const bin = join(root, "bin"); + mkdirSync(bin); + writeFileSync( + join(bin, "npx"), + `#!/bin/sh\necho "$@" > "${root}/args"\nwhile [ "$1" != "--dir" ]; do shift; done\n` + + `echo '[{"text":"a","start":0,"end":1},{"text":"b","start":1,"end":2}]' > "$2/transcript.json"\n` + + `echo '{"ok":true,"engine":"${cliEngine}"}'\n`, + ); + chmodSync(join(bin, "npx"), 0o755); + const input = join(root, "in.wav"); + writeFileSync(input, ""); + const script = fileURLToPath(new URL("./transcribe.mjs", import.meta.url)); + const env = { ...process.env, PATH: `${bin}:${process.env.PATH}`, HOME: root }; + delete env.HYPERFRAMES_PARAKEET; + const out = execFileSync( + process.execPath, + [script, "--input", input, "--engine", engine, "--json"], + { + encoding: "utf8", + env: { ...env, HYPERFRAMES_MEDIA_HOME: join(root, "home"), HYPERFRAMES_NO_TELEMETRY: "1" }, + }, + ); + return { result: JSON.parse(out.trim()), args: readFileSync(join(root, "args"), "utf8") }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +test( + "--engine whisper makes the CLI run whisper and reports the flat transcript's word count", + { skip: process.platform === "win32" }, + () => { + const { result, args } = runScript("whisper", "whisper"); + assert.match(args, /--engine whisper --json/); + assert.equal(result.words, 2); + assert.equal(result.engine, "whisper"); + }, +); + +test( + "auto reports the engine the CLI says it ran, Parakeet included", + { skip: process.platform === "win32" }, + () => { + const { result, args } = runScript("auto", "parakeet"); + assert.match(args, /--engine auto --json/); + assert.equal(result.engine, "parakeet"); + }, +); + +test( + "--engine parakeet without parakeet-mlx asks the CLI for its Parakeet", + { skip: process.platform === "win32" }, + () => { + const { result, args } = runScript("parakeet", "parakeet"); + assert.match(args, /--engine parakeet --json/); + assert.equal(result.engine, "parakeet"); + }, +); + +test( + "a failed --engine parakeet run reports the reason the CLI printed on stdout", + { skip: process.platform === "win32" }, + () => { + const root = mkdtempSync(join(tmpdir(), "media-use-transcribe-")); + try { + const bin = join(root, "bin"); + mkdirSync(bin); + const reason = + "Parakeet is not installed. Install it with: hyperframes models install parakeet"; + writeFileSync( + join(bin, "npx"), + `#!/bin/sh\necho '{"ok":false,"error":"${reason}"}'\nexit 1\n`, + ); + chmodSync(join(bin, "npx"), 0o755); + const input = join(root, "in.wav"); + writeFileSync(input, ""); + const script = fileURLToPath(new URL("./transcribe.mjs", import.meta.url)); + const res = spawnSync( + process.execPath, + [script, "--input", input, "--engine", "parakeet", "--json"], + { + encoding: "utf8", + env: { + ...process.env, + HYPERFRAMES_PARAKEET: "", + PATH: `${bin}:${process.env.PATH}`, + HOME: root, + HYPERFRAMES_MEDIA_HOME: join(root, "home"), + HYPERFRAMES_NO_TELEMETRY: "1", + }, + }, + ); + assert.equal(res.status, 1); + assert.match(JSON.parse(res.stdout.trim()).error, /hyperframes models install parakeet/); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, +);