diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index b6e68e6c29..ea84d77017 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -276,7 +276,7 @@ npx hyperframes transcribe transcript.json --to srt | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `--dir, -d` | Project directory (default: current directory) | | `--engine, -e` | ASR engine: `auto` (default — Parakeet if installed and the language is one of its 25, else whisper), `parakeet` (fails if Parakeet is missing, broken, or lacks the `--language`), or `whisper` | -| `--model, -m` | Whisper model (default `small`, multilingual): `tiny.en`, `base.en`, `small.en`, `medium.en`, `large-v3` | +| `--model, -m` | Whisper model (default `small`): `tiny`, `base`, `small`, `medium` (add `.en` for English only), `large-v3` | | `--language, -l` | Language code (`en`, `es`, `ja`, …). Whisper transcribes as this language; Parakeet, used when it covers it, detects the language itself. | | `--to` | Export a sidecar instead: `srt` or `vtt` | | `--output, -o` | Where the exported SRT/VTT goes | diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 2841470777..33a7f63e06 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -719,7 +719,7 @@ export default defineCommand({ model: { type: "string", description: - "Whisper model for transcription (e.g. tiny.en, base.en, small.en, medium.en, large)", + "Whisper model for transcription (default small; e.g. tiny, base, medium, large, or tiny.en for English only)", }, language: { type: "string", diff --git a/packages/cli/src/whisper/language.test.ts b/packages/cli/src/whisper/language.test.ts index f2db9c21df..6c3633fcac 100644 --- a/packages/cli/src/whisper/language.test.ts +++ b/packages/cli/src/whisper/language.test.ts @@ -2,8 +2,12 @@ import { describe, expect, it } from "vitest"; import { detectionWindows, parseDetection, pickLanguage } from "./language.js"; describe("detectionWindows", () => { - it("leaves a clip under a minute to whisper's own pick", () => { + it("leaves a clip under a minute with no intro, or under 30 s of speech, to whisper's own pick", () => { expect(detectionWindows(59, null)).toEqual([]); + expect(detectionWindows(69, 40)).toEqual([]); + }); + it("puts all three windows on the speech after an intro, overlapping when it is short", () => { + expect(detectionWindows(90, 40)).toEqual([40, 50, 60]); }); it("spreads three 30 s windows over the clip, centred at a sixth, a half and five sixths", () => { expect(detectionWindows(120, null)).toEqual([5, 45, 85]); @@ -30,8 +34,10 @@ describe("pickLanguage", () => { }); }); -it("reads whisper-cli's detection line", () => { - const line = "whisper_full_with_state: auto-detected language: es (p = 0.989864)"; - expect(parseDetection(line)).toEqual({ language: "es", p: 0.989864 }); - expect(parseDetection("whisper_init_from_file: loading model")).toBeNull(); +describe("parseDetection", () => { + it("reads whisper-cli's detection line", () => { + const line = "whisper_full_with_state: auto-detected language: es (p = 0.989864)"; + expect(parseDetection(line)).toEqual({ language: "es", p: 0.989864 }); + expect(parseDetection("whisper_init_from_file: loading model")).toBeNull(); + }); }); diff --git a/packages/cli/src/whisper/language.ts b/packages/cli/src/whisper/language.ts index 2a8fd520fb..dc275ea74a 100644 --- a/packages/cli/src/whisper/language.ts +++ b/packages/cli/src/whisper/language.ts @@ -3,12 +3,13 @@ export const WINDOW_SECONDS = 30; /** A music or noise window comes back near p = 0.5; speech in a language comes back above 0.9. */ const CONFIDENT = 0.7; -/** The starts of up to three windows spread over the clip after `onset`, so an intro alone cannot decide. A clip - * with under two windows of sound after it gets none: whisper's own pick already hears all of it. */ +/** The starts of three windows spread over the clip after `onset`, so an intro alone cannot decide; they overlap + * when the speech is short. None under 30 s of speech, or with no intro under a minute: whisper's own pick + * already hears that. */ export function detectionWindows(seconds: number, onset: number | null): number[] { const from = onset ?? 0; const span = seconds - from; - if (span < 2 * WINDOW_SECONDS) return []; + if (span < WINDOW_SECONDS || (onset === null && span < 2 * WINDOW_SECONDS)) return []; return [1 / 6, 1 / 2, 5 / 6].map((at) => { const start = from + span * at - WINDOW_SECONDS / 2; return Math.min(Math.max(start, from), seconds - WINDOW_SECONDS); @@ -22,11 +23,8 @@ export interface LanguageVote { /** The language at least two confident windows agree on, else null (whisper's own pick stands). */ export function pickLanguage(votes: readonly LanguageVote[]): string | null { - const counts = new Map(); - for (const { language, p } of votes) - if (p >= CONFIDENT) counts.set(language, (counts.get(language) ?? 0) + 1); - for (const [language, count] of counts) if (count >= 2) return language; - return null; + const sure = votes.filter((vote) => vote.p >= CONFIDENT).map((vote) => vote.language); + return sure.find((language, i) => sure.indexOf(language) !== i) ?? null; } /** whisper-cli --detect-language prints `auto-detected language: es (p = 0.983324)`. */ diff --git a/packages/cli/src/whisper/transcribe.metadata.test.ts b/packages/cli/src/whisper/transcribe.metadata.test.ts index 6e8398d2a1..11c75319ff 100644 --- a/packages/cli/src/whisper/transcribe.metadata.test.ts +++ b/packages/cli/src/whisper/transcribe.metadata.test.ts @@ -1,8 +1,9 @@ -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { transcribe, type TranscribeProgress } from "./transcribe.js"; +import { readWav } from "./wav.js"; import { encodeWav } from "./wav.test-helpers.js"; const native = vi.hoisted(() => ({ @@ -283,7 +284,7 @@ function detectsByWindow(answer: (sample: number) => string) { if (command !== "whisper-cli" || !args.includes("--detect-language")) return previous(command, args); // Whisper hears the whole window; the stand-in judges it by its middle. - const sample = readFileSync(args.at(-1)!).readInt16LE(44 + 15 * 32_000); + const sample = Math.round(readWav(args.at(-1)!).samples[15 * 16_000]! * 32768); return { stderr: `whisper_full_with_state: auto-detected language: ${answer(sample)}\n` }; }); } @@ -293,26 +294,42 @@ const languagePassed = () => .filter(([command, args]) => command === "whisper-cli" && !args.includes("--detect-language")) .map(([, args]) => args[args.indexOf("--language") + 1]); -it("a music intro does not decide the language: windows spread over the clip do", async () => { - musicThenSpanish(); - detectsByWindow((sample) => (sample === 2 ? "en (p = 0.534956)" : "es (p = 0.983324)")); - const result = await transcribe(join(dir, "audio.wav"), dir, { model: "small" }); - const detections = native.exec.mock.calls.filter(([, args]) => - args.includes("--detect-language"), - ); - expect(detections).toHaveLength(3); - expect(languagePassed()).toEqual(["es"]); - expect(result.detectedLanguage).toBe("es"); -}); +const detections = () => + native.exec.mock.calls.filter(([, args]) => args.includes("--detect-language")).length; + +it.each([undefined, "auto"])( + "a music intro does not decide the language (--language %s): windows spread over the clip do", + async (language) => { + musicThenSpanish(); + detectsByWindow((sample) => (sample === 2 ? "en (p = 0.534956)" : "es (p = 0.983324)")); + const result = await transcribe(join(dir, "audio.wav"), dir, { model: "small", language }); + expect(detections()).toBe(3); + expect(languagePassed()).toEqual(["es"]); + expect(result.detectedLanguage).toBe("es"); + }, +); it("windows that disagree leave the language to whisper's own pick", async () => { musicThenSpanish(); let n = 0; detectsByWindow(() => `${["en", "es", "fr"][n++]} (p = 0.95)`); await transcribe(join(dir, "audio.wav"), dir, { model: "small" }); + expect(detections()).toBe(3); expect(languagePassed()).toEqual(["auto"]); }); +it("a window whose detection fails loses only its own vote", async () => { + musicThenSpanish(); + let n = 0; + detectsByWindow((sample) => { + if (n++ === 0) throw new Error("whisper-cli exited 11"); + return sample === 2 ? "en (p = 0.534956)" : "es (p = 0.983324)"; + }); + await transcribe(join(dir, "audio.wav"), dir, { model: "small" }); + expect(detections()).toBe(3); + expect(languagePassed()).toEqual(["es"]); +}); + it("a detection that fails leaves the language to whisper's own pick", async () => { musicThenSpanish(); const previous = native.exec.getMockImplementation()!; @@ -327,6 +344,6 @@ it("a detection that fails leaves the language to whisper's own pick", async () }); expect(languagePassed()).toEqual(["auto"]); expect(said).toContain( - "Language detection failed (unknown argument: --detect-language); whisper picks it.", + "Language detection failed for one window (unknown argument: --detect-language).", ); }); diff --git a/packages/cli/src/whisper/transcribe.test.ts b/packages/cli/src/whisper/transcribe.test.ts index a8fa01c7df..f6a0d29d43 100644 --- a/packages/cli/src/whisper/transcribe.test.ts +++ b/packages/cli/src/whisper/transcribe.test.ts @@ -96,8 +96,8 @@ describe("whisperModelSlowdownFactor", () => { }); describe("resolveWhisperTimeoutMs with model factor", () => { - it("keeps small.en (default) at the historical baseline for 63s clips", () => { - // Regression guard: `small.en` (the CLI default model) must not lose + it("keeps small.en at the historical baseline for 63s clips", () => { + // Regression guard: `small.en` (the CLI default until `small`) must not lose // headroom when the caller passes a model. 63s * 10s/s * 1 = 630_000ms. expect(resolveWhisperTimeoutMs(63, { model: "small.en" })).toBe(630_000); }); diff --git a/packages/cli/src/whisper/transcribe.ts b/packages/cli/src/whisper/transcribe.ts index a47c919692..a52bd36594 100644 --- a/packages/cli/src/whisper/transcribe.ts +++ b/packages/cli/src/whisper/transcribe.ts @@ -3,15 +3,7 @@ import { execFile, execFileSync } from "node:child_process"; import { once } from "node:events"; import { createInterface } from "node:readline"; import type { Readable } from "node:stream"; -import { - existsSync, - readFileSync, - mkdirSync, - mkdtempSync, - rmSync, - statSync, - unlinkSync, -} from "node:fs"; +import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } from "node:fs"; import { join, extname } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; @@ -38,7 +30,7 @@ const AUDIO_PREPARATION_TIMEOUT_PER_MEDIA_SECOND_MS = 500; const AUDIO_PREPARATION_TIMEOUT_CAP_MS = 21_600_000; /** - * Model-specific slowdown factors relative to the `small.en` default. whisper.cpp's + * Model-specific slowdown factors relative to the `small` default. whisper.cpp's * per-token inference cost scales with model size — `medium` runs ~2x slower than * `small`, and the `large` family ~4x slower — so the 10x-realtime baseline that * comfortably covers `small.en` can still time out on `medium.en`/`large-v3` when @@ -65,13 +57,13 @@ const WHISPER_MODEL_SLOWDOWN_FACTORS: Readonly> = { "large-v3-turbo": 2, }; -// Unknown model names fall back to the `small.en` baseline so the returned +// Unknown model names fall back to the `small` baseline so the returned // timeout never dips below the historical safe window for a novel/custom model. const DEFAULT_MODEL_SLOWDOWN_FACTOR = 1; /** * Look up the auto-scale slowdown factor for a whisper model name. Case- - * insensitive. Unknown names fall back to the `small.en` baseline (factor 1) + * insensitive. Unknown names fall back to the `small` baseline (factor 1) * rather than a smaller factor so unknown models never accidentally shorten * the safety window. */ @@ -488,7 +480,9 @@ export async function transcribe( // 3. Prepare audio const wavPath = prepareWav(inputPath, options?.onProgress); - const automaticLanguage = options?.language === undefined && !model.endsWith(".en"); + // `--language auto` asks for detection too; Desktop always sends it. + const requested = options?.language === "auto" ? undefined : options?.language; + const automaticLanguage = requested === undefined && !model.endsWith(".en"); const wavSeconds = getPreparedWavDurationSeconds(wavPath); const speechOnsetSeconds = detectSpeechOnset(wavPath); const signal = options?.startCancellation?.(); @@ -500,7 +494,7 @@ export async function transcribe( onProgress: options?.onProgress, }) : null; - const language = options?.language ?? picked ?? (automaticLanguage ? "auto" : "en"); + const language = requested ?? picked ?? (automaticLanguage ? "auto" : "en"); options?.onProgress?.("Transcribing..."); options?.onEvent?.({ type: "progress", @@ -636,9 +630,8 @@ function segmentWords(line: string): { words: Word[]; end: number } | null { return { words, end }; } -/** Resolves once whisper exits and both its streams are read, so no printed line is lost. */ /** The language up to three windows spread over the speech agree on; null keeps whisper's own pick, which hears - * only the first 30 s (a music intro decided whole clips). A failed detection keeps it too, said aloud. */ + * only the first 30 s. A window whose detection fails loses its vote, said aloud. */ async function languageFromWindows( executable: string, { model, modelPath }: { model: string; modelPath: string }, @@ -653,37 +646,38 @@ async function languageFromWindows( const starts = clip.seconds === null ? [] : detectionWindows(clip.seconds, clip.onset); if (starts.length === 0) return null; clip.onProgress?.("Detecting the language..."); - const dir = mkdtempSync(join(tmpdir(), "hyperframes-language-")); + const window = tempWavPath(); + const votes: LanguageVote[] = []; try { - const votes: LanguageVote[] = []; - for (const [i, start] of starts.entries()) { - const window = join(dir, `${i}.wav`); - writeWavWindow(wavPath, window, start, WINDOW_SECONDS); - await runWhisper( - executable, - ["--model", modelPath, "--language", "auto", "--detect-language", window], - { - timeoutMs: resolveWhisperTimeoutMs(WINDOW_SECONDS, { model }), - signal: clip.signal, - onStderr: (line) => { - const vote = parseDetection(line); - if (vote) votes.push(vote); + for (const start of starts) { + try { + writeWavWindow(wavPath, window, start, WINDOW_SECONDS); + await runWhisper( + executable, + ["--model", modelPath, "--language", "auto", "--detect-language", window], + { + timeoutMs: resolveWhisperTimeoutMs(WINDOW_SECONDS, { model }), + signal: clip.signal, + onStderr: (line) => { + const vote = parseDetection(line); + if (vote) votes.push(vote); + }, }, - }, - ); + ); + } catch (err) { + if (clip.signal?.aborted) throw err; + clip.onProgress?.( + `Language detection failed for one window (${normalizeErrorMessage(err)}).`, + ); + } } - return pickLanguage(votes); - } catch (err) { - if (clip.signal?.aborted) throw err; - clip.onProgress?.( - `Language detection failed (${normalizeErrorMessage(err)}); whisper picks it.`, - ); - return null; } finally { - rmSync(dir, { recursive: true, force: true }); + rmSync(window, { force: true }); } + return pickLanguage(votes); } +/** Resolves once whisper exits and both its streams are read, so no printed line is lost. */ async function runWhisper( executable: string, args: string[],