From d73e5814b4c4635c404eadf62caf09f2bd03ff49 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Tue, 6 Oct 2026 17:04:42 -0700 Subject: [PATCH] fix(cli): auto picks the language from three spread windows of speech, not the first 30 s --- docs/packages/cli.mdx | 2 +- packages/cli/src/commands/transcribe.ts | 2 +- packages/cli/src/whisper/language.test.ts | 37 ++++++++ packages/cli/src/whisper/language.ts | 36 ++++++++ packages/cli/src/whisper/manager.ts | 3 +- .../src/whisper/transcribe.metadata.test.ts | 72 ++++++++++++++- packages/cli/src/whisper/transcribe.ts | 92 ++++++++++++++++--- packages/cli/src/whisper/wav.test-helpers.ts | 17 +--- packages/cli/src/whisper/wav.test.ts | 12 ++- packages/cli/src/whisper/wav.ts | 30 +++++- .../audio/references/captions/authoring.md | 2 +- .../media-use/audio/references/transcribe.md | 2 +- 12 files changed, 271 insertions(+), 36 deletions(-) create mode 100644 packages/cli/src/whisper/language.test.ts create mode 100644 packages/cli/src/whisper/language.ts diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 6e19367f92..b6e68e6c29 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -276,7 +276,7 @@ npx hyperframes transcribe transcript.json --to srt | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `--dir, -d` | Project directory (default: current directory) | | `--engine, -e` | ASR engine: `auto` (default — Parakeet if installed and the language is one of its 25, else whisper), `parakeet` (fails if Parakeet is missing, broken, or lacks the `--language`), or `whisper` | -| `--model, -m` | Whisper model (default `small.en`): `tiny.en`, `base.en`, `small.en`, `medium.en`, `large-v3` | +| `--model, -m` | Whisper model (default `small`, multilingual): `tiny.en`, `base.en`, `small.en`, `medium.en`, `large-v3` | | `--language, -l` | Language code (`en`, `es`, `ja`, …). Whisper transcribes as this language; Parakeet, used when it covers it, detects the language itself. | | `--to` | Export a sidecar instead: `srt` or `vtt` | | `--output, -o` | Where the exported SRT/VTT goes | diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.ts index c37e14b7b9..11ff090eba 100644 --- a/packages/cli/src/commands/transcribe.ts +++ b/packages/cli/src/commands/transcribe.ts @@ -67,7 +67,7 @@ export default defineCommand({ }, model: { type: "string", - description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny.en, base.en, small.en, medium.en, large-v3`, + description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny, base, small, medium (add .en for English only), large-v3`, alias: "m", }, language: { diff --git a/packages/cli/src/whisper/language.test.ts b/packages/cli/src/whisper/language.test.ts new file mode 100644 index 0000000000..f2db9c21df --- /dev/null +++ b/packages/cli/src/whisper/language.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { detectionWindows, parseDetection, pickLanguage } from "./language.js"; + +describe("detectionWindows", () => { + it("leaves a clip under a minute to whisper's own pick", () => { + expect(detectionWindows(59, null)).toEqual([]); + }); + it("spreads three 30 s windows over the clip, centred at a sixth, a half and five sixths", () => { + expect(detectionWindows(120, null)).toEqual([5, 45, 85]); + }); + it("starts after the onset of speech and never runs past the end", () => { + expect(detectionWindows(100, 40)).toEqual([40, 55, 70]); + }); +}); + +describe("pickLanguage", () => { + it("takes the language two confident windows agree on, over a music window", () => { + const votes = [ + { language: "en", p: 0.53 }, + { language: "es", p: 0.98 }, + { language: "es", p: 0.99 }, + ]; + expect(pickLanguage(votes)).toBe("es"); + }); + it("leaves the pick to whisper when the windows disagree or none is confident", () => { + const disagree = ["en", "es", "fr"].map((language) => ({ language, p: 0.95 })); + expect(pickLanguage(disagree)).toBeNull(); + const unsure = ["es", "es", "es"].map((language) => ({ language, p: 0.6 })); + expect(pickLanguage(unsure)).toBeNull(); + }); +}); + +it("reads whisper-cli's detection line", () => { + const line = "whisper_full_with_state: auto-detected language: es (p = 0.989864)"; + expect(parseDetection(line)).toEqual({ language: "es", p: 0.989864 }); + expect(parseDetection("whisper_init_from_file: loading model")).toBeNull(); +}); diff --git a/packages/cli/src/whisper/language.ts b/packages/cli/src/whisper/language.ts new file mode 100644 index 0000000000..2a8fd520fb --- /dev/null +++ b/packages/cli/src/whisper/language.ts @@ -0,0 +1,36 @@ +/** whisper.cpp hears 30 s to decide a language; it decides once, from the start of what it is given. */ +export const WINDOW_SECONDS = 30; +/** A music or noise window comes back near p = 0.5; speech in a language comes back above 0.9. */ +const CONFIDENT = 0.7; + +/** The starts of up to three windows spread over the clip after `onset`, so an intro alone cannot decide. A clip + * with under two windows of sound after it gets none: whisper's own pick already hears all of it. */ +export function detectionWindows(seconds: number, onset: number | null): number[] { + const from = onset ?? 0; + const span = seconds - from; + if (span < 2 * WINDOW_SECONDS) return []; + return [1 / 6, 1 / 2, 5 / 6].map((at) => { + const start = from + span * at - WINDOW_SECONDS / 2; + return Math.min(Math.max(start, from), seconds - WINDOW_SECONDS); + }); +} + +export interface LanguageVote { + language: string; + p: number; +} + +/** The language at least two confident windows agree on, else null (whisper's own pick stands). */ +export function pickLanguage(votes: readonly LanguageVote[]): string | null { + const counts = new Map(); + for (const { language, p } of votes) + if (p >= CONFIDENT) counts.set(language, (counts.get(language) ?? 0) + 1); + for (const [language, count] of counts) if (count >= 2) return language; + return null; +} + +/** whisper-cli --detect-language prints `auto-detected language: es (p = 0.983324)`. */ +export function parseDetection(line: string): LanguageVote | null { + const match = /auto-detected language: ([a-z]+) \(p = ([\d.]+)\)/.exec(line); + return match ? { language: match[1]!, p: Number(match[2]) } : null; +} diff --git a/packages/cli/src/whisper/manager.ts b/packages/cli/src/whisper/manager.ts index e9cf61304f..31670dec59 100644 --- a/packages/cli/src/whisper/manager.ts +++ b/packages/cli/src/whisper/manager.ts @@ -7,7 +7,8 @@ import { findFFmpeg } from "../browser/ffmpeg.js"; import { downloadFile } from "../utils/download.js"; const MODELS_DIR = join(homedir(), ".cache", "hyperframes", "whisper", "models"); -const DEFAULT_MODEL = "small.en"; +// Multilingual: an English-only default turned every other language into English text. +const DEFAULT_MODEL = "small"; export type WhisperSource = "env" | "system" | "brew" | "build"; diff --git a/packages/cli/src/whisper/transcribe.metadata.test.ts b/packages/cli/src/whisper/transcribe.metadata.test.ts index 1317129b95..6e8398d2a1 100644 --- a/packages/cli/src/whisper/transcribe.metadata.test.ts +++ b/packages/cli/src/whisper/transcribe.metadata.test.ts @@ -1,8 +1,9 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { transcribe, type TranscribeProgress } from "./transcribe.js"; +import { encodeWav } from "./wav.test-helpers.js"; const native = vi.hoisted(() => ({ exec: vi.fn(), @@ -24,14 +25,16 @@ vi.mock("node:child_process", async () => { const child = { stdout: new PassThrough(), stderr: new PassThrough() }; setImmediate(() => { let failure: unknown = null; + let said: unknown = null; try { - native.exec(command, args); + said = native.exec(command, args); } catch (err) { failure = err; } + const stderr = (said as { stderr?: string } | null)?.stderr ?? native.printed.stderr; child.stdout.end(native.printed.stdout); - child.stderr.end(native.printed.stderr); - done(failure, native.printed.stdout, native.printed.stderr); + child.stderr.end(stderr); + done(failure, native.printed.stdout, stderr); }); return child; }, @@ -266,3 +269,64 @@ it("names the timeout knob when whisper outlives its own timeout", async () => { "Whisper transcription exceeded", ); }); + +/** 100 s of prepared WAV: the first 40 s carry sample value 2 (loud music to the stand-in), the rest 1 (Spanish). */ +function musicThenSpanish() { + const samples = Array.from({ length: 100 * 16_000 }, (_, i) => (i < 40 * 16_000 ? 2 : 1) / 32768); + writeFileSync(join(dir, "audio.wav"), encodeWav(samples, 16_000)); +} + +/** whisper-cli's detection as the real one answers a window: music near p = 0.5, speech confidently. */ +function detectsByWindow(answer: (sample: number) => string) { + const previous = native.exec.getMockImplementation()!; + native.exec.mockImplementation((command: string, args: string[]) => { + if (command !== "whisper-cli" || !args.includes("--detect-language")) + return previous(command, args); + // Whisper hears the whole window; the stand-in judges it by its middle. + const sample = readFileSync(args.at(-1)!).readInt16LE(44 + 15 * 32_000); + return { stderr: `whisper_full_with_state: auto-detected language: ${answer(sample)}\n` }; + }); +} + +const languagePassed = () => + native.exec.mock.calls + .filter(([command, args]) => command === "whisper-cli" && !args.includes("--detect-language")) + .map(([, args]) => args[args.indexOf("--language") + 1]); + +it("a music intro does not decide the language: windows spread over the clip do", async () => { + musicThenSpanish(); + detectsByWindow((sample) => (sample === 2 ? "en (p = 0.534956)" : "es (p = 0.983324)")); + const result = await transcribe(join(dir, "audio.wav"), dir, { model: "small" }); + const detections = native.exec.mock.calls.filter(([, args]) => + args.includes("--detect-language"), + ); + expect(detections).toHaveLength(3); + expect(languagePassed()).toEqual(["es"]); + expect(result.detectedLanguage).toBe("es"); +}); + +it("windows that disagree leave the language to whisper's own pick", async () => { + musicThenSpanish(); + let n = 0; + detectsByWindow(() => `${["en", "es", "fr"][n++]} (p = 0.95)`); + await transcribe(join(dir, "audio.wav"), dir, { model: "small" }); + expect(languagePassed()).toEqual(["auto"]); +}); + +it("a detection that fails leaves the language to whisper's own pick", async () => { + musicThenSpanish(); + const previous = native.exec.getMockImplementation()!; + native.exec.mockImplementation((command: string, args: string[]) => { + if (args.includes("--detect-language")) throw new Error("unknown argument: --detect-language"); + return previous(command, args); + }); + const said: string[] = []; + await transcribe(join(dir, "audio.wav"), dir, { + model: "small", + onProgress: (m) => said.push(m), + }); + expect(languagePassed()).toEqual(["auto"]); + expect(said).toContain( + "Language detection failed (unknown argument: --detect-language); whisper picks it.", + ); +}); diff --git a/packages/cli/src/whisper/transcribe.ts b/packages/cli/src/whisper/transcribe.ts index caac298f65..a47c919692 100644 --- a/packages/cli/src/whisper/transcribe.ts +++ b/packages/cli/src/whisper/transcribe.ts @@ -3,16 +3,32 @@ import { execFile, execFileSync } from "node:child_process"; import { once } from "node:events"; import { createInterface } from "node:readline"; import type { Readable } from "node:stream"; -import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } from "node:fs"; +import { + existsSync, + readFileSync, + mkdirSync, + mkdtempSync, + rmSync, + statSync, + unlinkSync, +} from "node:fs"; import { join, extname } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js"; +import { normalizeErrorMessage } from "../utils/errorMessage.js"; import { stoppedByCancelSignal } from "../utils/renderCancellation.js"; import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js"; -import { findWavChunk } from "./wav.js"; +import { findWavChunk, writeWavWindow } from "./wav.js"; import type { Word } from "./normalize.js"; import { emitWords } from "./progress.js"; +import { + detectionWindows, + parseDetection, + pickLanguage, + WINDOW_SECONDS, + type LanguageVote, +} from "./language.js"; const WHISPER_TIMEOUT_FLOOR_MS = 300_000; const WHISPER_TIMEOUT_PER_AUDIO_SECOND_MS = 10_000; @@ -473,9 +489,19 @@ export async function transcribe( const wavPath = prepareWav(inputPath, options?.onProgress); const automaticLanguage = options?.language === undefined && !model.endsWith(".en"); - const language = options?.language ?? (automaticLanguage ? "auto" : "en"); - options?.onProgress?.("Transcribing..."); const wavSeconds = getPreparedWavDurationSeconds(wavPath); + const speechOnsetSeconds = detectSpeechOnset(wavPath); + const signal = options?.startCancellation?.(); + const picked = automaticLanguage + ? await languageFromWindows(whisper.executablePath, { model, modelPath }, wavPath, { + seconds: wavSeconds, + onset: speechOnsetSeconds, + signal, + onProgress: options?.onProgress, + }) + : null; + const language = options?.language ?? picked ?? (automaticLanguage ? "auto" : "en"); + options?.onProgress?.("Transcribing..."); options?.onEvent?.({ type: "progress", phase: "transcription", @@ -514,7 +540,7 @@ export async function transcribe( try { await runWhisper(whisper.executablePath, whisperArgs, { timeoutMs: whisperTimeoutMs, - signal: options?.startCancellation?.(), + signal, onStdout: onEvent && ((line) => { @@ -550,9 +576,10 @@ export async function transcribe( const transcript = JSON.parse(readFileSync(transcriptPath, "utf-8")); const reportedLanguage: unknown = transcript.result?.language; const detectedLanguage = - automaticLanguage && typeof reportedLanguage === "string" && reportedLanguage.length > 0 + picked ?? + (automaticLanguage && typeof reportedLanguage === "string" && reportedLanguage.length > 0 ? reportedLanguage - : null; + : null); const segments = transcript.transcription ?? []; let wordCount = 0; @@ -565,10 +592,6 @@ export async function transcribe( } } - // 7. Detect speech onset before cleaning up the WAV - options?.onProgress?.("Detecting speech onset..."); - const speechOnsetSeconds = detectSpeechOnset(wavPath); - // Clean up temp WAV if we created one if (wavPath !== inputPath) { try { @@ -614,6 +637,53 @@ function segmentWords(line: string): { words: Word[]; end: number } | null { } /** Resolves once whisper exits and both its streams are read, so no printed line is lost. */ +/** The language up to three windows spread over the speech agree on; null keeps whisper's own pick, which hears + * only the first 30 s (a music intro decided whole clips). A failed detection keeps it too, said aloud. */ +async function languageFromWindows( + executable: string, + { model, modelPath }: { model: string; modelPath: string }, + wavPath: string, + clip: { + seconds: number | null; + onset: number | null; + signal?: AbortSignal; + onProgress?: (message: string) => void; + }, +): Promise { + const starts = clip.seconds === null ? [] : detectionWindows(clip.seconds, clip.onset); + if (starts.length === 0) return null; + clip.onProgress?.("Detecting the language..."); + const dir = mkdtempSync(join(tmpdir(), "hyperframes-language-")); + try { + const votes: LanguageVote[] = []; + for (const [i, start] of starts.entries()) { + const window = join(dir, `${i}.wav`); + writeWavWindow(wavPath, window, start, WINDOW_SECONDS); + await runWhisper( + executable, + ["--model", modelPath, "--language", "auto", "--detect-language", window], + { + timeoutMs: resolveWhisperTimeoutMs(WINDOW_SECONDS, { model }), + signal: clip.signal, + onStderr: (line) => { + const vote = parseDetection(line); + if (vote) votes.push(vote); + }, + }, + ); + } + return pickLanguage(votes); + } catch (err) { + if (clip.signal?.aborted) throw err; + clip.onProgress?.( + `Language detection failed (${normalizeErrorMessage(err)}); whisper picks it.`, + ); + return null; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + async function runWhisper( executable: string, args: string[], diff --git a/packages/cli/src/whisper/wav.test-helpers.ts b/packages/cli/src/whisper/wav.test-helpers.ts index d0f924b63d..05717d203d 100644 --- a/packages/cli/src/whisper/wav.test-helpers.ts +++ b/packages/cli/src/whisper/wav.test-helpers.ts @@ -1,21 +1,10 @@ +import { wavHeader } from "./wav.js"; + /** A 16-bit mono PCM WAV of `samples` (-1..1), as prepareWav writes one. */ export function encodeWav(samples: ArrayLike, sampleRate: number): Buffer { const data = Buffer.alloc(samples.length * 2); for (let i = 0; i < samples.length; i++) { data.writeInt16LE(Math.max(-32768, Math.min(32767, Math.round(samples[i]! * 32768))), 2 * i); } - const head = Buffer.alloc(44); - head.write("RIFF", 0, "ascii"); - head.writeUInt32LE(36 + data.length, 4); - head.write("WAVEfmt ", 8, "ascii"); - head.writeUInt32LE(16, 16); - head.writeUInt16LE(1, 20); - head.writeUInt16LE(1, 22); - head.writeUInt32LE(sampleRate, 24); - head.writeUInt32LE(sampleRate * 2, 28); - head.writeUInt16LE(2, 32); - head.writeUInt16LE(16, 34); - head.write("data", 36, "ascii"); - head.writeUInt32LE(data.length, 40); - return Buffer.concat([head, data]); + return Buffer.concat([wavHeader(data.length, sampleRate), data]); } diff --git a/packages/cli/src/whisper/wav.test.ts b/packages/cli/src/whisper/wav.test.ts index 320abb4bd5..5ef4be1f71 100644 --- a/packages/cli/src/whisper/wav.test.ts +++ b/packages/cli/src/whisper/wav.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, expect, it } from "vitest"; -import { readWav } from "./wav.js"; +import { readWav, writeWavWindow } from "./wav.js"; import { encodeWav } from "./wav.test-helpers.js"; let dir: string; @@ -33,3 +33,13 @@ it("names a file that is not a 16-bit mono WAV", () => { ); expect(() => readWav(file("stereo.wav", stereo))).toThrow("is not a 16-bit mono PCM WAV"); }); + +it("writes a window of a prepared WAV as a WAV of its own", () => { + const second = (n: number) => new Array(16_000).fill(n / 32768); + const source = file("in.wav", encodeWav([...second(1), ...second(2), ...second(3)], 16_000)); + writeWavWindow(source, join(dir, "out.wav"), 1, 1); + const window = readWav(join(dir, "out.wav")); + expect(window.sampleRate).toBe(16_000); + expect(window.samples).toHaveLength(16_000); + expect(new Set(window.samples)).toEqual(new Set([2 / 32768])); +}); diff --git a/packages/cli/src/whisper/wav.ts b/packages/cli/src/whisper/wav.ts index e2d84e2d09..8e5f960be0 100644 --- a/packages/cli/src/whisper/wav.ts +++ b/packages/cli/src/whisper/wav.ts @@ -1,4 +1,4 @@ -import { readFileSync } from "node:fs"; +import { readFileSync, writeFileSync } from "node:fs"; /** Where a RIFF chunk's bytes start and how many there are, or null when the file has none. */ export function findWavChunk(buf: Buffer, want: string): { offset: number; size: number } | null { @@ -38,3 +38,31 @@ export function readWav(path: string): { samples: Float32Array; sampleRate: numb samples[i] = buf.readInt16LE(data.offset + 2 * i) / 32768; return { samples, sampleRate: buf.readUInt32LE(fmt.offset + 4) }; } + +/** The 44-byte header of a 16-bit mono PCM WAV holding `bytes` of samples. */ +export function wavHeader(bytes: number, sampleRate: number): Buffer { + const head = Buffer.alloc(44); + head.write("RIFF", 0, "ascii"); + head.writeUInt32LE(36 + bytes, 4); + head.write("WAVEfmt ", 8, "ascii"); + head.writeUInt32LE(16, 16); + head.writeUInt16LE(PCM, 20); + head.writeUInt16LE(1, 22); + head.writeUInt32LE(sampleRate, 24); + head.writeUInt32LE(sampleRate * 2, 28); + head.writeUInt16LE(2, 32); + head.writeUInt16LE(16, 34); + head.write("data", 36, "ascii"); + head.writeUInt32LE(bytes, 40); + return head; +} + +/** `seconds` of a prepared WAV (16 kHz mono s16) from `start`, written as a WAV of its own. */ +export function writeWavWindow(source: string, dest: string, start: number, seconds: number): void { + const buf = readFileSync(source); + const data = findWavChunk(buf, "data"); + if (!data) throw new Error(`${source} has no WAV data`); + const from = data.offset + Math.floor(start * 16_000) * 2; + const pcm = buf.subarray(from, Math.min(from + seconds * 32_000, data.offset + data.size)); + writeFileSync(dest, Buffer.concat([wavHeader(pcm.length, 16_000), pcm])); +} diff --git a/skills/media-use/audio/references/captions/authoring.md b/skills/media-use/audio/references/captions/authoring.md index 8992f1eff7..4b013eb19d 100644 --- a/skills/media-use/audio/references/captions/authoring.md +++ b/skills/media-use/audio/references/captions/authoring.md @@ -4,7 +4,7 @@ **The live search is the source of truth for what the registry has.** The table(s) below are a hand-maintained sample and under-cover by design: run `npx hyperframes catalog --query "" --json` — it needs nothing installed — before concluding the registry lacks something. Item names here are checked against `registry/registry.json` by `bun run lint:skills`. -Before authoring: confirm the transcript came from the right Whisper model. CLI default `small.en` silently translates non-English audio — see [`../transcribe.md`](../transcribe.md) → "Language Rule" and [`transcript-handling.md`](transcript-handling.md) for the mandatory quality check. +Before authoring: confirm the transcript came from the right Whisper model. An `.en` model silently translates non-English audio — see [`../transcribe.md`](../transcribe.md) → "Language Rule" and [`transcript-handling.md`](transcript-handling.md) for the mandatory quality check. Analyze spoken content to determine caption style. If user specifies a style, use that. Otherwise, detect tone from the transcript. diff --git a/skills/media-use/audio/references/transcribe.md b/skills/media-use/audio/references/transcribe.md index 06df2ce445..49e621e215 100644 --- a/skills/media-use/audio/references/transcribe.md +++ b/skills/media-use/audio/references/transcribe.md @@ -1,6 +1,6 @@ # Transcription -Create normalized word-level timestamps. **Always specify `--model` explicitly** — the CLI default is `small.en`, which silently translates non-English audio into English. +Create normalized word-level timestamps. **Always specify `--model` explicitly.** The CLI default is the multilingual `small`, which detects the language from three spread-out windows of the audio; an `.en` model silently turns non-English audio into English. ```bash npx hyperframes transcribe audio.mp3 --model small.en # known English