fix(cli): auto picks the language from three spread windows of speech, not the first 30 s

This commit is contained in:
Miguel Angel Simon Sierra
2026-10-06 17:32:59 -07:00
parent e8cfd6b73e
commit d73e5814b4
12 changed files with 271 additions and 36 deletions
+1 -1
View File
@@ -67,7 +67,7 @@ export default defineCommand({
},
model: {
type: "string",
description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny.en, base.en, small.en, medium.en, large-v3`,
description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny, base, small, medium (add .en for English only), large-v3`,
alias: "m",
},
language: {
+37
View File
@@ -0,0 +1,37 @@
import { describe, expect, it } from "vitest";
import { detectionWindows, parseDetection, pickLanguage } from "./language.js";
describe("detectionWindows", () => {
it("leaves a clip under a minute to whisper's own pick", () => {
expect(detectionWindows(59, null)).toEqual([]);
});
it("spreads three 30 s windows over the clip, centred at a sixth, a half and five sixths", () => {
expect(detectionWindows(120, null)).toEqual([5, 45, 85]);
});
it("starts after the onset of speech and never runs past the end", () => {
expect(detectionWindows(100, 40)).toEqual([40, 55, 70]);
});
});
describe("pickLanguage", () => {
it("takes the language two confident windows agree on, over a music window", () => {
const votes = [
{ language: "en", p: 0.53 },
{ language: "es", p: 0.98 },
{ language: "es", p: 0.99 },
];
expect(pickLanguage(votes)).toBe("es");
});
it("leaves the pick to whisper when the windows disagree or none is confident", () => {
const disagree = ["en", "es", "fr"].map((language) => ({ language, p: 0.95 }));
expect(pickLanguage(disagree)).toBeNull();
const unsure = ["es", "es", "es"].map((language) => ({ language, p: 0.6 }));
expect(pickLanguage(unsure)).toBeNull();
});
});
it("reads whisper-cli's detection line", () => {
const line = "whisper_full_with_state: auto-detected language: es (p = 0.989864)";
expect(parseDetection(line)).toEqual({ language: "es", p: 0.989864 });
expect(parseDetection("whisper_init_from_file: loading model")).toBeNull();
});
+36
View File
@@ -0,0 +1,36 @@
/** whisper.cpp hears 30 s to decide a language; it decides once, from the start of what it is given. */
export const WINDOW_SECONDS = 30;
/** A music or noise window comes back near p = 0.5; speech in a language comes back above 0.9. */
const CONFIDENT = 0.7;
/** The starts of up to three windows spread over the clip after `onset`, so an intro alone cannot decide. A clip
* with under two windows of sound after it gets none: whisper's own pick already hears all of it. */
export function detectionWindows(seconds: number, onset: number | null): number[] {
const from = onset ?? 0;
const span = seconds - from;
if (span < 2 * WINDOW_SECONDS) return [];
return [1 / 6, 1 / 2, 5 / 6].map((at) => {
const start = from + span * at - WINDOW_SECONDS / 2;
return Math.min(Math.max(start, from), seconds - WINDOW_SECONDS);
});
}
export interface LanguageVote {
language: string;
p: number;
}
/** The language at least two confident windows agree on, else null (whisper's own pick stands). */
export function pickLanguage(votes: readonly LanguageVote[]): string | null {
const counts = new Map<string, number>();
for (const { language, p } of votes)
if (p >= CONFIDENT) counts.set(language, (counts.get(language) ?? 0) + 1);
for (const [language, count] of counts) if (count >= 2) return language;
return null;
}
/** whisper-cli --detect-language prints `auto-detected language: es (p = 0.983324)`. */
export function parseDetection(line: string): LanguageVote | null {
const match = /auto-detected language: ([a-z]+) \(p = ([\d.]+)\)/.exec(line);
return match ? { language: match[1]!, p: Number(match[2]) } : null;
}
+2 -1
View File
@@ -7,7 +7,8 @@ import { findFFmpeg } from "../browser/ffmpeg.js";
import { downloadFile } from "../utils/download.js";
const MODELS_DIR = join(homedir(), ".cache", "hyperframes", "whisper", "models");
const DEFAULT_MODEL = "small.en";
// Multilingual: an English-only default turned every other language into English text.
const DEFAULT_MODEL = "small";
export type WhisperSource = "env" | "system" | "brew" | "build";
@@ -1,8 +1,9 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { transcribe, type TranscribeProgress } from "./transcribe.js";
import { encodeWav } from "./wav.test-helpers.js";
const native = vi.hoisted(() => ({
exec: vi.fn(),
@@ -24,14 +25,16 @@ vi.mock("node:child_process", async () => {
const child = { stdout: new PassThrough(), stderr: new PassThrough() };
setImmediate(() => {
let failure: unknown = null;
let said: unknown = null;
try {
native.exec(command, args);
said = native.exec(command, args);
} catch (err) {
failure = err;
}
const stderr = (said as { stderr?: string } | null)?.stderr ?? native.printed.stderr;
child.stdout.end(native.printed.stdout);
child.stderr.end(native.printed.stderr);
done(failure, native.printed.stdout, native.printed.stderr);
child.stderr.end(stderr);
done(failure, native.printed.stdout, stderr);
});
return child;
},
@@ -266,3 +269,64 @@ it("names the timeout knob when whisper outlives its own timeout", async () => {
"Whisper transcription exceeded",
);
});
/** 100 s of prepared WAV: the first 40 s carry sample value 2 (loud music to the stand-in), the rest 1 (Spanish). */
function musicThenSpanish() {
const samples = Array.from({ length: 100 * 16_000 }, (_, i) => (i < 40 * 16_000 ? 2 : 1) / 32768);
writeFileSync(join(dir, "audio.wav"), encodeWav(samples, 16_000));
}
/** whisper-cli's detection as the real one answers a window: music near p = 0.5, speech confidently. */
function detectsByWindow(answer: (sample: number) => string) {
const previous = native.exec.getMockImplementation()!;
native.exec.mockImplementation((command: string, args: string[]) => {
if (command !== "whisper-cli" || !args.includes("--detect-language"))
return previous(command, args);
// Whisper hears the whole window; the stand-in judges it by its middle.
const sample = readFileSync(args.at(-1)!).readInt16LE(44 + 15 * 32_000);
return { stderr: `whisper_full_with_state: auto-detected language: ${answer(sample)}\n` };
});
}
const languagePassed = () =>
native.exec.mock.calls
.filter(([command, args]) => command === "whisper-cli" && !args.includes("--detect-language"))
.map(([, args]) => args[args.indexOf("--language") + 1]);
it("a music intro does not decide the language: windows spread over the clip do", async () => {
musicThenSpanish();
detectsByWindow((sample) => (sample === 2 ? "en (p = 0.534956)" : "es (p = 0.983324)"));
const result = await transcribe(join(dir, "audio.wav"), dir, { model: "small" });
const detections = native.exec.mock.calls.filter(([, args]) =>
args.includes("--detect-language"),
);
expect(detections).toHaveLength(3);
expect(languagePassed()).toEqual(["es"]);
expect(result.detectedLanguage).toBe("es");
});
it("windows that disagree leave the language to whisper's own pick", async () => {
musicThenSpanish();
let n = 0;
detectsByWindow(() => `${["en", "es", "fr"][n++]} (p = 0.95)`);
await transcribe(join(dir, "audio.wav"), dir, { model: "small" });
expect(languagePassed()).toEqual(["auto"]);
});
it("a detection that fails leaves the language to whisper's own pick", async () => {
musicThenSpanish();
const previous = native.exec.getMockImplementation()!;
native.exec.mockImplementation((command: string, args: string[]) => {
if (args.includes("--detect-language")) throw new Error("unknown argument: --detect-language");
return previous(command, args);
});
const said: string[] = [];
await transcribe(join(dir, "audio.wav"), dir, {
model: "small",
onProgress: (m) => said.push(m),
});
expect(languagePassed()).toEqual(["auto"]);
expect(said).toContain(
"Language detection failed (unknown argument: --detect-language); whisper picks it.",
);
});
+81 -11
View File
@@ -3,16 +3,32 @@ import { execFile, execFileSync } from "node:child_process";
import { once } from "node:events";
import { createInterface } from "node:readline";
import type { Readable } from "node:stream";
import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } from "node:fs";
import {
existsSync,
readFileSync,
mkdirSync,
mkdtempSync,
rmSync,
statSync,
unlinkSync,
} from "node:fs";
import { join, extname } from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { stoppedByCancelSignal } from "../utils/renderCancellation.js";
import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js";
import { findWavChunk } from "./wav.js";
import { findWavChunk, writeWavWindow } from "./wav.js";
import type { Word } from "./normalize.js";
import { emitWords } from "./progress.js";
import {
detectionWindows,
parseDetection,
pickLanguage,
WINDOW_SECONDS,
type LanguageVote,
} from "./language.js";
const WHISPER_TIMEOUT_FLOOR_MS = 300_000;
const WHISPER_TIMEOUT_PER_AUDIO_SECOND_MS = 10_000;
@@ -473,9 +489,19 @@ export async function transcribe(
const wavPath = prepareWav(inputPath, options?.onProgress);
const automaticLanguage = options?.language === undefined && !model.endsWith(".en");
const language = options?.language ?? (automaticLanguage ? "auto" : "en");
options?.onProgress?.("Transcribing...");
const wavSeconds = getPreparedWavDurationSeconds(wavPath);
const speechOnsetSeconds = detectSpeechOnset(wavPath);
const signal = options?.startCancellation?.();
const picked = automaticLanguage
? await languageFromWindows(whisper.executablePath, { model, modelPath }, wavPath, {
seconds: wavSeconds,
onset: speechOnsetSeconds,
signal,
onProgress: options?.onProgress,
})
: null;
const language = options?.language ?? picked ?? (automaticLanguage ? "auto" : "en");
options?.onProgress?.("Transcribing...");
options?.onEvent?.({
type: "progress",
phase: "transcription",
@@ -514,7 +540,7 @@ export async function transcribe(
try {
await runWhisper(whisper.executablePath, whisperArgs, {
timeoutMs: whisperTimeoutMs,
signal: options?.startCancellation?.(),
signal,
onStdout:
onEvent &&
((line) => {
@@ -550,9 +576,10 @@ export async function transcribe(
const transcript = JSON.parse(readFileSync(transcriptPath, "utf-8"));
const reportedLanguage: unknown = transcript.result?.language;
const detectedLanguage =
automaticLanguage && typeof reportedLanguage === "string" && reportedLanguage.length > 0
picked ??
(automaticLanguage && typeof reportedLanguage === "string" && reportedLanguage.length > 0
? reportedLanguage
: null;
: null);
const segments = transcript.transcription ?? [];
let wordCount = 0;
@@ -565,10 +592,6 @@ export async function transcribe(
}
}
// 7. Detect speech onset before cleaning up the WAV
options?.onProgress?.("Detecting speech onset...");
const speechOnsetSeconds = detectSpeechOnset(wavPath);
// Clean up temp WAV if we created one
if (wavPath !== inputPath) {
try {
@@ -614,6 +637,53 @@ function segmentWords(line: string): { words: Word[]; end: number } | null {
}
/** Resolves once whisper exits and both its streams are read, so no printed line is lost. */
/** The language up to three windows spread over the speech agree on; null keeps whisper's own pick, which hears
* only the first 30 s (a music intro decided whole clips). A failed detection keeps it too, said aloud. */
async function languageFromWindows(
executable: string,
{ model, modelPath }: { model: string; modelPath: string },
wavPath: string,
clip: {
seconds: number | null;
onset: number | null;
signal?: AbortSignal;
onProgress?: (message: string) => void;
},
): Promise<string | null> {
const starts = clip.seconds === null ? [] : detectionWindows(clip.seconds, clip.onset);
if (starts.length === 0) return null;
clip.onProgress?.("Detecting the language...");
const dir = mkdtempSync(join(tmpdir(), "hyperframes-language-"));
try {
const votes: LanguageVote[] = [];
for (const [i, start] of starts.entries()) {
const window = join(dir, `${i}.wav`);
writeWavWindow(wavPath, window, start, WINDOW_SECONDS);
await runWhisper(
executable,
["--model", modelPath, "--language", "auto", "--detect-language", window],
{
timeoutMs: resolveWhisperTimeoutMs(WINDOW_SECONDS, { model }),
signal: clip.signal,
onStderr: (line) => {
const vote = parseDetection(line);
if (vote) votes.push(vote);
},
},
);
}
return pickLanguage(votes);
} catch (err) {
if (clip.signal?.aborted) throw err;
clip.onProgress?.(
`Language detection failed (${normalizeErrorMessage(err)}); whisper picks it.`,
);
return null;
} finally {
rmSync(dir, { recursive: true, force: true });
}
}
async function runWhisper(
executable: string,
args: string[],
+3 -14
View File
@@ -1,21 +1,10 @@
import { wavHeader } from "./wav.js";
/** A 16-bit mono PCM WAV of `samples` (-1..1), as prepareWav writes one. */
export function encodeWav(samples: ArrayLike<number>, sampleRate: number): Buffer {
const data = Buffer.alloc(samples.length * 2);
for (let i = 0; i < samples.length; i++) {
data.writeInt16LE(Math.max(-32768, Math.min(32767, Math.round(samples[i]! * 32768))), 2 * i);
}
const head = Buffer.alloc(44);
head.write("RIFF", 0, "ascii");
head.writeUInt32LE(36 + data.length, 4);
head.write("WAVEfmt ", 8, "ascii");
head.writeUInt32LE(16, 16);
head.writeUInt16LE(1, 20);
head.writeUInt16LE(1, 22);
head.writeUInt32LE(sampleRate, 24);
head.writeUInt32LE(sampleRate * 2, 28);
head.writeUInt16LE(2, 32);
head.writeUInt16LE(16, 34);
head.write("data", 36, "ascii");
head.writeUInt32LE(data.length, 40);
return Buffer.concat([head, data]);
return Buffer.concat([wavHeader(data.length, sampleRate), data]);
}
+11 -1
View File
@@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, expect, it } from "vitest";
import { readWav } from "./wav.js";
import { readWav, writeWavWindow } from "./wav.js";
import { encodeWav } from "./wav.test-helpers.js";
let dir: string;
@@ -33,3 +33,13 @@ it("names a file that is not a 16-bit mono WAV", () => {
);
expect(() => readWav(file("stereo.wav", stereo))).toThrow("is not a 16-bit mono PCM WAV");
});
it("writes a window of a prepared WAV as a WAV of its own", () => {
const second = (n: number) => new Array(16_000).fill(n / 32768);
const source = file("in.wav", encodeWav([...second(1), ...second(2), ...second(3)], 16_000));
writeWavWindow(source, join(dir, "out.wav"), 1, 1);
const window = readWav(join(dir, "out.wav"));
expect(window.sampleRate).toBe(16_000);
expect(window.samples).toHaveLength(16_000);
expect(new Set(window.samples)).toEqual(new Set([2 / 32768]));
});
+29 -1
View File
@@ -1,4 +1,4 @@
import { readFileSync } from "node:fs";
import { readFileSync, writeFileSync } from "node:fs";
/** Where a RIFF chunk's bytes start and how many there are, or null when the file has none. */
export function findWavChunk(buf: Buffer, want: string): { offset: number; size: number } | null {
@@ -38,3 +38,31 @@ export function readWav(path: string): { samples: Float32Array; sampleRate: numb
samples[i] = buf.readInt16LE(data.offset + 2 * i) / 32768;
return { samples, sampleRate: buf.readUInt32LE(fmt.offset + 4) };
}
/** The 44-byte header of a 16-bit mono PCM WAV holding `bytes` of samples. */
export function wavHeader(bytes: number, sampleRate: number): Buffer {
const head = Buffer.alloc(44);
head.write("RIFF", 0, "ascii");
head.writeUInt32LE(36 + bytes, 4);
head.write("WAVEfmt ", 8, "ascii");
head.writeUInt32LE(16, 16);
head.writeUInt16LE(PCM, 20);
head.writeUInt16LE(1, 22);
head.writeUInt32LE(sampleRate, 24);
head.writeUInt32LE(sampleRate * 2, 28);
head.writeUInt16LE(2, 32);
head.writeUInt16LE(16, 34);
head.write("data", 36, "ascii");
head.writeUInt32LE(bytes, 40);
return head;
}
/** `seconds` of a prepared WAV (16 kHz mono s16) from `start`, written as a WAV of its own. */
export function writeWavWindow(source: string, dest: string, start: number, seconds: number): void {
const buf = readFileSync(source);
const data = findWavChunk(buf, "data");
if (!data) throw new Error(`${source} has no WAV data`);
const from = data.offset + Math.floor(start * 16_000) * 2;
const pcm = buf.subarray(from, Math.min(from + seconds * 32_000, data.offset + data.size));
writeFileSync(dest, Buffer.concat([wavHeader(pcm.length, 16_000), pcm]));
}