mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-19 19:36:04 +00:00
fix(cli): auto picks the language from three spread windows of speech, not the first 30 s
This commit is contained in:
@@ -67,7 +67,7 @@ export default defineCommand({
|
||||
},
|
||||
model: {
|
||||
type: "string",
|
||||
description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny.en, base.en, small.en, medium.en, large-v3`,
|
||||
description: `Whisper model (default: ${DEFAULT_MODEL}). Options: tiny, base, small, medium (add .en for English only), large-v3`,
|
||||
alias: "m",
|
||||
},
|
||||
language: {
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { detectionWindows, parseDetection, pickLanguage } from "./language.js";
|
||||
|
||||
describe("detectionWindows", () => {
|
||||
it("leaves a clip under a minute to whisper's own pick", () => {
|
||||
expect(detectionWindows(59, null)).toEqual([]);
|
||||
});
|
||||
it("spreads three 30 s windows over the clip, centred at a sixth, a half and five sixths", () => {
|
||||
expect(detectionWindows(120, null)).toEqual([5, 45, 85]);
|
||||
});
|
||||
it("starts after the onset of speech and never runs past the end", () => {
|
||||
expect(detectionWindows(100, 40)).toEqual([40, 55, 70]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pickLanguage", () => {
|
||||
it("takes the language two confident windows agree on, over a music window", () => {
|
||||
const votes = [
|
||||
{ language: "en", p: 0.53 },
|
||||
{ language: "es", p: 0.98 },
|
||||
{ language: "es", p: 0.99 },
|
||||
];
|
||||
expect(pickLanguage(votes)).toBe("es");
|
||||
});
|
||||
it("leaves the pick to whisper when the windows disagree or none is confident", () => {
|
||||
const disagree = ["en", "es", "fr"].map((language) => ({ language, p: 0.95 }));
|
||||
expect(pickLanguage(disagree)).toBeNull();
|
||||
const unsure = ["es", "es", "es"].map((language) => ({ language, p: 0.6 }));
|
||||
expect(pickLanguage(unsure)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
it("reads whisper-cli's detection line", () => {
|
||||
const line = "whisper_full_with_state: auto-detected language: es (p = 0.989864)";
|
||||
expect(parseDetection(line)).toEqual({ language: "es", p: 0.989864 });
|
||||
expect(parseDetection("whisper_init_from_file: loading model")).toBeNull();
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
/** whisper.cpp hears 30 s to decide a language; it decides once, from the start of what it is given. */
|
||||
export const WINDOW_SECONDS = 30;
|
||||
/** A music or noise window comes back near p = 0.5; speech in a language comes back above 0.9. */
|
||||
const CONFIDENT = 0.7;
|
||||
|
||||
/** The starts of up to three windows spread over the clip after `onset`, so an intro alone cannot decide. A clip
|
||||
* with under two windows of sound after it gets none: whisper's own pick already hears all of it. */
|
||||
export function detectionWindows(seconds: number, onset: number | null): number[] {
|
||||
const from = onset ?? 0;
|
||||
const span = seconds - from;
|
||||
if (span < 2 * WINDOW_SECONDS) return [];
|
||||
return [1 / 6, 1 / 2, 5 / 6].map((at) => {
|
||||
const start = from + span * at - WINDOW_SECONDS / 2;
|
||||
return Math.min(Math.max(start, from), seconds - WINDOW_SECONDS);
|
||||
});
|
||||
}
|
||||
|
||||
export interface LanguageVote {
|
||||
language: string;
|
||||
p: number;
|
||||
}
|
||||
|
||||
/** The language at least two confident windows agree on, else null (whisper's own pick stands). */
|
||||
export function pickLanguage(votes: readonly LanguageVote[]): string | null {
|
||||
const counts = new Map<string, number>();
|
||||
for (const { language, p } of votes)
|
||||
if (p >= CONFIDENT) counts.set(language, (counts.get(language) ?? 0) + 1);
|
||||
for (const [language, count] of counts) if (count >= 2) return language;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** whisper-cli --detect-language prints `auto-detected language: es (p = 0.983324)`. */
|
||||
export function parseDetection(line: string): LanguageVote | null {
|
||||
const match = /auto-detected language: ([a-z]+) \(p = ([\d.]+)\)/.exec(line);
|
||||
return match ? { language: match[1]!, p: Number(match[2]) } : null;
|
||||
}
|
||||
@@ -7,7 +7,8 @@ import { findFFmpeg } from "../browser/ffmpeg.js";
|
||||
import { downloadFile } from "../utils/download.js";
|
||||
|
||||
const MODELS_DIR = join(homedir(), ".cache", "hyperframes", "whisper", "models");
|
||||
const DEFAULT_MODEL = "small.en";
|
||||
// Multilingual: an English-only default turned every other language into English text.
|
||||
const DEFAULT_MODEL = "small";
|
||||
|
||||
export type WhisperSource = "env" | "system" | "brew" | "build";
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import { transcribe, type TranscribeProgress } from "./transcribe.js";
|
||||
import { encodeWav } from "./wav.test-helpers.js";
|
||||
|
||||
const native = vi.hoisted(() => ({
|
||||
exec: vi.fn(),
|
||||
@@ -24,14 +25,16 @@ vi.mock("node:child_process", async () => {
|
||||
const child = { stdout: new PassThrough(), stderr: new PassThrough() };
|
||||
setImmediate(() => {
|
||||
let failure: unknown = null;
|
||||
let said: unknown = null;
|
||||
try {
|
||||
native.exec(command, args);
|
||||
said = native.exec(command, args);
|
||||
} catch (err) {
|
||||
failure = err;
|
||||
}
|
||||
const stderr = (said as { stderr?: string } | null)?.stderr ?? native.printed.stderr;
|
||||
child.stdout.end(native.printed.stdout);
|
||||
child.stderr.end(native.printed.stderr);
|
||||
done(failure, native.printed.stdout, native.printed.stderr);
|
||||
child.stderr.end(stderr);
|
||||
done(failure, native.printed.stdout, stderr);
|
||||
});
|
||||
return child;
|
||||
},
|
||||
@@ -266,3 +269,64 @@ it("names the timeout knob when whisper outlives its own timeout", async () => {
|
||||
"Whisper transcription exceeded",
|
||||
);
|
||||
});
|
||||
|
||||
/** 100 s of prepared WAV: the first 40 s carry sample value 2 (loud music to the stand-in), the rest 1 (Spanish). */
|
||||
function musicThenSpanish() {
|
||||
const samples = Array.from({ length: 100 * 16_000 }, (_, i) => (i < 40 * 16_000 ? 2 : 1) / 32768);
|
||||
writeFileSync(join(dir, "audio.wav"), encodeWav(samples, 16_000));
|
||||
}
|
||||
|
||||
/** whisper-cli's detection as the real one answers a window: music near p = 0.5, speech confidently. */
|
||||
function detectsByWindow(answer: (sample: number) => string) {
|
||||
const previous = native.exec.getMockImplementation()!;
|
||||
native.exec.mockImplementation((command: string, args: string[]) => {
|
||||
if (command !== "whisper-cli" || !args.includes("--detect-language"))
|
||||
return previous(command, args);
|
||||
// Whisper hears the whole window; the stand-in judges it by its middle.
|
||||
const sample = readFileSync(args.at(-1)!).readInt16LE(44 + 15 * 32_000);
|
||||
return { stderr: `whisper_full_with_state: auto-detected language: ${answer(sample)}\n` };
|
||||
});
|
||||
}
|
||||
|
||||
const languagePassed = () =>
|
||||
native.exec.mock.calls
|
||||
.filter(([command, args]) => command === "whisper-cli" && !args.includes("--detect-language"))
|
||||
.map(([, args]) => args[args.indexOf("--language") + 1]);
|
||||
|
||||
it("a music intro does not decide the language: windows spread over the clip do", async () => {
|
||||
musicThenSpanish();
|
||||
detectsByWindow((sample) => (sample === 2 ? "en (p = 0.534956)" : "es (p = 0.983324)"));
|
||||
const result = await transcribe(join(dir, "audio.wav"), dir, { model: "small" });
|
||||
const detections = native.exec.mock.calls.filter(([, args]) =>
|
||||
args.includes("--detect-language"),
|
||||
);
|
||||
expect(detections).toHaveLength(3);
|
||||
expect(languagePassed()).toEqual(["es"]);
|
||||
expect(result.detectedLanguage).toBe("es");
|
||||
});
|
||||
|
||||
it("windows that disagree leave the language to whisper's own pick", async () => {
|
||||
musicThenSpanish();
|
||||
let n = 0;
|
||||
detectsByWindow(() => `${["en", "es", "fr"][n++]} (p = 0.95)`);
|
||||
await transcribe(join(dir, "audio.wav"), dir, { model: "small" });
|
||||
expect(languagePassed()).toEqual(["auto"]);
|
||||
});
|
||||
|
||||
it("a detection that fails leaves the language to whisper's own pick", async () => {
|
||||
musicThenSpanish();
|
||||
const previous = native.exec.getMockImplementation()!;
|
||||
native.exec.mockImplementation((command: string, args: string[]) => {
|
||||
if (args.includes("--detect-language")) throw new Error("unknown argument: --detect-language");
|
||||
return previous(command, args);
|
||||
});
|
||||
const said: string[] = [];
|
||||
await transcribe(join(dir, "audio.wav"), dir, {
|
||||
model: "small",
|
||||
onProgress: (m) => said.push(m),
|
||||
});
|
||||
expect(languagePassed()).toEqual(["auto"]);
|
||||
expect(said).toContain(
|
||||
"Language detection failed (unknown argument: --detect-language); whisper picks it.",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -3,16 +3,32 @@ import { execFile, execFileSync } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import { createInterface } from "node:readline";
|
||||
import type { Readable } from "node:stream";
|
||||
import { existsSync, readFileSync, mkdirSync, rmSync, statSync, unlinkSync } from "node:fs";
|
||||
import {
|
||||
existsSync,
|
||||
readFileSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
unlinkSync,
|
||||
} from "node:fs";
|
||||
import { join, extname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
|
||||
import { normalizeErrorMessage } from "../utils/errorMessage.js";
|
||||
import { stoppedByCancelSignal } from "../utils/renderCancellation.js";
|
||||
import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js";
|
||||
import { findWavChunk } from "./wav.js";
|
||||
import { findWavChunk, writeWavWindow } from "./wav.js";
|
||||
import type { Word } from "./normalize.js";
|
||||
import { emitWords } from "./progress.js";
|
||||
import {
|
||||
detectionWindows,
|
||||
parseDetection,
|
||||
pickLanguage,
|
||||
WINDOW_SECONDS,
|
||||
type LanguageVote,
|
||||
} from "./language.js";
|
||||
|
||||
const WHISPER_TIMEOUT_FLOOR_MS = 300_000;
|
||||
const WHISPER_TIMEOUT_PER_AUDIO_SECOND_MS = 10_000;
|
||||
@@ -473,9 +489,19 @@ export async function transcribe(
|
||||
const wavPath = prepareWav(inputPath, options?.onProgress);
|
||||
|
||||
const automaticLanguage = options?.language === undefined && !model.endsWith(".en");
|
||||
const language = options?.language ?? (automaticLanguage ? "auto" : "en");
|
||||
options?.onProgress?.("Transcribing...");
|
||||
const wavSeconds = getPreparedWavDurationSeconds(wavPath);
|
||||
const speechOnsetSeconds = detectSpeechOnset(wavPath);
|
||||
const signal = options?.startCancellation?.();
|
||||
const picked = automaticLanguage
|
||||
? await languageFromWindows(whisper.executablePath, { model, modelPath }, wavPath, {
|
||||
seconds: wavSeconds,
|
||||
onset: speechOnsetSeconds,
|
||||
signal,
|
||||
onProgress: options?.onProgress,
|
||||
})
|
||||
: null;
|
||||
const language = options?.language ?? picked ?? (automaticLanguage ? "auto" : "en");
|
||||
options?.onProgress?.("Transcribing...");
|
||||
options?.onEvent?.({
|
||||
type: "progress",
|
||||
phase: "transcription",
|
||||
@@ -514,7 +540,7 @@ export async function transcribe(
|
||||
try {
|
||||
await runWhisper(whisper.executablePath, whisperArgs, {
|
||||
timeoutMs: whisperTimeoutMs,
|
||||
signal: options?.startCancellation?.(),
|
||||
signal,
|
||||
onStdout:
|
||||
onEvent &&
|
||||
((line) => {
|
||||
@@ -550,9 +576,10 @@ export async function transcribe(
|
||||
const transcript = JSON.parse(readFileSync(transcriptPath, "utf-8"));
|
||||
const reportedLanguage: unknown = transcript.result?.language;
|
||||
const detectedLanguage =
|
||||
automaticLanguage && typeof reportedLanguage === "string" && reportedLanguage.length > 0
|
||||
picked ??
|
||||
(automaticLanguage && typeof reportedLanguage === "string" && reportedLanguage.length > 0
|
||||
? reportedLanguage
|
||||
: null;
|
||||
: null);
|
||||
const segments = transcript.transcription ?? [];
|
||||
|
||||
let wordCount = 0;
|
||||
@@ -565,10 +592,6 @@ export async function transcribe(
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Detect speech onset before cleaning up the WAV
|
||||
options?.onProgress?.("Detecting speech onset...");
|
||||
const speechOnsetSeconds = detectSpeechOnset(wavPath);
|
||||
|
||||
// Clean up temp WAV if we created one
|
||||
if (wavPath !== inputPath) {
|
||||
try {
|
||||
@@ -614,6 +637,53 @@ function segmentWords(line: string): { words: Word[]; end: number } | null {
|
||||
}
|
||||
|
||||
/** Resolves once whisper exits and both its streams are read, so no printed line is lost. */
|
||||
/** The language up to three windows spread over the speech agree on; null keeps whisper's own pick, which hears
|
||||
* only the first 30 s (a music intro decided whole clips). A failed detection keeps it too, said aloud. */
|
||||
async function languageFromWindows(
|
||||
executable: string,
|
||||
{ model, modelPath }: { model: string; modelPath: string },
|
||||
wavPath: string,
|
||||
clip: {
|
||||
seconds: number | null;
|
||||
onset: number | null;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (message: string) => void;
|
||||
},
|
||||
): Promise<string | null> {
|
||||
const starts = clip.seconds === null ? [] : detectionWindows(clip.seconds, clip.onset);
|
||||
if (starts.length === 0) return null;
|
||||
clip.onProgress?.("Detecting the language...");
|
||||
const dir = mkdtempSync(join(tmpdir(), "hyperframes-language-"));
|
||||
try {
|
||||
const votes: LanguageVote[] = [];
|
||||
for (const [i, start] of starts.entries()) {
|
||||
const window = join(dir, `${i}.wav`);
|
||||
writeWavWindow(wavPath, window, start, WINDOW_SECONDS);
|
||||
await runWhisper(
|
||||
executable,
|
||||
["--model", modelPath, "--language", "auto", "--detect-language", window],
|
||||
{
|
||||
timeoutMs: resolveWhisperTimeoutMs(WINDOW_SECONDS, { model }),
|
||||
signal: clip.signal,
|
||||
onStderr: (line) => {
|
||||
const vote = parseDetection(line);
|
||||
if (vote) votes.push(vote);
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
return pickLanguage(votes);
|
||||
} catch (err) {
|
||||
if (clip.signal?.aborted) throw err;
|
||||
clip.onProgress?.(
|
||||
`Language detection failed (${normalizeErrorMessage(err)}); whisper picks it.`,
|
||||
);
|
||||
return null;
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function runWhisper(
|
||||
executable: string,
|
||||
args: string[],
|
||||
|
||||
@@ -1,21 +1,10 @@
|
||||
import { wavHeader } from "./wav.js";
|
||||
|
||||
/** A 16-bit mono PCM WAV of `samples` (-1..1), as prepareWav writes one. */
|
||||
export function encodeWav(samples: ArrayLike<number>, sampleRate: number): Buffer {
|
||||
const data = Buffer.alloc(samples.length * 2);
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
data.writeInt16LE(Math.max(-32768, Math.min(32767, Math.round(samples[i]! * 32768))), 2 * i);
|
||||
}
|
||||
const head = Buffer.alloc(44);
|
||||
head.write("RIFF", 0, "ascii");
|
||||
head.writeUInt32LE(36 + data.length, 4);
|
||||
head.write("WAVEfmt ", 8, "ascii");
|
||||
head.writeUInt32LE(16, 16);
|
||||
head.writeUInt16LE(1, 20);
|
||||
head.writeUInt16LE(1, 22);
|
||||
head.writeUInt32LE(sampleRate, 24);
|
||||
head.writeUInt32LE(sampleRate * 2, 28);
|
||||
head.writeUInt16LE(2, 32);
|
||||
head.writeUInt16LE(16, 34);
|
||||
head.write("data", 36, "ascii");
|
||||
head.writeUInt32LE(data.length, 40);
|
||||
return Buffer.concat([head, data]);
|
||||
return Buffer.concat([wavHeader(data.length, sampleRate), data]);
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, expect, it } from "vitest";
|
||||
import { readWav } from "./wav.js";
|
||||
import { readWav, writeWavWindow } from "./wav.js";
|
||||
import { encodeWav } from "./wav.test-helpers.js";
|
||||
|
||||
let dir: string;
|
||||
@@ -33,3 +33,13 @@ it("names a file that is not a 16-bit mono WAV", () => {
|
||||
);
|
||||
expect(() => readWav(file("stereo.wav", stereo))).toThrow("is not a 16-bit mono PCM WAV");
|
||||
});
|
||||
|
||||
it("writes a window of a prepared WAV as a WAV of its own", () => {
|
||||
const second = (n: number) => new Array(16_000).fill(n / 32768);
|
||||
const source = file("in.wav", encodeWav([...second(1), ...second(2), ...second(3)], 16_000));
|
||||
writeWavWindow(source, join(dir, "out.wav"), 1, 1);
|
||||
const window = readWav(join(dir, "out.wav"));
|
||||
expect(window.sampleRate).toBe(16_000);
|
||||
expect(window.samples).toHaveLength(16_000);
|
||||
expect(new Set(window.samples)).toEqual(new Set([2 / 32768]));
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
|
||||
/** Where a RIFF chunk's bytes start and how many there are, or null when the file has none. */
|
||||
export function findWavChunk(buf: Buffer, want: string): { offset: number; size: number } | null {
|
||||
@@ -38,3 +38,31 @@ export function readWav(path: string): { samples: Float32Array; sampleRate: numb
|
||||
samples[i] = buf.readInt16LE(data.offset + 2 * i) / 32768;
|
||||
return { samples, sampleRate: buf.readUInt32LE(fmt.offset + 4) };
|
||||
}
|
||||
|
||||
/** The 44-byte header of a 16-bit mono PCM WAV holding `bytes` of samples. */
|
||||
export function wavHeader(bytes: number, sampleRate: number): Buffer {
|
||||
const head = Buffer.alloc(44);
|
||||
head.write("RIFF", 0, "ascii");
|
||||
head.writeUInt32LE(36 + bytes, 4);
|
||||
head.write("WAVEfmt ", 8, "ascii");
|
||||
head.writeUInt32LE(16, 16);
|
||||
head.writeUInt16LE(PCM, 20);
|
||||
head.writeUInt16LE(1, 22);
|
||||
head.writeUInt32LE(sampleRate, 24);
|
||||
head.writeUInt32LE(sampleRate * 2, 28);
|
||||
head.writeUInt16LE(2, 32);
|
||||
head.writeUInt16LE(16, 34);
|
||||
head.write("data", 36, "ascii");
|
||||
head.writeUInt32LE(bytes, 40);
|
||||
return head;
|
||||
}
|
||||
|
||||
/** `seconds` of a prepared WAV (16 kHz mono s16) from `start`, written as a WAV of its own. */
|
||||
export function writeWavWindow(source: string, dest: string, start: number, seconds: number): void {
|
||||
const buf = readFileSync(source);
|
||||
const data = findWavChunk(buf, "data");
|
||||
if (!data) throw new Error(`${source} has no WAV data`);
|
||||
const from = data.offset + Math.floor(start * 16_000) * 2;
|
||||
const pcm = buf.subarray(from, Math.min(from + seconds * 32_000, data.offset + data.size));
|
||||
writeFileSync(dest, Buffer.concat([wavHeader(pcm.length, 16_000), pcm]));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user