diff --git a/packages/cli/src/whisper/parakeet.test.ts b/packages/cli/src/whisper/parakeet.test.ts index d6bc5c94df..ae27ae8361 100644 --- a/packages/cli/src/whisper/parakeet.test.ts +++ b/packages/cli/src/whisper/parakeet.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { mergeTokensToWords, mergeWindowsToWords, silenceCuts } from "./parakeet.js"; +import { + hasDroppedSpeech, + mergeTokensToWords, + mergeWindowsToWords, + silenceCuts, +} from "./parakeet.js"; describe("mergeTokensToWords", () => { it("joins Parakeet sub-word tokens into words on the space boundary", () => { @@ -64,4 +69,34 @@ describe("silenceCuts", () => { it("leaves audio shorter than one window whole", () => { expect(silenceCuts(loud(42), rate)).toEqual([0, 4200]); }); + + it("makes no window for empty audio", () => { + expect(silenceCuts(new Float32Array(0), rate)).toEqual([0]); + }); +}); + +describe("hasDroppedSpeech", () => { + const rate = 100; + const tokens = (timestamps: number[]) => ({ + tokens: timestamps.map(() => " w"), + timestamps, + durations: timestamps.map(() => 0.2), + }); + + it("flags a second of loud audio before the first token", () => { + const speech = new Float32Array(6 * rate).fill(0.5); + expect(hasDroppedSpeech(speech, rate, tokens([2.08, 2.5, 3, 3.5, 4, 4.5, 5, 5.5]))).toBe(true); + expect( + hasDroppedSpeech(speech, rate, tokens([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5])), + ).toBe(false); + }); + + it("ignores a quiet pause and silence with no tokens", () => { + const pause = new Float32Array(6 * rate).fill(0.5); + pause.fill(0, 2 * rate, 4 * rate); + expect(hasDroppedSpeech(pause, rate, tokens([0.1, 0.6, 1.1, 1.6, 4.1, 4.6, 5.1, 5.6]))).toBe( + false, + ); + expect(hasDroppedSpeech(new Float32Array(3 * rate), rate, tokens([]))).toBe(false); + }); }); diff --git a/packages/cli/src/whisper/parakeet.ts b/packages/cli/src/whisper/parakeet.ts index 71ba50cf0e..e1321517a6 100644 --- a/packages/cli/src/whisper/parakeet.ts +++ b/packages/cli/src/whisper/parakeet.ts @@ -130,12 +130,37 @@ export function mergeWindowsToWords(windows: SherpaWindow[]): Word[] { }); } +function meanEnergy(samples: Float32Array, from: number, to: number): number { + let energy = 0; + for (let i = from; i < to; i++) energy += samples[i]! * samples[i]!; + return energy / Math.max(1, to - from); +} + +/** + * True when a stretch of at least 1 s without tokens is about as loud as the window: the model + * skipped speech there. Measured: a 61.91 s window lost its first 9 words this way (ratio 0.94). + */ +export function hasDroppedSpeech( + samples: Float32Array, + sampleRate: number, + decoded: Omit, +): boolean { + const loud = 0.3 * meanEnergy(samples, 0, samples.length); + const { timestamps, durations } = decoded; + const gapStarts = [0, ...timestamps.map((t, i) => t + (durations?.[i] ?? 0))]; + const gapEnds = [...timestamps, samples.length / sampleRate]; + return gapStarts.some((from, i) => { + const to = gapEnds[i]!; + const at = (seconds: number) => Math.round(seconds * sampleRate); + return to - from >= 1 && meanEnergy(samples, at(from), at(to)) > loud; + }); +} + function quietestBlockCenter(samples: Float32Array, lo: number, hi: number, block: number): number { let best = hi; let bestEnergy = Infinity; for (let s = lo; s + block <= hi; s += block) { - let energy = 0; - for (let i = s; i < s + block; i++) energy += samples[i]! * samples[i]!; + const energy = meanEnergy(samples, s, s + block); if (energy < bestEnergy) { bestEnergy = energy; best = s + block / 2; @@ -162,7 +187,7 @@ export function silenceCuts( const hi = Math.min(target + slack, samples.length); cuts.push(quietestBlockCenter(samples, target - slack, hi, sampleRate / 10)); } - cuts.push(samples.length); + if (samples.length > 0) cuts.push(samples.length); return cuts; } diff --git a/packages/cli/src/whisper/sherpaWorker.test.ts b/packages/cli/src/whisper/sherpaWorker.test.ts new file mode 100644 index 0000000000..051b96b81f --- /dev/null +++ b/packages/cli/src/whisper/sherpaWorker.test.ts @@ -0,0 +1,78 @@ +import { execFile } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX } from "./parakeet.js"; + +const WORKER = fileURLToPath(new URL("./sherpaWorker.ts", import.meta.url)); + +// A stand-in sherpa-onnx-node: 3 s of steady sound. Unpadded, the recognizer "drops" the first +// 2 s the way the real model did on a 61.91 s window; with leading silence it hears everything. +const FAKE_SHERPA = ` +module.exports = { + readWave(path) { + if (path.endsWith("broken.wav")) throw new Error("Failed to read " + path); + return { sampleRate: 100, samples: new Float32Array(300).fill(0.5) }; + }, + OfflineRecognizer: class { + createStream() { return { acceptWaveform(w) { this.w = w; } }; } + decode() {} + getResult(stream) { + const padded = stream.w.samples[0] === 0; + return padded + ? { tokens: [" ask", " not"], timestamps: [0.75, 1.5], durations: [0.4, 0.4] } + : { tokens: [" not"], timestamps: [2.08], durations: [0.4] }; + } + }, +}; +`; + +function runWorker(runtimeDir: string, wavPath: string) { + const input = JSON.stringify({ wavPath, runtimeDir, config: {} }); + return new Promise<{ code: number | null; stdout: string; stderr: string }>((resolve) => { + execFile( + process.execPath, + ["--import", "tsx", WORKER], + { env: { ...process.env, HYPERFRAMES_PARAKEET_INPUT: input } }, + (err, stdout, stderr) => resolve({ code: err ? (err.code as number) : 0, stdout, stderr }), + ); + }); +} + +describe("sherpaWorker", () => { + let root: string; + afterEach(() => rmSync(root, { recursive: true, force: true })); + + function fakeRuntime(): string { + root = mkdtempSync(join(tmpdir(), "hf-sherpa-worker-")); + const pkg = join(root, "node_modules", "sherpa-onnx-node"); + mkdirSync(pkg, { recursive: true }); + writeFileSync(join(pkg, "package.json"), '{"name":"sherpa-onnx-node","main":"index.js"}'); + writeFileSync(join(pkg, "index.js"), FAKE_SHERPA); + return root; + } + + it("re-decodes a window that skipped loud audio, with leading silence", async () => { + const { code, stdout } = await runWorker(fakeRuntime(), "speech.wav"); + expect(code).toBe(0); + const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX))!; + expect(JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length))).toEqual([ + { offset: 0, tokens: [" ask", " not"], timestamps: [0.25, 1], durations: [0.4, 0.4] }, + ]); + }); + + it("exits non-zero with one prefixed error line when it cannot read the audio", async () => { + const { code, stderr } = await runWorker(fakeRuntime(), "broken.wav"); + expect(code).toBe(1); + expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}Failed to read broken.wav`); + }); + + it("names the missing runtime when sherpa-onnx-node is not installed", async () => { + root = mkdtempSync(join(tmpdir(), "hf-sherpa-worker-")); + const { code, stderr } = await runWorker(root, "speech.wav"); + expect(code).toBe(1); + expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}sherpa-onnx-node is not installed in ${root}`); + }); +}); diff --git a/packages/cli/src/whisper/sherpaWorker.ts b/packages/cli/src/whisper/sherpaWorker.ts index 93c4881040..aab053b5e2 100644 --- a/packages/cli/src/whisper/sherpaWorker.ts +++ b/packages/cli/src/whisper/sherpaWorker.ts @@ -1,5 +1,6 @@ import { loadInstalled } from "../utils/optionalPackages.js"; import { + hasDroppedSpeech, silenceCuts, SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX, @@ -22,6 +23,24 @@ interface SherpaOnnx { const { wavPath, runtimeDir, config } = JSON.parse(process.env.HYPERFRAMES_PARAKEET_INPUT ?? "{}"); +/** Leading silence moves the frame grid; 0.5 s recovered the dropped clause at every length tried. */ +const RETRY_PAD_SECONDS = 0.5; + +function decodeWindow( + recognizer: InstanceType, + wave: Wave, + padSeconds = 0, +): Omit { + const pad = Math.round(padSeconds * wave.sampleRate); + const samples = new Float32Array(pad + wave.samples.length); + samples.set(wave.samples, pad); + const stream = recognizer.createStream(); + stream.acceptWaveform({ sampleRate: wave.sampleRate, samples }); + recognizer.decode(stream); + const { tokens, timestamps, durations } = recognizer.getResult(stream); + return { tokens, timestamps: timestamps.map((t) => Math.max(0, t - padSeconds)), durations }; +} + try { const sherpa = loadInstalled(runtimeDir, "sherpa-onnx-node") as SherpaOnnx | null; if (!sherpa) throw new Error(`sherpa-onnx-node is not installed in ${runtimeDir}`); @@ -30,12 +49,16 @@ try { const cuts = silenceCuts(wave.samples, wave.sampleRate); const windows: SherpaWindow[] = []; for (let k = 0; k + 1 < cuts.length; k++) { - const stream = recognizer.createStream(); - const samples = wave.samples.subarray(cuts[k], cuts[k + 1]); - stream.acceptWaveform({ sampleRate: wave.sampleRate, samples }); - recognizer.decode(stream); - const { tokens, timestamps, durations } = recognizer.getResult(stream); - windows.push({ offset: cuts[k]! / wave.sampleRate, tokens, timestamps, durations }); + const slice = { + sampleRate: wave.sampleRate, + samples: wave.samples.subarray(cuts[k], cuts[k + 1]), + }; + let decoded = decodeWindow(recognizer, slice); + if (hasDroppedSpeech(slice.samples, slice.sampleRate, decoded)) { + const retry = decodeWindow(recognizer, slice, RETRY_PAD_SECONDS); + if (retry.tokens.length > decoded.tokens.length) decoded = retry; + } + windows.push({ offset: cuts[k]! / wave.sampleRate, ...decoded }); } process.stdout.write(`${SHERPA_RESULT_PREFIX}${JSON.stringify(windows)}\n`); } catch (err) {