fix(cli): re-decode a Parakeet window that skipped the start of its speech

At some window lengths the model silently drops the first clause: on a
looped speech clip, 4 of 167 lengths between 55 and 60 s, and a 61.91 s
window, lost their first 9 words. A window with a token gap of at least 1 s
that is as loud as speech is decoded again with 0.5 s of leading silence,
and the result with more tokens wins. Clean windows are unchanged.

Decoding with 1.5 s of overlap before each cut was measured too and was
worse on 573 s of audio (2.62% WER against 0.98%), so it is not used.

Empty audio no longer makes a zero-length window, which aborted the
decoder.

packages/cli/src/whisper/parakeet.test.ts

@@ -1,5 +1,10 @@
 import { describe, expect, it } from "vitest";
-import { mergeTokensToWords, mergeWindowsToWords, silenceCuts } from "./parakeet.js";
+import {
+  hasDroppedSpeech,
+  mergeTokensToWords,
+  mergeWindowsToWords,
+  silenceCuts,
+} from "./parakeet.js";
 
 describe("mergeTokensToWords", () => {
   it("joins Parakeet sub-word tokens into words on the space boundary", () => {
@@ -64,4 +69,34 @@ describe("silenceCuts", () => {
   it("leaves audio shorter than one window whole", () => {
     expect(silenceCuts(loud(42), rate)).toEqual([0, 4200]);
   });
+
+  it("makes no window for empty audio", () => {
+    expect(silenceCuts(new Float32Array(0), rate)).toEqual([0]);
+  });
+});
+
+describe("hasDroppedSpeech", () => {
+  const rate = 100;
+  const tokens = (timestamps: number[]) => ({
+    tokens: timestamps.map(() => " w"),
+    timestamps,
+    durations: timestamps.map(() => 0.2),
+  });
+
+  it("flags a second of loud audio before the first token", () => {
+    const speech = new Float32Array(6 * rate).fill(0.5);
+    expect(hasDroppedSpeech(speech, rate, tokens([2.08, 2.5, 3, 3.5, 4, 4.5, 5, 5.5]))).toBe(true);
+    expect(
+      hasDroppedSpeech(speech, rate, tokens([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5])),
+    ).toBe(false);
+  });
+
+  it("ignores a quiet pause and silence with no tokens", () => {
+    const pause = new Float32Array(6 * rate).fill(0.5);
+    pause.fill(0, 2 * rate, 4 * rate);
+    expect(hasDroppedSpeech(pause, rate, tokens([0.1, 0.6, 1.1, 1.6, 4.1, 4.6, 5.1, 5.6]))).toBe(
+      false,
+    );
+    expect(hasDroppedSpeech(new Float32Array(3 * rate), rate, tokens([]))).toBe(false);
+  });
 });

packages/cli/src/whisper/parakeet.ts

@@ -130,12 +130,37 @@ export function mergeWindowsToWords(windows: SherpaWindow[]): Word[] {
   });
 }
 
+function meanEnergy(samples: Float32Array, from: number, to: number): number {
+  let energy = 0;
+  for (let i = from; i < to; i++) energy += samples[i]! * samples[i]!;
+  return energy / Math.max(1, to - from);
+}
+
+/**
+ * True when a stretch of at least 1 s without tokens is about as loud as the window: the model
+ * skipped speech there. Measured: a 61.91 s window lost its first 9 words this way (ratio 0.94).
+ */
+export function hasDroppedSpeech(
+  samples: Float32Array,
+  sampleRate: number,
+  decoded: Omit<SherpaWindow, "offset">,
+): boolean {
+  const loud = 0.3 * meanEnergy(samples, 0, samples.length);
+  const { timestamps, durations } = decoded;
+  const gapStarts = [0, ...timestamps.map((t, i) => t + (durations?.[i] ?? 0))];
+  const gapEnds = [...timestamps, samples.length / sampleRate];
+  return gapStarts.some((from, i) => {
+    const to = gapEnds[i]!;
+    const at = (seconds: number) => Math.round(seconds * sampleRate);
+    return to - from >= 1 && meanEnergy(samples, at(from), at(to)) > loud;
+  });
+}
+
 function quietestBlockCenter(samples: Float32Array, lo: number, hi: number, block: number): number {
   let best = hi;
   let bestEnergy = Infinity;
   for (let s = lo; s + block <= hi; s += block) {
-    let energy = 0;
-    for (let i = s; i < s + block; i++) energy += samples[i]! * samples[i]!;
+    const energy = meanEnergy(samples, s, s + block);
     if (energy < bestEnergy) {
       bestEnergy = energy;
       best = s + block / 2;
@@ -162,7 +187,7 @@ export function silenceCuts(
     const hi = Math.min(target + slack, samples.length);
     cuts.push(quietestBlockCenter(samples, target - slack, hi, sampleRate / 10));
   }
-  cuts.push(samples.length);
+  if (samples.length > 0) cuts.push(samples.length);
   return cuts;
 }
 

packages/cli/src/whisper/sherpaWorker.test.ts

@@ -0,0 +1,78 @@
+import { execFile } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { afterEach, describe, expect, it } from "vitest";
+import { SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX } from "./parakeet.js";
+
+const WORKER = fileURLToPath(new URL("./sherpaWorker.ts", import.meta.url));
+
+// A stand-in sherpa-onnx-node: 3 s of steady sound. Unpadded, the recognizer "drops" the first
+// 2 s the way the real model did on a 61.91 s window; with leading silence it hears everything.
+const FAKE_SHERPA = `
+module.exports = {
+  readWave(path) {
+    if (path.endsWith("broken.wav")) throw new Error("Failed to read " + path);
+    return { sampleRate: 100, samples: new Float32Array(300).fill(0.5) };
+  },
+  OfflineRecognizer: class {
+    createStream() { return { acceptWaveform(w) { this.w = w; } }; }
+    decode() {}
+    getResult(stream) {
+      const padded = stream.w.samples[0] === 0;
+      return padded
+        ? { tokens: [" ask", " not"], timestamps: [0.75, 1.5], durations: [0.4, 0.4] }
+        : { tokens: [" not"], timestamps: [2.08], durations: [0.4] };
+    }
+  },
+};
+`;
+
+function runWorker(runtimeDir: string, wavPath: string) {
+  const input = JSON.stringify({ wavPath, runtimeDir, config: {} });
+  return new Promise<{ code: number | null; stdout: string; stderr: string }>((resolve) => {
+    execFile(
+      process.execPath,
+      ["--import", "tsx", WORKER],
+      { env: { ...process.env, HYPERFRAMES_PARAKEET_INPUT: input } },
+      (err, stdout, stderr) => resolve({ code: err ? (err.code as number) : 0, stdout, stderr }),
+    );
+  });
+}
+
+describe("sherpaWorker", () => {
+  let root: string;
+  afterEach(() => rmSync(root, { recursive: true, force: true }));
+
+  function fakeRuntime(): string {
+    root = mkdtempSync(join(tmpdir(), "hf-sherpa-worker-"));
+    const pkg = join(root, "node_modules", "sherpa-onnx-node");
+    mkdirSync(pkg, { recursive: true });
+    writeFileSync(join(pkg, "package.json"), '{"name":"sherpa-onnx-node","main":"index.js"}');
+    writeFileSync(join(pkg, "index.js"), FAKE_SHERPA);
+    return root;
+  }
+
+  it("re-decodes a window that skipped loud audio, with leading silence", async () => {
+    const { code, stdout } = await runWorker(fakeRuntime(), "speech.wav");
+    expect(code).toBe(0);
+    const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX))!;
+    expect(JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length))).toEqual([
+      { offset: 0, tokens: [" ask", " not"], timestamps: [0.25, 1], durations: [0.4, 0.4] },
+    ]);
+  });
+
+  it("exits non-zero with one prefixed error line when it cannot read the audio", async () => {
+    const { code, stderr } = await runWorker(fakeRuntime(), "broken.wav");
+    expect(code).toBe(1);
+    expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}Failed to read broken.wav`);
+  });
+
+  it("names the missing runtime when sherpa-onnx-node is not installed", async () => {
+    root = mkdtempSync(join(tmpdir(), "hf-sherpa-worker-"));
+    const { code, stderr } = await runWorker(root, "speech.wav");
+    expect(code).toBe(1);
+    expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}sherpa-onnx-node is not installed in ${root}`);
+  });
+});

packages/cli/src/whisper/sherpaWorker.ts

@@ -1,5 +1,6 @@
 import { loadInstalled } from "../utils/optionalPackages.js";
 import {
+  hasDroppedSpeech,
   silenceCuts,
   SHERPA_ERROR_PREFIX,
   SHERPA_RESULT_PREFIX,
@@ -22,6 +23,24 @@ interface SherpaOnnx {
 
 const { wavPath, runtimeDir, config } = JSON.parse(process.env.HYPERFRAMES_PARAKEET_INPUT ?? "{}");
 
+/** Leading silence moves the frame grid; 0.5 s recovered the dropped clause at every length tried. */
+const RETRY_PAD_SECONDS = 0.5;
+
+function decodeWindow(
+  recognizer: InstanceType<SherpaOnnx["OfflineRecognizer"]>,
+  wave: Wave,
+  padSeconds = 0,
+): Omit<SherpaWindow, "offset"> {
+  const pad = Math.round(padSeconds * wave.sampleRate);
+  const samples = new Float32Array(pad + wave.samples.length);
+  samples.set(wave.samples, pad);
+  const stream = recognizer.createStream();
+  stream.acceptWaveform({ sampleRate: wave.sampleRate, samples });
+  recognizer.decode(stream);
+  const { tokens, timestamps, durations } = recognizer.getResult(stream);
+  return { tokens, timestamps: timestamps.map((t) => Math.max(0, t - padSeconds)), durations };
+}
+
 try {
   const sherpa = loadInstalled(runtimeDir, "sherpa-onnx-node") as SherpaOnnx | null;
   if (!sherpa) throw new Error(`sherpa-onnx-node is not installed in ${runtimeDir}`);
@@ -30,12 +49,16 @@ try {
   const cuts = silenceCuts(wave.samples, wave.sampleRate);
   const windows: SherpaWindow[] = [];
   for (let k = 0; k + 1 < cuts.length; k++) {
-    const stream = recognizer.createStream();
-    const samples = wave.samples.subarray(cuts[k], cuts[k + 1]);
-    stream.acceptWaveform({ sampleRate: wave.sampleRate, samples });
-    recognizer.decode(stream);
-    const { tokens, timestamps, durations } = recognizer.getResult(stream);
-    windows.push({ offset: cuts[k]! / wave.sampleRate, tokens, timestamps, durations });
+    const slice = {
+      sampleRate: wave.sampleRate,
+      samples: wave.samples.subarray(cuts[k], cuts[k + 1]),
+    };
+    let decoded = decodeWindow(recognizer, slice);
+    if (hasDroppedSpeech(slice.samples, slice.sampleRate, decoded)) {
+      const retry = decodeWindow(recognizer, slice, RETRY_PAD_SECONDS);
+      if (retry.tokens.length > decoded.tokens.length) decoded = retry;
+    }
+    windows.push({ offset: cuts[k]! / wave.sampleRate, ...decoded });
   }
   process.stdout.write(`${SHERPA_RESULT_PREFIX}${JSON.stringify(windows)}\n`);
 } catch (err) {