fix(cli): read the WAV in JS so Built-in transcription runs inside Electron (#5129)

* fix(cli): read the WAV in JS so Built-in transcription runs inside Electron

* fix(cli): read an empty WAV as silence

packages/cli/src/whisper/sherpa.test.ts

@@ -25,6 +25,7 @@ import {
   transcribeWithSherpa,
   type ModelFile,
 } from "./sherpa.js";
+import { encodeWav } from "./wav.test-helpers.js";
 
 const file = (name: string, content: string): ModelFile => ({
   name,
@@ -183,6 +184,7 @@ describe("a pinned Sherpa copy beside the CLI", () => {
   let root: string;
   beforeEach(() => {
     root = mkdtempSync(join(tmpdir(), "hf-sherpa-beside-"));
+    writeFileSync(join(root, "speech.wav"), encodeWav(new Float32Array(100).fill(0.5), 100));
   });
   afterEach(() => {
     vi.restoreAllMocks();
@@ -204,7 +206,7 @@ describe("a pinned Sherpa copy beside the CLI", () => {
     return `
 if (process.pid === ${process.pid}) require("node:fs").writeFileSync(${JSON.stringify(parentMarker)}, "");
 module.exports = {
-  readWave() { return { sampleRate: 100, samples: new Float32Array(100).fill(0.5) }; },
+  readWave() { throw new Error("External buffers are not allowed"); },
   OfflineRecognizer: class {
     createStream() { return { acceptWaveform() {} }; }
     decode() {}
@@ -228,7 +230,8 @@ module.exports = {
     const event = { type: "progress", phase: "transcription", model: "parakeet-tdt-0.6b-v3" };
     // The worker's one 1 s window streams its words before the final transcript.
     expect(onEvent.mock.calls).toEqual([
-      [{ ...event, status: "started", durationSeconds: null }],
+      // The started line sizes the WAV as 16 kHz mono: 44 header bytes and 100 samples here.
+      [{ ...event, status: "started", durationSeconds: (44 + 2 * 100) / 32_000 }],
       [{ type: "words", model: event.model, words: result, through: 1 }],
       [{ ...event, status: "completed" }],
     ]);

packages/cli/src/whisper/sherpaWorker.test.ts

@@ -5,23 +5,20 @@ import { join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { afterEach, describe, expect, it } from "vitest";
 import { SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX, SHERPA_WINDOW_PREFIX } from "./parakeet.js";
+import { encodeWav } from "./wav.test-helpers.js";
 
 const WORKER = fileURLToPath(new URL("./sherpaWorker.ts", import.meta.url));
 
 // Stand-in sherpa-onnx-node, 3 s of sound: the whole window (300 samples) drops the first 2 s; the
 // padded window (350) hears it, and for speech.wav so does the padded gap +-0.5 s (308) alone.
 const FAKE_SHERPA = `
-let path;
+const path = JSON.parse(process.env.HYPERFRAMES_PARAKEET_INPUT).wavPath;
 module.exports = {
-  readWave(p) {
-    path = p;
-    if (path.endsWith("broken.wav")) throw new Error("Failed to read " + path);
-    if (path.endsWith("lines.wav")) throw new Error("Could not find it. Tried\\n\\n  ../a.node\\n  ./b.node\\n");
-    if (path.endsWith("long.wav")) return { sampleRate: 100, samples: new Float32Array(60000) };
-    return { sampleRate: 100, samples: new Float32Array(300).fill(0.5) };
-  },
+  // What Electron's V8 sandbox throws for this reader's native buffer: the worker reads the WAV itself.
+  readWave() { throw new Error("External buffers are not allowed"); },
   OfflineRecognizer: class {
     constructor() {
+      if (path.endsWith("lines.wav")) throw new Error("Could not find it. Tried\\n\\n  ../a.node\\n  ./b.node\\n");
       if (process.env.STARTED) require("fs").writeFileSync(process.env.STARTED, "");
     }
     createStream() { return { acceptWaveform(w) { this.w = w; } }; }
@@ -69,13 +66,20 @@ describe("sherpaWorker", () => {
     return root;
   }
 
+  /** 3 s of steady sound at 100 Hz, as the stand-in decoder expects; long.wav is 600 s of silence. */
+  function wav(name: string): string {
+    const samples = name === "long.wav" ? new Float32Array(60000) : new Float32Array(300).fill(0.5);
+    writeFileSync(join(root, name), encodeWav(samples, 100));
+    return join(root, name);
+  }
+
   function windowsOf(stdout: string) {
     const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX))!;
     return JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length));
   }
 
   it("decodes a gap that skipped loud audio on its own and splices in its new tokens", async () => {
-    const { code, stdout } = await runWorker(fakeRuntime(), "speech.wav");
+    const { code, stdout } = await runWorker(fakeRuntime(), wav("speech.wav"));
     expect(code).toBe(0);
     expect(windowsOf(stdout)).toEqual([
       { offset: 0, tokens: [" ask", " not"], timestamps: [0.5, 2.08], durations: [0.4, 0.4] },
@@ -83,7 +87,7 @@ describe("sherpaWorker", () => {
   });
 
   it("prints each window as it is decoded, with the audio seconds done", async () => {
-    const { stdout } = await runWorker(fakeRuntime(), "speech.wav");
+    const { stdout } = await runWorker(fakeRuntime(), wav("speech.wav"));
     const streamed = stdout
       .split("\n")
       .filter((l) => l.startsWith(SHERPA_WINDOW_PREFIX))
@@ -92,7 +96,7 @@ describe("sherpaWorker", () => {
   });
 
   it("re-decodes the window with leading silence when the gap alone gives nothing", async () => {
-    const { code, stdout } = await runWorker(fakeRuntime(), "hum.wav");
+    const { code, stdout } = await runWorker(fakeRuntime(), wav("hum.wav"));
     expect(code).toBe(0);
     expect(windowsOf(stdout)).toEqual([
       { offset: 0, tokens: [" ask", " not"], timestamps: [0.25, 1], durations: [0.4, 0.4] },
@@ -100,9 +104,11 @@ describe("sherpaWorker", () => {
   });
 
   it("exits non-zero with one prefixed error line when it cannot read the audio", async () => {
-    const { code, stderr } = await runWorker(fakeRuntime(), "broken.wav");
+    const broken = join(fakeRuntime(), "broken.wav");
+    writeFileSync(broken, "not audio");
+    const { code, stderr } = await runWorker(root, broken);
     expect(code).toBe(1);
-    expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}Failed to read broken.wav`);
+    expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}${broken} is not a 16-bit mono PCM WAV`);
   });
 
   it("keeps every line of a multi-line error on its one prefixed line", async () => {
@@ -116,7 +122,7 @@ describe("sherpaWorker", () => {
     async () => {
       const started = join(fakeRuntime(), "started");
       const input = JSON.stringify({
-        wavPath: "long.wav",
+        wavPath: wav("long.wav"),
         runtimePath: join(root, "node_modules", "sherpa-onnx-node", "index.js"),
         config: {},
       });

packages/cli/src/whisper/sherpaWorker.ts

@@ -8,6 +8,7 @@ import {
   SHERPA_WINDOW_PREFIX,
   type SherpaWindow,
 } from "./parakeet.js";
+import { readWav } from "./wav.js";
 
 interface Wave {
   samples: Float32Array;
@@ -15,7 +16,6 @@ interface Wave {
 }
 
 interface SherpaOnnx {
-  readWave(path: string): Wave;
   OfflineRecognizer: new (config: object) => {
     createStream(): { acceptWaveform(wave: Wave): void };
     decode(stream: unknown): void;
@@ -49,7 +49,7 @@ function decodeWindow(
 try {
   const sherpa = createRequire(import.meta.url)(runtimePath) as SherpaOnnx;
   const recognizer = new sherpa.OfflineRecognizer(config);
-  const wave = sherpa.readWave(wavPath);
+  const wave = readWav(wavPath);
   const cuts = silenceCuts(wave.samples, wave.sampleRate);
   const windows: SherpaWindow[] = [];
   for (let k = 0; k + 1 < cuts.length; k++) {

packages/cli/src/whisper/transcribe.ts

@@ -10,22 +10,10 @@ import { randomUUID } from "node:crypto";
 import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
 import { stoppedByCancelSignal } from "../utils/renderCancellation.js";
 import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js";
+import { findWavChunk } from "./wav.js";
 import type { Word } from "./normalize.js";
 import { emitWords } from "./progress.js";
 
-function findWavDataChunk(buf: Buffer): { offset: number; size: number } | null {
-  if (buf.length < 12) return null;
-  let pos = 12; // skip RIFF header
-  while (pos + 8 < buf.length) {
-    const id = buf.toString("ascii", pos, pos + 4);
-    const size = buf.readUInt32LE(pos + 4);
-    if (id === "data") return { offset: pos + 8, size: Math.min(size, buf.length - pos - 8) };
-    pos += 8 + size;
-    if (size % 2 !== 0) pos++; // RIFF chunks are word-aligned
-  }
-  return null;
-}
-
 const WHISPER_TIMEOUT_FLOOR_MS = 300_000;
 const WHISPER_TIMEOUT_PER_AUDIO_SECOND_MS = 10_000;
 const WHISPER_TIMEOUT_CAP_MS = 43_200_000;
@@ -196,7 +184,7 @@ export function detectSpeechOnset(wavPath: string): number | null {
 
   try {
     const buf = readFileSync(wavPath);
-    const dataChunk = findWavDataChunk(buf);
+    const dataChunk = findWavChunk(buf, "data");
     if (!dataChunk) return null;
     const pcm = new Int16Array(buf.buffer, buf.byteOffset + dataChunk.offset, dataChunk.size / 2);
     const totalWindows = Math.floor(pcm.length / WINDOW_SAMPLES);

packages/cli/src/whisper/wav.test-helpers.ts

@@ -0,0 +1,21 @@
+/** A 16-bit mono PCM WAV of `samples` (-1..1), as prepareWav writes one. */
+export function encodeWav(samples: ArrayLike<number>, sampleRate: number): Buffer {
+  const data = Buffer.alloc(samples.length * 2);
+  for (let i = 0; i < samples.length; i++) {
+    data.writeInt16LE(Math.max(-32768, Math.min(32767, Math.round(samples[i]! * 32768))), 2 * i);
+  }
+  const head = Buffer.alloc(44);
+  head.write("RIFF", 0, "ascii");
+  head.writeUInt32LE(36 + data.length, 4);
+  head.write("WAVEfmt ", 8, "ascii");
+  head.writeUInt32LE(16, 16);
+  head.writeUInt16LE(1, 20);
+  head.writeUInt16LE(1, 22);
+  head.writeUInt32LE(sampleRate, 24);
+  head.writeUInt32LE(sampleRate * 2, 28);
+  head.writeUInt16LE(2, 32);
+  head.writeUInt16LE(16, 34);
+  head.write("data", 36, "ascii");
+  head.writeUInt32LE(data.length, 40);
+  return Buffer.concat([head, data]);
+}

packages/cli/src/whisper/wav.test.ts

@@ -0,0 +1,35 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, expect, it } from "vitest";
+import { readWav } from "./wav.js";
+import { encodeWav } from "./wav.test-helpers.js";
+
+let dir: string;
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), "hf-wav-"));
+});
+afterEach(() => rmSync(dir, { recursive: true, force: true }));
+const file = (name: string, bytes: Buffer) => {
+  writeFileSync(join(dir, name), bytes);
+  return join(dir, name);
+};
+
+it("reads a 16-bit mono WAV's samples and rate in JS", () => {
+  const wav = readWav(file("a.wav", encodeWav([0, 0.5, -0.5, -1], 16_000)));
+  expect(wav.sampleRate).toBe(16_000);
+  expect([...wav.samples]).toEqual([0, 0.5, -0.5, -1]);
+});
+
+it("reads a WAV with no samples as silence, as ffmpeg writes for empty audio", () => {
+  expect(readWav(file("empty.wav", encodeWav([], 16_000))).samples).toHaveLength(0);
+});
+
+it("names a file that is not a 16-bit mono WAV", () => {
+  const stereo = encodeWav([0, 0], 16_000);
+  stereo.writeUInt16LE(2, 22);
+  expect(() => readWav(file("text.wav", Buffer.from("not audio")))).toThrow(
+    "text.wav is not a 16-bit mono PCM WAV",
+  );
+  expect(() => readWav(file("stereo.wav", stereo))).toThrow("is not a 16-bit mono PCM WAV");
+});

packages/cli/src/whisper/wav.ts

@@ -0,0 +1,40 @@
+import { readFileSync } from "node:fs";
+
+/** Where a RIFF chunk's bytes start and how many there are, or null when the file has none. */
+export function findWavChunk(buf: Buffer, want: string): { offset: number; size: number } | null {
+  if (buf.length < 12) return null;
+  let pos = 12; // skip RIFF header
+  while (pos + 8 <= buf.length) {
+    const id = buf.toString("ascii", pos, pos + 4);
+    const size = buf.readUInt32LE(pos + 4);
+    if (id === want) return { offset: pos + 8, size: Math.min(size, buf.length - pos - 8) };
+    pos += 8 + size;
+    if (size % 2 !== 0) pos++; // RIFF chunks are word-aligned
+  }
+  return null;
+}
+
+const PCM = 1;
+const EXTENSIBLE = 0xfffe;
+
+/** A prepared 16-bit mono WAV, read in JS: sherpa's readWave returns memory Electron refuses. */
+export function readWav(path: string): { samples: Float32Array; sampleRate: number } {
+  const buf = readFileSync(path);
+  const riff = buf.toString("ascii", 0, 4) === "RIFF" && buf.toString("ascii", 8, 12) === "WAVE";
+  const fmt = riff ? findWavChunk(buf, "fmt ") : null;
+  const data = fmt && findWavChunk(buf, "data");
+  const format = fmt && buf.readUInt16LE(fmt.offset);
+  if (
+    !fmt ||
+    !data ||
+    (format !== PCM && format !== EXTENSIBLE) ||
+    buf.readUInt16LE(fmt.offset + 2) !== 1 ||
+    buf.readUInt16LE(fmt.offset + 14) !== 16
+  ) {
+    throw new Error(`${path} is not a 16-bit mono PCM WAV`);
+  }
+  const samples = new Float32Array(data.size >> 1);
+  for (let i = 0; i < samples.length; i++)
+    samples[i] = buf.readInt16LE(data.offset + 2 * i) / 32768;
+  return { samples, sampleRate: buf.readUInt32LE(fmt.offset + 4) };
+}