diff --git a/CLAUDE.md b/CLAUDE.md index d71f20cd3..837fae642 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -95,13 +95,13 @@ Default is `small.en`. Upgrade for better accuracy: | Model | Size | Use case | | ----------- | ------ | -------------------------------- | -| `tiny.en` | 75 MB | Quick testing | -| `base.en` | 142 MB | Short clips, clear audio | -| `small.en` | 466 MB | **Default** — most content | -| `medium.en` | 1.5 GB | Important content, noisy audio | -| `large-v3` | 3.1 GB | Multilingual, production quality | +| `tiny` | 75 MB | Quick testing | +| `base` | 142 MB | Short clips, clear audio | +| `small` | 466 MB | **Default** — most content | +| `medium` | 1.5 GB | Important content, noisy audio | +| `large-v3` | 3.1 GB | Production quality | -Use `.en` suffix for English-only (more accurate). Drop it for multilingual content. +**Only use `.en` suffix when you know the audio is English.** `.en` models translate non-English audio into English instead of transcribing it. ### Supported transcript formats diff --git a/packages/cli/src/commands/transcribe.ts b/packages/cli/src/commands/transcribe.ts index c10aa34b4..ff12f15cb 100644 --- a/packages/cli/src/commands/transcribe.ts +++ b/packages/cli/src/commands/transcribe.ts @@ -103,7 +103,8 @@ async function transcribeAudio( opts: { model?: string; language?: string; json?: boolean }, ): Promise { const { transcribe } = await import("../whisper/transcribe.js"); - const { loadTranscript, patchCaptionHtml } = await import("../whisper/normalize.js"); + const { loadTranscript, patchCaptionHtml, stripBeforeOnset } = + await import("../whisper/normalize.js"); const model = opts.model ?? DEFAULT_MODEL; const spin = opts.json ? null : clack.spinner(); @@ -116,7 +117,19 @@ async function transcribeAudio( onProgress: spin ? (msg) => spin.message(msg) : undefined, }); - const { words } = loadTranscript(result.transcriptPath); + let { words } = loadTranscript(result.transcriptPath); + + if (result.speechOnsetSeconds != null) { + const before = words.length; + words = stripBeforeOnset(words, result.speechOnsetSeconds); + const stripped = before - words.length; + if (stripped > 0 && !opts.json) { + spin?.message( + `Stripped ${stripped} words before speech onset at ${result.speechOnsetSeconds.toFixed(1)}s`, + ); + } + } + writeFileSync(result.transcriptPath, JSON.stringify(words, null, 2)); patchCaptionHtml(dir, words); @@ -127,13 +140,18 @@ async function transcribeAudio( model, wordCount: words.length, durationSeconds: result.durationSeconds, + speechOnsetSeconds: result.speechOnsetSeconds, transcriptPath: result.transcriptPath, }), ); } else { - spin!.stop( + const onsetNote = + result.speechOnsetSeconds != null + ? ` — speech detected at ${result.speechOnsetSeconds.toFixed(1)}s` + : ""; + spin?.stop( c.success( - `Transcribed ${c.accent(String(words.length))} words (${result.durationSeconds.toFixed(1)}s)`, + `Transcribed ${c.accent(String(words.length))} words (${result.durationSeconds.toFixed(1)}s${onsetNote})`, ), ); } @@ -142,7 +160,7 @@ async function transcribeAudio( if (opts.json) { console.log(JSON.stringify({ ok: false, error: message })); } else { - spin!.stop(c.error(`Transcription failed: ${message}`)); + spin?.stop(c.error(`Transcription failed: ${message}`)); } process.exit(1); } diff --git a/packages/cli/src/whisper/normalize.test.ts b/packages/cli/src/whisper/normalize.test.ts index 8a6e78da3..64ce635e9 100644 --- a/packages/cli/src/whisper/normalize.test.ts +++ b/packages/cli/src/whisper/normalize.test.ts @@ -2,7 +2,8 @@ import { describe, it, expect, afterEach } from "vitest"; import { writeFileSync, readFileSync, mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import { loadTranscript, detectFormat, patchCaptionHtml } from "./normalize.js"; +import { loadTranscript, detectFormat, patchCaptionHtml, stripBeforeOnset } from "./normalize.js"; +import { detectSpeechOnset } from "./transcribe.js"; function tmpFile(name: string, content: string): string { const dir = join(tmpdir(), `hf-normalize-test-${Date.now()}`); @@ -117,7 +118,7 @@ describe("loadTranscript", () => { ); const { words } = loadTranscript(path); expect(words).toHaveLength(1); - expect(words[0]!.text).toBe("Hello"); + expect(words[0]?.text).toBe("Hello"); }); it("parses OpenAI Whisper API response", () => { @@ -183,8 +184,8 @@ Short format `; const path = tmpFile("short.vtt", vtt); const { words } = loadTranscript(path); - expect(words[0]!.start).toBeCloseTo(83.456, 2); - expect(words[0]!.end).toBe(120.0); + expect(words[0]?.start).toBeCloseTo(83.456, 2); + expect(words[0]?.end).toBe(120.0); }); it("strips HTML tags from SRT/VTT", () => { @@ -194,7 +195,7 @@ Short format `; const path = tmpFile("tags.srt", srt); const { words } = loadTranscript(path); - expect(words[0]!.text).toBe("Bold and italic"); + expect(words[0]?.text).toBe("Bold and italic"); }); it("passes through normalized word arrays", () => { @@ -209,6 +210,172 @@ Short format }); }); +describe("whisper-cpp contraction merging", () => { + it("merges didn + 't into didn't", () => { + const path = tmpFile( + "contractions.json", + JSON.stringify({ + transcription: [ + { + tokens: [ + { text: " I", offsets: { from: 0, to: 200 } }, + { text: " didn", offsets: { from: 200, to: 500 } }, + { text: "'t", offsets: { from: 500, to: 700 } }, + { text: " know", offsets: { from: 700, to: 1000 } }, + ], + }, + ], + }), + ); + const { words } = loadTranscript(path); + expect(words).toEqual([ + { text: "I", start: 0, end: 0.2 }, + { text: "didn't", start: 0.2, end: 0.7 }, + { text: "know", start: 0.7, end: 1 }, + ]); + }); + + it("merges I + 'm into I'm", () => { + const path = tmpFile( + "im.json", + JSON.stringify({ + transcription: [ + { + tokens: [ + { text: " I", offsets: { from: 0, to: 100 } }, + { text: "'m", offsets: { from: 100, to: 300 } }, + { text: " done", offsets: { from: 300, to: 600 } }, + ], + }, + ], + }), + ); + const { words } = loadTranscript(path); + expect(words[0]?.text).toBe("I'm"); + expect(words[0]?.end).toBe(0.3); + }); + + it("merges could + 've into could've", () => { + const path = tmpFile( + "couldve.json", + JSON.stringify({ + transcription: [ + { + tokens: [ + { text: " could", offsets: { from: 0, to: 400 } }, + { text: "'ve", offsets: { from: 400, to: 600 } }, + { text: " been", offsets: { from: 600, to: 900 } }, + ], + }, + ], + }), + ); + const { words } = loadTranscript(path); + expect(words[0]?.text).toBe("could've"); + }); +}); + +describe("whisper-cpp fragment merging", () => { + it("merges single capital + lowercase: C + aught -> Caught", () => { + const path = tmpFile( + "fragments.json", + JSON.stringify({ + transcription: [ + { + tokens: [ + { text: " C", offsets: { from: 0, to: 100 } }, + { text: "aught", offsets: { from: 100, to: 500 } }, + { text: " a", offsets: { from: 500, to: 600 } }, + ], + }, + ], + }), + ); + const { words } = loadTranscript(path); + expect(words[0]?.text).toBe("Caught"); + expect(words[0]?.end).toBe(0.5); + expect(words).toHaveLength(2); + }); + + it("merges consonant + in': shin + in' -> shinin'", () => { + const path = tmpFile( + "dropg.json", + JSON.stringify({ + transcription: [ + { + tokens: [ + { text: " shin", offsets: { from: 0, to: 300 } }, + { text: "in'", offsets: { from: 300, to: 500 } }, + ], + }, + ], + }), + ); + const { words } = loadTranscript(path); + expect(words).toHaveLength(1); + expect(words[0]?.text).toBe("shinin'"); + }); +}); + +describe("whisper-cpp zero-duration interpolation", () => { + it("interpolates a cluster of zero-duration words", () => { + const path = tmpFile( + "zerodur.json", + JSON.stringify({ + transcription: [ + { + tokens: [ + { text: " hello", offsets: { from: 0, to: 500 } }, + { text: " we", offsets: { from: 1000, to: 1000 } }, + { text: " are", offsets: { from: 1000, to: 1000 } }, + { text: " here", offsets: { from: 1000, to: 1000 } }, + { text: " now", offsets: { from: 1500, to: 2000 } }, + ], + }, + ], + }), + ); + const { words } = loadTranscript(path); + expect(words).toHaveLength(5); + // The three zero-duration words should be spread between 0.5 and 1.5 + const we = words[1] ?? { start: 0, end: 0, text: "" }; + const are = words[2] ?? { start: 0, end: 0, text: "" }; + const here = words[3] ?? { start: 0, end: 0, text: "" }; + expect(we.start).toBeCloseTo(0.5, 1); + expect(we.end).toBeCloseTo(0.833, 1); + expect(are.start).toBeCloseTo(0.833, 1); + expect(are.end).toBeCloseTo(1.167, 1); + expect(here.start).toBeCloseTo(1.167, 1); + expect(here.end).toBeCloseTo(1.5, 1); + // Each should have positive duration + expect(we.end).toBeGreaterThan(we.start); + expect(are.end).toBeGreaterThan(are.start); + expect(here.end).toBeGreaterThan(here.start); + }); + + it("handles isolated zero-duration word", () => { + const path = tmpFile( + "singlezero.json", + JSON.stringify({ + transcription: [ + { + tokens: [ + { text: " hello", offsets: { from: 0, to: 500 } }, + { text: " I", offsets: { from: 800, to: 800 } }, + { text: " know", offsets: { from: 1000, to: 1500 } }, + ], + }, + ], + }), + ); + const { words } = loadTranscript(path); + const iWord = words[1] ?? { start: 0, end: 0, text: "" }; + expect(iWord.end).toBeGreaterThan(iWord.start); + expect(iWord.start).toBeCloseTo(0.5, 1); + expect(iWord.end).toBeCloseTo(1, 1); + }); +}); + describe("patchCaptionHtml", () => { it("replaces const script = [] in HTML files", () => { const dir = join(tmpdir(), `hf-patch-test-${Date.now()}`); @@ -276,3 +443,105 @@ describe("patchCaptionHtml", () => { expect(result).toBe(html); }); }); + +describe("detectSpeechOnset", () => { + function makeSyntheticWav( + sampleRate: number, + durationSeconds: number, + energyFn: (t: number) => number, + ): string { + const numSamples = Math.floor(sampleRate * durationSeconds); + const dataSize = numSamples * 2; + const buf = Buffer.alloc(44 + dataSize); + // RIFF header + buf.write("RIFF", 0); + buf.writeUInt32LE(36 + dataSize, 4); + buf.write("WAVE", 8); + buf.write("fmt ", 12); + buf.writeUInt32LE(16, 16); // chunk size + buf.writeUInt16LE(1, 20); // PCM + buf.writeUInt16LE(1, 22); // mono + buf.writeUInt32LE(sampleRate, 24); + buf.writeUInt32LE(sampleRate * 2, 28); // byte rate + buf.writeUInt16LE(2, 32); // block align + buf.writeUInt16LE(16, 34); // bits per sample + buf.write("data", 36); + buf.writeUInt32LE(dataSize, 40); + for (let i = 0; i < numSamples; i++) { + const t = i / sampleRate; + const amplitude = energyFn(t); + buf.writeInt16LE(Math.round(amplitude * 32767), 44 + i * 2); + } + const path = join(tmpdir(), `hf-wav-test-${Date.now()}-${Math.floor(Math.random() * 1e6)}.wav`); + writeFileSync(path, buf); + dirs.push(path); + return path; + } + + it("detects onset when silence transitions to loud", () => { + const wavPath = makeSyntheticWav(16000, 15, (t) => (t < 5 ? 0.01 : 0.8)); + const onset = detectSpeechOnset(wavPath); + expect(onset).not.toBeNull(); + expect(onset!).toBeGreaterThanOrEqual(4); + expect(onset!).toBeLessThanOrEqual(7); + }); + + it("returns null for consistent energy throughout", () => { + const wavPath = makeSyntheticWav(16000, 10, () => 0.5); + const onset = detectSpeechOnset(wavPath); + expect(onset).toBeNull(); + }); + + it("returns null for very short audio", () => { + const wavPath = makeSyntheticWav(16000, 2, () => 0.5); + const onset = detectSpeechOnset(wavPath); + expect(onset).toBeNull(); + }); + + it("returns null when onset is too early (< 3s)", () => { + const wavPath = makeSyntheticWav(16000, 10, (t) => (t < 1 ? 0.01 : 0.8)); + const onset = detectSpeechOnset(wavPath); + expect(onset).toBeNull(); + }); +}); + +describe("stripBeforeOnset", () => { + it("removes words before onset time", () => { + const words = [ + { text: "ghost", start: 0.5, end: 2.0 }, + { text: "alone", start: 3.0, end: 5.0 }, + { text: "Given", start: 19.0, end: 19.5 }, + { text: "the", start: 19.5, end: 20.0 }, + ]; + const result = stripBeforeOnset(words, 18.5); + expect(result).toHaveLength(2); + expect(result[0]!.text).toBe("Given"); + }); + + it("keeps words within 0.5s tolerance of onset", () => { + const words = [ + { text: "hello", start: 18.2, end: 18.8 }, + { text: "world", start: 19.0, end: 19.5 }, + ]; + const result = stripBeforeOnset(words, 18.5); + expect(result).toHaveLength(2); + }); + + it("keeps everything when onset is 0", () => { + const words = [ + { text: "hello", start: 0.1, end: 0.5 }, + { text: "world", start: 0.6, end: 1.0 }, + ]; + const result = stripBeforeOnset(words, 0); + expect(result).toHaveLength(2); + }); + + it("returns empty array when all words are before onset", () => { + const words = [ + { text: "ghost", start: 0.5, end: 2.0 }, + { text: "alone", start: 3.0, end: 5.0 }, + ]; + const result = stripBeforeOnset(words, 20.0); + expect(result).toHaveLength(0); + }); +}); diff --git a/packages/cli/src/whisper/normalize.ts b/packages/cli/src/whisper/normalize.ts index 4975501a2..7b2d4fce9 100644 --- a/packages/cli/src/whisper/normalize.ts +++ b/packages/cli/src/whisper/normalize.ts @@ -43,6 +43,67 @@ function detectJsonFormat(raw: unknown): TranscriptFormat { // Parsers // --------------------------------------------------------------------------- +/** + * Rejoin word fragments that whisper splits across tokens: + * - Single capital + lowercase continuation: C + aught -> Caught, G + onna -> Gonna + * - Word ending in consonant + in': shin + in' -> shinin', hid + in' -> hidin' + */ +function mergeFragments(words: Word[]): void { + for (let i = 0; i < words.length - 1; i++) { + const curr = words[i]; + const next = words[i + 1]; + if (!curr || !next) continue; + const isSingleLetterFragment = + curr.text.length === 1 && + /^[A-Z]$/.test(curr.text) && + !/^[IAO]$/.test(curr.text) && + /^[a-z]/.test(next.text); + const shouldMerge = + isSingleLetterFragment || (/[a-z]$/.test(curr.text) && /^in'$/i.test(next.text)); + if (shouldMerge) { + curr.text += next.text; + curr.end = next.end; + words.splice(i + 1, 1); + i--; + } + } +} + +/** + * Distribute timestamps evenly across zero-duration word clusters. + * Whisper sometimes assigns identical start/end to sequences of words, + * making karaoke highlights flash through them instantly. + * + * Also handles malformed timestamps where start > end — these are treated + * the same as zero-duration and get interpolated from surrounding words. + */ +function interpolateZeroDuration(words: Word[]): void { + for (let i = 0; i < words.length; i++) { + const wi = words[i]; + if (!wi || wi.start < wi.end) continue; + let j = i; + while (j < words.length) { + const wj = words[j]; + if (!wj || wj.start < wj.end) break; + j++; + } + const clusterLen = j - i; + const prev = i > 0 ? words[i - 1] : undefined; + const prevEnd = prev ? prev.end : wi.start; + const nextWord = j < words.length ? words[j] : undefined; + const nextStart = nextWord ? nextWord.start : prevEnd + clusterLen * 0.3; + const span = nextStart - prevEnd; + const perWord = span / clusterLen; + for (let k = i; k < j; k++) { + const wk = words[k]; + if (!wk) continue; + wk.start = round3(prevEnd + (k - i) * perWord); + wk.end = round3(prevEnd + (k - i + 1) * perWord); + } + i = j - 1; + } +} + function parseWhisperCpp(data: Record): Word[] { const words: Word[] = []; const transcription = data.transcription as Array<{ @@ -54,13 +115,21 @@ function parseWhisperCpp(data: Record): Word[] { for (const seg of transcription ?? []) { for (const token of seg.tokens ?? []) { - const text = (token.text ?? "").trim(); + const rawText = token.text ?? ""; + const text = rawText.trim(); if (!text || text.startsWith("[_") || text.startsWith("[BLANK")) continue; - // Merge punctuation with the previous word - const isPunctuation = /^[.,!?;:'")\]}>…–—-]+$/.test(text); const lastWord = words[words.length - 1]; - if (isPunctuation && lastWord) { + + // Merge into previous word when the token is a sub-word continuation, + // trailing punctuation, or a contraction suffix. + // Whisper uses leading spaces to mark word boundaries in all languages. + const shouldMerge = + lastWord && + (!rawText.startsWith(" ") || + /^[.,!?;:'")\]}>…–—¡¿-]+$/.test(text) || + /^'(t|m|s|ve|re|ll|d)$/i.test(text)); + if (shouldMerge) { lastWord.text += text; lastWord.end = round3((token.offsets?.to ?? 0) / 1000); continue; @@ -73,6 +142,10 @@ function parseWhisperCpp(data: Record): Word[] { }); } } + + mergeFragments(words); + interpolateZeroDuration(words); + return words; } @@ -229,9 +302,15 @@ export function loadTranscript(filePath: string): { words: Word[]; format: Trans } /** - * Patch caption HTML files in a project directory with transcript words. - * Replaces `const script = [...]` or `const TRANSCRIPT = [...]` in