mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-01 19:42:03 +00:00
feat(whisper): language detection, speech onset, contraction merging
- Auto-detect language and switch from .en to multilingual model when needed - Detect speech onset in WAV to strip hallucinated words before speech begins - Merge whisper-cpp token fragments: contractions (didn + 't → didn't), split capitals (C + aught → Caught), dropped-g (shin + in' → shinin') - Interpolate zero-duration word clusters for reliable karaoke timing - Replace non-null assertions with optional chaining in tests Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Vance Ingalls
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
afa5021b02
commit
4b0ce61438
@@ -103,7 +103,8 @@ async function transcribeAudio(
|
||||
opts: { model?: string; language?: string; json?: boolean },
|
||||
): Promise<void> {
|
||||
const { transcribe } = await import("../whisper/transcribe.js");
|
||||
const { loadTranscript, patchCaptionHtml } = await import("../whisper/normalize.js");
|
||||
const { loadTranscript, patchCaptionHtml, stripBeforeOnset } =
|
||||
await import("../whisper/normalize.js");
|
||||
|
||||
const model = opts.model ?? DEFAULT_MODEL;
|
||||
const spin = opts.json ? null : clack.spinner();
|
||||
@@ -116,7 +117,19 @@ async function transcribeAudio(
|
||||
onProgress: spin ? (msg) => spin.message(msg) : undefined,
|
||||
});
|
||||
|
||||
const { words } = loadTranscript(result.transcriptPath);
|
||||
let { words } = loadTranscript(result.transcriptPath);
|
||||
|
||||
if (result.speechOnsetSeconds != null) {
|
||||
const before = words.length;
|
||||
words = stripBeforeOnset(words, result.speechOnsetSeconds);
|
||||
const stripped = before - words.length;
|
||||
if (stripped > 0 && !opts.json) {
|
||||
spin?.message(
|
||||
`Stripped ${stripped} words before speech onset at ${result.speechOnsetSeconds.toFixed(1)}s`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync(result.transcriptPath, JSON.stringify(words, null, 2));
|
||||
patchCaptionHtml(dir, words);
|
||||
|
||||
@@ -127,13 +140,18 @@ async function transcribeAudio(
|
||||
model,
|
||||
wordCount: words.length,
|
||||
durationSeconds: result.durationSeconds,
|
||||
speechOnsetSeconds: result.speechOnsetSeconds,
|
||||
transcriptPath: result.transcriptPath,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
spin!.stop(
|
||||
const onsetNote =
|
||||
result.speechOnsetSeconds != null
|
||||
? ` — speech detected at ${result.speechOnsetSeconds.toFixed(1)}s`
|
||||
: "";
|
||||
spin?.stop(
|
||||
c.success(
|
||||
`Transcribed ${c.accent(String(words.length))} words (${result.durationSeconds.toFixed(1)}s)`,
|
||||
`Transcribed ${c.accent(String(words.length))} words (${result.durationSeconds.toFixed(1)}s${onsetNote})`,
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -142,7 +160,7 @@ async function transcribeAudio(
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify({ ok: false, error: message }));
|
||||
} else {
|
||||
spin!.stop(c.error(`Transcription failed: ${message}`));
|
||||
spin?.stop(c.error(`Transcription failed: ${message}`));
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -2,7 +2,8 @@ import { describe, it, expect, afterEach } from "vitest";
|
||||
import { writeFileSync, readFileSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { loadTranscript, detectFormat, patchCaptionHtml } from "./normalize.js";
|
||||
import { loadTranscript, detectFormat, patchCaptionHtml, stripBeforeOnset } from "./normalize.js";
|
||||
import { detectSpeechOnset } from "./transcribe.js";
|
||||
|
||||
function tmpFile(name: string, content: string): string {
|
||||
const dir = join(tmpdir(), `hf-normalize-test-${Date.now()}`);
|
||||
@@ -117,7 +118,7 @@ describe("loadTranscript", () => {
|
||||
);
|
||||
const { words } = loadTranscript(path);
|
||||
expect(words).toHaveLength(1);
|
||||
expect(words[0]!.text).toBe("Hello");
|
||||
expect(words[0]?.text).toBe("Hello");
|
||||
});
|
||||
|
||||
it("parses OpenAI Whisper API response", () => {
|
||||
@@ -183,8 +184,8 @@ Short format
|
||||
`;
|
||||
const path = tmpFile("short.vtt", vtt);
|
||||
const { words } = loadTranscript(path);
|
||||
expect(words[0]!.start).toBeCloseTo(83.456, 2);
|
||||
expect(words[0]!.end).toBe(120.0);
|
||||
expect(words[0]?.start).toBeCloseTo(83.456, 2);
|
||||
expect(words[0]?.end).toBe(120.0);
|
||||
});
|
||||
|
||||
it("strips HTML tags from SRT/VTT", () => {
|
||||
@@ -194,7 +195,7 @@ Short format
|
||||
`;
|
||||
const path = tmpFile("tags.srt", srt);
|
||||
const { words } = loadTranscript(path);
|
||||
expect(words[0]!.text).toBe("Bold and italic");
|
||||
expect(words[0]?.text).toBe("Bold and italic");
|
||||
});
|
||||
|
||||
it("passes through normalized word arrays", () => {
|
||||
@@ -209,6 +210,172 @@ Short format
|
||||
});
|
||||
});
|
||||
|
||||
describe("whisper-cpp contraction merging", () => {
|
||||
it("merges didn + 't into didn't", () => {
|
||||
const path = tmpFile(
|
||||
"contractions.json",
|
||||
JSON.stringify({
|
||||
transcription: [
|
||||
{
|
||||
tokens: [
|
||||
{ text: " I", offsets: { from: 0, to: 200 } },
|
||||
{ text: " didn", offsets: { from: 200, to: 500 } },
|
||||
{ text: "'t", offsets: { from: 500, to: 700 } },
|
||||
{ text: " know", offsets: { from: 700, to: 1000 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const { words } = loadTranscript(path);
|
||||
expect(words).toEqual([
|
||||
{ text: "I", start: 0, end: 0.2 },
|
||||
{ text: "didn't", start: 0.2, end: 0.7 },
|
||||
{ text: "know", start: 0.7, end: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("merges I + 'm into I'm", () => {
|
||||
const path = tmpFile(
|
||||
"im.json",
|
||||
JSON.stringify({
|
||||
transcription: [
|
||||
{
|
||||
tokens: [
|
||||
{ text: " I", offsets: { from: 0, to: 100 } },
|
||||
{ text: "'m", offsets: { from: 100, to: 300 } },
|
||||
{ text: " done", offsets: { from: 300, to: 600 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const { words } = loadTranscript(path);
|
||||
expect(words[0]?.text).toBe("I'm");
|
||||
expect(words[0]?.end).toBe(0.3);
|
||||
});
|
||||
|
||||
it("merges could + 've into could've", () => {
|
||||
const path = tmpFile(
|
||||
"couldve.json",
|
||||
JSON.stringify({
|
||||
transcription: [
|
||||
{
|
||||
tokens: [
|
||||
{ text: " could", offsets: { from: 0, to: 400 } },
|
||||
{ text: "'ve", offsets: { from: 400, to: 600 } },
|
||||
{ text: " been", offsets: { from: 600, to: 900 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const { words } = loadTranscript(path);
|
||||
expect(words[0]?.text).toBe("could've");
|
||||
});
|
||||
});
|
||||
|
||||
describe("whisper-cpp fragment merging", () => {
|
||||
it("merges single capital + lowercase: C + aught -> Caught", () => {
|
||||
const path = tmpFile(
|
||||
"fragments.json",
|
||||
JSON.stringify({
|
||||
transcription: [
|
||||
{
|
||||
tokens: [
|
||||
{ text: " C", offsets: { from: 0, to: 100 } },
|
||||
{ text: "aught", offsets: { from: 100, to: 500 } },
|
||||
{ text: " a", offsets: { from: 500, to: 600 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const { words } = loadTranscript(path);
|
||||
expect(words[0]?.text).toBe("Caught");
|
||||
expect(words[0]?.end).toBe(0.5);
|
||||
expect(words).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("merges consonant + in': shin + in' -> shinin'", () => {
|
||||
const path = tmpFile(
|
||||
"dropg.json",
|
||||
JSON.stringify({
|
||||
transcription: [
|
||||
{
|
||||
tokens: [
|
||||
{ text: " shin", offsets: { from: 0, to: 300 } },
|
||||
{ text: "in'", offsets: { from: 300, to: 500 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const { words } = loadTranscript(path);
|
||||
expect(words).toHaveLength(1);
|
||||
expect(words[0]?.text).toBe("shinin'");
|
||||
});
|
||||
});
|
||||
|
||||
describe("whisper-cpp zero-duration interpolation", () => {
|
||||
it("interpolates a cluster of zero-duration words", () => {
|
||||
const path = tmpFile(
|
||||
"zerodur.json",
|
||||
JSON.stringify({
|
||||
transcription: [
|
||||
{
|
||||
tokens: [
|
||||
{ text: " hello", offsets: { from: 0, to: 500 } },
|
||||
{ text: " we", offsets: { from: 1000, to: 1000 } },
|
||||
{ text: " are", offsets: { from: 1000, to: 1000 } },
|
||||
{ text: " here", offsets: { from: 1000, to: 1000 } },
|
||||
{ text: " now", offsets: { from: 1500, to: 2000 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const { words } = loadTranscript(path);
|
||||
expect(words).toHaveLength(5);
|
||||
// The three zero-duration words should be spread between 0.5 and 1.5
|
||||
const we = words[1] ?? { start: 0, end: 0, text: "" };
|
||||
const are = words[2] ?? { start: 0, end: 0, text: "" };
|
||||
const here = words[3] ?? { start: 0, end: 0, text: "" };
|
||||
expect(we.start).toBeCloseTo(0.5, 1);
|
||||
expect(we.end).toBeCloseTo(0.833, 1);
|
||||
expect(are.start).toBeCloseTo(0.833, 1);
|
||||
expect(are.end).toBeCloseTo(1.167, 1);
|
||||
expect(here.start).toBeCloseTo(1.167, 1);
|
||||
expect(here.end).toBeCloseTo(1.5, 1);
|
||||
// Each should have positive duration
|
||||
expect(we.end).toBeGreaterThan(we.start);
|
||||
expect(are.end).toBeGreaterThan(are.start);
|
||||
expect(here.end).toBeGreaterThan(here.start);
|
||||
});
|
||||
|
||||
it("handles isolated zero-duration word", () => {
|
||||
const path = tmpFile(
|
||||
"singlezero.json",
|
||||
JSON.stringify({
|
||||
transcription: [
|
||||
{
|
||||
tokens: [
|
||||
{ text: " hello", offsets: { from: 0, to: 500 } },
|
||||
{ text: " I", offsets: { from: 800, to: 800 } },
|
||||
{ text: " know", offsets: { from: 1000, to: 1500 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const { words } = loadTranscript(path);
|
||||
const iWord = words[1] ?? { start: 0, end: 0, text: "" };
|
||||
expect(iWord.end).toBeGreaterThan(iWord.start);
|
||||
expect(iWord.start).toBeCloseTo(0.5, 1);
|
||||
expect(iWord.end).toBeCloseTo(1, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("patchCaptionHtml", () => {
|
||||
it("replaces const script = [] in HTML files", () => {
|
||||
const dir = join(tmpdir(), `hf-patch-test-${Date.now()}`);
|
||||
@@ -276,3 +443,105 @@ describe("patchCaptionHtml", () => {
|
||||
expect(result).toBe(html);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectSpeechOnset", () => {
|
||||
function makeSyntheticWav(
|
||||
sampleRate: number,
|
||||
durationSeconds: number,
|
||||
energyFn: (t: number) => number,
|
||||
): string {
|
||||
const numSamples = Math.floor(sampleRate * durationSeconds);
|
||||
const dataSize = numSamples * 2;
|
||||
const buf = Buffer.alloc(44 + dataSize);
|
||||
// RIFF header
|
||||
buf.write("RIFF", 0);
|
||||
buf.writeUInt32LE(36 + dataSize, 4);
|
||||
buf.write("WAVE", 8);
|
||||
buf.write("fmt ", 12);
|
||||
buf.writeUInt32LE(16, 16); // chunk size
|
||||
buf.writeUInt16LE(1, 20); // PCM
|
||||
buf.writeUInt16LE(1, 22); // mono
|
||||
buf.writeUInt32LE(sampleRate, 24);
|
||||
buf.writeUInt32LE(sampleRate * 2, 28); // byte rate
|
||||
buf.writeUInt16LE(2, 32); // block align
|
||||
buf.writeUInt16LE(16, 34); // bits per sample
|
||||
buf.write("data", 36);
|
||||
buf.writeUInt32LE(dataSize, 40);
|
||||
for (let i = 0; i < numSamples; i++) {
|
||||
const t = i / sampleRate;
|
||||
const amplitude = energyFn(t);
|
||||
buf.writeInt16LE(Math.round(amplitude * 32767), 44 + i * 2);
|
||||
}
|
||||
const path = join(tmpdir(), `hf-wav-test-${Date.now()}-${Math.floor(Math.random() * 1e6)}.wav`);
|
||||
writeFileSync(path, buf);
|
||||
dirs.push(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
it("detects onset when silence transitions to loud", () => {
|
||||
const wavPath = makeSyntheticWav(16000, 15, (t) => (t < 5 ? 0.01 : 0.8));
|
||||
const onset = detectSpeechOnset(wavPath);
|
||||
expect(onset).not.toBeNull();
|
||||
expect(onset!).toBeGreaterThanOrEqual(4);
|
||||
expect(onset!).toBeLessThanOrEqual(7);
|
||||
});
|
||||
|
||||
it("returns null for consistent energy throughout", () => {
|
||||
const wavPath = makeSyntheticWav(16000, 10, () => 0.5);
|
||||
const onset = detectSpeechOnset(wavPath);
|
||||
expect(onset).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for very short audio", () => {
|
||||
const wavPath = makeSyntheticWav(16000, 2, () => 0.5);
|
||||
const onset = detectSpeechOnset(wavPath);
|
||||
expect(onset).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when onset is too early (< 3s)", () => {
|
||||
const wavPath = makeSyntheticWav(16000, 10, (t) => (t < 1 ? 0.01 : 0.8));
|
||||
const onset = detectSpeechOnset(wavPath);
|
||||
expect(onset).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("stripBeforeOnset", () => {
|
||||
it("removes words before onset time", () => {
|
||||
const words = [
|
||||
{ text: "ghost", start: 0.5, end: 2.0 },
|
||||
{ text: "alone", start: 3.0, end: 5.0 },
|
||||
{ text: "Given", start: 19.0, end: 19.5 },
|
||||
{ text: "the", start: 19.5, end: 20.0 },
|
||||
];
|
||||
const result = stripBeforeOnset(words, 18.5);
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0]!.text).toBe("Given");
|
||||
});
|
||||
|
||||
it("keeps words within 0.5s tolerance of onset", () => {
|
||||
const words = [
|
||||
{ text: "hello", start: 18.2, end: 18.8 },
|
||||
{ text: "world", start: 19.0, end: 19.5 },
|
||||
];
|
||||
const result = stripBeforeOnset(words, 18.5);
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("keeps everything when onset is 0", () => {
|
||||
const words = [
|
||||
{ text: "hello", start: 0.1, end: 0.5 },
|
||||
{ text: "world", start: 0.6, end: 1.0 },
|
||||
];
|
||||
const result = stripBeforeOnset(words, 0);
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("returns empty array when all words are before onset", () => {
|
||||
const words = [
|
||||
{ text: "ghost", start: 0.5, end: 2.0 },
|
||||
{ text: "alone", start: 3.0, end: 5.0 },
|
||||
];
|
||||
const result = stripBeforeOnset(words, 20.0);
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -43,6 +43,67 @@ function detectJsonFormat(raw: unknown): TranscriptFormat {
|
||||
// Parsers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Rejoin word fragments that whisper splits across tokens:
|
||||
* - Single capital + lowercase continuation: C + aught -> Caught, G + onna -> Gonna
|
||||
* - Word ending in consonant + in': shin + in' -> shinin', hid + in' -> hidin'
|
||||
*/
|
||||
function mergeFragments(words: Word[]): void {
|
||||
for (let i = 0; i < words.length - 1; i++) {
|
||||
const curr = words[i];
|
||||
const next = words[i + 1];
|
||||
if (!curr || !next) continue;
|
||||
const isSingleLetterFragment =
|
||||
curr.text.length === 1 &&
|
||||
/^[A-Z]$/.test(curr.text) &&
|
||||
!/^[IAO]$/.test(curr.text) &&
|
||||
/^[a-z]/.test(next.text);
|
||||
const shouldMerge =
|
||||
isSingleLetterFragment || (/[a-z]$/.test(curr.text) && /^in'$/i.test(next.text));
|
||||
if (shouldMerge) {
|
||||
curr.text += next.text;
|
||||
curr.end = next.end;
|
||||
words.splice(i + 1, 1);
|
||||
i--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Distribute timestamps evenly across zero-duration word clusters.
|
||||
* Whisper sometimes assigns identical start/end to sequences of words,
|
||||
* making karaoke highlights flash through them instantly.
|
||||
*
|
||||
* Also handles malformed timestamps where start > end — these are treated
|
||||
* the same as zero-duration and get interpolated from surrounding words.
|
||||
*/
|
||||
function interpolateZeroDuration(words: Word[]): void {
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
const wi = words[i];
|
||||
if (!wi || wi.start < wi.end) continue;
|
||||
let j = i;
|
||||
while (j < words.length) {
|
||||
const wj = words[j];
|
||||
if (!wj || wj.start < wj.end) break;
|
||||
j++;
|
||||
}
|
||||
const clusterLen = j - i;
|
||||
const prev = i > 0 ? words[i - 1] : undefined;
|
||||
const prevEnd = prev ? prev.end : wi.start;
|
||||
const nextWord = j < words.length ? words[j] : undefined;
|
||||
const nextStart = nextWord ? nextWord.start : prevEnd + clusterLen * 0.3;
|
||||
const span = nextStart - prevEnd;
|
||||
const perWord = span / clusterLen;
|
||||
for (let k = i; k < j; k++) {
|
||||
const wk = words[k];
|
||||
if (!wk) continue;
|
||||
wk.start = round3(prevEnd + (k - i) * perWord);
|
||||
wk.end = round3(prevEnd + (k - i + 1) * perWord);
|
||||
}
|
||||
i = j - 1;
|
||||
}
|
||||
}
|
||||
|
||||
function parseWhisperCpp(data: Record<string, unknown>): Word[] {
|
||||
const words: Word[] = [];
|
||||
const transcription = data.transcription as Array<{
|
||||
@@ -54,13 +115,21 @@ function parseWhisperCpp(data: Record<string, unknown>): Word[] {
|
||||
|
||||
for (const seg of transcription ?? []) {
|
||||
for (const token of seg.tokens ?? []) {
|
||||
const text = (token.text ?? "").trim();
|
||||
const rawText = token.text ?? "";
|
||||
const text = rawText.trim();
|
||||
if (!text || text.startsWith("[_") || text.startsWith("[BLANK")) continue;
|
||||
|
||||
// Merge punctuation with the previous word
|
||||
const isPunctuation = /^[.,!?;:'")\]}>…–—-]+$/.test(text);
|
||||
const lastWord = words[words.length - 1];
|
||||
if (isPunctuation && lastWord) {
|
||||
|
||||
// Merge into previous word when the token is a sub-word continuation,
|
||||
// trailing punctuation, or a contraction suffix.
|
||||
// Whisper uses leading spaces to mark word boundaries in all languages.
|
||||
const shouldMerge =
|
||||
lastWord &&
|
||||
(!rawText.startsWith(" ") ||
|
||||
/^[.,!?;:'")\]}>…–—¡¿-]+$/.test(text) ||
|
||||
/^'(t|m|s|ve|re|ll|d)$/i.test(text));
|
||||
if (shouldMerge) {
|
||||
lastWord.text += text;
|
||||
lastWord.end = round3((token.offsets?.to ?? 0) / 1000);
|
||||
continue;
|
||||
@@ -73,6 +142,10 @@ function parseWhisperCpp(data: Record<string, unknown>): Word[] {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
mergeFragments(words);
|
||||
interpolateZeroDuration(words);
|
||||
|
||||
return words;
|
||||
}
|
||||
|
||||
@@ -229,9 +302,15 @@ export function loadTranscript(filePath: string): { words: Word[]; format: Trans
|
||||
}
|
||||
|
||||
/**
|
||||
* Patch caption HTML files in a project directory with transcript words.
|
||||
* Replaces `const script = [...]` or `const TRANSCRIPT = [...]` in <script> blocks.
|
||||
* Remove words that fall before the detected speech onset.
|
||||
* Whisper can hallucinate words over non-speech sections at the start of audio.
|
||||
*/
|
||||
export function stripBeforeOnset(words: Word[], onsetSeconds: number): Word[] {
|
||||
// 0.5s tolerance: keep words whose timestamps straddle the onset boundary,
|
||||
// since whisper may assign a slightly early start to the first spoken word.
|
||||
return words.filter((w) => w.start >= onsetSeconds - 0.5);
|
||||
}
|
||||
|
||||
export function patchCaptionHtml(dir: string, words: Word[]): void {
|
||||
if (words.length === 0) return;
|
||||
|
||||
|
||||
@@ -4,6 +4,98 @@ import { join, extname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js";
|
||||
|
||||
/**
|
||||
* Detect the language of a WAV file using whisper's built-in language detection.
|
||||
* Returns an ISO 639-1 code (e.g. "en", "es", "hi") or null if detection fails.
|
||||
*/
|
||||
function detectLanguage(whisperPath: string, modelPath: string, wavPath: string): string | null {
|
||||
try {
|
||||
const output = execFileSync(whisperPath, ["--model", modelPath, "--detect-language", wavPath], {
|
||||
encoding: "utf-8",
|
||||
timeout: 30_000,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const match = output.match(/auto-detected language:\s*(\w+)/);
|
||||
return match?.[1] ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function findWavDataChunk(buf: Buffer): { offset: number; size: number } | null {
|
||||
if (buf.length < 12) return null;
|
||||
let pos = 12; // skip RIFF header
|
||||
while (pos + 8 < buf.length) {
|
||||
const id = buf.toString("ascii", pos, pos + 4);
|
||||
const size = buf.readUInt32LE(pos + 4);
|
||||
if (id === "data") return { offset: pos + 8, size: Math.min(size, buf.length - pos - 8) };
|
||||
pos += 8 + size;
|
||||
if (size % 2 !== 0) pos++; // RIFF chunks are word-aligned
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect when speech begins in a 16kHz mono WAV by finding the first
|
||||
* sustained energy jump above the track's median RMS. Returns onset time in
|
||||
* seconds, or null if the track has consistent energy throughout.
|
||||
*/
|
||||
export function detectSpeechOnset(wavPath: string): number | null {
|
||||
const SAMPLE_RATE = 16000;
|
||||
const WINDOW_SECONDS = 0.5;
|
||||
const WINDOW_SAMPLES = SAMPLE_RATE * WINDOW_SECONDS;
|
||||
const SUSTAINED_WINDOWS = 3; // 1.5s above threshold to count as onset
|
||||
const SILENCE_THRESHOLD_RATIO = 0.6;
|
||||
const MIN_INTRO_SECONDS = 3; // don't strip if onset is very early
|
||||
|
||||
try {
|
||||
const buf = readFileSync(wavPath);
|
||||
const dataChunk = findWavDataChunk(buf);
|
||||
if (!dataChunk) return null;
|
||||
const pcm = new Int16Array(buf.buffer, buf.byteOffset + dataChunk.offset, dataChunk.size / 2);
|
||||
const totalWindows = Math.floor(pcm.length / WINDOW_SAMPLES);
|
||||
if (totalWindows < 10) return null;
|
||||
|
||||
const rmsValues: number[] = [];
|
||||
for (let i = 0; i < totalWindows; i++) {
|
||||
const start = i * WINDOW_SAMPLES;
|
||||
let sumSq = 0;
|
||||
for (let j = start; j < start + WINDOW_SAMPLES; j++) {
|
||||
const sample = pcm[j] ?? 0;
|
||||
sumSq += sample * sample;
|
||||
}
|
||||
rmsValues.push(Math.sqrt(sumSq / WINDOW_SAMPLES));
|
||||
}
|
||||
|
||||
const sorted = [...rmsValues].sort((a, b) => a - b);
|
||||
const median = sorted[Math.floor(sorted.length / 2)] ?? 0;
|
||||
const threshold = median * SILENCE_THRESHOLD_RATIO;
|
||||
|
||||
// Check if energy is fairly consistent (no clear intro) — ratio of
|
||||
// first 10s average to median. If it's already close, no onset to detect.
|
||||
const introAvg =
|
||||
rmsValues.slice(0, Math.min(20, rmsValues.length)).reduce((a, b) => a + b, 0) /
|
||||
Math.min(20, rmsValues.length);
|
||||
if (introAvg >= threshold) return null;
|
||||
|
||||
let consecutive = 0;
|
||||
for (let i = 0; i < rmsValues.length; i++) {
|
||||
if ((rmsValues[i] ?? 0) >= threshold) {
|
||||
consecutive++;
|
||||
if (consecutive >= SUSTAINED_WINDOWS) {
|
||||
const onsetSeconds = (i - SUSTAINED_WINDOWS + 1) * WINDOW_SECONDS;
|
||||
return onsetSeconds >= MIN_INTRO_SECONDS ? onsetSeconds : null;
|
||||
}
|
||||
} else {
|
||||
consecutive = 0;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Can't read WAV — skip onset detection
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const AUDIO_EXTENSIONS = new Set([".mp3", ".wav", ".m4a", ".aac", ".ogg", ".flac"]);
|
||||
const VIDEO_EXTENSIONS = new Set([".mp4", ".webm", ".mov", ".mkv", ".avi"]);
|
||||
|
||||
@@ -17,6 +109,7 @@ export interface TranscribeResult {
|
||||
transcriptPath: string;
|
||||
wordCount: number;
|
||||
durationSeconds: number;
|
||||
speechOnsetSeconds: number | null;
|
||||
}
|
||||
|
||||
function isAudioFile(filePath: string): boolean {
|
||||
@@ -121,29 +214,53 @@ export async function transcribe(
|
||||
throw new Error(`Unsupported file type: ${ext}`);
|
||||
}
|
||||
|
||||
// 4. Run whisper
|
||||
// 4. Detect language and ensure correct model
|
||||
let effectiveModel = model;
|
||||
let effectiveModelPath = modelPath;
|
||||
let detectedLanguage = options?.language ?? null;
|
||||
|
||||
// Only auto-detect language when using a multilingual model.
|
||||
// .en models always report "en" regardless of actual language, so detection
|
||||
// would be a no-op. If the user chose .en, they want English.
|
||||
if (!detectedLanguage && !effectiveModel.endsWith(".en")) {
|
||||
options?.onProgress?.("Detecting language...");
|
||||
detectedLanguage = detectLanguage(whisper.executablePath, effectiveModelPath, wavPath);
|
||||
}
|
||||
|
||||
if (detectedLanguage && detectedLanguage !== "en" && effectiveModel.endsWith(".en")) {
|
||||
const multilingualModel = effectiveModel.replace(/\.en$/, "");
|
||||
options?.onProgress?.(
|
||||
`Detected ${detectedLanguage} — switching to ${multilingualModel} model...`,
|
||||
);
|
||||
effectiveModelPath = await ensureModel(multilingualModel, {
|
||||
onProgress: options?.onProgress,
|
||||
});
|
||||
effectiveModel = multilingualModel;
|
||||
}
|
||||
|
||||
// 5. Run whisper
|
||||
options?.onProgress?.("Transcribing...");
|
||||
const outputBase = join(outputDir, "transcript");
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const whisperArgs = [
|
||||
"--model",
|
||||
modelPath,
|
||||
effectiveModelPath,
|
||||
"--output-json-full",
|
||||
"--output-file",
|
||||
outputBase,
|
||||
"--dtw",
|
||||
model,
|
||||
effectiveModel,
|
||||
"--suppress-nst",
|
||||
];
|
||||
if (options?.language) {
|
||||
whisperArgs.push("--language", options.language);
|
||||
if (detectedLanguage) {
|
||||
whisperArgs.push("--language", detectedLanguage);
|
||||
}
|
||||
whisperArgs.push(wavPath);
|
||||
|
||||
execFileSync(whisper.executablePath, whisperArgs, { stdio: "ignore", timeout: 300_000 });
|
||||
|
||||
// 5. Read and validate output
|
||||
// 6. Read and validate output
|
||||
const transcriptPath = `${outputBase}.json`;
|
||||
if (!existsSync(transcriptPath)) {
|
||||
throw new Error("Whisper did not produce output. Check the input file.");
|
||||
@@ -162,6 +279,10 @@ export async function transcribe(
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Detect speech onset before cleaning up the WAV
|
||||
options?.onProgress?.("Detecting speech onset...");
|
||||
const speechOnsetSeconds = detectSpeechOnset(wavPath);
|
||||
|
||||
// Clean up temp WAV if we created one
|
||||
if (wavPath !== inputPath) {
|
||||
try {
|
||||
@@ -175,6 +296,7 @@ export async function transcribe(
|
||||
transcriptPath,
|
||||
wordCount,
|
||||
durationSeconds: maxEnd / 1000,
|
||||
speechOnsetSeconds,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user