diff --git a/packages/cli/src/whisper/normalize.test.ts b/packages/cli/src/whisper/normalize.test.ts index f016b5c48f..73717da8af 100644 --- a/packages/cli/src/whisper/normalize.test.ts +++ b/packages/cli/src/whisper/normalize.test.ts @@ -591,6 +591,89 @@ describe("whisper-cpp fragment merging", () => { }); }); +describe("whisper-cpp CJK tokens", () => { + const token = (text: string, from: number, to: number) => ({ text, offsets: { from, to } }); + + // Shaped like whisper-cli --output-json-full on Mandarin speech: no token + // carries a leading space, not even the first one of a segment. + function chineseTranscript(): string { + return tmpFile( + "transcript.json", + JSON.stringify({ + transcription: [ + { + offsets: { from: 0, to: 3990 }, + text: "大家好,今天用AI剪輯。", + tokens: [ + token("[_BEG_]", 0, 0), + token("大家好", 10, 730), + token(",", 1110, 1290), + token("今天", 1360, 1680), + token("用", 1680, 2010), + token("AI", 2010, 2500), + token("剪", 2970, 3460), + token("輯", 3460, 3830), + token("。", 3830, 3990), + ], + }, + { + offsets: { from: 4000, to: 5600 }, + text: "謝謝觀看。", + tokens: [ + token("謝謝", 4000, 4500), + token("觀", 4500, 4900), + token("看", 4900, 5300), + token("。", 5300, 5600), + token("[_TT_280]", 5600, 5600), + ], + }, + ], + }), + ); + } + + it("keeps each CJK token as its own timed word", () => { + const { words } = loadTranscript(chineseTranscript()); + expect(words.map((w) => [w.text, w.start, w.end])).toEqual([ + ["大家好,", 0.01, 1.29], + ["今天", 1.36, 1.68], + ["用", 1.68, 2.01], + ["AI", 2.01, 2.5], + ["剪", 2.97, 3.46], + ["輯。", 3.46, 3.99], + ["謝謝", 4, 4.5], + ["觀", 4.5, 4.9], + ["看。", 4.9, 5.6], + ]); + }); + + it("splits Japanese kana tokens and merges full-width punctuation", () => { + const path = tmpFile( + "transcript.json", + JSON.stringify({ + transcription: [ + { + tokens: [ + token("こんにち", 0, 400), + token("は", 400, 600), + token("、", 600, 700), + token("世界", 700, 1200), + token("!", 1200, 1300), + ], + }, + ], + }), + ); + const { words } = loadTranscript(path); + expect(words.map((w) => w.text)).toEqual(["こんにち", "は、", "世界!"]); + }); + + it("ends a cue at a CJK sentence mark", () => { + const { words } = loadTranscript(chineseTranscript()); + expect(wordsToCues(words).map((c) => c.text)).toEqual(["大家好,今天用AI剪輯。", "謝謝觀看。"]); + }); +}); + describe("whisper-cpp zero-duration interpolation", () => { it("interpolates a cluster of zero-duration words", () => { const path = tmpFile( diff --git a/packages/cli/src/whisper/normalize.ts b/packages/cli/src/whisper/normalize.ts index f6c3ce0c56..497b8bb3fb 100644 --- a/packages/cli/src/whisper/normalize.ts +++ b/packages/cli/src/whisper/normalize.ts @@ -126,6 +126,26 @@ function interpolateZeroDuration(words: Word[]): void { } } +const TRAILING_PUNCTUATION = /^[.,!?;:'")\]}>…–—¡¿\-,。!?、;:」』)】》〉]+$/; + +function crossesCjkBoundary(left: string, right: string): boolean { + return CJK_CHAR.test(left.at(-1) ?? "") || CJK_CHAR.test(right[0] ?? ""); +} + +/** + * Whether a token extends the previous word: a sub-word continuation, trailing + * punctuation, or a contraction suffix. Whisper marks word boundaries with a + * leading space, except in CJK, where no token has one: there each token is + * its own word. + */ +function continuesWord(previous: string, rawText: string, text: string): boolean { + return ( + TRAILING_PUNCTUATION.test(text) || + /^'(t|m|s|ve|re|ll|d)$/i.test(text) || + (!rawText.startsWith(" ") && !crossesCjkBoundary(previous, text)) + ); +} + function parseWhisperCpp(data: Record): Word[] { const words: Word[] = []; const transcription = data.transcription as Array<{ @@ -142,16 +162,7 @@ function parseWhisperCpp(data: Record): Word[] { if (!text || text.startsWith("[_") || text.startsWith("[BLANK")) continue; const lastWord = words[words.length - 1]; - - // Merge into previous word when the token is a sub-word continuation, - // trailing punctuation, or a contraction suffix. - // Whisper uses leading spaces to mark word boundaries in all languages. - const shouldMerge = - lastWord && - (!rawText.startsWith(" ") || - /^[.,!?;:'")\]}>…–—¡¿-]+$/.test(text) || - /^'(t|m|s|ve|re|ll|d)$/i.test(text)); - if (shouldMerge) { + if (lastWord && continuesWord(lastWord.text, rawText, text)) { lastWord.text += text; lastWord.end = round3((token.offsets?.to ?? 0) / 1000); continue; @@ -330,7 +341,7 @@ function pad3(n: number): string { } function endsSentence(text: string): boolean { - return /[.!?][)"'\]}]*$/.test(text); + return /[.!?。!?][)"'\]}」』)]*$/.test(text); } function pushCue(cues: Cue[], cue: Cue | undefined): void { @@ -411,10 +422,7 @@ function inferPreGrouped(words: Word[]): boolean { /** Join two adjacent tokens, omitting the space across a CJK boundary. */ function joinTokens(left: string, right: string): string { - const a = left.at(-1) ?? ""; - const b = right[0] ?? ""; - const sep = CJK_CHAR.test(a) || CJK_CHAR.test(b) ? "" : " "; - return `${left}${sep}${right}`; + return `${left}${crossesCjkBoundary(left, right) ? "" : " "}${right}`; } export function wordsToCues(words: Word[], opts: WordsToCuesOptions = {}): Cue[] { diff --git a/scripts/verify-packed-manifests.mjs b/scripts/verify-packed-manifests.mjs index c24408182f..bc57ed57d7 100644 --- a/scripts/verify-packed-manifests.mjs +++ b/scripts/verify-packed-manifests.mjs @@ -479,7 +479,7 @@ function writeConsumerFixture(packDir, packedWorkspaces) { export const CONSUMER_MIN_RELEASE_AGE_SECONDS = 24 * 60 * 60; const BLOCKED_BY_AGE = - /No version matching "([^"]+)" found for specifier "[^"]*" \(blocked by minimum-release-age/g; + /No version matching "[^"]+" found for specifier "([^"]+)" \(blocked by minimum-release-age/g; export function installConsumer(dir, { args = [], env = process.env } = {}) { const install = () =>