From 18409c9f2742e491ef7ff6a1454675a295e5d0a0 Mon Sep 17 00:00:00 2001 From: Rajan Pantha Date: Thu, 27 Aug 2026 02:02:25 +0545 Subject: [PATCH] fix(cli): keep phrase-level CJK and Thai transcripts as separate cues (#3436) * fix(cli): keep phrase-level CJK and Thai transcripts as separate cues wordsToCues inferred whether entries were already grouped into phrases by testing for internal whitespace. Chinese, Japanese, Thai and the other scripts written without inter-word spaces never satisfy that test, so their phrase-level transcripts were treated as word-level and re-grouped into a single cue covering the whole clip. A three-phrase Chinese transcript produced one cue; the same transcript in English produced three. The failure was silent: the export succeeded, and the user found out by watching the captions. For entries with no whitespace at all, fall back to entry length when they are in a spaceless script. Whisper emits word-level tokens for those scripts one or two characters at a time, while a phrase-level cue runs to several times that. The median is used so one long token cannot declare word-level input pre-grouped, and a couple of short cues cannot declare a real transcript word-level. --preserve-cues still forces the same thing, and behaviour for space-separated scripts is unchanged. Fixes #3353 * test(cli): pin the spaceless phrase length threshold --- packages/cli/src/whisper/normalize.test.ts | 47 +++++++++++++++++++++ packages/cli/src/whisper/normalize.ts | 49 ++++++++++++++++++++-- 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/whisper/normalize.test.ts b/packages/cli/src/whisper/normalize.test.ts index 8ee305a61..885306fff 100644 --- a/packages/cli/src/whisper/normalize.test.ts +++ b/packages/cli/src/whisper/normalize.test.ts @@ -292,6 +292,53 @@ Render video. Built for agents. expect(cues).toEqual([{ text: "你好世界", start: 0, end: 1 }]); }); + it("keeps phrase-level CJK entries as separate cues", () => { + // Chinese has no inter-word spaces, so the whitespace test cannot see + // that these are phrases; they used to collapse into one cue spanning + // the whole transcript. + const cues = wordsToCues([ + { text: "这是第一个句子", start: 0, end: 2 }, + { text: "这是第二个句子", start: 2, end: 4 }, + { text: "这是第三个句子", start: 4, end: 6 }, + ]); + expect(cues).toHaveLength(3); + expect(cues[0]).toEqual({ + text: "这是第一个句子", + start: 0, + end: 2, + }); + }); + + it("keeps phrase-level Thai entries as separate cues", () => { + const cues = wordsToCues([ + { text: "สวัสดีครับ", start: 0, end: 2 }, + { text: "ยินดีต้อนรับ", start: 2, end: 4 }, + ]); + expect(cues).toHaveLength(2); + }); + + it("treats entries at the length threshold as phrases", () => { + // Four characters is the boundary: at or above it the entries are read as + // phrase-level cues, below it as word-level tokens. + const cues = wordsToCues([ + { text: "你好世界", start: 0, end: 2 }, + { text: "谢谢大家", start: 2, end: 4 }, + ]); + expect(cues).toHaveLength(2); + }); + + it("still groups word-level CJK tokens into cues", () => { + // The mirror of the case above: short per-token entries are word-level + // whisper output and must still be joined. + const cues = wordsToCues([ + { text: "你", start: 0, end: 0.3 }, + { text: "好", start: 0.3, end: 0.6 }, + { text: "世", start: 0.6, end: 0.9 }, + { text: "界", start: 0.9, end: 1.2 }, + ]); + expect(cues).toEqual([{ text: "你好世界", start: 0, end: 1.2 }]); + }); + it("preserves single-word cue boundaries when preGrouped", () => { // Phrase-level cues without internal whitespace (one-word or CJK captions) // must not merge — auto-detection can't see them, so the caller forces it. diff --git a/packages/cli/src/whisper/normalize.ts b/packages/cli/src/whisper/normalize.ts index 2599d8ab9..4ad7747d3 100644 --- a/packages/cli/src/whisper/normalize.ts +++ b/packages/cli/src/whisper/normalize.ts @@ -356,6 +356,47 @@ function entriesToCues(words: Word[]): Cue[] { // inter-word spaces. const CJK_CHAR = /[　-〿぀-ヿ㐀-䶿一-鿿豈-﫿＀-￯]/; +// Scripts written without inter-word spaces: the CJK ranges above plus Thai, +// Lao, Myanmar and Khmer. Used only to decide whether entries are already +// phrase-level — the whitespace test below cannot answer that for these +// scripts. Hangul is excluded for the same reason as in CJK_CHAR. +const SPACELESS_SCRIPT_CHAR = /[฀-๿຀-໿က-႟ក-៿　-〿぀-ヿ㐀-䶿一-鿿豈-﫿＀-￯]/; + +// Whisper emits word-level tokens for spaceless scripts one or two characters +// at a time, while a phrase-level cue runs to several times that. Four is +// comfortably above the token case and below any real caption. +const SPACELESS_PHRASE_MIN_CHARS = 4; + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length === 0) return 0; + if (sorted.length % 2 === 1) return sorted[mid] ?? 0; + return ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2; +} + +/** + * Whether the entries are already grouped into phrases. + * + * Any entry containing internal whitespace is a multi-word phrase, which + * settles it for space-separated scripts. Chinese, Japanese, Thai and the + * other spaceless scripts never satisfy that test, so their phrase-level + * transcripts used to be re-grouped into a single cue covering the whole + * clip. For those, fall back to entry length instead. + * + * The median, rather than `some` or `every`, keeps one long token from + * declaring word-level input pre-grouped and a couple of short cues (a bare + * yes or no) from declaring a real transcript word-level. + */ +function inferPreGrouped(words: Word[]): boolean { + if (words.some((w) => /\s/.test(w.text.trim()))) return true; + + const spaceless = words.filter((w) => SPACELESS_SCRIPT_CHAR.test(w.text)); + if (spaceless.length === 0) return false; + + return median(spaceless.map((w) => w.text.trim().length)) >= SPACELESS_PHRASE_MIN_CHARS; +} + /** Join two adjacent tokens, omitting the space across a CJK boundary. */ function joinTokens(left: string, right: string): string { const a = left.at(-1) ?? ""; @@ -367,10 +408,10 @@ function joinTokens(left: string, right: string): string { export function wordsToCues(words: Word[], opts: WordsToCuesOptions = {}): Cue[] { // Phrase-level transcripts (imported .srt/.vtt cues) must keep their existing // cue boundaries — re-grouping would merge distinct captions and lose timing. - // The caller can force this via `preGrouped`; otherwise infer it from the data - // (any entry containing internal whitespace is a multi-word phrase, so the - // whole transcript is phrase-level rather than word-level whisper output). - const preGrouped = opts.preGrouped ?? words.some((w) => /\s/.test(w.text.trim())); + // The caller can force this via `preGrouped`; otherwise infer it from the + // data — internal whitespace for space-separated scripts, entry length for + // the scripts that have no inter-word spaces to look for. + const preGrouped = opts.preGrouped ?? inferPreGrouped(words); if (preGrouped) return entriesToCues(words); const maxChars = opts.maxChars ?? 42;