fix(cli): keep whisper CJK tokens as separate timed words (#6058)

* fix(cli): keep whisper CJK tokens as separate timed words

whisper.cpp marks word boundaries with a leading space, but Chinese and
Japanese tokens never carry one, so parseWhisperCpp merged every CJK token
into the previous word, across segments too. A 9.5-minute Mandarin episode
came out as two "words" of about 280 s each, leaving captions and
sentence-building nothing to split.

A token now starts its own word when it crosses a CJK boundary, full-width
punctuation still attaches to the word before it, and cues end at 。！？.

* fix(ci): capture package name in release-age exclusion

packages/cli/src/whisper/normalize.test.ts

@@ -591,6 +591,89 @@ describe("whisper-cpp fragment merging", () => {
   });
 });
 
+describe("whisper-cpp CJK tokens", () => {
+  const token = (text: string, from: number, to: number) => ({ text, offsets: { from, to } });
+
+  // Shaped like whisper-cli --output-json-full on Mandarin speech: no token
+  // carries a leading space, not even the first one of a segment.
+  function chineseTranscript(): string {
+    return tmpFile(
+      "transcript.json",
+      JSON.stringify({
+        transcription: [
+          {
+            offsets: { from: 0, to: 3990 },
+            text: "大家好,今天用AI剪輯。",
+            tokens: [
+              token("[_BEG_]", 0, 0),
+              token("大家好", 10, 730),
+              token(",", 1110, 1290),
+              token("今天", 1360, 1680),
+              token("用", 1680, 2010),
+              token("AI", 2010, 2500),
+              token("剪", 2970, 3460),
+              token("輯", 3460, 3830),
+              token("。", 3830, 3990),
+            ],
+          },
+          {
+            offsets: { from: 4000, to: 5600 },
+            text: "謝謝觀看。",
+            tokens: [
+              token("謝謝", 4000, 4500),
+              token("觀", 4500, 4900),
+              token("看", 4900, 5300),
+              token("。", 5300, 5600),
+              token("[_TT_280]", 5600, 5600),
+            ],
+          },
+        ],
+      }),
+    );
+  }
+
+  it("keeps each CJK token as its own timed word", () => {
+    const { words } = loadTranscript(chineseTranscript());
+    expect(words.map((w) => [w.text, w.start, w.end])).toEqual([
+      ["大家好,", 0.01, 1.29],
+      ["今天", 1.36, 1.68],
+      ["用", 1.68, 2.01],
+      ["AI", 2.01, 2.5],
+      ["剪", 2.97, 3.46],
+      ["輯。", 3.46, 3.99],
+      ["謝謝", 4, 4.5],
+      ["觀", 4.5, 4.9],
+      ["看。", 4.9, 5.6],
+    ]);
+  });
+
+  it("splits Japanese kana tokens and merges full-width punctuation", () => {
+    const path = tmpFile(
+      "transcript.json",
+      JSON.stringify({
+        transcription: [
+          {
+            tokens: [
+              token("こんにち", 0, 400),
+              token("は", 400, 600),
+              token("、", 600, 700),
+              token("世界", 700, 1200),
+              token("！", 1200, 1300),
+            ],
+          },
+        ],
+      }),
+    );
+    const { words } = loadTranscript(path);
+    expect(words.map((w) => w.text)).toEqual(["こんにち", "は、", "世界！"]);
+  });
+
+  it("ends a cue at a CJK sentence mark", () => {
+    const { words } = loadTranscript(chineseTranscript());
+    expect(wordsToCues(words).map((c) => c.text)).toEqual(["大家好,今天用AI剪輯。", "謝謝觀看。"]);
+  });
+});
+
 describe("whisper-cpp zero-duration interpolation", () => {
   it("interpolates a cluster of zero-duration words", () => {
     const path = tmpFile(

packages/cli/src/whisper/normalize.ts

@@ -126,6 +126,26 @@ function interpolateZeroDuration(words: Word[]): void {
   }
 }
 
+const TRAILING_PUNCTUATION = /^[.,!?;:'")\]}>…–—¡¿\-，。！？、；：」』）】》〉]+$/;
+
+function crossesCjkBoundary(left: string, right: string): boolean {
+  return CJK_CHAR.test(left.at(-1) ?? "") || CJK_CHAR.test(right[0] ?? "");
+}
+
+/**
+ * Whether a token extends the previous word: a sub-word continuation, trailing
+ * punctuation, or a contraction suffix. Whisper marks word boundaries with a
+ * leading space, except in CJK, where no token has one: there each token is
+ * its own word.
+ */
+function continuesWord(previous: string, rawText: string, text: string): boolean {
+  return (
+    TRAILING_PUNCTUATION.test(text) ||
+    /^'(t|m|s|ve|re|ll|d)$/i.test(text) ||
+    (!rawText.startsWith(" ") && !crossesCjkBoundary(previous, text))
+  );
+}
+
 function parseWhisperCpp(data: Record<string, unknown>): Word[] {
   const words: Word[] = [];
   const transcription = data.transcription as Array<{
@@ -142,16 +162,7 @@ function parseWhisperCpp(data: Record<string, unknown>): Word[] {
       if (!text || text.startsWith("[_") || text.startsWith("[BLANK")) continue;
 
       const lastWord = words[words.length - 1];
-
-      // Merge into previous word when the token is a sub-word continuation,
-      // trailing punctuation, or a contraction suffix.
-      // Whisper uses leading spaces to mark word boundaries in all languages.
-      const shouldMerge =
-        lastWord &&
-        (!rawText.startsWith(" ") ||
-          /^[.,!?;:'")\]}>…–—¡¿-]+$/.test(text) ||
-          /^'(t|m|s|ve|re|ll|d)$/i.test(text));
-      if (shouldMerge) {
+      if (lastWord && continuesWord(lastWord.text, rawText, text)) {
         lastWord.text += text;
         lastWord.end = round3((token.offsets?.to ?? 0) / 1000);
         continue;
@@ -330,7 +341,7 @@ function pad3(n: number): string {
 }
 
 function endsSentence(text: string): boolean {
-  return /[.!?][)"'\]}]*$/.test(text);
+  return /[.!?。！？][)"'\]}」』）]*$/.test(text);
 }
 
 function pushCue(cues: Cue[], cue: Cue | undefined): void {
@@ -411,10 +422,7 @@ function inferPreGrouped(words: Word[]): boolean {
 
 /** Join two adjacent tokens, omitting the space across a CJK boundary. */
 function joinTokens(left: string, right: string): string {
-  const a = left.at(-1) ?? "";
-  const b = right[0] ?? "";
-  const sep = CJK_CHAR.test(a) || CJK_CHAR.test(b) ? "" : " ";
-  return `${left}${sep}${right}`;
+  return `${left}${crossesCjkBoundary(left, right) ? "" : " "}${right}`;
 }
 
 export function wordsToCues(words: Word[], opts: WordsToCuesOptions = {}): Cue[] {

scripts/verify-packed-manifests.mjs

@@ -479,7 +479,7 @@ function writeConsumerFixture(packDir, packedWorkspaces) {
 export const CONSUMER_MIN_RELEASE_AGE_SECONDS = 24 * 60 * 60;
 
 const BLOCKED_BY_AGE =
-  /No version matching "([^"]+)" found for specifier "[^"]*" \(blocked by minimum-release-age/g;
+  /No version matching "[^"]+" found for specifier "([^"]+)" \(blocked by minimum-release-age/g;
 
 export function installConsumer(dir, { args = [], env = process.env } = {}) {
   const install = () =>