diff --git a/packages/cli/src/utils/craftAudit.test.ts b/packages/cli/src/utils/craftAudit.test.ts index 1b4af12f93..4d298a4fce 100644 --- a/packages/cli/src/utils/craftAudit.test.ts +++ b/packages/cli/src/utils/craftAudit.test.ts @@ -85,6 +85,25 @@ describe("readingWords", () => { expect(readingWords("今天天气很好")).toBeGreaterThan(1); }); + it("counts Korean by its spaces and a number with its counter as one unit", () => { + expect(readingWords("실패 12회")).toBe(2); + expect(readingWords("오류 3건 발생")).toBe(3); + expect(readingWords("테스트 실패 12/40")).toBe(4); + expect(readingWords("커피12잔")).toBe(1); + }); + + it("still segments scripts written without spaces", () => { + expect(readingWords("ฉันรักภาษาไทยมาก")).toBeGreaterThan(1); + expect(readingWords("ខ្ញុំស្រលាញ់ភាសាខ្មែរ")).toBeGreaterThan(1); + expect(readingWords("ຂ້ອຍຮັກພາສາລາວຫຼາຍ")).toBeGreaterThan(1); + expect(readingWords("ကျွန်တော်မြန်မာစကားကိုချစ်တယ်")).toBeGreaterThan(1); + }); + + it("keeps counting a Japanese number and its counter as two", () => { + expect(readingWords("12回")).toBe(2); + expect(readingWords("実行12回失敗")).toBe(4); + }); + it("counts a contraction as one word", () => { expect(readingWords("a tooth can't get in")).toBe(5); expect(readingWords("a tooth can’t get in")).toBe(5); diff --git a/packages/cli/src/utils/craftAudit.ts b/packages/cli/src/utils/craftAudit.ts index 242cd867bb..24dddbf55f 100644 --- a/packages/cli/src/utils/craftAudit.ts +++ b/packages/cli/src/utils/craftAudit.ts @@ -32,13 +32,18 @@ const ZERO_BBOX: CheckBbox = { x: 0, y: 0, width: 0, height: 0 }; const WORD_RUNS = /[\p{L}\p{M}\p{N}%$]+(?:['’][\p{L}\p{M}\p{N}]+)*/gu; // Chinese and Japanese have no spaces; a dictionary segmenter counts their words like an English line's. const WORD_SEGMENTER = new Intl.Segmenter("ja", { granularity: "word" }); +const UNSPACED_SCRIPT = + /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u; /** Words a viewer reads: letter/number runs, with CJK runs split into dictionary words. */ export function readingWords(text: string): number { const runs = text.toLowerCase().normalize("NFC").match(WORD_RUNS) ?? []; return runs.reduce( (count, run) => - count + Math.max(1, [...WORD_SEGMENTER.segment(run)].filter((s) => s.isWordLike).length), + count + + (UNSPACED_SCRIPT.test(run) + ? Math.max(1, [...WORD_SEGMENTER.segment(run)].filter((s) => s.isWordLike).length) + : 1), 0, ); }