mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(check): a Korean number and its counter count as one word in line reading time (#7343)
* fix(check): count a Korean number and its counter as one word in line reading time * fix(check): keep segmenting Thai, Lao, Khmer and Burmese when counting reading words
This commit is contained in:
@@ -85,6 +85,25 @@ describe("readingWords", () => {
|
||||
expect(readingWords("今天天气很好")).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("counts Korean by its spaces and a number with its counter as one unit", () => {
|
||||
expect(readingWords("실패 12회")).toBe(2);
|
||||
expect(readingWords("오류 3건 발생")).toBe(3);
|
||||
expect(readingWords("테스트 실패 12/40")).toBe(4);
|
||||
expect(readingWords("커피12잔")).toBe(1);
|
||||
});
|
||||
|
||||
it("still segments scripts written without spaces", () => {
|
||||
expect(readingWords("ฉันรักภาษาไทยมาก")).toBeGreaterThan(1);
|
||||
expect(readingWords("ខ្ញុំស្រលាញ់ភាសាខ្មែរ")).toBeGreaterThan(1);
|
||||
expect(readingWords("ຂ້ອຍຮັກພາສາລາວຫຼາຍ")).toBeGreaterThan(1);
|
||||
expect(readingWords("ကျွန်တော်မြန်မာစကားကိုချစ်တယ်")).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("keeps counting a Japanese number and its counter as two", () => {
|
||||
expect(readingWords("12回")).toBe(2);
|
||||
expect(readingWords("実行12回失敗")).toBe(4);
|
||||
});
|
||||
|
||||
it("counts a contraction as one word", () => {
|
||||
expect(readingWords("a tooth can't get in")).toBe(5);
|
||||
expect(readingWords("a tooth can’t get in")).toBe(5);
|
||||
|
||||
@@ -32,13 +32,18 @@ const ZERO_BBOX: CheckBbox = { x: 0, y: 0, width: 0, height: 0 };
|
||||
const WORD_RUNS = /[\p{L}\p{M}\p{N}%$]+(?:['’][\p{L}\p{M}\p{N}]+)*/gu;
|
||||
// Chinese and Japanese have no spaces; a dictionary segmenter counts their words like an English line's.
|
||||
const WORD_SEGMENTER = new Intl.Segmenter("ja", { granularity: "word" });
|
||||
const UNSPACED_SCRIPT =
|
||||
/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
|
||||
|
||||
/** Words a viewer reads: letter/number runs, with CJK runs split into dictionary words. */
|
||||
export function readingWords(text: string): number {
|
||||
const runs = text.toLowerCase().normalize("NFC").match(WORD_RUNS) ?? [];
|
||||
return runs.reduce(
|
||||
(count, run) =>
|
||||
count + Math.max(1, [...WORD_SEGMENTER.segment(run)].filter((s) => s.isWordLike).length),
|
||||
count +
|
||||
(UNSPACED_SCRIPT.test(run)
|
||||
? Math.max(1, [...WORD_SEGMENTER.segment(run)].filter((s) => s.isWordLike).length)
|
||||
: 1),
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user