From 92d9af274fd61db3e58a012dc103fff1e9080ea9 Mon Sep 17 00:00:00 2001 From: miga-heygen Date: Mon, 31 Aug 2026 23:09:56 +0000 Subject: [PATCH] fix(fonts): make Google Fonts subsetting CSS text-transform aware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the subset character closure to cover locale/context-sensitive case transforms and non-case CSS text-transform values: - Parse lang attributes from authored HTML and apply toLocaleUpperCase/ toLocaleLowerCase for each detected locale (covers Turkish İ/ı, Azeri, German ẞ, and other locale-dependent casing) - Map ASCII U+0021–U+007E to fullwidth equivalents U+FF01–U+FF5E when full-width appears in the source - Map small hiragana/katakana to full-size equivalents when full-size-kana appears in the source - Preserve the existing 1700-char encoded URL budget and full-font fallback Closes #3496 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../deterministicFonts-textSubset.test.ts | 114 ++++++++++++++++++ .../src/services/deterministicFonts.ts | 50 +++++++- 2 files changed, 159 insertions(+), 5 deletions(-) diff --git a/packages/producer/src/services/deterministicFonts-textSubset.test.ts b/packages/producer/src/services/deterministicFonts-textSubset.test.ts index abcc811e1..01cdda411 100644 --- a/packages/producer/src/services/deterministicFonts-textSubset.test.ts +++ b/packages/producer/src/services/deterministicFonts-textSubset.test.ts @@ -81,4 +81,118 @@ describe("Google Fonts text subsetting", () => { expect(url.searchParams.has("text")).toBe(false); }); + + it("includes Turkish İ and ı when lang=tr is present", async () => { + const url = await requestedGoogleFontUrl( + `

istanbul

`, + ); + + const text = url.searchParams.get("text") ?? ""; + expect(text).toContain("İ"); + expect(text).toContain("ı"); + }); + + it("does not include Turkish İ/ı without a Turkish lang attribute", async () => { + const url = await requestedGoogleFontUrl( + `

istanbul

`, + ); + + const text = url.searchParams.get("text") ?? ""; + expect(text).not.toContain("İ"); + expect(text).not.toContain("ı"); + }); + + it("includes Azeri locale variants when lang=az is present", async () => { + const url = await requestedGoogleFontUrl( + `

iyi

`, + ); + + const text = url.searchParams.get("text") ?? ""; + expect(text).toContain("İ"); + expect(text).toContain("ı"); + }); + + it("maps ASCII to fullwidth equivalents when full-width appears in the source", async () => { + const url = await requestedGoogleFontUrl( + `

ABC

`, + ); + + const text = url.searchParams.get("text") ?? ""; + expect(text).toContain("Ａ"); + expect(text).toContain("Ｂ"); + expect(text).toContain("Ｃ"); + }); + + it("does not add fullwidth variants without full-width in the source", async () => { + const url = await requestedGoogleFontUrl( + `

ABC

`, + ); + + const text = url.searchParams.get("text") ?? ""; + expect(text).not.toContain("Ａ"); + }); + + it("maps small kana to full-size equivalents when full-size-kana appears in the source", async () => { + const url = await requestedGoogleFontUrl( + `

ぁっょ

`, + ); + + const text = url.searchParams.get("text") ?? ""; + expect(text).toContain("あ"); + expect(text).toContain("つ"); + expect(text).toContain("よ"); + }); + + it("maps small katakana to full-size when full-size-kana appears in the source", async () => { + const url = await requestedGoogleFontUrl( + `

ァヵ

`, + ); + + const text = url.searchParams.get("text") ?? ""; + expect(text).toContain("ア"); + expect(text).toContain("カ"); + }); + + it("stays within the URL budget for a realistic mixed-script composition with all transforms", async () => { + const latin = "The Quick Brown Fox Jumps Over The Lazy Dog — Your Kidney Transplant: What Happens Next"; + const cjk = "旅行ランキング東京大阪京都名古屋福岡"; + const kana = "ぁぃぅぇぉっゃゅょゎァィゥェォッャュョヮヵヶ"; + + const url = await requestedGoogleFontUrl( + `

${latin}

${cjk}${kana}

`, + ); + + const text = url.searchParams.get("text") ?? ""; + expect(text.length).toBeGreaterThan(0); + expect(encodeURIComponent(text).length).toBeLessThanOrEqual(1700); + }); + + it("collects lang from nested elements, not just the root", async () => { + const url = await requestedGoogleFontUrl( + `

hello

istanbul

`, + ); + + const text = url.searchParams.get("text") ?? ""; + expect(text).toContain("İ"); + expect(text).toContain("ı"); + }); }); diff --git a/packages/producer/src/services/deterministicFonts.ts b/packages/producer/src/services/deterministicFonts.ts index 7b99aad6e..0cce2ca69 100644 --- a/packages/producer/src/services/deterministicFonts.ts +++ b/packages/producer/src/services/deterministicFonts.ts @@ -1198,22 +1198,62 @@ export interface InjectDeterministicFontFacesOptions { // collapsing repeated prose and base64 assets to a tiny set. const GOOGLE_FONTS_TEXT_MAX_ENCODED_LENGTH = 1_700; +const SMALL_TO_FULL_KANA: ReadonlyMap = new Map([ + ["ぁ", "あ"], ["ぃ", "い"], ["ぅ", "う"], ["ぇ", "え"], ["ぉ", "お"], + ["っ", "つ"], ["ゃ", "や"], ["ゅ", "ゆ"], ["ょ", "よ"], ["ゎ", "わ"], + ["ァ", "ア"], ["ィ", "イ"], ["ゥ", "ウ"], ["ェ", "エ"], ["ォ", "オ"], + ["ッ", "ツ"], ["ャ", "ヤ"], ["ュ", "ユ"], ["ョ", "ヨ"], ["ヮ", "ワ"], + ["ヵ", "カ"], ["ヶ", "ケ"], +]); + +function collectLangAttributes(document: { querySelectorAll(selector: string): Iterable<{ getAttribute(name: string): string | null }> }): Set { + const locales = new Set(); + for (const element of document.querySelectorAll("[lang]")) { + const lang = element.getAttribute("lang"); + if (lang) { + locales.add(lang.split("-")[0]!.toLowerCase()); + } + } + return locales; +} + function extractGoogleFontsText(html: string): string | undefined { const { document } = parseHTML(html); const decodedBodyText = document.body?.textContent ?? ""; - // Source + decoded text is an intentional over-approximation: base64, scripts, and class names - // collapse in the Set, while decoded entities contribute the glyphs the browser actually paints. + const locales = collectLangAttributes(document); + const hasFullWidth = html.includes("full-width"); + const hasFullSizeKana = html.includes("full-size-kana"); + const characters = [...Array.from(html), ...Array.from(decodedBodyText)]; const uniqueCharacters = new Set(); for (const character of characters) { uniqueCharacters.add(character); - // This closes locale-independent Unicode casing, including multi-code-point expansions such as - // ß -> SS. Locale/context transforms (for example Turkish İ) and CSS full-width/full-size-kana - // need a transform-aware follow-up rather than pretending this code-point closure is exhaustive. for (const variant of `${character.toUpperCase()}${character.toLowerCase()}`) { uniqueCharacters.add(variant); } + for (const locale of locales) { + for (const variant of `${character.toLocaleUpperCase(locale)}${character.toLocaleLowerCase(locale)}`) { + uniqueCharacters.add(variant); + } + } } + + if (hasFullWidth) { + for (const character of [...uniqueCharacters]) { + const code = character.codePointAt(0) ?? 0; + if (code >= 0x0021 && code <= 0x007e) { + uniqueCharacters.add(String.fromCodePoint(code + 0xfee0)); + } + } + } + + if (hasFullSizeKana) { + for (const character of [...uniqueCharacters]) { + const full = SMALL_TO_FULL_KANA.get(character); + if (full) uniqueCharacters.add(full); + } + } + const fontText = [...uniqueCharacters].join(""); return encodeURIComponent(fontText).length <= GOOGLE_FONTS_TEXT_MAX_ENCODED_LENGTH ? fontText