Compare commits

...
Author SHA1 Message Date
Teknium b21e322702 fix(tools): normalize Unicode space family and minus sign in patch fuzzy matching
Port from anomalyco/opencode#38133/#38134 (patch Unicode matching corpus):
extend UNICODE_MAP with the Zs space-separator family (en/em quad, en/em/
three-per-em/four-per-em/six-per-em/figure/punctuation/thin/hair spaces,
narrow NBSP, medium mathematical space, CJK ideographic space) and the
Unicode minus sign U+2212.

Before: a file containing typographic spacing (French narrow NBSP, CJK
ideographic spaces, math minus) never matched a model's ASCII old_string
via the precise strategies — the edit only succeeded through the
similarity-based context_aware fallback, which (a) can pick the wrong
region (#54572 family) and (b) silently flattens the file's Unicode to
ASCII on replacement. After: these match at unicode_normalized (strategy
7), whose _preserve_unicode_in_replacement keeps the file's typographic
characters in unchanged spans.

All additions are 1:1 mappings, so the existing position-mapping and
preservation logic apply unchanged. Proven live before/after with a
multi-line probe; 59 fuzzy-match + 188 file-tools/patch/skill-manager
tests pass.
2026-07-23 17:12:04 -07:00
2 changed files with 69 additions and 0 deletions
+52
View File
@@ -320,6 +320,58 @@ class TestUnicodeNormalized:
assert new == "plain text there"
class TestUnicodeSpaceAndMinusNormalized:
"""Space-separator family + Unicode minus normalization.
Port of the anomalyco/opencode#38133 patch-matching corpus: files with
typographic spacing (en/em/thin spaces, narrow NBSP, CJK ideographic
space) or the Unicode minus sign must match a model's ASCII old_string
at the precise unicode_normalized strategy — not fall through to the
similarity-based context_aware fallback.
"""
def test_unicode_minus_matched_and_preserved(self):
content = "offset = value \u2212 1\nprint(offset)\n"
new, count, strategy, err = fuzzy_find_and_replace(
content, "offset = value - 1", "offset = delta - 1"
)
assert count == 1, f"Expected match, got err={err}"
assert strategy == "unicode_normalized"
# The untouched minus keeps its Unicode form
assert "delta \u2212 1" in new, f"Got {new!r}"
def test_space_variants_match_at_unicode_strategy(self):
# en space, em space, thin space, narrow NBSP, medium math space,
# ideographic space, figure space, hair space
for space in ["\u2002", "\u2003", "\u2009", "\u202f",
"\u205f", "\u3000", "\u2007", "\u200a"]:
content = f"# wait{space}30{space}seconds\nrun()\n"
new, count, strategy, err = fuzzy_find_and_replace(
content, "# wait 30 seconds", "# wait 60 seconds"
)
assert count == 1, (
f"space U+{ord(space):04X}: expected match, err={err}"
)
assert strategy == "unicode_normalized", (
f"space U+{ord(space):04X}: matched via {strategy}, "
"expected unicode_normalized"
)
# Unchanged spaces keep their typographic form; only the
# digits change.
assert f"60{space}seconds" in new, (
f"space U+{ord(space):04X}: got {new!r}"
)
def test_ideographic_space_cjk_line(self):
content = "標題\u3000第一章\nbody text\n"
new, count, strategy, err = fuzzy_find_and_replace(
content, "標題 第一章", "標題 第二章"
)
assert count == 1, f"Expected match, got err={err}"
assert strategy == "unicode_normalized"
assert "標題\u3000第二章" in new, f"Got {new!r}"
class TestBlockAnchorThreshold:
"""Tests for the raised block_anchor threshold (Bug 4)."""
+17
View File
@@ -38,6 +38,23 @@ UNICODE_MAP = {
"\u2018": "'", "\u2019": "'", # smart single quotes
"\u2014": "--", "\u2013": "-", # em/en dashes
"\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space
# Unicode minus sign — models type ASCII '-' for file content that uses
# the typographic minus (math/scientific docs).
"\u2212": "-",
# Space-separator family (Zs) beyond NBSP. Files with typographic
# spacing (en/em/thin spaces, narrow NBSP in French text, ideographic
# space in CJK text) never match a model's ASCII-space old_string via
# the precise strategies, falling through to the similarity-based
# context_aware fallback — which can pick the wrong region and flattens
# the file's Unicode on replacement. (anomalyco/opencode#38133 corpus)
"\u2000": " ", "\u2001": " ", # en/em quad
"\u2002": " ", "\u2003": " ", # en/em space
"\u2004": " ", "\u2005": " ", "\u2006": " ", # three/four/six-per-em
"\u2007": " ", "\u2008": " ", # figure/punctuation space
"\u2009": " ", "\u200a": " ", # thin/hair space
"\u202f": " ", # narrow no-break space
"\u205f": " ", # medium mathematical space
"\u3000": " ", # ideographic (CJK full-width) space
}
def _unicode_normalize(text: str) -> str: