fix(agent): tolerate lone UTF-16 surrogates in tool-guardrail hashing
Tool results scraped from the web/social platforms can carry unpaired UTF-16 surrogates (e.g. half of a mathematical-bold character pair). _sha256() did a strict utf-8 encode, which raises UnicodeEncodeError on that input and took down the whole conversation loop — the hash only needs deterministic bytes, not valid UTF-8, so encode with surrogatepass instead.
This commit is contained in:
committed by
Teknium
parent
33300cdc7f
commit
fb0217c656
@@ -472,4 +472,8 @@ def _positive_int(value: Any, default: int) -> int:
|
||||
|
||||
|
||||
def _sha256(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
# surrogatepass: tool results scraped from the web can carry unpaired
|
||||
# UTF-16 surrogates (e.g. half of a mathematical-bold pair); a strict
|
||||
# encode raises and takes down the whole conversation loop. The hash only
|
||||
# needs deterministic bytes, not valid UTF-8.
|
||||
return hashlib.sha256(value.encode("utf-8", "surrogatepass")).hexdigest()
|
||||
|
||||
@@ -256,3 +256,24 @@ def test_reset_for_turn_clears_bounded_guardrail_state():
|
||||
|
||||
assert controller.before_call("web_search", {"query": "same"}).action == "allow"
|
||||
assert controller.before_call("read_file", {"path": "/tmp/x"}).action == "allow"
|
||||
|
||||
|
||||
def test_after_call_survives_lone_surrogates_in_result_and_args():
|
||||
# Scraped web/social text can contain unpaired UTF-16 surrogates (e.g. the
|
||||
# first half of a mathematical-bold pair, '\ud835'). str.encode('utf-8')
|
||||
# rejects them, and the result hasher crashed the whole conversation loop
|
||||
# (live outage: "Outer loop error in API call #34 ... surrogates not
|
||||
# allowed"). Weird text must never take down the loop.
|
||||
controller = ToolCallGuardrailController(
|
||||
ToolCallGuardrailConfig(hard_stop_enabled=True, exact_failure_block_after=2, no_progress_block_after=2)
|
||||
)
|
||||
dirty = "price \ud835 update"
|
||||
|
||||
decision = controller.after_call("web_search", {"query": dirty}, dirty, failed=False)
|
||||
assert decision.action in {"allow", "warn"}
|
||||
|
||||
# hashing stays deterministic: the same dirty failure twice still trips
|
||||
# the exact-failure guard, proving the hash is stable across calls
|
||||
controller.after_call("web_search", {"query": dirty}, '{"error":"\ud835 boom"}', failed=True)
|
||||
controller.after_call("web_search", {"query": dirty}, '{"error":"\ud835 boom"}', failed=True)
|
||||
assert controller.before_call("web_search", {"query": dirty}).action == "block"
|
||||
|
||||
Reference in New Issue
Block a user