mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
The title call sends reasoning_effort "none". Endpoints whose reasoning is mandatory refuse it (OpenRouter GLM 5.3: 400 "Reasoning is mandatory for this endpoint and cannot be disabled"), so those sessions never got a title. When the refusal is about reasoning or effort, the title is asked once more without the effort and with room for the reasoning. Other failures are not retried. The native Anthropic KeyError first reported here is already fixed on main (effort.NO_EFFORT handling).
247 lines
11 KiB
Python
247 lines
11 KiB
Python
"""FB-010 — LLM auto-titles.
|
|
|
|
After a user turn completes, the manager fires ONE fire-and-forget completion on the
|
|
session's own provider/model to generate a 4-5 word title, stored in `auto_title` —
|
|
never in `title`, so a manual rename always wins. The small-talk sentinel earns exactly
|
|
one retry (with both openers) after the next user turn; every failure is swallowed."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
|
|
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient
|
|
from coworker.server.manager import SessionManager
|
|
|
|
|
|
class TitleAwareProvider(ProviderClient):
|
|
"""Chat turns come from one queue; title calls (recognized by the titling system
|
|
prompt) are answered from another — so the fire-and-forget title call can never
|
|
steal a scripted chat turn."""
|
|
|
|
def __init__(self, chat_turns, titles):
|
|
self._chat = list(chat_turns)
|
|
self._titles = list(titles)
|
|
self.title_calls: list[list[dict]] = []
|
|
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
if messages and "title chat sessions" in str(messages[0].get("content", "")):
|
|
self.title_calls.append([dict(m) for m in messages])
|
|
item = self._titles.pop(0)
|
|
if isinstance(item, Exception):
|
|
raise item
|
|
return AssistantTurn(text=item, finish_reason="stop")
|
|
return self._chat.pop(0)
|
|
|
|
def capabilities(self, model):
|
|
return ModelCapabilities()
|
|
|
|
|
|
def _text(text):
|
|
return AssistantTurn(text=text, finish_reason="stop")
|
|
|
|
|
|
async def _turn(mgr: SessionManager, sid: str, text: str) -> None:
|
|
"""Drive one user turn the way the server does: run, save, mark idle (the
|
|
auto-title hook), then wait for the fire-and-forget call to settle."""
|
|
engine = mgr.get_engine(sid, agent="chat")
|
|
mgr.mark_running(sid)
|
|
async for _ in engine.run(text):
|
|
pass
|
|
mgr.save(sid, engine)
|
|
mgr.mark_idle(sid)
|
|
deadline = time.time() + 5.0
|
|
while sid in mgr._autotitle_inflight and time.time() < deadline:
|
|
await asyncio.sleep(0.005)
|
|
|
|
|
|
def _mgr(tmp_path, chat_turns, titles):
|
|
provider = TitleAwareProvider(chat_turns, titles)
|
|
return SessionManager(workspace=tmp_path, provider=provider), provider
|
|
|
|
|
|
async def test_title_set_after_first_turn(tmp_path):
|
|
mgr, provider = _mgr(tmp_path, [_text("sure")], ["Japan Trip Planning Help"])
|
|
await _turn(mgr, "s1", "help me plan a trip to japan")
|
|
|
|
assert len(provider.title_calls) == 1
|
|
# The agent's first reply rides along as titling evidence (owner ruling 2026-08-24).
|
|
assert provider.title_calls[0][-1]["content"] == (
|
|
"help me plan a trip to japan\n\n[the assistant's first reply]\nsure"
|
|
)
|
|
assert mgr.session_store.title_state("s1")["auto_title"] == (
|
|
"Japan Trip Planning Help"
|
|
)
|
|
# display precedence: the auto title wins over the title_from snapshot everywhere
|
|
assert mgr.session_store.load("s1").title == "Japan Trip Planning Help"
|
|
row = next(s for s in mgr.list_sessions() if s["session_id"] == "s1")
|
|
assert row["title"] == "Japan Trip Planning Help"
|
|
|
|
|
|
async def test_small_talk_sentinel_then_retry_with_both_openers(tmp_path):
|
|
mgr, provider = _mgr(
|
|
tmp_path,
|
|
[_text("hey!"), _text("on it")],
|
|
["small-talk", "Quarterly Report Draft"],
|
|
)
|
|
await _turn(mgr, "s2", "hey")
|
|
assert mgr.session_store.title_state("s2")["auto_title"] is None
|
|
assert mgr.session_store.load("s2").title == "hey" # title_from fallback stands
|
|
|
|
await _turn(mgr, "s2", "help me draft the quarterly report")
|
|
assert len(provider.title_calls) == 2
|
|
retry_opener = provider.title_calls[1][-1]["content"]
|
|
assert "hey" in retry_opener and "quarterly report" in retry_opener
|
|
assert mgr.session_store.load("s2").title == "Quarterly Report Draft"
|
|
|
|
|
|
async def test_gives_up_after_second_sentinel(tmp_path):
|
|
mgr, provider = _mgr(
|
|
tmp_path,
|
|
[_text("hi"), _text("hello"), _text("yo")],
|
|
["small-talk", "small-talk", "Should Never Be Asked"],
|
|
)
|
|
await _turn(mgr, "s3", "hey")
|
|
await _turn(mgr, "s3", "how are you")
|
|
await _turn(mgr, "s3", "good morning")
|
|
assert len(provider.title_calls) == 2 # attempt 1 + the single retry, then give up
|
|
assert mgr.session_store.title_state("s3")["auto_title"] is None
|
|
assert mgr.session_store.load("s3").title == "hey"
|
|
|
|
|
|
async def test_manual_rename_always_wins(tmp_path):
|
|
# rename AFTER the auto title landed
|
|
mgr, _ = _mgr(tmp_path, [_text("ok")], ["Login Bug Investigation"])
|
|
await _turn(mgr, "s4", "the login page 500s")
|
|
assert mgr.session_store.load("s4").title == "Login Bug Investigation"
|
|
mgr.rename_session("s4", "Prod incident 42")
|
|
assert mgr.session_store.load("s4").title == "Prod incident 42"
|
|
# ...and a late-landing generated title can't displace it
|
|
assert mgr.session_store.set_auto_title("s4", "sneaky") is False
|
|
assert mgr.session_store.load("s4").title == "Prod incident 42"
|
|
|
|
|
|
async def test_rename_before_generation_blocks_it(tmp_path):
|
|
mgr, provider = _mgr(
|
|
tmp_path, [_text("hi"), _text("ok")], ["small-talk", "Should Never Be Asked"]
|
|
)
|
|
await _turn(mgr, "s5", "hey") # sentinel — no title yet
|
|
mgr.rename_session("s5", "My Thread")
|
|
await _turn(mgr, "s5", "help me plan the offsite")
|
|
assert len(provider.title_calls) == 1 # the retry never fired
|
|
assert mgr.session_store.load("s5").title == "My Thread"
|
|
|
|
|
|
async def test_provider_failure_is_swallowed(tmp_path):
|
|
mgr, provider = _mgr(tmp_path, [_text("ok")], [RuntimeError("model down")])
|
|
await _turn(mgr, "s6", "summarize this doc") # must not raise
|
|
assert mgr.session_store.title_state("s6")["auto_title"] is None
|
|
assert mgr.session_store.load("s6").title == "summarize this doc"
|
|
|
|
|
|
async def test_sanitizes_and_rejects_absurd_output(tmp_path):
|
|
# surrounding quotes stripped + whitespace collapsed
|
|
mgr, _ = _mgr(tmp_path, [_text("ok")], ['"Fix Login Bug"'])
|
|
await _turn(mgr, "s7", "the login page 500s")
|
|
assert mgr.session_store.load("s7").title == "Fix Login Bug"
|
|
|
|
# absurdly long output (>80 chars) is a failure, not a title
|
|
mgr2, _ = _mgr(tmp_path / "b", [_text("ok")], ["x" * 90])
|
|
await _turn(mgr2, "s8", "the login page 500s")
|
|
assert mgr2.session_store.title_state("s8")["auto_title"] is None
|
|
|
|
|
|
def test_turn_start_titles_before_the_turn_completes(tmp_path):
|
|
# Owner catch 2026-08-24: titling used to ride ONLY turn completion, so a long
|
|
# agentic turn left the session titled by its first message for the whole run. The
|
|
# turn-start hook titles on the user's words the moment they land.
|
|
async def go():
|
|
mgr, provider = _mgr(tmp_path, [_text("working on it…")], ["Scan the API for secrets"])
|
|
sid = "early1"
|
|
engine = mgr.get_engine(sid, agent="chat")
|
|
engine.messages.append({"role": "user", "content": "scan code for vuln and secrets"})
|
|
mgr.save(sid, engine)
|
|
mgr.mark_running(sid) # the turn is RUNNING — no completion in sight
|
|
mgr._maybe_autotitle(sid) # the app's turn_start hook
|
|
deadline = time.time() + 5.0
|
|
while sid in mgr._autotitle_inflight and time.time() < deadline:
|
|
await asyncio.sleep(0.005)
|
|
state = mgr.session_store.title_state(sid)
|
|
assert state["auto_title"] == "Scan the API for secrets"
|
|
# The completion hook later, same openers: skipped WITHOUT burning attempt 2.
|
|
mgr.mark_idle(sid)
|
|
assert mgr._autotitle_attempts[sid] == 1
|
|
|
|
asyncio.run(go())
|
|
|
|
|
|
async def test_title_call_pins_the_opener_language(tmp_path):
|
|
# With no language pinned in the titling instruction, an English system prompt
|
|
# ("4-5 word title") biases the model toward English titles even for non-English
|
|
# sessions. The prompt must tell the model to match the opener's language.
|
|
mgr, provider = _mgr(tmp_path, [_text("好的")], ["日本行程规划"])
|
|
await _turn(mgr, "lang1", "帮我规划一下日本的旅行")
|
|
|
|
system = provider.title_calls[0][0]["content"]
|
|
assert "same language as the user's opening message" in system
|
|
|
|
|
|
async def test_non_english_opener_keeps_non_english_title(tmp_path):
|
|
# End to end: a CJK title survives the sanitizer (whitespace collapse and quote
|
|
# strip don't touch CJK) and the 80-char absurdity cap, and is stored verbatim.
|
|
mgr, _ = _mgr(tmp_path, [_text("好的,我来帮你安排")], ["日本行程规划"])
|
|
await _turn(mgr, "lang2", "帮我规划一下日本的旅行")
|
|
assert mgr.session_store.load("lang2").title == "日本行程规划"
|
|
|
|
|
|
async def test_non_english_small_talk_still_hits_the_sentinel(tmp_path):
|
|
# The sentinel is a fixed English token regardless of the session's language;
|
|
# normalization must keep catching it after the prompt rewording.
|
|
mgr, provider = _mgr(
|
|
tmp_path,
|
|
[_text("你好呀"), _text("在的")],
|
|
["small-talk", "季度汇报整理"],
|
|
)
|
|
await _turn(mgr, "lang3", "你好")
|
|
assert mgr.session_store.title_state("lang3")["auto_title"] is None
|
|
|
|
await _turn(mgr, "lang3", "帮我整理季度汇报")
|
|
assert len(provider.title_calls) == 2
|
|
assert mgr.session_store.load("lang3").title == "季度汇报整理"
|
|
|
|
|
|
class _SettingsRecorder(TitleAwareProvider):
|
|
def __init__(self, chat_turns, titles):
|
|
super().__init__(chat_turns, titles)
|
|
self.title_settings: list[dict] = []
|
|
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
if messages and "title chat sessions" in str(messages[0].get("content", "")):
|
|
self.title_settings.append(dict(settings))
|
|
return super().complete(model=model, messages=messages, tools=tools, **settings)
|
|
|
|
|
|
async def test_a_model_that_cannot_turn_reasoning_off_still_gets_a_title(tmp_path):
|
|
# OPE-201: an endpoint whose reasoning is mandatory refuses effort "none" (OpenRouter
|
|
# GLM 5.3 answers 400). The title is asked once more without the effort, with room for
|
|
# the reasoning the model spends first.
|
|
refused = RuntimeError("Error code: 400 - {'error': {'message': 'Reasoning is mandatory for "
|
|
"this endpoint and cannot be disabled.', 'code': 400}}")
|
|
provider = _SettingsRecorder([_text("sure")], [refused, "Release Notes Draft"])
|
|
mgr = SessionManager(workspace=tmp_path, provider=provider)
|
|
await _turn(mgr, "s20", "draft the release notes")
|
|
|
|
first, second = provider.title_settings
|
|
assert first["reasoning_effort"] == "none" and first["max_tokens"] == 64
|
|
assert "reasoning_effort" not in second
|
|
assert second["max_tokens"] == SessionManager._AUTOTITLE_REASONING_MAX_TOKENS
|
|
assert mgr.session_store.title_state("s20")["auto_title"] == "Release Notes Draft"
|
|
|
|
|
|
async def test_other_title_failures_are_not_retried(tmp_path):
|
|
provider = _SettingsRecorder([_text("ok")], [RuntimeError("model down"), "Never Used"])
|
|
mgr = SessionManager(workspace=tmp_path, provider=provider)
|
|
await _turn(mgr, "s21", "summarize this doc")
|
|
assert len(provider.title_settings) == 1
|
|
assert mgr.session_store.title_state("s21")["auto_title"] is None
|