mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
A turn that dies mid-stream (provider 503, user interrupt) appends its partial
assistant message to history so the transcript still shows what streamed, then
an error notice. `_outbound_messages()` drops notices, so the partial becomes
the FINAL message sent to the provider, and `retry()` replays it verbatim.
Several providers reject that outright:
- Gemini: "Requests ending with a model turn are not supported"
- Claude 4.6+ / Fable 5: "does not support assistant message prefill"
- Claude 4.5 and older: "final assistant content cannot end with trailing
whitespace" -- streamed deltas routinely end in a space
So every retry replays the same bad tail and the session is wedged until the
user sends a new message or gives up on it. Reproduced end-to-end against live
APIs on Gemini 3.6 Flash and Claude Sonnet 4.6; three of the four Claude models
in the curated picker reject the payload unconditionally.
Drop the abandoned partial in `_outbound_messages()`, alongside the notices it
already treats as display-only. The trailing error/interrupted notice is what
identifies it: a COMPLETED turn's assistant message is structurally identical
(trailing, no tool_calls) and must be kept, since its `extras` sidecar is
replayed back to the owning provider. Trailing tool_calls mean a turn suspended
awaiting durable resume, never an abandoned one.
Because the notices persist, this also unwedges sessions already broken on disk.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
166 lines
5.3 KiB
Python
166 lines
5.3 KiB
Python
"""Durable resume: a prompt pending when the process 'restarts' is answered later and the turn
|
|
continues — rebuilt from the persisted thread, with no live await."""
|
|
|
|
import asyncio
|
|
|
|
from coworker.providers import (
|
|
AssistantTurn,
|
|
ModelCapabilities,
|
|
ProviderClient,
|
|
ToolCall,
|
|
)
|
|
from coworker.server.manager import SessionManager
|
|
|
|
|
|
class ScriptedProvider(ProviderClient):
|
|
def __init__(self, turns):
|
|
self._turns = list(turns)
|
|
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
return self._turns.pop(0)
|
|
|
|
def capabilities(self, model):
|
|
return ModelCapabilities()
|
|
|
|
|
|
def _tool(name, args, call_id):
|
|
return AssistantTurn(tool_calls=[ToolCall(id=call_id, name=name, arguments=args)])
|
|
|
|
|
|
def _text(text):
|
|
return AssistantTurn(text=text, finish_reason="stop")
|
|
|
|
|
|
async def _run_until_pending(mgr, sid, engine):
|
|
async def first():
|
|
async for _ in engine.run("go"):
|
|
pass
|
|
|
|
task = asyncio.create_task(first())
|
|
pend = []
|
|
for _ in range(100):
|
|
await asyncio.sleep(0.02)
|
|
pend = mgr.inbox.pending(sid)
|
|
if pend:
|
|
break
|
|
assert pend, "prompt never became a pending Inbox item"
|
|
# simulate a restart: cancel the suspended turn + drop the live engine
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
mgr._engines.pop(sid, None)
|
|
mgr.mark_idle(sid)
|
|
return pend[0]
|
|
|
|
|
|
def _final_assistant_texts(mgr, sid):
|
|
rec = mgr.session_store.load(sid)
|
|
return [
|
|
m.get("content")
|
|
for m in rec.messages
|
|
if m.get("role") == "assistant" and m.get("content")
|
|
]
|
|
|
|
|
|
def test_durable_resume_question(tmp_path):
|
|
mgr = SessionManager(
|
|
workspace=tmp_path,
|
|
provider=ScriptedProvider(
|
|
[
|
|
_tool(
|
|
"ask_user",
|
|
{
|
|
"question": "Which region?",
|
|
"options": ["us-east-1", "us-west-2"],
|
|
},
|
|
"call_q",
|
|
),
|
|
_text("You chose us-west-2."),
|
|
]
|
|
),
|
|
)
|
|
sid = "dur-q"
|
|
|
|
async def scenario():
|
|
engine = mgr.get_engine(sid, agent="cowork", workspace=str(tmp_path))
|
|
item = await _run_until_pending(mgr, sid, engine)
|
|
assert item.kind == "question" and item.tool_call_id == "call_q"
|
|
await mgr.resolve_inbox(item.id, "us-west-2") # restart-style resume
|
|
|
|
asyncio.run(scenario())
|
|
assert any("us-west-2" in (t or "") for t in _final_assistant_texts(mgr, sid))
|
|
assert mgr.inbox.pending(sid) == [] # nothing left pending
|
|
|
|
|
|
def test_resume_payload_keeps_the_suspended_tool_call(tmp_path):
|
|
# `_outbound_messages` drops a TRAILING assistant turn (the partial a mid-stream death
|
|
# leaves behind). A suspended turn is also stored ending on an assistant turn, so this
|
|
# pins that resume is unaffected: the pending calls are answered first, making the tail a
|
|
# `tool` message, and the assistant turn that made the call still reaches the model.
|
|
class Recording(ScriptedProvider):
|
|
def __init__(self, turns):
|
|
super().__init__(turns)
|
|
self.payloads = []
|
|
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
self.payloads.append(messages)
|
|
return super().complete(
|
|
model=model, messages=messages, tools=tools, **settings
|
|
)
|
|
|
|
provider = Recording(
|
|
[
|
|
_tool("ask_user", {"question": "Which region?", "options": ["a", "b"]}, "call_q"),
|
|
_text("You chose b."),
|
|
]
|
|
)
|
|
mgr = SessionManager(workspace=tmp_path, provider=provider)
|
|
sid = "dur-payload"
|
|
|
|
async def scenario():
|
|
engine = mgr.get_engine(sid, agent="cowork", workspace=str(tmp_path))
|
|
item = await _run_until_pending(mgr, sid, engine)
|
|
await mgr.resolve_inbox(item.id, "b")
|
|
|
|
asyncio.run(scenario())
|
|
|
|
resumed = next(
|
|
p for p in provider.payloads if any(m.get("role") == "tool" for m in p)
|
|
)
|
|
assert [m.get("role") for m in resumed][-2:] == ["assistant", "tool"]
|
|
assert any(
|
|
m.get("role") == "assistant" and m.get("tool_calls") for m in resumed
|
|
), "the assistant turn that made the call must still reach the model"
|
|
|
|
|
|
def test_durable_resume_approval_executes_tool(tmp_path):
|
|
# The model wants a write (needs approval); on durable resume "allow" must RE-EXECUTE the tool.
|
|
target = tmp_path / "scratch_marker.txt"
|
|
mgr = SessionManager(
|
|
workspace=tmp_path,
|
|
provider=ScriptedProvider(
|
|
[
|
|
_tool("write_file", {"path": str(target), "content": "ok"}, "call_w"),
|
|
_text("Done — file written."),
|
|
]
|
|
),
|
|
)
|
|
sid = "dur-a"
|
|
|
|
async def scenario():
|
|
engine = mgr.get_engine(sid, agent="cowork", workspace=str(tmp_path))
|
|
item = await _run_until_pending(mgr, sid, engine)
|
|
assert item.kind == "approval" and item.tool_call_id == "call_w"
|
|
assert not target.exists() # not executed before approval
|
|
await mgr.resolve_inbox(
|
|
item.id, "allow"
|
|
) # restart-style resume → re-execute the tool
|
|
|
|
asyncio.run(scenario())
|
|
assert (
|
|
target.exists() and target.read_text() == "ok"
|
|
) # the approved write actually ran
|
|
assert any("Done" in (t or "") for t in _final_assistant_texts(mgr, sid))
|