mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
Phase 2, OB-3. Every finished turn is checked against fixed rules (the shared signals.yaml, 19 checks) and the result is stored with the turn. The Steps panel gets an Issues tab, and the sidebar an amber ring for warnings not yet looked at. - The engine records the recording format 1.2 fields the checks read: the arguments' hash (RFC 8785), repeat_ok, result_error, error_kind, reaches_network and the todo plan on tool steps; max_iterations on the turn; max_output on model steps; inbox_at on waits; whether the machine expects a sandbox in the stamp. - coworker/observability/signals.py holds the checks, the same ones that were validated on the in-app recordings and the shared golden test. - The checks run when a turn ends; a result made by an older signals.yaml is worked out again on read. New endpoints serve a turn's issues and mark a session's issues seen. - The Issues tab groups issues by check, warnings first, with the step's command and output line and Go to step; "Checked and fine" is folded. - tests: the Phase 2 golden test (38 scenarios and hash vectors), the 1.2 fields from a real turn, and the endpoints through the server.
283 lines
10 KiB
Python
283 lines
10 KiB
Python
"""Observability E20 and E4: wait steps, decision paths and summary steps.
|
|
|
|
- E20: an approval card is a wait step from the moment it is shown to the answer, linked to
|
|
its tool call; its outcome and who answered are recorded.
|
|
- E4: every tool call that went through the permission flow carries its decision path
|
|
(`attrs.decision = {path, origin}`); a summary or trim is a summary step with its model
|
|
call nested inside it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from types import SimpleNamespace
|
|
|
|
from coworker import compaction as _compaction
|
|
from coworker.engine import ApprovalOutcome, TurnEngine
|
|
from coworker.observability import Recorder
|
|
from coworker.permissions import Mode, PermissionEngine
|
|
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
|
|
from coworker.providers.base import TokenUsage
|
|
from coworker.tools import ToolRegistry
|
|
|
|
|
|
class _Meta:
|
|
category: str = ""
|
|
risk_level: str = "high"
|
|
requires_approval: bool = False
|
|
|
|
|
|
class _ReadMeta:
|
|
category: str = ""
|
|
risk_level: str = "low"
|
|
requires_approval: bool = False
|
|
|
|
|
|
class _Scripted(ProviderClient):
|
|
def __init__(self, turns):
|
|
self._turns = list(turns)
|
|
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
return self._turns.pop(0)
|
|
|
|
def capabilities(self, model):
|
|
return ModelCapabilities()
|
|
|
|
|
|
def run_shell(command: str) -> dict:
|
|
"""Run a command.
|
|
|
|
Args:
|
|
command: the command line
|
|
"""
|
|
return {"command": command, "exit_code": 0, "output": ""}
|
|
|
|
|
|
def look(path: str) -> dict:
|
|
"""Read something.
|
|
|
|
Args:
|
|
path: what to read
|
|
"""
|
|
return {"path": path, "text": "ok"}
|
|
|
|
|
|
def _engine(tmp_path, turns, *, answer=ApprovalOutcome.ONCE, mode=Mode.INTERACTIVE):
|
|
registry = ToolRegistry()
|
|
registry.register(run_shell, metadata=_Meta())
|
|
registry.register(look, metadata=_ReadMeta())
|
|
|
|
async def approver(request):
|
|
return answer
|
|
|
|
engine = TurnEngine(
|
|
provider=_Scripted(turns),
|
|
registry=registry,
|
|
permissions=PermissionEngine(workspace_root=tmp_path, mode=mode),
|
|
model="openai:model-a",
|
|
approver=approver,
|
|
)
|
|
engine.audit_context = {"session_id": "sess_w"}
|
|
engine.is_attended = lambda: True
|
|
engine.resolver_of = lambda call_id: "devika"
|
|
rec = Recorder()
|
|
engine.recorder = rec
|
|
return engine, rec
|
|
|
|
|
|
def _call(cid, name, args):
|
|
return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls")
|
|
|
|
|
|
def _run(engine, text="go"):
|
|
async def go():
|
|
return [ev async for ev in engine.run(text, trigger="user")]
|
|
|
|
return asyncio.run(go())
|
|
|
|
|
|
def _steps(rec, kind):
|
|
return [s for s in rec.as_dict()["steps"] if s["kind"] == kind]
|
|
|
|
|
|
def test_an_approved_card_is_a_wait_step_and_the_call_records_its_path(tmp_path):
|
|
engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make build"}),
|
|
AssistantTurn(text="built")])
|
|
_run(engine)
|
|
[wait] = _steps(rec, "wait")
|
|
assert (wait["name"], wait["call_id"], wait["status"]) == ("approval", "c1", "ok")
|
|
assert wait["attrs"] == {"for_call_id": "c1", "kind": "approval", "attended": True,
|
|
"where": "session", "outcome": "approved", "inbox_at": None}
|
|
assert wait["content"] == {"resolved_by": "devika"}
|
|
[tool] = _steps(rec, "tool")
|
|
assert tool["status"] == "ok"
|
|
assert tool["attrs"]["decision"] == {
|
|
"path": [{"stage": "permission", "outcome": "needs_user"},
|
|
{"stage": "card", "outcome": "shown_session"},
|
|
{"stage": "resolved", "outcome": "approved_once"}],
|
|
"origin": "user",
|
|
}
|
|
# The wait comes before the tool in the canonical order.
|
|
kinds = [s["kind"] for s in rec.as_dict()["steps"]]
|
|
assert kinds.index("wait") < kinds.index("tool")
|
|
|
|
|
|
def test_a_denied_card_is_a_zero_length_denied_step_refused_by_the_user(tmp_path):
|
|
engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "rm -rf build"}),
|
|
AssistantTurn(text="ok, not doing it")],
|
|
answer=ApprovalOutcome.DENY)
|
|
_run(engine)
|
|
[wait] = _steps(rec, "wait")
|
|
assert wait["attrs"]["outcome"] == "denied"
|
|
[tool] = _steps(rec, "tool")
|
|
assert (tool["status"], tool["duration_ms"]) == ("denied", 0)
|
|
assert tool["attrs"]["decision"]["origin"] == "user"
|
|
assert tool["attrs"]["decision"]["path"][-1] == {"stage": "resolved", "outcome": "denied"}
|
|
|
|
|
|
def test_a_session_grant_is_recorded_as_its_origin(tmp_path):
|
|
engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "pytest"}),
|
|
_call("c2", "run_shell", {"command": "pytest"}),
|
|
AssistantTurn(text="done")],
|
|
answer=ApprovalOutcome.ALWAYS_COMMAND)
|
|
_run(engine)
|
|
first, second = _steps(rec, "tool")
|
|
assert first["attrs"]["decision"]["path"][-1] == {
|
|
"stage": "resolved", "outcome": "approved_session", "detail": "always_command"}
|
|
assert second["attrs"]["decision"] == {
|
|
"path": [{"stage": "permission", "outcome": "allowed"},
|
|
{"stage": "grant", "outcome": "session_grant"}],
|
|
"origin": "session_grant",
|
|
}
|
|
assert len(_steps(rec, "wait")) == 1 # no card the second time
|
|
|
|
|
|
def test_a_low_risk_read_needs_no_one(tmp_path):
|
|
engine, rec = _engine(tmp_path, [_call("c1", "look", {"path": "x"}), AssistantTurn(text="ok")])
|
|
_run(engine)
|
|
[tool] = _steps(rec, "tool")
|
|
assert tool["attrs"]["decision"] == {
|
|
"path": [{"stage": "permission", "outcome": "allowed", "detail": "low_risk"}],
|
|
"origin": "low_risk",
|
|
}
|
|
assert _steps(rec, "wait") == []
|
|
|
|
|
|
def test_full_access_is_bypass(tmp_path):
|
|
engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="ok")],
|
|
mode=Mode.BYPASS_APPROVALS)
|
|
_run(engine)
|
|
[tool] = _steps(rec, "tool")
|
|
assert tool["attrs"]["decision"]["origin"] == "bypass"
|
|
|
|
|
|
def test_a_summary_is_a_step_with_its_model_call_inside(tmp_path, monkeypatch):
|
|
engine, rec = _engine(tmp_path, [
|
|
AssistantTurn(text="a short summary", usage=TokenUsage(input=900, output=40)),
|
|
AssistantTurn(text="next", usage=TokenUsage(input=300, output=5, cache_read=200)),
|
|
])
|
|
engine._last_context_tokens = 47000
|
|
|
|
def fake_build_state(messages, *, provider, model, **kw):
|
|
provider.complete(model=model, messages=[{"role": "user", "content": "summarize"}])
|
|
return SimpleNamespace(boundary_index=0, carries_outside=False, transcript_path=None)
|
|
|
|
monkeypatch.setattr(_compaction, "build_state", fake_build_state)
|
|
|
|
async def go():
|
|
return await engine._compact_now()
|
|
|
|
assert asyncio.run(go()) is not None
|
|
engine.provider.complete(model=engine.model, messages=[]) # the next main call
|
|
[summary] = _steps(rec, "summary")
|
|
[model] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model" and s["attrs"]["purpose"] == "summary"]
|
|
assert (summary["name"], summary["status"]) == ("summary", "ok")
|
|
assert model["parent_step_id"] == summary["step_id"]
|
|
assert summary["attrs"]["model_step_id"] == model["step_id"]
|
|
assert summary["attrs"]["reason"] == "threshold"
|
|
assert summary["attrs"]["tokens_before"] == 47000
|
|
assert summary["attrs"]["tokens_after"] == 500
|
|
|
|
|
|
def test_no_summary_step_when_nothing_was_summarised(tmp_path, monkeypatch):
|
|
engine, rec = _engine(tmp_path, [])
|
|
monkeypatch.setattr(_compaction, "build_state", lambda messages, **kw: None)
|
|
|
|
async def go():
|
|
return await engine._compact_now()
|
|
|
|
assert asyncio.run(go()) is None
|
|
assert _steps(rec, "summary") == []
|
|
|
|
|
|
def test_a_stop_while_a_plan_card_waits_is_interrupted_not_refused(tmp_path):
|
|
# Round-2 in-app check: the plan handler answers "denied" when the turn is stopped, so
|
|
# the propose_plan step read as refused. A Stop is recorded as interrupted.
|
|
engine, rec = _engine(tmp_path, [_call("p1", "propose_plan", {"plan": "do it"}), AssistantTurn(text="x")],
|
|
mode=Mode.PLAN)
|
|
|
|
async def never(args, call_id):
|
|
await asyncio.sleep(30)
|
|
|
|
engine.plan_approver = never
|
|
|
|
async def go():
|
|
async def stop_soon():
|
|
await asyncio.sleep(0.3)
|
|
engine.request_interrupt()
|
|
task = asyncio.create_task(stop_soon())
|
|
[ev async for ev in engine.run("plan", trigger="user")]
|
|
await task
|
|
|
|
asyncio.run(go())
|
|
[wait] = _steps(rec, "wait")
|
|
[plan] = [s for s in _steps(rec, "tool") if s["name"] == "propose_plan"]
|
|
assert wait["attrs"]["outcome"] == "interrupted"
|
|
assert plan["status"] == "interrupted"
|
|
|
|
|
|
def test_a_stop_while_an_approval_card_waits_is_interrupted_not_refused(tmp_path):
|
|
# Round-2 in-app check (F3): the wait and the path said interrupted, the tool step denied.
|
|
engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="x")])
|
|
|
|
async def never(request):
|
|
await asyncio.sleep(30)
|
|
|
|
engine.approver = never
|
|
|
|
async def go():
|
|
async def stop_soon():
|
|
await asyncio.sleep(0.3)
|
|
engine.request_interrupt()
|
|
task = asyncio.create_task(stop_soon())
|
|
[ev async for ev in engine.run("go", trigger="user")]
|
|
await task
|
|
|
|
asyncio.run(go())
|
|
[tool] = _steps(rec, "tool")
|
|
assert tool["status"] == "interrupted"
|
|
assert tool["attrs"]["decision"]["path"][-1] == {"stage": "resolved", "outcome": "interrupted"}
|
|
|
|
|
|
def test_a_tool_killed_by_stop_is_interrupted(tmp_path):
|
|
# Round-2 in-app check (F2): a shell command killed by Stop answers "interrupted by
|
|
# user" and the step said ok.
|
|
engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "sleep 40"}), AssistantTurn(text="x")],
|
|
mode=Mode.BYPASS_APPROVALS)
|
|
|
|
def run_shell(command: str) -> dict:
|
|
"""Run a command.
|
|
|
|
Args:
|
|
command: the command line
|
|
"""
|
|
engine.request_interrupt() # the Stop arrives while it runs; it is killed
|
|
return {"command": command, "exit_code": None, "error": "interrupted by user"}
|
|
|
|
registry = ToolRegistry()
|
|
registry.register(run_shell, metadata=_Meta())
|
|
engine.registry = registry
|
|
_run(engine)
|
|
[tool] = _steps(rec, "tool")
|
|
assert tool["status"] == "interrupted"
|