mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
"Turn N of M" opens a list of the session's turns: number, start time, the first words of your message, tokens, an issue badge and the wake reason; click a line, or type a number and press Enter. A search box keeps the turns whose message, answer, tool names, file names or issue titles match, ignoring case, and says "No turns match." when none do. It reads only what the GUI already holds: the conversation, split at each turn's answer by its message id, and the recorded turns, which now list their warning checks. ← and → move between turns while the panel has focus and no text box does. Under the picker, a thin bar per turn: its height is the turn's tokens, amber when it has issues, the current one marked, click to jump, and a hover line "Turn N · 12.3k tokens · 1 issue"; past 120 turns the bars are thin and the strip scrolls.
239 lines
10 KiB
Python
239 lines
10 KiB
Python
"""Observability Phase 2, OB-3: the Issues checks (E8b) and the 1.2 fields they read (E8a).
|
|
|
|
The scenarios in `tests/corpora/observability/golden-phase2/` are the shared test of the checks:
|
|
a copy of ocw-context docs/observability/golden/phase2 (signals.yaml 0.7, recording format 1.2).
|
|
Every recording must give exactly its expected issues, and the hash vectors must hash the same,
|
|
in this engine and in the Rust harness."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from coworker.engine import TurnEngine
|
|
from coworker.observability import Recorder, signals
|
|
from coworker.observability.canonical import jcs, short_hash, text_hash
|
|
from coworker.permissions import Mode, PermissionEngine
|
|
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
|
|
from coworker.tools import ToolRegistry
|
|
|
|
GOLDEN = Path(__file__).parent / "corpora" / "observability" / "golden-phase2"
|
|
INDEX = json.loads((GOLDEN / "index.json").read_text(encoding="utf-8"))
|
|
|
|
|
|
def _expected(rec: dict) -> dict:
|
|
res = signals.check_turn(rec["turns"][0], rec["steps"])
|
|
fired: dict[str, list[int]] = {}
|
|
for i in res["issues"]:
|
|
fired.setdefault(i["check"], []).append(i["step_seq"])
|
|
return {"signals_version": res["signals_version"], "warn": res["warn"],
|
|
"fired": {k: sorted(x for x in v if x is not None) for k, v in sorted(fired.items())},
|
|
"fine": sorted(f["check"] for f in res["fine"]),
|
|
"not_checked": sorted(n["check"] for n in res["not_checked"])}
|
|
|
|
|
|
def test_the_shipped_signals_file_is_the_golden_tests_version():
|
|
assert signals.version() == INDEX["signals_version"]
|
|
from coworker.observability import scopes
|
|
|
|
known = set(signals.CHECKS) | set(scopes.SERIES_CHECKS) | set(scopes.TASK_CHECKS)
|
|
assert set(signals.signals()["checks"]) <= known
|
|
for cid, check in signals.signals()["checks"].items():
|
|
assert (signals.scope_of(check) == "turn") == (cid in signals.CHECKS), cid
|
|
|
|
|
|
@pytest.mark.parametrize("sid", [s["id"] for s in INDEX["scenarios"]])
|
|
def test_golden_scenario_gives_exactly_its_issues(sid):
|
|
rec = json.loads((GOLDEN / sid / "recording.json").read_text(encoding="utf-8"))
|
|
want = json.loads((GOLDEN / sid / "expected-issues.json").read_text(encoding="utf-8"))
|
|
assert _expected(rec) == want
|
|
|
|
|
|
def test_hash_vectors():
|
|
v = json.loads((GOLDEN / "vectors" / "hashes.json").read_text(encoding="utf-8"))
|
|
for case in v["args"]:
|
|
assert jcs(case["value"]) == case["canonical"], case["name"]
|
|
assert short_hash(case["value"]) == case["args_hash"], case["name"]
|
|
for item in v["plan_items"]:
|
|
assert text_hash(item["text"]) == item["h"]
|
|
|
|
|
|
# -- E8a: what the engine records for the checks ---------------------------------------------
|
|
|
|
class _Scripted(ProviderClient):
|
|
def __init__(self, turns):
|
|
self._turns = list(turns)
|
|
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
return self._turns.pop(0)
|
|
|
|
def capabilities(self, model):
|
|
return ModelCapabilities()
|
|
|
|
|
|
class _Meta:
|
|
category: str = ""
|
|
risk_level: str = "low"
|
|
requires_approval: bool = False
|
|
|
|
|
|
def read_file(path: str) -> dict:
|
|
"""Read a file.
|
|
|
|
Args:
|
|
path: the file
|
|
"""
|
|
return {"error": "file not found"} if path == "missing.txt" else {"text": "hi"}
|
|
|
|
|
|
def todo_write(todos: list = None) -> dict:
|
|
"""Replace the task list.
|
|
|
|
Args:
|
|
todos: the items
|
|
"""
|
|
return {"count": len(todos or [])}
|
|
|
|
|
|
def _call(cid, name, args):
|
|
return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls")
|
|
|
|
|
|
def _run(tmp_path, turns, *, max_iterations=12):
|
|
registry = ToolRegistry()
|
|
registry.register(read_file, metadata=_Meta())
|
|
registry.register(todo_write, metadata=_Meta())
|
|
engine = TurnEngine(
|
|
provider=_Scripted(turns), registry=registry,
|
|
permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),
|
|
model="openai:model-a", instructions="You are a test coworker.", max_iterations=max_iterations,
|
|
)
|
|
engine.audit_context = {"session_id": "sess_i"}
|
|
rec = Recorder()
|
|
engine.recorder = rec
|
|
|
|
async def go():
|
|
return [ev async for ev in engine.run("go", trigger="user")]
|
|
|
|
asyncio.run(go())
|
|
return rec.as_dict()
|
|
|
|
|
|
def test_tool_steps_carry_the_12_fields(tmp_path):
|
|
d = _run(tmp_path, [
|
|
_call("c1", "todo_write", {"todos": [{"content": "read the notes", "status": "in_progress"},
|
|
{"content": "answer", "status": "completed"}]}),
|
|
_call("c2", "read_file", {"path": "missing.txt"}),
|
|
_call("c3", "nope_tool", {"x": 1}),
|
|
AssistantTurn(text="done"),
|
|
])
|
|
assert d["turns"][0]["max_iterations"] == 12
|
|
tools = {s["call_id"]: s for s in d["steps"] if s["kind"] == "tool"}
|
|
plan = tools["c1"]["attrs"]
|
|
assert plan["args_hash"] == short_hash({"todos": [{"content": "read the notes", "status": "in_progress"},
|
|
{"content": "answer", "status": "completed"}]})
|
|
assert plan["plan"] == {"items": [{"h": text_hash("read the notes"), "s": "in_progress"},
|
|
{"h": text_hash("answer"), "s": "done"}]}
|
|
assert plan["reaches_network"] is False and plan["repeat_ok"] is False and plan["error_kind"] is None
|
|
missing = tools["c2"]["attrs"]
|
|
assert tools["c2"]["status"] == "ok" and missing["result_error"] is True
|
|
unknown = tools["c3"]
|
|
assert unknown["status"] == "error" and unknown["attrs"]["error_kind"] == "unknown_tool"
|
|
assert unknown["attrs"]["reaches_network"] is None # it never ran
|
|
|
|
|
|
def test_a_recorded_turn_gives_its_issues(tmp_path):
|
|
d = _run(tmp_path, [
|
|
_call("c1", "read_file", {"path": "a"}), _call("c2", "read_file", {"path": "a"}),
|
|
_call("c3", "read_file", {"path": "a"}), _call("c4", "nope_tool", {}),
|
|
AssistantTurn(text="done"),
|
|
])
|
|
res = signals.check_turn(d["turns"][0], d["steps"])
|
|
fired = {i["check"] for i in res["issues"]}
|
|
assert {"repeated_step", "made_up_tool"} <= fired
|
|
assert "tool_failed" not in fired # the unknown tool is made_up_tool's, not a tool failure
|
|
assert any(f["check"] == "step_limit" and "of 12" in f["text"] for f in res["fine"])
|
|
|
|
|
|
def test_arguments_that_do_not_fit_the_tool_are_bad_arguments(tmp_path):
|
|
"""Labelling (T11): a model gave replace_in_file a name it does not take. The call fails at
|
|
the call itself: the model's mistake, so made_up_tool, not "A tool failed"."""
|
|
d = _run(tmp_path, [
|
|
_call("c1", "read_file", {"pth": "a"}), # a name the tool does not take
|
|
_call("c2", "read_file", {}), # the required one missing
|
|
_call("c3", "read_file", {"path": "a"}), # fits
|
|
AssistantTurn(text="done"),
|
|
])
|
|
tools = {s["call_id"]: s for s in d["steps"] if s["kind"] == "tool"}
|
|
assert tools["c1"]["status"] == "error" and tools["c1"]["attrs"]["error_kind"] == "bad_arguments"
|
|
assert tools["c2"]["attrs"]["error_kind"] == "bad_arguments"
|
|
assert tools["c3"]["attrs"]["error_kind"] is None
|
|
res = signals.check_turn(d["turns"][0], d["steps"])
|
|
found = [i for i in res["issues"] if i["check"] == "made_up_tool"]
|
|
assert len(found) == 2 and found[0]["text"] == "The model sent arguments that do not fit read_file."
|
|
assert "tool_failed" not in {i["check"] for i in res["issues"]}
|
|
|
|
|
|
def test_a_tool_that_fails_by_itself_is_still_raised(tmp_path):
|
|
def broken(path: str) -> dict:
|
|
"""Break.
|
|
|
|
Args:
|
|
path: anything
|
|
"""
|
|
raise RuntimeError("disk on fire")
|
|
|
|
from coworker.engine import _args_fit
|
|
from coworker.tools import ToolRegistry as _R
|
|
|
|
reg = _R()
|
|
reg.register(broken, metadata=_Meta())
|
|
assert _args_fit(reg.get("broken"), {"path": "x"}) is True
|
|
assert _args_fit(reg.get("broken"), {"path": "x", "extra": 1}) is False
|
|
assert _args_fit(None, {"anything": 1}) is True # nothing to check against
|
|
|
|
|
|
# -- E8c: stored with the turn and served ------------------------------------------------------
|
|
|
|
def test_issues_are_stored_at_turn_end_and_served(tmp_path):
|
|
from fastapi.testclient import TestClient
|
|
|
|
from coworker.server import SessionManager, create_app
|
|
|
|
(tmp_path / "notes.txt").write_text("Friday", encoding="utf-8")
|
|
|
|
class _Provider(_Scripted):
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
if str(messages[0].get("content") or "").startswith("You title"):
|
|
return AssistantTurn(text="Notes")
|
|
return super().complete(model=model, messages=messages, tools=tools, **settings)
|
|
|
|
provider = _Provider([
|
|
_call("c1", "read_file", {"path": "missing.txt"}),
|
|
AssistantTurn(text="It is missing.", finish_reason="stop"),
|
|
])
|
|
manager = SessionManager(workspace=tmp_path, provider=provider)
|
|
client = TestClient(create_app(manager))
|
|
with client.websocket_connect("/ws/session/i1") as ws:
|
|
assert ws.receive_json()["type"] == "ready"
|
|
ws.send_json({"type": "user_message", "text": "read missing.txt"})
|
|
while ws.receive_json()["type"] != "turn_done":
|
|
pass
|
|
|
|
[turn] = client.get("/v1/sessions/i1/turns").json()["turns"]
|
|
assert turn["issues"] == {"warn": 1, "info": 0}
|
|
assert turn["issue_checks"] == ["tool_failed"] # T3-14: the turn list searches by issue title
|
|
got = client.get(f"/v1/sessions/i1/turns/{turn['turn_id']}/issues").json()
|
|
assert got["signals_version"] == signals.version() and got["seen"] is False
|
|
[failed] = got["issues"]
|
|
assert failed["check"] == "tool_failed" and failed["facts"]["how"] == "result_error"
|
|
assert {f["check"] for f in got["fine"]} >= {"repeated_step", "cut_off", "step_limit"}
|
|
row = next(s for s in client.get("/v1/sessions").json()["sessions"] if s["session_id"] == "i1")
|
|
assert row["issues_unseen"] == 1
|
|
assert client.post("/v1/sessions/i1/issues/seen").json() == {"ok": True}
|
|
row = next(s for s in client.get("/v1/sessions").json()["sessions"] if s["session_id"] == "i1")
|
|
assert row["issues_unseen"] == 0
|