mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
- P3-1: the team rail's token total adds up the lead's and workers' recordings, so it matches their Steps pills (titles and auto-approver calls included). Sessions with no recording keep the old count. - P3-3: the backend closes a manual run when its turn ends, so it no longer depends on the window. At start, runs an earlier app left "running" are closed, or marked failed if the app stopped mid-turn. - P3-4: at start, a background pass works out Issues for finished turns with no result or one from an older signals.yaml (TD3-8). Results are stored as seen. - P3-5: the store's usage_reported column is filled for model steps only. - P3-6: "sent today" counts from the viewer's midnight, sent by the desktop as day_start_ms (TD3-2). - P3-7: a model call with no purpose is "unattributed", not "main".
142 lines
6.1 KiB
Python
142 lines
6.1 KiB
Python
"""Phase 1 and 2 problems found by the Phase 3 design (observability/FIX-LIST.md, P3-n):
|
|
P3-3 a manual run is closed when its turn ends, and at start; P3-4 Issues are re-checked at
|
|
start; P3-5 `usage_reported` is a model step's column only; P3-6 "today" is the viewer's day;
|
|
P3-7 a model call with no purpose is "unattributed"."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
|
|
from coworker.observability import calculator, read_api
|
|
from coworker.observability.store import _step_row
|
|
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient
|
|
|
|
auto = importlib.import_module("test_automation")
|
|
|
|
|
|
class _Scripted(ProviderClient):
|
|
def __init__(self, n=4):
|
|
self._turns = [AssistantTurn(text=f"Done {i}.", finish_reason="stop") for i in range(n)]
|
|
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
return self._turns.pop(0)
|
|
|
|
def capabilities(self, model):
|
|
return ModelCapabilities()
|
|
|
|
|
|
def _manager(tmp_path, monkeypatch):
|
|
from coworker.server.manager import SessionManager
|
|
|
|
monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
|
|
ws = tmp_path / "ws"
|
|
ws.mkdir(exist_ok=True)
|
|
return SessionManager(data_dir=tmp_path / "data", provider=_Scripted()), ws
|
|
|
|
|
|
async def _manual_turn(manager, ws):
|
|
task = auto._task(workspace=str(ws), agent="cowork")
|
|
manager.task_store.save(task)
|
|
prep = manager.prepare_manual_run(task.id)
|
|
engine = manager.get_engine(prep["session_id"], workspace=str(ws), agent="cowork")
|
|
async for _ in engine.run(prep["prompt"]):
|
|
pass
|
|
return task, prep
|
|
|
|
|
|
async def test_a_manual_run_is_closed_when_its_turn_ends(tmp_path, monkeypatch):
|
|
manager, ws = _manager(tmp_path, monkeypatch)
|
|
task, prep = await _manual_turn(manager, ws)
|
|
assert manager.task_store.runs(task.id)[0].status == "running"
|
|
# No window asks for it: the turn's end is enough.
|
|
await manager.broadcast_session(prep["session_id"], {"type": "turn_done", "data": {}})
|
|
run = manager.task_store.runs(task.id)[0]
|
|
assert run.status == "ok" and run.result_text == "Done 0."
|
|
assert manager.task_store.get(task.id).run_count == 1
|
|
# The desktop's own finalize, arriving later, changes nothing.
|
|
manager.finalize_manual_run(task.id, prep["run_id"])
|
|
assert manager.task_store.get(task.id).run_count == 1
|
|
|
|
|
|
async def test_stale_manual_runs_are_closed_at_start(tmp_path, monkeypatch):
|
|
manager, ws = _manager(tmp_path, monkeypatch)
|
|
done, _ = await _manual_turn(manager, ws)
|
|
cut, cut_prep = await _manual_turn(manager, ws)
|
|
waiting = auto._task(workspace=str(ws), agent="cowork")
|
|
manager.task_store.save(waiting)
|
|
manager.prepare_manual_run(waiting.id) # opened, never run
|
|
store = manager.recordings_store
|
|
store.flush()
|
|
with store._lock, store._conn: # the app stopped mid-turn: start closed it as unknown
|
|
store._conn.execute("UPDATE turns SET status = 'unknown' WHERE session_id = ?", (cut_prep["session_id"],))
|
|
|
|
assert manager.finish_stale_manual_runs() == 2
|
|
assert manager.task_store.runs(done.id)[0].status == "ok"
|
|
failed = manager.task_store.runs(cut.id)[0]
|
|
assert failed.status == "error" and failed.error
|
|
assert manager.task_store.get(cut.id).last_status == "error"
|
|
assert manager.task_store.runs(waiting.id)[0].status == "running"
|
|
assert manager.finish_stale_manual_runs() == 0
|
|
|
|
|
|
async def test_issues_are_rechecked_at_start(tmp_path, monkeypatch):
|
|
from coworker.observability import signals
|
|
|
|
manager, ws = _manager(tmp_path, monkeypatch)
|
|
engine = manager.get_engine("s1", workspace=str(ws))
|
|
async for _ in engine.run("hello"):
|
|
pass
|
|
store = manager.recordings_store
|
|
store.flush()
|
|
version = signals.version()
|
|
with store._lock, store._conn: # a turn that ended before checks existed, or an older version
|
|
store._conn.execute("UPDATE turns SET issues = NULL WHERE session_id = 's1'")
|
|
assert store.sessions_needing_check(version) == ["s1"]
|
|
|
|
assert manager.recheck_issues() == 1
|
|
store.flush()
|
|
[result] = store.read_issues("s1").values()
|
|
assert result["signals_version"] == version
|
|
assert result["seen"] is True # an old turn's result never lights the sidebar dot
|
|
assert store.sessions_needing_check(version) == []
|
|
assert manager.recheck_issues() == 0
|
|
|
|
|
|
async def test_background_checks_run_in_a_thread(tmp_path, monkeypatch):
|
|
manager, ws = _manager(tmp_path, monkeypatch)
|
|
task, _ = await _manual_turn(manager, ws)
|
|
t = manager.start_background_checks()
|
|
t.join(timeout=10)
|
|
assert not t.is_alive()
|
|
assert manager.task_store.runs(task.id)[0].status == "ok"
|
|
|
|
|
|
def test_usage_reported_is_a_model_steps_column_only():
|
|
cols = __import__("coworker.observability.store", fromlist=["_STEP_COLUMNS"])._STEP_COLUMNS
|
|
at = cols.index("usage_reported")
|
|
model = {"step_id": "m", "kind": "model", "attrs": {"usage": {"reported": True, "input_fresh": 5, "output": 1}}}
|
|
check = {"step_id": "a", "kind": "auto_approver", "attrs": {"usage_reported": False}}
|
|
assert _step_row(model)[at] == 1
|
|
assert _step_row(check)[at] is None # it read as a call of 0 tokens to any sum over the column
|
|
|
|
|
|
def test_a_model_call_with_no_purpose_is_unattributed():
|
|
u = {"reported": True, "input_fresh": 10, "cache_read": 0, "cache_write": 0, "output": 2}
|
|
rec = {"turns": [{"turn_id": "t", "status": "ok"}],
|
|
"steps": [{"step_id": "m", "turn_id": "t", "kind": "model", "status": "ok",
|
|
"attrs": {"model": "model-a", "usage": u}}]}
|
|
assert calculator.usage(rec)["by_purpose"] == {"unattributed": 12}
|
|
assert list(read_api.usage(rec, "s")["by_model_parts"]) == ["model-a (unattributed)"]
|
|
|
|
|
|
def test_today_is_the_viewers_day(tmp_path, monkeypatch):
|
|
manager, _ = _manager(tmp_path, monkeypatch)
|
|
obs = manager.observability
|
|
seen = []
|
|
monkeypatch.setattr(obs.store, "outside_calls", lambda since_ms: seen.append(since_ms) or {"calls": 0, "tokens": 0})
|
|
obs.status(day_start_ms=1_700_000_000_000)
|
|
obs.status() # no viewer's day: this machine's midnight
|
|
obs.status(day_start_ms=-5) # nonsense: the same
|
|
assert seen[0] == 1_700_000_000_000
|
|
assert seen[1] == seen[2] != seen[0]
|