mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
Phase 1 read side (ocw-context docs/observability/phase-1/technical-design.md
§6.3, §7.1, §12).
- coworker/observability/calculator.py: the token pill, the "at least"
marker, failed and over-limit calls, by purpose and model, cache share,
and each turn's time buckets. The shared golden test now also checks
expected-numbers.json: 30 of 30.
- GET /v1/sessions/{id}/turns (turns with stamp, buckets and usage),
/turns/{turn_id}/steps (a light list in canonical order),
/steps/{step_id} (one step in full: content, the exchange rebuilt from
its pieces, the transcript message it points at cut at 4 KB, its audit
rows and child steps) and /usage (the session's numbers and context
fill). A session recorded before Phase 1 answers recorded: false.
- While a turn runs, the recording events go to the session's sockets as
{"type": "step", "events": [...]}, batched every 100 ms.
Not yet: sending an explore helper's steps only when its step ends, and
computing /usage from the store's columns instead of the loaded recording.
107 lines
4.2 KiB
Python
107 lines
4.2 KiB
Python
"""Observability golden test (Phase 1): recording events in, recording out.
|
|
|
|
The scenarios in `tests/corpora/observability/golden-phase1/` are the shared test of
|
|
recording format 1.0 (`ocw-context/docs/observability/golden/phase1/`, frozen 2026-10-31).
|
|
The Rust harness SDK runs the same files, so both engines fold the same events into the
|
|
same recording. Each scenario is replayed into a fresh recorder and compared with its
|
|
`expected-recording.json` exactly, after the one renaming the golden README describes:
|
|
step ids become `s1`, `s2`, ... by their position in the file.
|
|
|
|
The same replay through a `RecordingStore` must give back the same recording from
|
|
`recordings.db`, so nothing is lost or changed on the way to disk.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from coworker.observability import Recorder, RecordingStore
|
|
|
|
GOLDEN = Path(__file__).parent / "corpora" / "observability" / "golden-phase1"
|
|
SCENARIOS = [s["id"] for s in json.loads((GOLDEN / "index.json").read_text("utf-8"))["scenarios"]]
|
|
|
|
|
|
def _events(scenario: str) -> list[dict[str, Any]]:
|
|
lines = (GOLDEN / scenario / "events.jsonl").read_text("utf-8").splitlines()
|
|
return [json.loads(line) for line in lines if line.strip()]
|
|
|
|
|
|
def _renamed(recording: dict[str, Any]) -> dict[str, Any]:
|
|
"""Step ids renamed s1, s2, ... in file order, with every field that holds one."""
|
|
ids = {s["step_id"]: f"s{i}" for i, s in enumerate(recording["steps"], 1)}
|
|
steps = []
|
|
for s in recording["steps"]:
|
|
s = json.loads(json.dumps(s))
|
|
s["step_id"] = ids[s["step_id"]]
|
|
s["parent_step_id"] = ids.get(s["parent_step_id"], s["parent_step_id"])
|
|
if s["kind"] == "summary" and s["attrs"].get("model_step_id"):
|
|
s["attrs"]["model_step_id"] = ids.get(
|
|
s["attrs"]["model_step_id"], s["attrs"]["model_step_id"]
|
|
)
|
|
steps.append(s)
|
|
return {"turns": recording["turns"], "steps": steps}
|
|
|
|
|
|
def _replay(events: list[dict[str, Any]], store: RecordingStore | None = None) -> Recorder:
|
|
rec = Recorder(store=store)
|
|
for ev in events:
|
|
rec.emit(ev)
|
|
return rec
|
|
|
|
|
|
def test_every_scenario_is_listed():
|
|
folders = sorted(p.name for p in GOLDEN.iterdir() if (p / "events.jsonl").exists())
|
|
assert sorted(SCENARIOS) == folders
|
|
assert len(SCENARIOS) == 30
|
|
|
|
|
|
@pytest.mark.parametrize("scenario", SCENARIOS)
|
|
def test_events_fold_to_the_expected_recording(scenario):
|
|
expected = json.loads((GOLDEN / scenario / "expected-recording.json").read_text("utf-8"))
|
|
got = _renamed(_replay(_events(scenario)).as_dict())
|
|
assert got == expected
|
|
|
|
|
|
@pytest.mark.parametrize("scenario", SCENARIOS)
|
|
def test_the_store_gives_back_the_same_recording(scenario, tmp_path):
|
|
events = _events(scenario)
|
|
in_memory = _replay(events).as_dict()
|
|
store = RecordingStore(tmp_path / "recordings.db", background=False)
|
|
try:
|
|
_replay(events, store)
|
|
sessions = {t["session_id"] for t in in_memory["turns"]}
|
|
for session in sessions:
|
|
want = {
|
|
"turns": [t for t in in_memory["turns"] if t["session_id"] == session],
|
|
"steps": [s for s in in_memory["steps"] if s["session_id"] == session],
|
|
}
|
|
assert store.load_recording(session) == want
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
@pytest.mark.parametrize("scenario", SCENARIOS)
|
|
def test_the_screen_numbers(scenario):
|
|
"""`expected-numbers.json`: every token number and time bucket the screens show."""
|
|
from coworker.observability import calculator
|
|
|
|
expected = json.loads((GOLDEN / scenario / "expected-numbers.json").read_text("utf-8"))
|
|
rec = _replay(_events(scenario)).as_dict()
|
|
sessions = sorted({t["session_id"] for t in rec["turns"] if t["session_id"]}
|
|
| {s["session_id"] for s in rec["steps"] if s["session_id"]})
|
|
|
|
def usage(sid=None):
|
|
u = calculator.usage(rec, sid)
|
|
u.pop("unreported_step_ids") # an addition for the "≥" links (M-R19), not in the golden
|
|
return u
|
|
|
|
got = {"usage": usage()}
|
|
if len(sessions) > 1:
|
|
got["usage_by_session"] = {sid: usage(sid) for sid in sessions}
|
|
got["buckets_by_turn"] = {t["turn_id"]: calculator.buckets(rec, t) for t in rec["turns"]}
|
|
assert got == expected
|