mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
Turns now record the stretches the machine could not be seen working: - nobody_watching, on the crossings between no viewer and one; - controller_offline, from the link's loss until it comes back, saved in the state folder so a restart in between still writes it; - clock_jump, when the wall clock runs over 5 s ahead of the turn's clock: later steps follow the wall clock again; - recording_gap gains why the turn was found open and when. Each is a notice with a duration in every running turn, cut at its start and end. Export sends a gap as its own span. Signals 0.4: "Waited while nobody was watching" also counts the wait's overlap with nobody_watching, so a card left in an open session while the desktop was closed counts (golden S2-17d). The Steps panel draws gaps as dashed bands with their times, with Machine offline and Nobody watching totals.
302 lines
12 KiB
Python
302 lines
12 KiB
Python
"""OB-14: gaps in the timeline (phase-2 design §11, recording format 1.2): the machine
|
|
offline from its controller, nobody watching the session, the machine asleep, and the
|
|
app restarted mid-turn. Each is a notice; the first three have a duration."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import jsonschema
|
|
import pytest
|
|
|
|
from coworker.observability import Recorder, RecordingStore, gaps, signals
|
|
from coworker.observability.engine_hooks import TurnRecording
|
|
from coworker.observability.export_spans import Masker, build_turn_spans
|
|
from coworker.server.manager import SessionManager
|
|
|
|
SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
|
|
|
|
|
|
def _check(recording):
|
|
validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)
|
|
errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))
|
|
assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_machine_conditions():
|
|
gaps._machine.clear()
|
|
yield
|
|
gaps._machine.clear()
|
|
|
|
|
|
def _recording(session="sess_g"):
|
|
rec = Recorder()
|
|
obs = TurnRecording(rec, lambda: session)
|
|
return rec, obs
|
|
|
|
|
|
def _begin(obs):
|
|
return obs.begin_turn(kind="run", trigger="user", attrs={
|
|
"approval_mode": "ask", "attended": True,
|
|
"reviewer": {"active": False, "shadow": False, "kind": None, "model": None}}, stamp={}, first_message_id=None)
|
|
|
|
|
|
def _gaps(rec, name):
|
|
return [s for s in rec.as_dict()["steps"] if s["kind"] == "notice" and s["name"] == name]
|
|
|
|
|
|
# -- nobody watching (E15b-2) ------------------------------------------------------------------
|
|
|
|
|
|
def test_nobody_watching_mid_turn_is_a_band_from_leaving_to_coming_back():
|
|
rec, obs = _recording()
|
|
_begin(obs)
|
|
start = obs._turn_started_ms
|
|
obs.condition("nobody_watching", True, at_ms=start + 1000)
|
|
obs.condition("nobody_watching", False, at_ms=start + 61_000)
|
|
obs.end_turn("completed")
|
|
[gap] = _gaps(rec, "nobody_watching")
|
|
assert (gap["started_at"], gap["duration_ms"], gap["status"], gap["attrs"]) == (start + 1000, 60_000, "ok", {})
|
|
assert gap["parent_step_id"] is None
|
|
|
|
|
|
def test_a_gap_still_on_is_cut_at_each_turns_start_and_end():
|
|
rec, obs = _recording()
|
|
obs.condition("nobody_watching", True, at_ms=1) # left long before the turn: a schedule ran it
|
|
import time
|
|
|
|
t1 = _begin(obs)
|
|
time.sleep(0.01)
|
|
obs.end_turn("completed")
|
|
t2 = _begin(obs)
|
|
time.sleep(0.01)
|
|
obs.end_turn("completed")
|
|
one, two = _gaps(rec, "nobody_watching")
|
|
assert one["turn_id"] == t1 and two["turn_id"] == t2
|
|
turns = {t["turn_id"]: t for t in rec.as_dict()["turns"]}
|
|
assert one["started_at"] == turns[t1]["started_at"] # not before its turn
|
|
assert one["started_at"] + one["duration_ms"] <= turns[t1]["started_at"] + turns[t1]["duration_ms"] + 1
|
|
_check(rec.as_dict())
|
|
|
|
|
|
def test_a_condition_outside_any_turn_writes_nothing():
|
|
rec, obs = _recording()
|
|
obs.condition("nobody_watching", True)
|
|
obs.condition("nobody_watching", False)
|
|
assert rec.as_dict()["steps"] == []
|
|
|
|
|
|
def test_only_the_crossings_between_none_and_one_viewer_count():
|
|
told = []
|
|
fake = SimpleNamespace(_session_clients={}, _engines={})
|
|
fake._note_watching = lambda sid, watching: told.append((sid, watching))
|
|
register = SessionManager.register_session_client.__get__(fake)
|
|
unregister = SessionManager.unregister_session_client.__get__(fake)
|
|
a, b = object(), object()
|
|
register("s1", a)
|
|
register("s1", b) # a second window: no row
|
|
unregister("s1", a)
|
|
unregister("s1", b)
|
|
assert told == [("s1", True), ("s1", False)]
|
|
|
|
|
|
def test_the_manager_tells_the_engines_recording():
|
|
rec, obs = _recording("s1")
|
|
fake = SimpleNamespace(_engines={"s1": SimpleNamespace(_obs=obs)})
|
|
note = SessionManager._note_watching.__get__(fake)
|
|
_begin(obs)
|
|
note("s1", False)
|
|
note("s1", True)
|
|
obs.end_turn("completed")
|
|
assert len(_gaps(rec, "nobody_watching")) <= 1 # a band only if any time passed
|
|
note("missing", False) # no engine: nothing, no error
|
|
|
|
|
|
# -- the controller link (E15b-1) --------------------------------------------------------------
|
|
|
|
|
|
def test_machine_offline_reaches_every_running_turn(tmp_path):
|
|
rec1, obs1 = _recording("a")
|
|
rec2, obs2 = _recording("b")
|
|
_begin(obs1)
|
|
_begin(obs2)
|
|
start = max(obs1._turn_started_ms, obs2._turn_started_ms)
|
|
gaps.machine_condition("controller_offline", True, at_ms=start + 10, attrs={"reason": "ping_timeout"})
|
|
gaps.machine_condition("controller_offline", False, at_ms=start + 420_010)
|
|
obs1.end_turn("completed")
|
|
obs2.end_turn("completed")
|
|
for rec in (rec1, rec2):
|
|
[gap] = _gaps(rec, "controller_offline")
|
|
assert gap["duration_ms"] == 420_000 and gap["attrs"] == {"reason": "ping_timeout"}
|
|
_check(rec.as_dict())
|
|
|
|
|
|
def test_an_engine_made_while_offline_hears_it():
|
|
gaps.machine_condition("controller_offline", True, at_ms=5, attrs={"reason": "failed"})
|
|
rec, obs = _recording()
|
|
_begin(obs)
|
|
gaps.machine_condition("controller_offline", False, at_ms=obs._turn_started_ms + 1_000)
|
|
[gap] = _gaps(rec, "controller_offline")
|
|
assert gap["started_at"] == obs._turn_started_ms and gap["attrs"] == {"reason": "failed"}
|
|
|
|
|
|
def test_the_lost_moment_survives_a_restart(tmp_path):
|
|
gaps.link_lost(tmp_path, "closed")
|
|
saved = json.loads((tmp_path / "controller-link-lost.json").read_text("utf-8"))
|
|
assert saved["reason"] == "closed"
|
|
gaps._machine.clear() # the app restarts: the memory is gone, the file is not
|
|
rec, obs = _recording()
|
|
_begin(obs)
|
|
import time
|
|
time.sleep(0.01) # the link comes back a moment after the turn started
|
|
gaps.link_back(tmp_path)
|
|
[gap] = _gaps(rec, "controller_offline")
|
|
assert gap["attrs"] == {"reason": "closed"}
|
|
assert not (tmp_path / "controller-link-lost.json").exists()
|
|
gaps.link_back(tmp_path) # a welcome with nothing lost: nothing
|
|
assert len(_gaps(rec, "controller_offline")) == 1
|
|
|
|
|
|
def test_link_reasons():
|
|
assert gaps.link_reason(None) == "closed"
|
|
assert gaps.link_reason(TimeoutError("keepalive ping timeout")) == "ping_timeout"
|
|
assert gaps.link_reason(OSError("connection refused")) == "failed"
|
|
|
|
|
|
# -- the machine asleep (E15b-3) ---------------------------------------------------------------
|
|
|
|
|
|
def _ev(typ, **kw):
|
|
return {"v": 1, "type": typ, "session_id": "sess_c", "turn_id": "t1", **kw}
|
|
|
|
|
|
def _step(step_id, at, mono, end_at, end_mono):
|
|
return [
|
|
_ev("step_started", step_id=step_id, at_ms=at, mono_ms=mono, kind="tool", name="look",
|
|
parent_step_id=None, call_id=step_id, call_index=0, shadow=False, after_turn=False,
|
|
attrs={}, content={}),
|
|
_ev("step_ended", step_id=step_id, at_ms=end_at, mono_ms=end_mono, status="ok", message_id=None,
|
|
attrs={}, content={}, kind="tool", name="look", parent_step_id=None, call_id=step_id,
|
|
call_index=0, shadow=False, after_turn=False, started_at=at),
|
|
]
|
|
|
|
|
|
def test_a_sleep_mid_turn_is_a_clock_jump_and_later_steps_follow_the_wall_clock():
|
|
rec = Recorder()
|
|
rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
|
|
attrs={}, stamp={}))
|
|
for ev in _step("a", 1_100, 100, 1_200, 200):
|
|
rec.emit(ev)
|
|
# The machine slept 60 s: the wall clock moved on, the monotonic clock did not.
|
|
for ev in _step("b", 61_300, 300, 61_400, 400):
|
|
rec.emit(ev)
|
|
rec.emit(_ev("turn_ended", at_ms=61_500, mono_ms=500, status="completed", last_message_id=None))
|
|
d = rec.as_dict()
|
|
by = {s["name"]: s for s in d["steps"]}
|
|
jump = by["clock_jump"]
|
|
assert jump["started_at"] == 1_200 and jump["duration_ms"] == 60_000
|
|
assert jump["attrs"] == {"skipped_ms": 60_000}
|
|
assert by["look"]["started_at"] in (1_100, 61_300)
|
|
starts = sorted(s["started_at"] for s in d["steps"] if s["kind"] == "tool")
|
|
assert starts == [1_100, 61_300] # the second step is placed by the wall clock again
|
|
assert d["turns"][0]["duration_ms"] == 60_500 # the turn's length counts the sleep
|
|
assert [s["name"] for s in d["steps"]] == ["look", "clock_jump", "look"]
|
|
|
|
|
|
def test_small_drift_is_not_a_jump():
|
|
rec = Recorder()
|
|
rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
|
|
attrs={}, stamp={}))
|
|
for ev in _step("a", 4_000, 100, 4_100, 200): # 2.9 s ahead: within the 5 s allowed
|
|
rec.emit(ev)
|
|
assert not [s for s in rec.as_dict()["steps"] if s["name"] == "clock_jump"]
|
|
|
|
|
|
# -- the app restarted (E15b-4) ----------------------------------------------------------------
|
|
|
|
|
|
def _open_turn(path):
|
|
store = RecordingStore(path, background=False)
|
|
rec = Recorder(store=store)
|
|
rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
|
|
attrs={}, stamp={}))
|
|
store.flush()
|
|
return store
|
|
|
|
|
|
def _gap_attrs(path):
|
|
s = RecordingStore(path, background=False)
|
|
try:
|
|
assert s.close_open_turns() == 1
|
|
[gap] = [x for x in s.load_recording("sess_c")["steps"] if x["name"] == "recording_gap"]
|
|
return gap["attrs"]
|
|
finally:
|
|
s.close()
|
|
|
|
|
|
def test_a_turn_cut_by_a_crash_says_restart(tmp_path):
|
|
path = tmp_path / "recordings.db"
|
|
first = RecordingStore(path, background=False)
|
|
first.close_open_turns() # this run owns the folder
|
|
first._conn.close() # killed: never closed properly
|
|
s = _open_turn(path)
|
|
s._conn.close()
|
|
attrs = _gap_attrs(path)
|
|
assert attrs["cause"] == "restart" and isinstance(attrs["until"], int)
|
|
|
|
|
|
def test_a_turn_cut_by_closing_the_app_says_stopped(tmp_path):
|
|
path = tmp_path / "recordings.db"
|
|
first = RecordingStore(path, background=False)
|
|
first.close_open_turns()
|
|
rec = Recorder(store=first)
|
|
rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
|
|
attrs={}, stamp={}))
|
|
first.close() # a clean stop while the turn ran
|
|
assert _gap_attrs(path)["cause"] == "stopped"
|
|
|
|
|
|
def test_with_no_word_of_the_last_run_the_cause_is_unknown(tmp_path):
|
|
path = tmp_path / "recordings.db"
|
|
s = _open_turn(path)
|
|
s.close() # not the owner: leaves no word
|
|
assert _gap_attrs(path)["cause"] == "unknown"
|
|
|
|
|
|
# -- what reads the gaps -----------------------------------------------------------------------
|
|
|
|
|
|
def test_a_gap_is_its_own_span_and_other_notices_stay_events():
|
|
rec, obs = _recording()
|
|
_begin(obs)
|
|
start = obs._turn_started_ms
|
|
obs.condition("nobody_watching", True, at_ms=start)
|
|
obs.condition("nobody_watching", False, at_ms=start + 5_000)
|
|
obs.instant_step(kind="notice", name="interrupted", status="ok")
|
|
obs.end_turn("interrupted")
|
|
d = rec.as_dict()
|
|
turn = d["turns"][0]
|
|
spans = build_turn_spans(turn, d["steps"], sw={}, dialect="otel", get_piece=lambda k: None, masker=Masker())
|
|
[gap] = [s for s in spans if s["name"] == "openworker.gap.nobody_watching"]
|
|
assert gap["end_ms"] - gap["start_ms"] == 5_000
|
|
[root] = [s for s in spans if s["name"] == "invoke_agent coworker"]
|
|
assert [e["name"] for e in root["events"]] == ["openworker.notice.interrupted"]
|
|
|
|
|
|
def test_a_card_left_in_the_session_while_nobody_watched_is_an_issue():
|
|
turn = {"turn_id": "t1", "started_at": 0, "duration_ms": 1_300_000, "status": "completed", "stamp": {},
|
|
"reviewer": None, "attended": True}
|
|
wait = {"step_id": "w", "kind": "wait", "name": "approval", "status": "ok", "parent_step_id": None,
|
|
"shadow": False, "after_turn": False, "started_at": 0, "duration_ms": 1_200_000, "seq": 1,
|
|
"attrs": {"kind": "approval", "where": "session", "outcome": "approved"}}
|
|
gap = {"step_id": "g", "kind": "notice", "name": "nobody_watching", "status": "ok", "parent_step_id": None,
|
|
"shadow": False, "after_turn": False, "started_at": 100_000, "duration_ms": 1_000_000, "seq": 2, "attrs": {}}
|
|
found = signals.check_turn(turn, [wait, gap])
|
|
[i] = [x for x in found["issues"] if x["check"] == "waited_unwatched"]
|
|
assert "17 min for an answer" in i["text"] and "Inbox" not in i["text"]
|
|
quiet = signals.check_turn(turn, [wait])
|
|
assert not [x for x in quiet["issues"] if x["check"] == "waited_unwatched"]
|