mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
stamp_changed was in the schema but never written. The engine now compares attendance, the reviewer and the sandbox with the turn's start at each model call and after the sandbox starts, and writes the notice with only what changed. The sandbox a first turn makes now reaches the recording, with its runner version. The sandbox stamp's site count read a list no provider has, so it was always empty; it now counts the profile's hosts, the machine's extra hosts and the hosts the copied logins need. A lead's steer_worker woke the worker as a channel turn; it is a team wake. The Steps panel shows what changed when a Settings changed row is opened.
312 lines
13 KiB
Python
312 lines
13 KiB
Python
"""Observability E15a, E16, E5, E13: stamps, the coworker fingerprint, the context step,
|
|
notices and per-call sandbox facts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
from coworker.engine import TurnEngine
|
|
from coworker.observability import Recorder
|
|
from coworker.observability import stamps
|
|
from coworker.permissions import Mode, PermissionEngine
|
|
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
|
|
from coworker.tools import ToolRegistry
|
|
|
|
|
|
class _Scripted(ProviderClient):
|
|
def __init__(self, turns):
|
|
self._turns = list(turns)
|
|
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
turn = self._turns.pop(0)
|
|
if isinstance(turn, Exception):
|
|
raise turn
|
|
return turn
|
|
|
|
def capabilities(self, model):
|
|
return ModelCapabilities()
|
|
|
|
|
|
class _ReadMeta:
|
|
category: str = ""
|
|
risk_level: str = "low"
|
|
requires_approval: bool = False
|
|
|
|
|
|
_BLOCK = {"text": ""}
|
|
|
|
|
|
def look(path: str) -> dict:
|
|
"""Read something.
|
|
|
|
Args:
|
|
path: what to read
|
|
"""
|
|
if _BLOCK["text"]:
|
|
_BLOCK["text"] += ", b" # a skill became available mid-turn
|
|
return {"path": path}
|
|
|
|
|
|
def _engine(tmp_path, turns, *, blocks=None, max_iterations=12):
|
|
registry = ToolRegistry()
|
|
registry.register(look, metadata=_ReadMeta())
|
|
_BLOCK["text"] = blocks[0] if blocks else ""
|
|
engine = TurnEngine(
|
|
provider=_Scripted(turns), registry=registry,
|
|
permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),
|
|
model="openai:model-a", instructions="You are a test coworker.",
|
|
context_provider=(lambda: _BLOCK["text"]) if blocks else None,
|
|
max_iterations=max_iterations,
|
|
)
|
|
engine.audit_context = {"session_id": "sess_c"}
|
|
engine.persona_fp = "9f3a21"
|
|
engine.context_facts = {"memories": [12, 15], "memory_mode": "full", "rules": 2, "agents_md": 1}
|
|
rec = Recorder()
|
|
engine.recorder = rec
|
|
return engine, rec
|
|
|
|
|
|
def _run(engine, text="go"):
|
|
async def go():
|
|
return [ev async for ev in engine.run(text, trigger="user")]
|
|
|
|
return asyncio.run(go())
|
|
|
|
|
|
def _call(cid, name, args):
|
|
return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls")
|
|
|
|
|
|
def test_the_turn_is_stamped(tmp_path):
|
|
engine, rec = _engine(tmp_path, [AssistantTurn(text="hi")])
|
|
_run(engine)
|
|
stamp = rec.as_dict()["turns"][0]["stamp"]
|
|
assert stamp["machine"]["kind"] in ("this_machine", "own_machine", "cloud")
|
|
assert stamp["machine"]["app_version"] != "0.0.0"
|
|
assert stamp["model"]["id"] == "openai:model-a"
|
|
assert stamp["persona_fp"] == "9f3a21"
|
|
|
|
|
|
def test_machine_kind():
|
|
assert stamps.machine_kind(None) == "this_machine"
|
|
assert stamps.machine_kind("https://machines.openworker.com/") == "cloud"
|
|
assert stamps.machine_kind("https://my-box.example:8443") == "own_machine"
|
|
|
|
|
|
def test_fingerprint_changes_only_with_the_coworkers_own_files(tmp_path):
|
|
skills = tmp_path / "skills"
|
|
(skills / "a").mkdir(parents=True)
|
|
(skills / "a" / "SKILL.md").write_text("one", encoding="utf-8")
|
|
fp = stamps.persona_fingerprint("helper", "manifest text", [skills])
|
|
assert len(fp) == 6
|
|
assert stamps.persona_fingerprint("helper", "manifest text", [skills]) == fp
|
|
(skills / "a" / "SKILL.md").write_text("two", encoding="utf-8")
|
|
assert stamps.persona_fingerprint("helper", "manifest text", [skills]) != fp
|
|
assert stamps.persona_fingerprint("other", "manifest text", [skills]) != stamps.persona_fingerprint(
|
|
"helper", "manifest text", [skills])
|
|
|
|
|
|
def test_a_context_step_at_the_first_model_call_and_a_notice_when_it_changes(tmp_path):
|
|
engine, rec = _engine(tmp_path, [_call("c1", "look", {"path": "x"}), AssistantTurn(text="ok")],
|
|
blocks=["Folders: /w\nSkills: a"])
|
|
_run(engine)
|
|
steps = rec.as_dict()["steps"]
|
|
[context] = [s for s in steps if s["kind"] == "context"]
|
|
a = context["attrs"]
|
|
assert context["duration_ms"] == 0
|
|
assert a["size_chars"] == len("Folders: /w\nSkills: a") and len(a["block_hash"]) == 6
|
|
assert len(a["session_prompt_hash"]) == 6
|
|
assert a["session_start"] == {"memories": [12, 15], "memory_mode": "full", "rules": 2, "agents_md": 1}
|
|
[changed] = [s for s in steps if s["kind"] == "notice"]
|
|
assert changed["name"] == "context_changed"
|
|
assert changed["attrs"]["block_hash"] != a["block_hash"]
|
|
assert steps[0]["kind"] == "context"
|
|
|
|
|
|
def test_session_start_facts_only_on_the_first_turn(tmp_path):
|
|
engine, rec = _engine(tmp_path, [AssistantTurn(text="a"), AssistantTurn(text="b")])
|
|
_run(engine, "one")
|
|
_run(engine, "two")
|
|
contexts = [s for s in rec.as_dict()["steps"] if s["kind"] == "context"]
|
|
assert len(contexts) == 2
|
|
assert "session_start" in contexts[0]["attrs"] and "session_start" not in contexts[1]["attrs"]
|
|
|
|
|
|
def test_error_and_step_limit_are_notices(tmp_path):
|
|
engine, rec = _engine(tmp_path, [RuntimeError("boom")])
|
|
_run(engine)
|
|
assert [s["name"] for s in rec.as_dict()["steps"] if s["kind"] == "notice"] == ["error"]
|
|
|
|
engine, rec = _engine(tmp_path, [_call("c1", "look", {"path": "x"}), _call("c2", "look", {"path": "y"})],
|
|
max_iterations=1)
|
|
_run(engine)
|
|
assert "step_limit" in [s["name"] for s in rec.as_dict()["steps"] if s["kind"] == "notice"]
|
|
|
|
|
|
def test_blocked_sites_go_to_the_tool_step_not_the_transcript(tmp_path):
|
|
class _Sandbox:
|
|
reports_blocked = True
|
|
|
|
def describe(self):
|
|
return {"provider": "openshell", "enforcement": "full"}
|
|
|
|
def fetch(url: str) -> dict:
|
|
"""Fetch a page.
|
|
|
|
Args:
|
|
url: the page
|
|
"""
|
|
return {"exit_code": 1, "blocked_sites": ["semgrep.dev:443"], "network_note": "blocked",
|
|
"_blocked": [{"host": "semgrep.dev", "port": 443, "count": 2, "first_at": 1, "late": False,
|
|
"uncertain": True}]}
|
|
|
|
engine, rec = _engine(tmp_path, [_call("c1", "fetch", {"url": "https://semgrep.dev"}), AssistantTurn(text="ok")])
|
|
engine.registry.register(fetch, metadata=_ReadMeta())
|
|
engine.sandbox_workspace = _Sandbox()
|
|
_run(engine)
|
|
[tool] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]
|
|
assert tool["attrs"]["sandbox"] == {"blocked_known": True, "blocked_count": 2, "notice": None}
|
|
assert tool["content"]["blocked"][0]["host"] == "semgrep.dev"
|
|
result = next(m for m in engine.messages if m.get("role") == "tool")
|
|
assert "_blocked" not in str(result["content"])
|
|
assert rec.as_dict()["turns"][0]["stamp"]["sandbox"] == {"provider": "openshell", "enforcement": "full"}
|
|
|
|
|
|
def test_network_note_keeps_the_counts(tmp_path, monkeypatch):
|
|
from coworker.sandbox.workspace import RunnerWorkspace
|
|
|
|
ws = RunnerWorkspace.__new__(RunnerWorkspace)
|
|
ws._blocked_told = 0.0
|
|
ws.provider = type("P", (), {"blocked_lag": 0.0})()
|
|
monkeypatch.setattr(RunnerWorkspace, "reports_blocked", property(lambda self: True), raising=False)
|
|
monkeypatch.setattr(ws, "blocked_since", lambda t: [(100.0, "semgrep.dev:443"), (101.0, "semgrep.dev:443")], raising=False)
|
|
monkeypatch.setattr(ws, "_asking", lambda: False, raising=False)
|
|
note = ws.network_note(started=100.0, result={"exit_code": 0})
|
|
assert note["blocked_sites"] == ["semgrep.dev:443"]
|
|
assert note["_blocked"] == [{"host": "semgrep.dev", "port": 443, "count": 2, "first_at": 100000,
|
|
"late": False, "uncertain": True}]
|
|
|
|
|
|
def test_the_skills_menu_is_the_one_offered_this_turn(tmp_path):
|
|
# Round-2 in-app check: a skill disabled mid-session left the menu, but the context step
|
|
# still counted every skill on disk. The step counts what the turn's block offered.
|
|
from coworker.agent import build_engine
|
|
from coworker.agents.chat import chat_agent
|
|
|
|
skills = tmp_path / "skills"
|
|
for name in ("alpha", "beta"):
|
|
(skills / name).mkdir(parents=True)
|
|
(skills / name / "SKILL.md").write_text(
|
|
f"---\nname: {name}\ndescription: The {name} skill.\n---\nDo {name}.\n", encoding="utf-8")
|
|
offered = {"alpha", "beta"}
|
|
(tmp_path / "w").mkdir()
|
|
engine = build_engine(agent=chat_agent(), model="openai:model-a", workspace=tmp_path / "w",
|
|
provider=_Scripted([AssistantTurn(text="a"), AssistantTurn(text="b")]),
|
|
extra_skill_dirs=[skills], skill_filter=lambda: set(offered))
|
|
engine.recorder = rec = Recorder()
|
|
_run(engine, "one")
|
|
offered.discard("beta") # turned off in Settings between the two messages
|
|
_run(engine, "two")
|
|
menus = [s["attrs"]["per_turn"] for s in rec.as_dict()["steps"] if s["kind"] == "context"]
|
|
assert [m["skills_menu"] for m in menus] == [2, 1]
|
|
assert menus[0]["skills_hash"] != menus[1]["skills_hash"]
|
|
|
|
|
|
def test_sites_count_the_profile_the_machine_list_and_the_logins_hosts():
|
|
"""V2-28: the stamp's `sites` read a list no provider has, so it was always None."""
|
|
provider = type("P", (), {"profile": "allowlist", "extra_hosts": ["a.example:443", "b.example:443"],
|
|
"copied": type("C", (), {"hosts": ["a.example:443", "c.example:443"]})()})()
|
|
ws = type("W", (), {"provider": provider})()
|
|
assert stamps._site_count(ws) == 3
|
|
provider.profile = "open"
|
|
assert stamps._site_count(ws) is None # any host: no count
|
|
assert stamps._site_count(type("W", (), {"provider": object()})()) is None # keeps no list
|
|
|
|
|
|
class _LateSandbox:
|
|
"""A sandbox the first turn makes after the turn's stamp was taken."""
|
|
|
|
reports_blocked = False
|
|
|
|
def __init__(self):
|
|
self.started = False
|
|
self.provider = type("P", (), {"name": "openshell", "profile": "allowlist", "extra_hosts": ["x.example:443"]})()
|
|
|
|
def ensure_started(self):
|
|
self.started = True
|
|
|
|
def describe(self):
|
|
info = {"provider": "openshell", "enforcement": "full"}
|
|
if self.started:
|
|
info["runner"] = {"runner_version": "0.4.1"}
|
|
return info
|
|
|
|
|
|
def test_the_sandbox_made_by_the_first_turn_is_restamped(tmp_path):
|
|
"""V2-28: the runner's version is known only once the sandbox started, after the stamp."""
|
|
engine, rec = _engine(tmp_path, [_call("c1", "look", {"path": "x"}), AssistantTurn(text="ok"),
|
|
AssistantTurn(text="again")])
|
|
engine.sandbox_workspace = _LateSandbox()
|
|
_run(engine)
|
|
d = rec.as_dict()
|
|
assert "runner_version" not in d["turns"][0]["stamp"]["sandbox"]
|
|
[notice] = [s for s in d["steps"] if s["kind"] == "notice"]
|
|
assert notice["name"] == "stamp_changed"
|
|
assert notice["attrs"] == {"stamp": {"sandbox": {"provider": "openshell", "enforcement": "full",
|
|
"profile": "allowlist", "sites": 1,
|
|
"runner_version": "0.4.1"}}}
|
|
_run(engine, "again") # the next turn's stamp has it, and nothing changes in it
|
|
d = rec.as_dict()
|
|
assert d["turns"][1]["stamp"]["sandbox"]["runner_version"] == "0.4.1"
|
|
assert [s["name"] for s in d["steps"] if s["kind"] == "notice"] == ["stamp_changed"]
|
|
|
|
|
|
def test_attendance_changing_mid_turn_is_a_stamp_changed_notice(tmp_path):
|
|
"""V2-27: the Unattended toggle applies mid-turn; the turn keeps its start value and a
|
|
notice carries the new one, with only the key that changed."""
|
|
attended = {"now": True}
|
|
engine, rec = _engine(tmp_path, [_call("c1", "look", {"path": "x"}), AssistantTurn(text="ok")])
|
|
engine.is_attended = lambda: attended["now"]
|
|
original = engine._execute_sync
|
|
|
|
def run_tool(*a, **kw):
|
|
attended["now"] = False # you switched to Unattended while the tool ran
|
|
return original(*a, **kw)
|
|
|
|
engine._execute_sync = run_tool
|
|
_run(engine)
|
|
d = rec.as_dict()
|
|
assert d["turns"][0]["attended"] is True
|
|
[notice] = [s for s in d["steps"] if s["kind"] == "notice"]
|
|
assert notice["name"] == "stamp_changed" and notice["attrs"] == {"attended": False}
|
|
|
|
|
|
def test_a_turn_where_nothing_changes_has_no_stamp_notice(tmp_path):
|
|
engine, rec = _engine(tmp_path, [_call("c1", "look", {"path": "x"}), _call("c2", "look", {"path": "y"}),
|
|
AssistantTurn(text="ok")])
|
|
engine.is_attended = lambda: True
|
|
_run(engine)
|
|
assert not [s for s in rec.as_dict()["steps"] if s["kind"] == "notice"]
|
|
|
|
|
|
def test_a_leads_steering_wakes_the_worker_as_a_team(tmp_path, monkeypatch):
|
|
"""V2-30: `steer_worker` delivers through the `team` connector; it was recorded as `channel`."""
|
|
from coworker.providers import AssistantTurn as _T
|
|
from coworker.server.manager import SessionManager
|
|
|
|
mgr = SessionManager(workspace=tmp_path, provider=_Scripted([_T(text="on it"), _T(text="hi")]))
|
|
engine = mgr.get_engine("W", agent="chat")
|
|
seen: list[str] = []
|
|
real_run = engine.run
|
|
|
|
def run(*a, trigger="user", **kw):
|
|
seen.append(trigger)
|
|
return real_run(*a, trigger=trigger, **kw)
|
|
|
|
monkeypatch.setattr(engine, "run", run)
|
|
asyncio.run(mgr.deliver_to_session("W", "[Lead] switch to the API docs",
|
|
source={"connector": "team", "sender_name": "lead"}))
|
|
asyncio.run(mgr.deliver_to_session("W", "hello", source={"connector": "slack", "sender_name": "pat"}))
|
|
assert seen == ["team_wake", "channel"]
|