mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
A store with the background writer runs its retention sweep on its first tick. A turn dated 1970 is past the 30 days, so when that sweep landed between a test's write and its read, the turn was gone (CI: test_turns_ recorded_before_the_ledger_are_filled_in, `assert [] == ['T0']`). The four tests that write such turns to a SessionManager's store now date them an hour ago: the ledger backfill, lead polling, the team summary cache and the resumed automation turn. The product is unchanged.
454 lines
24 KiB
Python
454 lines
24 KiB
Python
"""Phase 3 engine items, one test each: the scanner facts on a real shell step (E17b), the
|
|
1.3 turn facts an entry point passes (E18a, E18b, E26a, E25a), the wake reason of a team
|
|
delivery (TD3-18), the journal's evidence links (E17a), the `verifies` link (E17d), the
|
|
hand-off coverage report (E17c), stored series and task results, the ledger's 90 days and
|
|
the feedback export (E27b)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import jsonschema
|
|
import pytest
|
|
|
|
from coworker.engine import TurnEngine
|
|
from coworker.observability import Recorder, RecordingStore, scopes, signals
|
|
from coworker.observability import export as ex
|
|
from coworker.observability import export_spans as spans
|
|
from coworker.permissions import Mode, PermissionEngine
|
|
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
|
|
from coworker.providers.base import TokenUsage
|
|
from coworker.server.manager import SessionManager
|
|
from coworker.teams.journal import JournalStore
|
|
from coworker.teams.model import Actor, Role
|
|
from coworker.teams.store import BoardError, TeamStore
|
|
from coworker.tools import ToolRegistry
|
|
|
|
SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
|
|
|
|
# A turn an hour old, not from 1970: a store with the background writer sweeps on its first tick, and
|
|
# a 1970 turn is past the 30 days, so it could vanish between the write and the read.
|
|
RECENT = int(time.time() * 1000) - 3_600_000
|
|
|
|
|
|
class _Scripted(ProviderClient):
|
|
def __init__(self, turns):
|
|
self._turns = list(turns)
|
|
|
|
def complete(self, *, model, messages, tools=None, **settings):
|
|
return self._turns.pop(0)
|
|
|
|
def capabilities(self, model):
|
|
return ModelCapabilities()
|
|
|
|
|
|
class _Meta:
|
|
category: str = ""
|
|
risk_level: str = "low"
|
|
requires_approval: bool = False
|
|
|
|
|
|
def run_shell(command: str) -> dict:
|
|
"""Run a shell command.
|
|
|
|
Args:
|
|
command: the command
|
|
"""
|
|
return {"exit_code": 1 if "gitleaks" in command else 0, "stdout": "found 2 leaks"}
|
|
|
|
|
|
def _call(cid, name, args):
|
|
return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls",
|
|
usage=TokenUsage(input=10, output=2))
|
|
|
|
|
|
def _engine(tmp_path, turns):
|
|
registry = ToolRegistry()
|
|
registry.register(run_shell, metadata=_Meta())
|
|
engine = TurnEngine(
|
|
provider=_Scripted(turns), registry=registry,
|
|
permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),
|
|
model="openai:model-x", instructions="You are a test coworker.", max_iterations=12,
|
|
)
|
|
engine.audit_context = {"session_id": "sess_p3"}
|
|
engine.persona_id, engine.persona_fp, engine.persona_coverage = "security", "a1b2c3", ["secrets"]
|
|
rec = Recorder()
|
|
engine.recorder = rec
|
|
return engine, rec
|
|
|
|
|
|
def test_a_shell_step_records_its_scanner_and_result_hash(tmp_path):
|
|
engine, rec = _engine(tmp_path, [
|
|
_call("c1", "run_shell", {"command": "npx gitleaks detect --source ."}),
|
|
_call("c2", "run_shell", {"command": "grep -r semgrep docs"}),
|
|
AssistantTurn(text="done", usage=TokenUsage(input=10, output=2)),
|
|
])
|
|
|
|
async def go():
|
|
return [ev async for ev in engine.run("scan", trigger="team_wake", team={"team_id": "t1", "role": "worker"},
|
|
wake={"reason": "assigned", "item_ids": [7], "board_seq": 41})]
|
|
|
|
asyncio.run(go())
|
|
d = rec.as_dict()
|
|
tools = {s["call_id"]: s["attrs"] for s in d["steps"] if s["kind"] == "tool"}
|
|
assert tools["c1"]["scanner"] == {"name": "gitleaks", "action": "scan"} and tools["c1"]["exit_code"] == 1
|
|
assert tools["c2"]["scanner"] is None
|
|
raw = json.dumps({"exit_code": 1, "stdout": "found 2 leaks"}, ensure_ascii=False).encode("utf-8")
|
|
assert tools["c1"]["result_sha256"] == hashlib.sha256(raw).hexdigest()
|
|
turn = d["turns"][0]
|
|
assert turn["stamp"]["persona_id"] == "security" and turn["stamp"]["persona_coverage"] == ["secrets"]
|
|
assert turn["stamp"]["team"] == {"team_id": "t1", "role": "worker"}
|
|
assert turn["wake"] == {"reason": "assigned", "kinds": [], "item_ids": ["7"], "board_seq": 41,
|
|
"batched": None, "deferred": None, "by": None}
|
|
errs = list(jsonschema.Draft202012Validator({**SCHEMA, "$ref": "#/$defs/recording"}).iter_errors(d))
|
|
assert not errs, errs[0].message
|
|
|
|
|
|
def test_an_automation_turn_carries_its_run_and_a_bad_wake_reason_reads_unknown(tmp_path):
|
|
engine, rec = _engine(tmp_path, [AssistantTurn(text="brief", usage=TokenUsage(input=10, output=2))])
|
|
|
|
async def go():
|
|
return [ev async for ev in engine.run("go", trigger="schedule", wake={"reason": "bogus"},
|
|
automation={"task_id": "task-1", "run_id": "run-1", "trigger": "catch_up"})]
|
|
|
|
asyncio.run(go())
|
|
turn = rec.as_dict()["turns"][0]
|
|
assert turn["automation"] == {"task_id": "task-1", "run_id": "run-1", "trigger": "catch_up"}
|
|
assert turn["wake"]["reason"] == "unknown" and turn["trigger"] == "schedule"
|
|
|
|
|
|
@pytest.mark.parametrize("rows,lead,want", [
|
|
([{"kind": "moved", "to": "review", "item": 3}, {"kind": "assigned", "item": 4}], True, "review"),
|
|
([{"kind": "moved", "to": "blocked", "item": 3}, {"kind": "waiting", "item": 4}], True, "blocker"),
|
|
([{"kind": "chat"}, {"kind": "chat"}], False, "mention"),
|
|
([{"kind": "moved", "to": "done", "item": 3}], False, "prereq_done"),
|
|
([{"kind": "moved", "to": "done", "item": 3}], True, "board"),
|
|
([{"kind": "claimed", "item": 5}, {"kind": "chat"}], True, "assigned"),
|
|
])
|
|
def test_one_wake_reason_by_fixed_priority(rows, lead, want):
|
|
reason, kinds, items = SessionManager._wake_from_rows(rows, is_lead=lead)
|
|
assert reason == want
|
|
assert kinds == list(dict.fromkeys(r["kind"] for r in rows))
|
|
|
|
|
|
def _team(tmp_path):
|
|
journal = JournalStore(tmp_path / "journal.db")
|
|
store = TeamStore(tmp_path / "board.db", journal=journal)
|
|
lead = Actor(id="lead:1", role=Role.LEAD, persona="devsecops-lead", session_id="lead-s")
|
|
worker = Actor(id="appsec:1", role=Role.WORKER, persona="appsec-worker", session_id="w-s")
|
|
return store, journal, lead, worker
|
|
|
|
|
|
def test_verifies_is_a_board_link_both_ways(tmp_path):
|
|
store, _j, lead, _w = _team(tmp_path)
|
|
fix = store.create_item("space", lead, title="Fix SQLi", criteria="query is parameterised")
|
|
check = store.create_item("space", lead, title="Verify the fix", criteria="rerun semgrep", verifies=fix["id"])
|
|
assert {"kind": "verifies", "item": fix["id"]} in check["links"]
|
|
again = store.get_item("space", fix["id"], actor=lead)
|
|
assert {"kind": "verified_by", "item": check["id"]} in again["links"]
|
|
with pytest.raises(BoardError):
|
|
store.create_item("space", lead, title="x", criteria="y", verifies=999)
|
|
|
|
|
|
def test_a_hand_off_carries_a_checked_coverage_report(tmp_path):
|
|
store, _j, lead, worker = _team(tmp_path)
|
|
item = store.create_item("space", lead, title="Scan", criteria="all areas")
|
|
store.assign("space", lead, item["id"], worker.id)
|
|
store.transition("space", worker, item["id"], "in_progress")
|
|
store.transition("space", worker, item["id"], "review", comment="done",
|
|
coverage={"secrets": {"ran": False, "findings": 0, "reason": "gitleaks missing"},
|
|
"code": {"ran": True, "findings": 3}})
|
|
ev = [e for e in store.events("space", item_id=item["id"]) if e["kind"] == "item_transitioned"][-1]
|
|
assert ev["payload"]["coverage"] == {"secrets": {"ran": False, "findings": 0, "reason": "gitleaks missing"},
|
|
"code": {"ran": True, "findings": 3}}
|
|
for bad in ({"network": {"ran": True}}, {"code": {"ran": "yes"}}, {"code": {"ran": True, "findings": -1}}):
|
|
with pytest.raises(BoardError):
|
|
store.transition("space", worker, item["id"], "in_progress", coverage=bad)
|
|
|
|
|
|
def test_journal_evidence_is_outside_the_hash_chain(tmp_path):
|
|
_s, journal, lead, worker = _team(tmp_path)
|
|
plain = journal.append(worker, "case-1", "an old-style note")
|
|
found = journal.append(worker, "case-1", "SQLi in login", kind="finding", severity="high",
|
|
step_call_id="call_9", evidence=[{"session_id": "w-s", "call_id": "call_4"}])
|
|
assert found["evidence"] == [{"session_id": "w-s", "call_id": "call_4"}] and found["step_call_id"] == "call_9"
|
|
rows = journal.read(worker, "case-1")
|
|
assert rows[0]["evidence"] is None and rows[0]["severity"] is None
|
|
assert rows[1]["evidence"] == [{"session_id": "w-s", "call_id": "call_4"}] and rows[1]["severity"] == "high"
|
|
# changing the pointer does not touch the chain
|
|
with journal._lock:
|
|
journal._conn.execute("UPDATE journal_entries SET evidence = '[]' WHERE seq = ?", (found["seq"],))
|
|
assert journal.read(worker, "case-1")[1]["hash"] == found["hash"]
|
|
assert plain["hash"] != found["hash"]
|
|
|
|
|
|
def test_journal_append_links_the_sessions_scans_by_default(tmp_path):
|
|
from coworker.teams.tools import journal_tools
|
|
|
|
_s, journal, _lead, worker = _team(tmp_path)
|
|
tools = {t.__name__: t for t in journal_tools(
|
|
journal, actor=worker, space="space",
|
|
scan_evidence=lambda item: [{"session_id": "w-s", "call_id": "call_2"}])}
|
|
out = tools["journal_append"]("case-2", "a leaked key", kind="finding", severity="critical")
|
|
assert out["evidence"] == [{"session_id": "w-s", "call_id": "call_2"}]
|
|
named = tools["journal_append"]("case-2", "another", kind="finding", evidence=["call_7"])
|
|
assert named["evidence"] == [{"session_id": "w-s", "call_id": "call_7"}]
|
|
note = tools["journal_append"]("case-2", "just a note")
|
|
assert note["evidence"] is None
|
|
assert "error" in tools["journal_append"]("case-2", "x", kind="finding", severity="severe")
|
|
|
|
|
|
def _write(store, turns, steps, issues=None):
|
|
for t in turns:
|
|
store.write_turn(t)
|
|
for s in steps:
|
|
store.write_step(s)
|
|
for t in turns:
|
|
store.write_issues(t["turn_id"], (issues or {}).get(t["turn_id"]) or
|
|
{"signals_version": signals.version(), "warn": 0, "info": 0, "issues": [], "fine": []}, seen=True)
|
|
store.flush()
|
|
|
|
|
|
def _turn(tid, sid, seq, at, wake, team=None, status="completed"):
|
|
stamp = {"machine": {"kind": "this_machine", "id": None}, "persona_id": "devsecops-lead"}
|
|
if team:
|
|
stamp["team"] = team
|
|
return {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "team_wake", "started_at": at,
|
|
"duration_ms": 1000, "status": status, "incomplete": False, "approval_mode": "interactive",
|
|
"attended": True, "reviewer": None, "stamp": stamp, "first_message_id": None,
|
|
"last_message_id": None, "schema_version": 1, "wake": wake}
|
|
|
|
|
|
def _step(tid, sid, n, at):
|
|
return {"step_id": f"{tid}-s{n}", "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": n,
|
|
"kind": "model", "name": "model", "started_at": at, "duration_ms": 100, "status": "ok",
|
|
"call_id": None, "call_index": None, "message_id": None, "shadow": False, "after_turn": False,
|
|
"attrs": {"purpose": "main", "model": "model-x", "usage": {"reported": True, "input_fresh": 10,
|
|
"cache_read": 0, "cache_write": 0, "output": 1}},
|
|
"content": {}}
|
|
|
|
|
|
def test_lead_polling_is_stored_once_on_the_last_turn_of_the_run(tmp_path):
|
|
manager = SessionManager(workspace=tmp_path, provider=_Scripted([]))
|
|
store = manager.recordings_store
|
|
team = {"team_id": "t1", "role": "lead"}
|
|
turns = [_turn(f"L{i}", "lead-s", i, RECENT + i * 60_000, {"reason": "timer", "board_seq": 5}, team)
|
|
for i in range(1, 5)]
|
|
_write(store, turns, [_step(t["turn_id"], "lead-s", 1, t["started_at"]) for t in turns])
|
|
manager._scope_lead_polling("lead-s")
|
|
got = {tid: [i["check"] for i in (store.read_turn_issues(tid) or {}).get("issues", [])] for tid in ("L3", "L4")}
|
|
assert got == {"L3": [], "L4": ["lead_polling"]}
|
|
issue = store.read_turn_issues("L4")
|
|
assert issue["warn"] == 1 and issue["seen"] is False
|
|
assert issue["issues"][0]["text"] == "The lead woke on its timer 4 times with no board change."
|
|
# the ledger follows the stored result
|
|
[t4] = [t for t in store.usage_turns(0, 2**62) if t["turn_id"] == "L4"]
|
|
assert t4["warn"] == 1 and "lead_polling" in t4["warn_checks"]
|
|
# a re-check of the turn alone keeps the series result
|
|
rec = store.load_recording("lead-s")
|
|
store.write_issues("L4", {**issue, "signals_version": "old"})
|
|
store.flush()
|
|
out = manager._recording_issues("lead-s", rec)
|
|
assert [i["check"] for i in out["L4"]["issues"]] == ["lead_polling"]
|
|
manager.observability.close()
|
|
|
|
|
|
def test_the_ledger_is_kept_90_days(tmp_path):
|
|
store = RecordingStore(tmp_path / "recordings.db", background=False)
|
|
now = 200 * 24 * 3600 * 1000
|
|
old = _turn("old", "s", 1, now - 91 * 24 * 3600 * 1000, {"reason": "user"})
|
|
new = _turn("new", "s", 2, now - 40 * 24 * 3600 * 1000, {"reason": "user"})
|
|
_write(store, [old, new], [_step("old", "s", 1, old["started_at"]), _step("new", "s", 1, new["started_at"])])
|
|
store.sweep(now_ms=now) # recordings: 30 days
|
|
assert store.load_recording("s")["turns"] == []
|
|
store.sweep_ledger(now_ms=now) # the ledger: 90 days
|
|
assert [t["turn_id"] for t in store.usage_turns(0, 2**62)] == ["new"]
|
|
store.close()
|
|
|
|
|
|
def test_ratings_go_out_as_a_langfuse_score_and_an_otlp_span(tmp_path):
|
|
store = RecordingStore(tmp_path / "recordings.db", background=False)
|
|
t = _turn("T1", "s", 1, 5_000, {"reason": "user"})
|
|
_write(store, [t], [_step("T1", "s", 1, 5_000)])
|
|
store.put_feedback({"turn_id": "T1", "session_id": "s", "rating": "down", "reasons": ["did_unasked"],
|
|
"note": "used my key sk-PLANT", "auto_approved": []})
|
|
posts = []
|
|
|
|
def post(url, body, headers, timeout):
|
|
posts.append((url, json.loads(body)))
|
|
return 200, ""
|
|
|
|
for kind, prompts in (("langfuse", False), ("otlp", True)):
|
|
posts.clear()
|
|
dest = ex.Destination(kind="langfuse", host="http://lf.test") if kind == "langfuse" else \
|
|
ex.Destination(kind="otlp", url="http://otel.test/v1/traces")
|
|
cfg = ex.ExportConfig(destination=dest, headers={}, include={"messages": prompts}, since_ms=0)
|
|
exp = ex.Exporter(store, cfg, post=post, secret_values=lambda: ["sk-PLANT"])
|
|
exp.flush_once()
|
|
fb = [p for p in posts if "scores" in p[0] or "feedback" in json.dumps(p[1])]
|
|
assert len(fb) == 1
|
|
if kind == "langfuse":
|
|
body = fb[0][1]
|
|
assert fb[0][0] == "http://lf.test/api/public/scores"
|
|
assert body["name"] == "user_feedback" and body["value"] == -1
|
|
assert body["id"] == spans.feedback_id("T1") and body["traceId"] == spans.trace_id("T1")
|
|
# the note stays out without capture.prompts, and an empty comment replaces an earlier one
|
|
assert body["metadata"] == {"reasons": ["did_unasked"]} and body["comment"] == ""
|
|
else:
|
|
span = fb[0][1]["resourceSpans"][0]["scopeSpans"][0]["spans"][0]
|
|
attrs = {a["key"]: a["value"] for a in span["attributes"]}
|
|
assert span["name"] == "openworker.feedback" and span["parentSpanId"] == spans.root_span_id("T1")
|
|
assert attrs["openworker.feedback.rating"] == {"stringValue": "down"}
|
|
assert "PLANT" not in json.dumps(attrs) and "openworker.feedback.note" in attrs
|
|
# sent once; a changed rating goes again, under the same fixed id
|
|
posts.clear()
|
|
exp.flush_once()
|
|
assert not [p for p in posts if "scores" in p[0] or "feedback" in json.dumps(p[1])]
|
|
store.close()
|
|
|
|
|
|
def test_the_turn_span_names_the_coworker_and_keeps_agent_name(tmp_path):
|
|
t = _turn("T1", "s", 1, 5_000, {"reason": "review"}, {"team_id": "t1", "role": "lead"})
|
|
t["stamp"]["persona_fp"] = "a1b2c3"
|
|
t["automation"] = {"task_id": "task-1", "run_id": "run-1", "trigger": "manual"}
|
|
built = spans.build_turn_spans(t, [_step("T1", "s", 1, 5_000)], sw=spans.switches_from({}), dialect="semconv",
|
|
get_piece=lambda k: None, masker=spans.Masker(), include_root=True,
|
|
only=None, session_title=None)
|
|
root = built[0]["attrs"]
|
|
assert root["gen_ai.agent.name"] == "coworker" # TD3-25: the 1.0 value stays
|
|
assert root["openworker.persona.id"] == "devsecops-lead" and root["openworker.persona.fp"] == "a1b2c3"
|
|
assert root["openworker.wake.reason"] == "review" and root["openworker.team.role"] == "lead"
|
|
assert root["openworker.automation.run_id"] == "run-1" and root["openworker.automation.trigger"] == "manual"
|
|
|
|
|
|
def test_run_health_counts_a_stuck_run_as_failed():
|
|
turn = {"status": "completed", "warn_checks": [], "ended_at": 1_000}
|
|
assert scopes.run_health({"status": "running"}, turn, now_ms=1_000 + 600_001) == "failed"
|
|
assert scopes.run_health({"status": "running"}, turn, now_ms=1_000 + 60_000) == "ok"
|
|
assert scopes.run_health({"status": "ok"}, {**turn, "warn_checks": ["model_failed"]}) == "failed"
|
|
assert scopes.run_health({"status": "ok"}, None) == "unknown"
|
|
|
|
|
|
def test_turns_recorded_before_the_ledger_are_filled_in(tmp_path):
|
|
manager = SessionManager(workspace=tmp_path, provider=_Scripted([]))
|
|
store = manager.recordings_store
|
|
t = _turn("T0", "s-old", 1, RECENT, {"reason": "user"})
|
|
_write(store, [t], [_step("T0", "s-old", 1, RECENT)])
|
|
with store._lock, store._conn:
|
|
store._conn.execute("DELETE FROM turn_usage") # as a Phase 2 store has none
|
|
assert store.turns_without_usage() == ["T0"]
|
|
assert manager.backfill_usage() == 1
|
|
[row] = store.usage_turns(0, 2**62)
|
|
assert row["total"] == 11 and store.turns_without_usage() == []
|
|
manager.observability.close()
|
|
|
|
|
|
def test_a_wake_that_was_not_delivered_is_given_back_to_the_cap(tmp_path):
|
|
from coworker.teams.registry import TeamRegistry, TeamWorker
|
|
|
|
reg = TeamRegistry(tmp_path / "teams.json")
|
|
team = reg.create(space="sp", lead_session="lead-s", lead_actor="lead-1",
|
|
workers=[TeamWorker(actor="w", persona="appsec-worker", session_id="w-s")])
|
|
assert all(reg.count_wake(team.team_id, cap=2) for _ in range(2))
|
|
assert reg.count_wake(team.team_id, cap=2) is False
|
|
reg.refund_wake(team.team_id) # TD3-22: counted only once delivered
|
|
assert reg.count_wake(team.team_id, cap=2) is True
|
|
|
|
|
|
def test_the_wake_cap_event_wakes_nobody_and_deferrals_are_recorded(tmp_path):
|
|
from coworker.teams.registry import TeamWorker
|
|
|
|
manager = SessionManager(workspace=tmp_path, provider=_Scripted([]))
|
|
team = manager.teams.create(space=str(tmp_path), lead_session="lead-s", lead_actor="lead-1",
|
|
workers=[TeamWorker(actor="w", persona="appsec-worker", session_id="w-s")])
|
|
manager._note_held("lead-s", "cap")
|
|
manager._note_wake_cap(team)
|
|
manager._note_wake_cap(team) # once per hour
|
|
caps = [e for e in manager.team_store.events(team.space) if e["kind"] == "wake_cap_reached"]
|
|
assert len(caps) == 1 and caps[0]["payload"]["held"] == ["lead-s"]
|
|
assert manager._actionable_team_events(team, caps, actor="lead-1", is_lead=True) == []
|
|
wake = manager._turn_wake("lead-s", source={"connector": "board"}, trigger="team_wake",
|
|
rows=[{"kind": "moved", "to": "review", "item": 3}])
|
|
assert wake["reason"] == "review" and wake["deferred"]["by"] == "cap" and wake["deferred"]["ms"] >= 0
|
|
assert wake["board_seq"] == caps[0]["seq"]
|
|
assert manager._team_stamp("w-s") == {"team_id": team.team_id, "role": "worker"}
|
|
manager.observability.close()
|
|
|
|
|
|
def test_a_turn_on_a_new_event_loop_is_not_read_as_stopped(tmp_path):
|
|
"""Two turns of one engine on two event loops (each test-client socket has its own): the
|
|
second turn's answer and its model call stay in that turn (gate 1)."""
|
|
engine, rec = _engine(tmp_path, [AssistantTurn(text="one", usage=TokenUsage(input=10, output=2)),
|
|
AssistantTurn(text="two", usage=TokenUsage(input=10, output=3))])
|
|
|
|
async def go(text):
|
|
return [ev async for ev in engine.run(text, trigger="user")]
|
|
|
|
asyncio.run(go("first"))
|
|
asyncio.run(go("second"))
|
|
d = rec.as_dict()
|
|
assert [m.get("content") for m in engine.messages if m.get("role") == "assistant"] == ["one", "two"]
|
|
models = [s for s in d["steps"] if s["kind"] == "model"]
|
|
assert len(models) == 2 and all(s["turn_id"] for s in models)
|
|
assert {s["turn_id"] for s in models} == {t["turn_id"] for t in d["turns"]}
|
|
|
|
|
|
POWERSHELL_NOT_FOUND = (
|
|
"semgrep : The term 'semgrep' is not recognized as the name of a cmdlet, function, script file, or operable\n"
|
|
"program. Check the spelling of the name, or if a path was included, verify that the path is correct and try again.\n"
|
|
" + CategoryInfo : ObjectNotFound: (semgrep:String) [], CommandNotFoundException\n")
|
|
|
|
|
|
@pytest.mark.parametrize("name,code,output,missing", [
|
|
("semgrep", 1, POWERSHELL_NOT_FOUND, True), # Windows PowerShell
|
|
("gitleaks", 1, "'gitleaks' is not recognized as an internal or external command,", True), # cmd.exe
|
|
("gitleaks", 127, "bash: gitleaks: command not found", True), # sh
|
|
("gitleaks", 127, "", True), # the code alone
|
|
("trivy", 9009, "", True), # cmd.exe's code
|
|
("pip-audit", 1, "/usr/bin/python3: No module named pip_audit", True), # python -m
|
|
("semgrep", 1, "Ran 312 rules on 40 files: 25 findings.", False), # found something
|
|
("gitleaks", 1, "leaks found: 2", False),
|
|
("semgrep", 2, "semgrep: error: invalid configuration", False), # a real failure
|
|
("semgrep", 1, "warning: rule 'x' not found in registry", False), # names no scanner word with the phrase
|
|
])
|
|
def test_a_scanner_that_is_not_installed_did_not_scan(name, code, output, missing):
|
|
assert scopes.scanner_not_found({"name": name, "action": "scan"}, code, output) is missing
|
|
|
|
|
|
def test_a_missing_scanner_on_windows_reads_as_a_scan_that_did_not_run(tmp_path):
|
|
"""P3B-1: PowerShell exits 1 for an unknown command, the same as semgrep's "found something".
|
|
The step records no scanner, so "A promised scan did not run" fires instead of nothing."""
|
|
registry = ToolRegistry()
|
|
|
|
def run_shell(command: str) -> dict:
|
|
"""Run a shell command.
|
|
|
|
Args:
|
|
command: the command
|
|
"""
|
|
return {"exit_code": 1, "output": POWERSHELL_NOT_FOUND}
|
|
|
|
registry.register(run_shell, metadata=_Meta())
|
|
engine = TurnEngine(provider=_Scripted([_call("c1", "run_shell", {"command": "semgrep scan --config auto"}),
|
|
AssistantTurn(text="done", usage=TokenUsage(input=10, output=2))]),
|
|
registry=registry,
|
|
permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),
|
|
model="openai:model-x", instructions="test", max_iterations=12)
|
|
engine.audit_context = {"session_id": "sess_p3b1"}
|
|
rec = Recorder()
|
|
engine.recorder = rec
|
|
|
|
async def go():
|
|
return [ev async for ev in engine.run("scan", trigger="user")]
|
|
|
|
asyncio.run(go())
|
|
[step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]
|
|
assert step["attrs"]["scanner"] is None and step["attrs"]["exit_code"] == 1
|
|
fired = scopes.coverage_checks({"promised": ["code"], "steps": [
|
|
{"id": step["step_id"], "scanner": step["attrs"]["scanner"], "exit_code": step["attrs"]["exit_code"]}]})
|
|
assert fired == {"scan_missing": [{"area": "code"}]}
|