Files
openworker/coworker/audit.py
T
Devika Verma c59c5deae1 Feature 4: reviewer metering - badge, mode-menu summary, durable stats
Spec 1.7: the cost of Auto-Approve is visible while it accrues, not
discovered later. This is also where "uses your session model" gets
communicated (picker copy decision A): as a real accruing number.

Audit store:
- New columns call_id / tokens_in / tokens_out, with an idempotent ALTER
  migration for existing databases. This also fixes a feature-1 gap found
  in the process: the engine passed call_id and token counts on reviewer
  rows but the fixed column set silently dropped them, which would have
  broken the shadow-eval join and made token metering impossible.
- reviewer_stats(session_id): SQL aggregation of reviewer_verdict (live)
  and reviewer_shadow rows into checks/allow/deny/unsure + token sums.
  Durable - survives restarts and engine rebuilds.

Server: GET /v1/sessions/{id}/reviewer-stats (same shape as /unattended).

GUI:
- Polled with the existing 4s per-session poller.
- Mode button gains the badge when the session is in auto-approve and has
  checks: "Auto-Approve . 12 checks".
- Mode menu gains the session summary line: "This session: 12 checks . 10
  cleared . 0 blocked . 2 asked you . ~1k tokens". Only the LIVE bucket
  surfaces in the composer; shadow counts are a Settings/analysis concern.

Verified live against the running sidecar: the store already held 9 real
verdicts from manual testing of the mode, the endpoint aggregates them,
and both badge and summary render with real data.

Tests: stats aggregation (per-stage, per-session isolation, token sums),
legacy-DB migration (old schema opens, migrates, and round-trips call_id),
and the endpoint's empty shape. 113 backend + 114 GUI green.
2026-08-12 17:40:51 -07:00

226 lines
7.9 KiB
Python

"""Durable local audit log for connector/tool actions."""
from __future__ import annotations
import json
import sqlite3
import threading
from pathlib import Path
from typing import Any, Optional
from .connectors import connector_for_tool
_SECRET_KEYS = (
"token",
"secret",
"password",
"api_key",
"access_token",
"bot_token",
"app_token",
"raw",
)
_BODY_KEYS = ("body", "content", "html")
class AuditStore:
def __init__(self, db_path: str | Path) -> None:
self.db_path = Path(db_path).expanduser()
self._lock = threading.RLock()
self._conn = sqlite3.connect(self.db_path, check_same_thread=False)
self._conn.row_factory = sqlite3.Row
self._conn.execute("""
CREATE TABLE IF NOT EXISTS audit_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
session_id TEXT,
agent TEXT,
workspace TEXT,
connector TEXT,
tool TEXT,
stage TEXT,
status TEXT,
approval TEXT,
args TEXT,
result_preview TEXT,
reason TEXT,
resource TEXT,
call_id TEXT,
tokens_in INTEGER DEFAULT 0,
tokens_out INTEGER DEFAULT 0
)
""")
# Existing databases predate the reviewer columns (2026-08-12): call_id joins a
# shadow verdict to the human's decision on the same tool call, tokens_in/out are
# the reviewer metering (§1.7). ALTER is idempotent-by-error: "duplicate column"
# means an already-migrated file.
for column, decl in (
("call_id", "TEXT"),
("tokens_in", "INTEGER DEFAULT 0"),
("tokens_out", "INTEGER DEFAULT 0"),
):
try:
self._conn.execute(
f"ALTER TABLE audit_events ADD COLUMN {column} {decl}"
)
except sqlite3.OperationalError:
pass # column already exists
self._conn.commit()
def append(self, event: dict[str, Any]) -> None:
tool = str(event.get("tool") or event.get("tool_name") or "")
connector = str(event.get("connector") or connector_for_tool(tool) or "")
args = _sanitize_args(tool, event.get("arguments") or {})
resource = _resource(
tool, event.get("arguments") or {}, event.get("result") or {}
)
with self._lock:
self._conn.execute(
"""
INSERT INTO audit_events
(session_id, agent, workspace, connector, tool, stage, status, approval, args, result_preview, reason, resource, call_id, tokens_in, tokens_out)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
event.get("session_id") or "",
event.get("agent") or "",
event.get("workspace") or "",
connector,
tool,
event.get("stage") or "",
event.get("status") or "",
event.get("approval") or "",
json.dumps(args, default=str),
_truncate(str(event.get("result_preview") or "")),
_truncate(str(event.get("reason") or "")),
_truncate(str(resource or "")),
str(event.get("call_id") or ""),
int(event.get("tokens_in") or 0),
int(event.get("tokens_out") or 0),
),
)
self._conn.commit()
def reviewer_stats(self, session_id: str) -> dict[str, Any]:
"""Per-session Auto-Approve metering (§1.7), computed from the durable rows so it
survives restarts and engine rebuilds. `live` counts stage=reviewer_verdict (the
mode actually deciding); `shadow` counts stage=reviewer_shadow (recording only)."""
def _bucket(stage: str) -> dict[str, int]:
with self._lock:
rows = self._conn.execute(
"""
SELECT status, COUNT(*) AS n,
COALESCE(SUM(tokens_in), 0) AS tin,
COALESCE(SUM(tokens_out), 0) AS tout
FROM audit_events
WHERE session_id = ? AND stage = ?
GROUP BY status
""",
(session_id, stage),
).fetchall()
out = {"checks": 0, "allow": 0, "deny": 0, "unsure": 0, "tokens_in": 0, "tokens_out": 0}
for row in rows:
status = str(row["status"])
if status in ("allow", "deny", "unsure"):
out[status] += int(row["n"])
out["checks"] += int(row["n"])
out["tokens_in"] += int(row["tin"])
out["tokens_out"] += int(row["tout"])
return out
return {"live": _bucket("reviewer_verdict"), "shadow": _bucket("reviewer_shadow")}
def list(
self,
*,
limit: int = 100,
session_id: Optional[str] = None,
connector: Optional[str] = None,
tool: Optional[str] = None,
) -> list[dict[str, Any]]:
where = []
params: list[Any] = []
if session_id:
where.append("session_id = ?")
params.append(session_id)
if connector:
where.append("connector = ?")
params.append(connector)
if tool:
where.append("tool = ?")
params.append(tool)
sql = "SELECT * FROM audit_events"
if where:
sql += " WHERE " + " AND ".join(where)
sql += " ORDER BY id DESC LIMIT ?"
params.append(max(1, min(int(limit or 100), 500)))
with self._lock:
rows = self._conn.execute(sql, params).fetchall()
out = []
for row in rows:
item = dict(row)
try:
item["args"] = json.loads(item.get("args") or "{}")
except json.JSONDecodeError:
item["args"] = {}
out.append(item)
return out
def close(self) -> None:
self._conn.close()
def _sanitize_args(tool: str, args: dict[str, Any]) -> dict[str, Any]:
if not isinstance(args, dict):
return {}
out: dict[str, Any] = {}
for key, value in args.items():
lk = str(key).lower()
if any(s in lk for s in _SECRET_KEYS):
out[key] = "[redacted]"
elif tool == "browser_type" and lk == "text":
out[key] = "[redacted input]"
elif any(b == lk or lk.endswith("_" + b) for b in _BODY_KEYS):
out[key] = "[redacted body]"
else:
out[key] = _summarize(value)
return out
def _summarize(value: Any) -> Any:
if isinstance(value, str):
return _truncate(value)
if isinstance(value, (int, float, bool)) or value is None:
return value
if isinstance(value, list):
return [_summarize(v) for v in value[:10]]
if isinstance(value, dict):
return {str(k): _summarize(v) for k, v in list(value.items())[:20]}
return _truncate(str(value))
def _resource(tool: str, args: dict[str, Any], result: Any) -> str:
for key in (
"url",
"owner",
"repo",
"issue_key",
"page_id",
"ticket_id",
"calendar_id",
"message_id",
):
if isinstance(args, dict) and args.get(key):
return str(args[key])
if isinstance(args, dict) and args.get("subdomain"):
return f"{args['subdomain']}.zendesk.com"
if isinstance(result, dict) and result.get("url"):
return str(result["url"])
return ""
def _truncate(text: str, limit: int = 500) -> str:
text = text.replace("\n", "\\n")
return text if len(text) <= limit else text[: limit - 3] + "..."