ext/openworker
2
0
Fork 0
mirror of https://github.com/andrewyng/openworker.git synced 2026-09-03 04:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature 4: reviewer metering - badge, mode-menu summary, durable stats

Browse Source 
Spec 1.7: the cost of Auto-Approve is visible while it accrues, not
discovered later. This is also where "uses your session model" gets
communicated (picker copy decision A): as a real accruing number.

Audit store:
- New columns call_id / tokens_in / tokens_out, with an idempotent ALTER
  migration for existing databases. This also fixes a feature-1 gap found
  in the process: the engine passed call_id and token counts on reviewer
  rows but the fixed column set silently dropped them, which would have
  broken the shadow-eval join and made token metering impossible.
- reviewer_stats(session_id): SQL aggregation of reviewer_verdict (live)
  and reviewer_shadow rows into checks/allow/deny/unsure + token sums.
  Durable - survives restarts and engine rebuilds.

Server: GET /v1/sessions/{id}/reviewer-stats (same shape as /unattended).

GUI:
- Polled with the existing 4s per-session poller.
- Mode button gains the badge when the session is in auto-approve and has
  checks: "Auto-Approve . 12 checks".
- Mode menu gains the session summary line: "This session: 12 checks . 10
  cleared . 0 blocked . 2 asked you . ~1k tokens". Only the LIVE bucket
  surfaces in the composer; shadow counts are a Settings/analysis concern.

Verified live against the running sidecar: the store already held 9 real
verdicts from manual testing of the mode, the endpoint aggregates them,
and both badge and summary render with real data.

Tests: stats aggregation (per-stage, per-session isolation, token sums),
legacy-DB migration (old schema opens, migrates, and round-trips call_id),
and the endpoint's empty shape. 113 backend + 114 GUI green.
This commit is contained in:
Devika Verma
2026-08-12 17:40:51 -07:00
parent 29c9f2b4a8
commit c59c5deae1
6 changed files with 176 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coworker/audit.py
+54 -3
View File
@@ -44,9 +44,27 @@ class AuditStore:
args TEXT,
result_preview TEXT,
reason TEXT,
resource TEXT
resource TEXT,
call_id TEXT,
tokens_in INTEGER DEFAULT 0,
tokens_out INTEGER DEFAULT 0
)
""")
# Existing databases predate the reviewer columns (2026-08-12): call_id joins a
# shadow verdict to the human's decision on the same tool call, tokens_in/out are
# the reviewer metering (§1.7). ALTER is idempotent-by-error: "duplicate column"
# means an already-migrated file.
for column, decl in (
("call_id", "TEXT"),
("tokens_in", "INTEGER DEFAULT 0"),
("tokens_out", "INTEGER DEFAULT 0"),
):
try:
self._conn.execute(
f"ALTER TABLE audit_events ADD COLUMN {column} {decl}"
)
except sqlite3.OperationalError:
pass # column already exists
self._conn.commit()
def append(self, event: dict[str, Any]) -> None:
@@ -60,8 +78,8 @@ class AuditStore:
self._conn.execute(
"""
INSERT INTO audit_events
(session_id, agent, workspace, connector, tool, stage, status, approval, args, result_preview, reason, resource)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
(session_id, agent, workspace, connector, tool, stage, status, approval, args, result_preview, reason, resource, call_id, tokens_in, tokens_out)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
event.get("session_id") or "",
@@ -76,10 +94,43 @@ class AuditStore:
_truncate(str(event.get("result_preview") or "")),
_truncate(str(event.get("reason") or "")),
_truncate(str(resource or "")),
str(event.get("call_id") or ""),
int(event.get("tokens_in") or 0),
int(event.get("tokens_out") or 0),
),
)
self._conn.commit()
def reviewer_stats(self, session_id: str) -> dict[str, Any]:
"""Per-session Auto-Approve metering (§1.7), computed from the durable rows so it
survives restarts and engine rebuilds. `live` counts stage=reviewer_verdict (the
mode actually deciding); `shadow` counts stage=reviewer_shadow (recording only)."""
def _bucket(stage: str) -> dict[str, int]:
with self._lock:
rows = self._conn.execute(
"""
SELECT status, COUNT(*) AS n,
COALESCE(SUM(tokens_in), 0) AS tin,
COALESCE(SUM(tokens_out), 0) AS tout
FROM audit_events
WHERE session_id = ? AND stage = ?
GROUP BY status
""",
(session_id, stage),
).fetchall()
out = {"checks": 0, "allow": 0, "deny": 0, "unsure": 0, "tokens_in": 0, "tokens_out": 0}
for row in rows:
status = str(row["status"])
if status in ("allow", "deny", "unsure"):
out[status] += int(row["n"])
out["checks"] += int(row["n"])
out["tokens_in"] += int(row["tin"])
out["tokens_out"] += int(row["tout"])
return out
return {"live": _bucket("reviewer_verdict"), "shadow": _bucket("reviewer_shadow")}
def list(
self,
*,
coworker/server/app.py
+6
View File
@@ -383,6 +383,12 @@ def create_app(manager: SessionManager) -> FastAPI:
def get_unattended(session_id: str) -> dict[str, Any]:
return {"unattended": manager.unattended.is_unattended(session_id)}
@app.get("/v1/sessions/{session_id}/reviewer-stats")
def get_reviewer_stats(session_id: str) -> dict[str, Any]:
# Auto-Approve metering (§1.7): checks/verdicts/tokens from the durable audit rows.
# Drives the composer's "Auto-Approve · N checks" badge and the mode-menu summary.
return manager.audit_store.reviewer_stats(session_id)
@app.post("/v1/sessions/{session_id}/unattended")
def set_unattended(session_id: str, body: dict) -> dict[str, Any]:
# The GUI gates the on-transition behind a one-tap confirm; the manager records the
surfaces/gui/src/App.tsx
+6
View File
@@ -15,6 +15,7 @@ import {
getSettings,
getPersonas,
getInbox,
getReviewerStats,
getUnattended,
PERSONAS_CHANGED,
resolveInboxItem,
@@ -315,6 +316,9 @@ export function App() {
// expanded sidebar owns its own instance; this one exists so search never disappears with it.
const [searchOpen, setSearchOpen] = useState(false);
// A pending composer prefill (text + attachments) pushed from the session start panel.
// Auto-Approve metering (§1.7): live reviewer counts for the composer badge. Polled with
// the session inbox; null until the first fetch (badge hidden).
const [reviewerStats, setReviewerStats] = useState<import("./api").ReviewerStats | null>(null);
const [composerPrefill, setComposerPrefill] = useState<{ text: string; attachments?: Attachment[]; nonce: number }>();
// Persona metadata drives workspace behavior by FAMILY, not by hardcoded id (so a DevOps/SecOps
@@ -885,6 +889,7 @@ export function App() {
const load = () => {
getInbox(sessionId, "pending").then(setSessionInbox).catch(() => setSessionInbox([]));
getUnattended(sessionId).then(markUnattended).catch(() => markUnattended(false));
getReviewerStats(sessionId).then(setReviewerStats).catch(() => setReviewerStats(null));
};
load();
const t = setInterval(load, 4000);
@@ -1652,6 +1657,7 @@ export function App() {
sessionId={sessionId}
workspace={needsWorkspace(agent) ? workspace || "" : undefined}
unattended={unattended}
reviewerStats={reviewerStats?.live ?? null}
onUnattendedChange={agent !== "chat" ? toggleUnattended : undefined}
prefill={composerPrefill}
resetKey={sessionId}
surfaces/gui/src/api.ts
+19
View File
@@ -1420,6 +1420,25 @@ export async function setUnattended(
return res.json();
}
// Auto-Approve metering (§1.7): per-session reviewer counts from the durable audit rows.
export interface ReviewerBucket {
checks: number;
allow: number;
deny: number;
unsure: number;
tokens_in: number;
tokens_out: number;
}
export interface ReviewerStats {
live: ReviewerBucket; // the mode actually deciding (Mode.AUTO_APPROVE)
shadow: ReviewerBucket; // shadow evaluation: recorded next to the human's own decisions
}
export async function getReviewerStats(sessionId: string): Promise<ReviewerStats> {
const res = await fetch(`${httpBase()}/v1/sessions/${sessionId}/reviewer-stats`);
return res.json();
}
export async function getSettings(): Promise<ModelSettings> {
const res = await fetch(`${httpBase()}/v1/settings`);
return res.json();
surfaces/gui/src/components/Composer.tsx
+23
View File
@@ -97,6 +97,9 @@ interface Props {
// when" is one mental model. Absent handler = no toggle (e.g. Chat).
unattended?: boolean;
onUnattendedChange?: (on: boolean) => void;
// Auto-Approve metering (§1.7): the "Auto-Approve · N checks" badge + mode-menu summary.
// Absent/zero hides both; only the LIVE bucket surfaces here (shadow is a Settings concern).
reviewerStats?: { checks: number; allow: number; deny: number; unsure: number; tokens_in: number; tokens_out: number } | null;
approvalSlot?: ReactNode;
// Push text + attachments into the composer (e.g. a start-panel task card). The `nonce` makes
// repeated identical prefills re-apply; the user can still edit before sending.
@@ -599,6 +602,7 @@ export function Composer(props: Props) {
) : props.workspace !== undefined ? (
<ModeMenu
mode={props.mode}
reviewerStats={props.reviewerStats}
onModeChange={props.onModeChange}
unattended={props.unattended}
onUnattendedChange={props.onUnattendedChange}
@@ -851,11 +855,13 @@ function ModeMenu({
onModeChange,
unattended,
onUnattendedChange,
reviewerStats,
}: {
mode: string;
onModeChange: (mode: string) => void;
unattended?: boolean;
onUnattendedChange?: (on: boolean) => void;
reviewerStats?: { checks: number; allow: number; deny: number; unsure: number; tokens_in: number; tokens_out: number } | null;
}) {
const [open, setOpen] = useState(false);
// The Auto-Approve entry is gated on the server flag. Fetch once on first open; a session
@@ -889,6 +895,11 @@ function ModeMenu({
}
>
{current?.label || mode}
{mode === "auto-approve" && !!reviewerStats?.checks && (
<span className="text-faint" data-testid="reviewer-badge">
· {reviewerStats.checks} {reviewerStats.checks === 1 ? "check" : "checks"}
</span>
)}
<Icon name="chevronDown" size={11} className="text-faint" />
</button>
{open && (
@@ -923,6 +934,18 @@ function ModeMenu({
<span className="text-[11px] text-faint leading-snug">{o.description}</span>
</button>
))}
{mode === "auto-approve" && !!reviewerStats?.checks && (
<>
<div className="my-1 border-t border-line" />
<div className="px-2.5 py-1.5 text-[11px] text-faint leading-snug" data-testid="reviewer-summary">
This session: {reviewerStats.checks} {reviewerStats.checks === 1 ? "check" : "checks"} ·{" "}
{reviewerStats.allow} cleared · {reviewerStats.deny} blocked · {reviewerStats.unsure} asked you
{reviewerStats.tokens_in + reviewerStats.tokens_out > 0 && (
<> · ~{Math.round((reviewerStats.tokens_in + reviewerStats.tokens_out) / 100) / 10}k tokens</>
)}
</div>
</>
)}
{onUnattendedChange && (
<>
<div className="my-1 border-t border-line" />
tests/test_auto_approve_settings.py
+68
View File
@@ -78,3 +78,71 @@ def test_build_engine_override_beats_config(tmp_path, monkeypatch):
assert engine.reviewer is not None
engine2 = build_engine(agent=chat_agent(), auto_approve=False, auto_approve_shadow=False)
assert engine2.reviewer is None
# -- metering (§1.7): durable reviewer stats from the audit store ------------------
def test_reviewer_stats_aggregates_by_stage(tmp_path):
from coworker.audit import AuditStore
store = AuditStore(tmp_path / "audit.db")
sid = "s1"
rows = [
{"session_id": sid, "tool": "run_shell", "stage": "reviewer_verdict", "status": "allow", "tokens_in": 100, "tokens_out": 20},
{"session_id": sid, "tool": "run_shell", "stage": "reviewer_verdict", "status": "allow", "tokens_in": 110, "tokens_out": 25},
{"session_id": sid, "tool": "web_fetch", "stage": "reviewer_verdict", "status": "deny", "tokens_in": 90, "tokens_out": 30},
{"session_id": sid, "tool": "write_file", "stage": "reviewer_verdict", "status": "unsure", "tokens_in": 80, "tokens_out": 15},
{"session_id": sid, "tool": "write_file", "stage": "reviewer_shadow", "status": "allow", "tokens_in": 70, "tokens_out": 10},
# Other sessions and stages must not leak in.
{"session_id": "other", "tool": "run_shell", "stage": "reviewer_verdict", "status": "allow", "tokens_in": 999, "tokens_out": 999},
{"session_id": sid, "tool": "run_shell", "stage": "finished", "status": "ok"},
]
for r in rows:
store.append(r)
stats = store.reviewer_stats(sid)
assert stats["live"] == {
"checks": 4, "allow": 2, "deny": 1, "unsure": 1,
"tokens_in": 380, "tokens_out": 90,
}
assert stats["shadow"]["checks"] == 1 and stats["shadow"]["allow"] == 1
store.close()
def test_audit_migration_adds_columns_to_a_legacy_db(tmp_path):
import sqlite3
from coworker.audit import AuditStore
# A pre-2026-08-12 database without the reviewer columns.
db = tmp_path / "legacy.db"
conn = sqlite3.connect(db)
conn.execute(
"""CREATE TABLE audit_events (
id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
session_id TEXT, agent TEXT, workspace TEXT, connector TEXT, tool TEXT,
stage TEXT, status TEXT, approval TEXT, args TEXT, result_preview TEXT,
reason TEXT, resource TEXT)"""
)
conn.execute(
"INSERT INTO audit_events (session_id, tool, stage, status) VALUES ('s1','t','finished','ok')"
)
conn.commit()
conn.close()
store = AuditStore(db) # opening migrates
store.append(
{"session_id": "s1", "tool": "run_shell", "stage": "reviewer_verdict",
"status": "allow", "tokens_in": 50, "tokens_out": 9, "call_id": "c1"}
)
stats = store.reviewer_stats("s1")
assert stats["live"]["checks"] == 1 and stats["live"]["tokens_in"] == 50
row = [r for r in store.list(session_id="s1") if r["stage"] == "reviewer_verdict"][0]
assert row["call_id"] == "c1"
store.close()
def test_reviewer_stats_endpoint(client):
empty = client.get("/v1/sessions/nope/reviewer-stats").json()
assert empty["live"]["checks"] == 0 and empty["shadow"]["checks"] == 0