From 07f394db990e1d0dc80ff5cd6344ac7dceb321d2 Mon Sep 17 00:00:00 2001 From: Devika Verma Date: Thu, 5 Nov 2026 13:53:00 +0530 Subject: [PATCH] fix(observability): Auto-approve is one table, a row per reviewer (T3-6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each reviewer is named in full (Session model, Decision model · OpenAI Decisions, Decision model · Jev) with its own checks, overrides, median time and model tokens, and a Total row carries the shadow note. The server gives overridden and tokens per reviewer in by_reviewer: an override counts for the reviewer its turn ran with. --- coworker/observability/store.py | 23 +++++- coworker/observability/usage_api.py | 16 +++- surfaces/gui/src/api.ts | 4 +- .../gui/src/components/UsagePage.test.tsx | 30 +++++++- surfaces/gui/src/components/UsagePage.tsx | 77 ++++++++++++------- surfaces/gui/src/locales/en.json | 35 +++++---- surfaces/gui/src/locales/zh.json | 34 ++++---- surfaces/gui/src/styles.css | 3 + .../test_observability_phase3_manual_fixes.py | 71 +++++++++++++++++ 9 files changed, 224 insertions(+), 69 deletions(-) create mode 100644 tests/test_observability_phase3_manual_fixes.py diff --git a/coworker/observability/store.py b/coworker/observability/store.py index 308423ae..35d889f4 100644 --- a/coworker/observability/store.py +++ b/coworker/observability/store.py @@ -901,7 +901,7 @@ class RecordingStore: self.flush() with self._lock: rows = self._conn.execute( - "SELECT s.kind, s.shadow, s.duration_ms, s.attrs, s.parent_step_id FROM steps s " + "SELECT s.kind, s.shadow, s.duration_ms, s.attrs, s.parent_step_id, t.reviewer FROM steps s " "JOIN turns t ON t.turn_id = s.turn_id WHERE t.started_at >= ? AND t.started_at < ? " "AND (s.kind = 'auto_approver' OR (s.kind = 'tool' AND s.attrs LIKE '%allow_anyway%'))" # one coworker: by the ledger's coworker id, which stands in for 1.2 turns @@ -909,7 +909,26 @@ class RecordingStore: if persona_id else ""), (start, end, persona_id) if persona_id else (start, end)).fetchall() return [{"kind": r["kind"], "shadow": bool(r["shadow"]), "duration_ms": r["duration_ms"], - "parent_step_id": r["parent_step_id"], "attrs": json.loads(r["attrs"] or "{}")} for r in rows] + "parent_step_id": r["parent_step_id"], "attrs": json.loads(r["attrs"] or "{}"), + "turn_reviewer": json.loads(r["reviewer"]) if r["reviewer"] else None} for r in rows] + + def approver_tokens_by_reviewer(self, start: int, end: int, *, + persona_id: Optional[str] = None) -> dict[str, int]: + """The live auto-approver model calls' tokens of turns that started in the period, by the + reviewer kind the turn ran with (T3-6, one row per reviewer). Shadow checks are left out: + the strip's total, from the ledger, has them.""" + self.flush() + with self._lock: + rows = self._conn.execute( + "SELECT json_extract(t.reviewer, '$.kind') AS rkind, SUM(COALESCE(s.input_fresh,0)" + "+COALESCE(s.cache_read,0)+COALESCE(s.cache_write,0)+COALESCE(s.output,0)) AS tok " + "FROM steps s JOIN turns t ON t.turn_id = s.turn_id WHERE t.started_at >= ? AND t.started_at < ? " + "AND s.kind = 'model' AND s.shadow = 0 AND s.usage_reported = 1 " + "AND s.purpose = 'auto_approver'" + + (" AND EXISTS (SELECT 1 FROM turn_usage u WHERE u.turn_id = t.turn_id AND u.persona_id = ?)" + if persona_id else "") + " GROUP BY rkind", + (start, end, persona_id) if persona_id else (start, end)).fetchall() + return {r["rkind"] or "unknown": int(r["tok"] or 0) for r in rows} def outside_calls_between(self, start: int, end: int) -> dict[str, int]: """Model steps with no turn (proxy calls) in [start, end): in no pill, so outside the diff --git a/coworker/observability/usage_api.py b/coworker/observability/usage_api.py index 96b2e5b5..f10fe01e 100644 --- a/coworker/observability/usage_api.py +++ b/coworker/observability/usage_api.py @@ -101,9 +101,15 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = No steps = store.approver_steps(start, end, persona_id=persona_id) live = [s for s in steps if s["kind"] == "auto_approver" and not s["shadow"] and s["parent_step_id"] is None] shadow = [s for s in steps if s["kind"] == "auto_approver" and s["shadow"]] - overridden = sum(1 for s in steps if s["kind"] == "tool" and any( + overrides = [s for s in steps if s["kind"] == "tool" and any( x.get("stage") == "grant" and x.get("outcome") == "allow_anyway" - for x in (s["attrs"].get("decision") or {}).get("path") or [])) + for x in (s["attrs"].get("decision") or {}).get("path") or [])] + # T3-6: an override counts for the reviewer its turn ran with (the one whose block was + # let through); the step itself names none, as no reviewer is asked again. + overridden_by: dict[str, int] = {} + for s in overrides: + k = (s.get("turn_reviewer") or {}).get("kind") or "unknown" + overridden_by[k] = overridden_by.get(k, 0) + 1 def counts(rows: list[dict]) -> dict[str, Any]: res = [r["attrs"].get("result") for r in rows] @@ -117,8 +123,10 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = No tokens = sum(g["total"] for g in store.usage_rollup(["purpose"], start, end, where={"persona_id": persona_id} if persona_id else None) if g["key"]["purpose"] == "auto_approver") - return {**counts(live), "overridden": overridden, "tokens": tokens, "shadow_checks": len(shadow), - "by_reviewer": [{"kind": k, **counts(v)} for k, v in sorted(by_kind.items())], + tokens_by = store.approver_tokens_by_reviewer(start, end, persona_id=persona_id) + return {**counts(live), "overridden": len(overrides), "tokens": tokens, "shadow_checks": len(shadow), + "by_reviewer": [{"kind": k, **counts(v), "overridden": overridden_by.get(k, 0), + "tokens": tokens_by.get(k, 0)} for k, v in sorted(by_kind.items())], "shown": bool(live)} diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.ts index 39a91eeb..9dc672f8 100644 --- a/surfaces/gui/src/api.ts +++ b/surfaces/gui/src/api.ts @@ -4733,7 +4733,9 @@ export interface ApproverStrip extends ApproverCounts { overridden: number; tokens: number; shadow_checks: number; - by_reviewer: (ApproverCounts & { kind: string })[]; + // T3-6: each reviewer's own overrides and model tokens (live checks only; the strip's + // `tokens` total has the shadow checks too) + by_reviewer: (ApproverCounts & { kind: string; overridden?: number; tokens?: number })[]; shown: boolean; } diff --git a/surfaces/gui/src/components/UsagePage.test.tsx b/surfaces/gui/src/components/UsagePage.test.tsx index c490328c..09ed2527 100644 --- a/surfaces/gui/src/components/UsagePage.test.tsx +++ b/surfaces/gui/src/components/UsagePage.test.tsx @@ -91,13 +91,37 @@ describe("ApproverStripView", () => { tokens: 3796, shadow_checks: 2, by_reviewer: [{ kind: "session_model", checks: 1, allowed: 1, asked: 0, blocked: 0, could_not_run: 0, median_ms: 1000 }], shown: true }; render(); const text = screen.getByTestId("usage-approver").textContent || ""; - expect(text).toContain("allowed 1"); expect(text).not.toContain("%"); - expect(text).toContain("session model"); - expect(text).toContain("incl. 2 shadow checks"); + expect(screen.getByTestId("usage-approver-session_model").textContent).toContain("Session model"); + expect(screen.getByTestId("usage-approver-total").textContent).toContain("incl. 2 shadow checks"); + expect(screen.getByTestId("usage-approver-total").textContent).toContain("3.8k"); expect(text).toContain("Auto-approve"); expect(text).not.toContain("Safety reviewer"); }); + + it("one row per reviewer, named in full, with its own overrides and tokens, and a Total row (T3-6)", () => { + const row = (kind: string, checks: number, allowed: number, asked: number, blocked: number, overridden: number, tokens: number) => + ({ kind, checks, allowed, asked, blocked, could_not_run: 0, median_ms: 1000, overridden, tokens }); + const ap = { checks: 98, allowed: 80, asked: 10, blocked: 8, could_not_run: 0, median_ms: 1000, overridden: 3, + tokens: 457_000, shadow_checks: 2, shown: true, + by_reviewer: [row("openai_decisions", 2, 1, 1, 0, 0, 0), row("session_model", 95, 79, 8, 8, 3, 450_000), row("typesafe_jev", 1, 0, 1, 0, 0, 0)] }; + render(); + const cells = (id: string) => [...screen.getByTestId(id).querySelectorAll("td")].map((c) => c.textContent); + expect(cells("usage-approver-openai_decisions")[0]).toBe("Decision model · OpenAI Decisions"); + expect(cells("usage-approver-typesafe_jev")[0]).toBe("Decision model · Jev"); + // overridden is each reviewer's own, not the period's total on every line + expect(cells("usage-approver-session_model")[6]).toBe("3"); + expect(cells("usage-approver-typesafe_jev")[6]).toBe("0"); + // 95 checks: shares; 2 checks: counts + expect(cells("usage-approver-session_model")[2]).toBe("83%"); + expect(cells("usage-approver-openai_decisions")[2]).toBe("1"); + const total = cells("usage-approver-total"); + expect(total[0]).toContain("Total"); + expect(total[0]).toContain("incl. 2 shadow checks"); + expect(total[1]).toBe("98"); + expect(total[6]).toBe("3"); + expect(total[8]).toBe("457k"); + }); }); describe("one coworker", () => { diff --git a/surfaces/gui/src/components/UsagePage.tsx b/surfaces/gui/src/components/UsagePage.tsx index a84e9f7b..8b7361ad 100644 --- a/surfaces/gui/src/components/UsagePage.tsx +++ b/surfaces/gui/src/components/UsagePage.tsx @@ -322,36 +322,57 @@ export function coworkerTable(groups: RollupGroup[], names: Record b.total - a.total); } +/** T3-6: one row per reviewer, named in full, and a Total row; the point is comparing + * reviewers, so they stay apart. Under 10 checks a cell is a count, from 10 a share (M3-R5). */ export function ApproverStripView({ ap, t }: { ap: ApproverStrip; t: T }) { - const lines = ap.by_reviewer.length ? ap.by_reviewer : [{ ...ap, kind: "" }]; + const rows = ap.by_reviewer.length ? ap.by_reviewer : [{ ...ap, kind: "unknown", tokens: undefined }]; + const part = (n: number, checks: number) => { + const p = approverPart(n, checks); + return p.share ? t("p3.ap.share", { n: p.value }) : String(p.value); + }; + const cells = (l: { checks: number; allowed: number; asked: number; blocked: number; could_not_run: number; median_ms: number | null }, + overridden: number | undefined, tokens: number | undefined) => ( + <> + {l.checks} + {part(l.allowed, l.checks)} + {part(l.asked, l.checks)} + {part(l.blocked, l.checks)} + {l.could_not_run} + {overridden ?? "—"} + {formatDuration(l.median_ms)} + {tokens == null ? "—" : formatTokens(tokens)} + + ); + const anyShare = rows.some((r) => r.checks >= 10) || ap.checks >= 10; return ( -
- {lines.map((l) => { - const part = (n: number, key: string) => { - const p = approverPart(n, l.checks); - return t(`p3.ap.${key}_${p.share ? "share" : "count"}`, { n: p.value }); - }; - return ( -
- {t("p3.ap.title")} - - {t("p3.ap.checks", { count: l.checks })} - {l.kind ? " · " + t(`p3.ap.kind.${l.kind}`, { defaultValue: l.kind }) : ""} - - {part(l.allowed, "allowed")} - {part(l.asked, "asked")} - {part(l.blocked, "blocked")} - - {t("p3.ap.could_not_run", { n: l.could_not_run })} · {t("p3.ap.overridden", { n: ap.overridden })} ·{" "} - {t("p3.ap.median", { time: formatDuration(l.median_ms) })} - -
- ); - })} -
- {t("p3.ap.tokens", { tokens: formatTokens(ap.tokens) })} - {ap.shadow_checks > 0 ? " · " + t("p3.ap.incl_shadow", { count: ap.shadow_checks }) : ""} -
+
+ {t("p3.ap.title")} + + + + + {(["checks", "allowed", "asked", "blocked", "could_not_run", "overridden", "median", "tokens"] as const).map((c) => ( + + ))} + + + + {rows.map((l) => ( + + + {cells(l, l.overridden, l.tokens)} + + ))} + + + {cells(ap, ap.overridden, ap.tokens)} + + +
{t("p3.ap.col.reviewer")}{t(`p3.ap.col.${c}`)}
{t(`p3.ap.name.${l.kind}`, { defaultValue: l.kind })}
+ {t("p3.ap.total")} + {ap.shadow_checks > 0 &&
{t("p3.ap.incl_shadow", { count: ap.shadow_checks })}
} +
+ {anyShare &&
{t("p3.ap.shares_note")}
}
); } diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.json index 2c6367e0..5978fd88 100644 --- a/surfaces/gui/src/locales/en.json +++ b/surfaces/gui/src/locales/en.json @@ -3968,23 +3968,26 @@ }, "ap": { "title": "Auto-approve", - "checks_one": "{{count}} check", - "checks_other": "{{count}} checks", - "kind": { - "session_model": "session model", - "openai_decisions": "decision model", - "typesafe_jev": "decision model" + "col": { + "reviewer": "Reviewer", + "checks": "Checks", + "allowed": "Allowed", + "asked": "Asked you", + "blocked": "Blocked", + "could_not_run": "Could not run", + "overridden": "Overridden", + "median": "Median time", + "tokens": "Tokens" }, - "allowed_count": "allowed {{n}}", - "allowed_share": "allowed {{n}}%", - "asked_count": "asked you {{n}}", - "asked_share": "asked you {{n}}%", - "blocked_count": "blocked {{n}}", - "blocked_share": "blocked {{n}}%", - "could_not_run": "could not run {{n}}", - "overridden": "overridden {{n}}", - "median": "{{time}} median", - "tokens": "{{tokens}} tokens", + "name": { + "session_model": "Session model", + "openai_decisions": "Decision model · OpenAI Decisions", + "typesafe_jev": "Decision model · Jev", + "unknown": "Unknown reviewer" + }, + "total": "Total", + "share": "{{n}}%", + "shares_note": "From 10 checks, a row shows shares of its checks; under 10, counts.", "incl_shadow_one": "incl. {{count}} shadow check", "incl_shadow_other": "incl. {{count}} shadow checks" }, diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.json index fe779169..e8f837c9 100644 --- a/surfaces/gui/src/locales/zh.json +++ b/surfaces/gui/src/locales/zh.json @@ -3908,22 +3908,26 @@ }, "ap": { "title": "自动审批", - "checks_other": "{{count}} 次检查", - "kind": { - "session_model": "会话模型", - "openai_decisions": "决策模型", - "typesafe_jev": "决策模型" + "col": { + "reviewer": "审核者", + "checks": "检查", + "allowed": "允许", + "asked": "询问你", + "blocked": "阻止", + "could_not_run": "无法运行", + "overridden": "被你放行", + "median": "中位用时", + "tokens": "令牌" }, - "allowed_count": "允许 {{n}}", - "allowed_share": "允许 {{n}}%", - "asked_count": "询问你 {{n}}", - "asked_share": "询问你 {{n}}%", - "blocked_count": "阻止 {{n}}", - "blocked_share": "阻止 {{n}}%", - "could_not_run": "无法运行 {{n}}", - "overridden": "被你放行 {{n}}", - "median": "中位数 {{time}}", - "tokens": "{{tokens}} 令牌", + "name": { + "session_model": "会话模型", + "openai_decisions": "决策模型 · OpenAI Decisions", + "typesafe_jev": "决策模型 · Jev", + "unknown": "未知审核者" + }, + "total": "合计", + "share": "{{n}}%", + "shares_note": "一行有 10 次及以上检查时显示占比,不足 10 次时显示次数。", "incl_shadow_other": "含 {{count}} 次影子检查" }, "foot": { diff --git a/surfaces/gui/src/styles.css b/surfaces/gui/src/styles.css index d5ceb078..ce359554 100644 --- a/surfaces/gui/src/styles.css +++ b/surfaces/gui/src/styles.css @@ -2644,6 +2644,9 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3 .p3-chip.ok { background:var(--ok-soft); color:var(--ok); } .p3-chip.warn { background:var(--warn-soft); color:var(--warn-ink); } .p3-chip.err { background:var(--danger-soft); color:var(--danger); } +/* T3-6: the Auto-approve table, one row per reviewer and a Total row */ +.p3-ap td.c-ok { color:var(--ok); } .p3-ap td.c-warn { color:var(--warn-ink); } .p3-ap td.c-err { color:var(--danger); } +.p3-ap tr.p3-total td { border-top:1px solid var(--line-strong, var(--line)); } .p3-callout { border-radius:9px; padding:8px 12px; font-size:var(--fs-meta); margin:8px 0; } .p3-callout.err { background:var(--danger-soft); color:var(--danger); } .p3-event { display:flex; align-items:center; gap:8px; width:100%; background:none; border:0; padding:3px 0; font-size:var(--fs-meta); color:var(--ink); cursor:pointer; } diff --git a/tests/test_observability_phase3_manual_fixes.py b/tests/test_observability_phase3_manual_fixes.py new file mode 100644 index 00000000..479e9a5e --- /dev/null +++ b/tests/test_observability_phase3_manual_fixes.py @@ -0,0 +1,71 @@ +"""Phase 3 manual testing (5 Nov), the server side of rows T3-6 and T3-10: the Auto-approve table +counts "overridden" per reviewer, and the Ratings card's numbers.""" + +from __future__ import annotations + +import time + +from coworker.observability import RecordingStore, usage_api + +# Turns an hour old: a store with the background writer sweeps on its first tick. +RECENT = int(time.time() * 1000) - 3_600_000 + + +def _turn(tid, sid, seq, at, reviewer_kind=None): + return {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "user", "started_at": at, + "duration_ms": 1000, "status": "completed", "incomplete": False, "approval_mode": "auto_approve", + "attended": True, "reviewer": {"kind": reviewer_kind, "active": True, "shadow": False} if reviewer_kind else None, + "stamp": {"machine": {"kind": "this_machine", "id": None}}, "first_message_id": None, + "last_message_id": None, "schema_version": 1, "wake": {"reason": "user"}} + + +def _step(tid, sid, n, at, kind, attrs): + return {"step_id": f"{tid}-s{n}", "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": n, + "kind": kind, "name": kind, "started_at": at, "duration_ms": 100, "status": "ok", "call_id": None, + "call_index": None, "message_id": None, "shadow": False, "after_turn": False, "attrs": attrs, + "content": {}} + + +def _check(result, kind): + return {"result": result, "reviewer": {"kind": kind}} + + +OVERRIDE = {"decision": {"path": [{"stage": "auto_approver", "outcome": "deny"}, {"stage": "grant", "outcome": "allow_anyway"}]}} + + +def test_overridden_is_counted_per_reviewer(tmp_path): + store = RecordingStore(tmp_path / "recordings.db", background=False) + turns = [_turn("A", "s1", 1, RECENT, "session_model"), _turn("B", "s2", 1, RECENT + 10, "typesafe_jev"), + _turn("C", "s2", 2, RECENT + 20, "typesafe_jev"), _turn("D", "s3", 1, RECENT + 30, "openai_decisions")] + steps = [ + _step("A", "s1", 1, RECENT, "auto_approver", _check("allow", "session_model")), + _step("A", "s1", 2, RECENT + 1, "auto_approver", _check("allow", "session_model")), + _step("B", "s2", 1, RECENT + 10, "auto_approver", _check("deny", "typesafe_jev")), + # the person let Jev's block through, twice + _step("C", "s2", 1, RECENT + 20, "tool", OVERRIDE), + _step("C", "s2", 2, RECENT + 21, "tool", OVERRIDE), + _step("D", "s3", 1, RECENT + 30, "auto_approver", _check("unsure", "openai_decisions")), + # the reviewers' own model calls: tokens per reviewer + _step("A", "s1", 3, RECENT + 2, "model", {"purpose": "auto_approver", "model": "model-x", "usage": { + "reported": True, "input_fresh": 300, "cache_read": 0, "cache_write": 0, "output": 20}}), + _step("D", "s3", 2, RECENT + 31, "model", {"purpose": "auto_approver", "model": "model-y", "usage": { + "reported": True, "input_fresh": 50, "cache_read": 0, "cache_write": 0, "output": 5}}), + _step("A", "s1", 4, RECENT + 3, "model", {"purpose": "main", "model": "model-x", "usage": { + "reported": True, "input_fresh": 9000, "cache_read": 0, "cache_write": 0, "output": 100}}), + ] + for t in turns: + store.write_turn(t) + for s in steps: + store.write_step(s) + store.flush() + got = usage_api.approver(store, start=RECENT - 1, end=RECENT + 1000) + assert got["overridden"] == 2 # the total, as before + rows = {r["kind"]: r for r in got["by_reviewer"]} + assert set(rows) == {"session_model", "typesafe_jev", "openai_decisions"} + assert rows["typesafe_jev"]["overridden"] == 2 + assert rows["session_model"]["overridden"] == 0 and rows["openai_decisions"]["overridden"] == 0 + assert rows["session_model"]["checks"] == 2 and rows["openai_decisions"]["asked"] == 1 + # each reviewer's own tokens; main work is not in them + assert rows["session_model"]["tokens"] == 320 and rows["openai_decisions"]["tokens"] == 55 + assert rows["typesafe_jev"]["tokens"] == 0 + store.close()