Feature 4: reviewer metering - badge, mode-menu summary, durable stats

Spec 1.7: the cost of Auto-Approve is visible while it accrues, not
discovered later. This is also where "uses your session model" gets
communicated (picker copy decision A): as a real accruing number.

Audit store:
- New columns call_id / tokens_in / tokens_out, with an idempotent ALTER
  migration for existing databases. This also fixes a feature-1 gap found
  in the process: the engine passed call_id and token counts on reviewer
  rows but the fixed column set silently dropped them, which would have
  broken the shadow-eval join and made token metering impossible.
- reviewer_stats(session_id): SQL aggregation of reviewer_verdict (live)
  and reviewer_shadow rows into checks/allow/deny/unsure + token sums.
  Durable - survives restarts and engine rebuilds.

Server: GET /v1/sessions/{id}/reviewer-stats (same shape as /unattended).

GUI:
- Polled with the existing 4s per-session poller.
- Mode button gains the badge when the session is in auto-approve and has
  checks: "Auto-Approve . 12 checks".
- Mode menu gains the session summary line: "This session: 12 checks . 10
  cleared . 0 blocked . 2 asked you . ~1k tokens". Only the LIVE bucket
  surfaces in the composer; shadow counts are a Settings/analysis concern.

Verified live against the running sidecar: the store already held 9 real
verdicts from manual testing of the mode, the endpoint aggregates them,
and both badge and summary render with real data.

Tests: stats aggregation (per-stage, per-session isolation, token sums),
legacy-DB migration (old schema opens, migrates, and round-trips call_id),
and the endpoint's empty shape. 113 backend + 114 GUI green.
This commit is contained in:
Devika Verma
2026-08-12 17:40:51 -07:00
parent 29c9f2b4a8
commit c59c5deae1
6 changed files with 176 additions and 3 deletions
+6
View File
@@ -15,6 +15,7 @@ import {
getSettings,
getPersonas,
getInbox,
getReviewerStats,
getUnattended,
PERSONAS_CHANGED,
resolveInboxItem,
@@ -315,6 +316,9 @@ export function App() {
// expanded sidebar owns its own instance; this one exists so search never disappears with it.
const [searchOpen, setSearchOpen] = useState(false);
// A pending composer prefill (text + attachments) pushed from the session start panel.
// Auto-Approve metering (§1.7): live reviewer counts for the composer badge. Polled with
// the session inbox; null until the first fetch (badge hidden).
const [reviewerStats, setReviewerStats] = useState<import("./api").ReviewerStats | null>(null);
const [composerPrefill, setComposerPrefill] = useState<{ text: string; attachments?: Attachment[]; nonce: number }>();
// Persona metadata drives workspace behavior by FAMILY, not by hardcoded id (so a DevOps/SecOps
@@ -885,6 +889,7 @@ export function App() {
const load = () => {
getInbox(sessionId, "pending").then(setSessionInbox).catch(() => setSessionInbox([]));
getUnattended(sessionId).then(markUnattended).catch(() => markUnattended(false));
getReviewerStats(sessionId).then(setReviewerStats).catch(() => setReviewerStats(null));
};
load();
const t = setInterval(load, 4000);
@@ -1652,6 +1657,7 @@ export function App() {
sessionId={sessionId}
workspace={needsWorkspace(agent) ? workspace || "" : undefined}
unattended={unattended}
reviewerStats={reviewerStats?.live ?? null}
onUnattendedChange={agent !== "chat" ? toggleUnattended : undefined}
prefill={composerPrefill}
resetKey={sessionId}
+19
View File
@@ -1420,6 +1420,25 @@ export async function setUnattended(
return res.json();
}
// Auto-Approve metering (§1.7): per-session reviewer counts from the durable audit rows.
export interface ReviewerBucket {
checks: number;
allow: number;
deny: number;
unsure: number;
tokens_in: number;
tokens_out: number;
}
export interface ReviewerStats {
live: ReviewerBucket; // the mode actually deciding (Mode.AUTO_APPROVE)
shadow: ReviewerBucket; // shadow evaluation: recorded next to the human's own decisions
}
export async function getReviewerStats(sessionId: string): Promise<ReviewerStats> {
const res = await fetch(`${httpBase()}/v1/sessions/${sessionId}/reviewer-stats`);
return res.json();
}
export async function getSettings(): Promise<ModelSettings> {
const res = await fetch(`${httpBase()}/v1/settings`);
return res.json();
+23
View File
@@ -97,6 +97,9 @@ interface Props {
// when" is one mental model. Absent handler = no toggle (e.g. Chat).
unattended?: boolean;
onUnattendedChange?: (on: boolean) => void;
// Auto-Approve metering (§1.7): the "Auto-Approve · N checks" badge + mode-menu summary.
// Absent/zero hides both; only the LIVE bucket surfaces here (shadow is a Settings concern).
reviewerStats?: { checks: number; allow: number; deny: number; unsure: number; tokens_in: number; tokens_out: number } | null;
approvalSlot?: ReactNode;
// Push text + attachments into the composer (e.g. a start-panel task card). The `nonce` makes
// repeated identical prefills re-apply; the user can still edit before sending.
@@ -599,6 +602,7 @@ export function Composer(props: Props) {
) : props.workspace !== undefined ? (
<ModeMenu
mode={props.mode}
reviewerStats={props.reviewerStats}
onModeChange={props.onModeChange}
unattended={props.unattended}
onUnattendedChange={props.onUnattendedChange}
@@ -851,11 +855,13 @@ function ModeMenu({
onModeChange,
unattended,
onUnattendedChange,
reviewerStats,
}: {
mode: string;
onModeChange: (mode: string) => void;
unattended?: boolean;
onUnattendedChange?: (on: boolean) => void;
reviewerStats?: { checks: number; allow: number; deny: number; unsure: number; tokens_in: number; tokens_out: number } | null;
}) {
const [open, setOpen] = useState(false);
// The Auto-Approve entry is gated on the server flag. Fetch once on first open; a session
@@ -889,6 +895,11 @@ function ModeMenu({
}
>
{current?.label || mode}
{mode === "auto-approve" && !!reviewerStats?.checks && (
<span className="text-faint" data-testid="reviewer-badge">
· {reviewerStats.checks} {reviewerStats.checks === 1 ? "check" : "checks"}
</span>
)}
<Icon name="chevronDown" size={11} className="text-faint" />
</button>
{open && (
@@ -923,6 +934,18 @@ function ModeMenu({
<span className="text-[11px] text-faint leading-snug">{o.description}</span>
</button>
))}
{mode === "auto-approve" && !!reviewerStats?.checks && (
<>
<div className="my-1 border-t border-line" />
<div className="px-2.5 py-1.5 text-[11px] text-faint leading-snug" data-testid="reviewer-summary">
This session: {reviewerStats.checks} {reviewerStats.checks === 1 ? "check" : "checks"} ·{" "}
{reviewerStats.allow} cleared · {reviewerStats.deny} blocked · {reviewerStats.unsure} asked you
{reviewerStats.tokens_in + reviewerStats.tokens_out > 0 && (
<> · ~{Math.round((reviewerStats.tokens_in + reviewerStats.tokens_out) / 100) / 10}k tokens</>
)}
</div>
</>
)}
{onUnattendedChange && (
<>
<div className="my-1 border-t border-line" />