mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
fix(observability): Auto-approve is one table, a row per reviewer (T3-6)
Each reviewer is named in full (Session model, Decision model · OpenAI Decisions, Decision model · Jev) with its own checks, overrides, median time and model tokens, and a Total row carries the shadow note. The server gives overridden and tokens per reviewer in by_reviewer: an override counts for the reviewer its turn ran with.
This commit is contained in:
@@ -901,7 +901,7 @@ class RecordingStore:
|
||||
self.flush()
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"SELECT s.kind, s.shadow, s.duration_ms, s.attrs, s.parent_step_id FROM steps s "
|
||||
"SELECT s.kind, s.shadow, s.duration_ms, s.attrs, s.parent_step_id, t.reviewer FROM steps s "
|
||||
"JOIN turns t ON t.turn_id = s.turn_id WHERE t.started_at >= ? AND t.started_at < ? "
|
||||
"AND (s.kind = 'auto_approver' OR (s.kind = 'tool' AND s.attrs LIKE '%allow_anyway%'))"
|
||||
# one coworker: by the ledger's coworker id, which stands in for 1.2 turns
|
||||
@@ -909,7 +909,26 @@ class RecordingStore:
|
||||
if persona_id else ""),
|
||||
(start, end, persona_id) if persona_id else (start, end)).fetchall()
|
||||
return [{"kind": r["kind"], "shadow": bool(r["shadow"]), "duration_ms": r["duration_ms"],
|
||||
"parent_step_id": r["parent_step_id"], "attrs": json.loads(r["attrs"] or "{}")} for r in rows]
|
||||
"parent_step_id": r["parent_step_id"], "attrs": json.loads(r["attrs"] or "{}"),
|
||||
"turn_reviewer": json.loads(r["reviewer"]) if r["reviewer"] else None} for r in rows]
|
||||
|
||||
def approver_tokens_by_reviewer(self, start: int, end: int, *,
|
||||
persona_id: Optional[str] = None) -> dict[str, int]:
|
||||
"""The live auto-approver model calls' tokens of turns that started in the period, by the
|
||||
reviewer kind the turn ran with (T3-6, one row per reviewer). Shadow checks are left out:
|
||||
the strip's total, from the ledger, has them."""
|
||||
self.flush()
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"SELECT json_extract(t.reviewer, '$.kind') AS rkind, SUM(COALESCE(s.input_fresh,0)"
|
||||
"+COALESCE(s.cache_read,0)+COALESCE(s.cache_write,0)+COALESCE(s.output,0)) AS tok "
|
||||
"FROM steps s JOIN turns t ON t.turn_id = s.turn_id WHERE t.started_at >= ? AND t.started_at < ? "
|
||||
"AND s.kind = 'model' AND s.shadow = 0 AND s.usage_reported = 1 "
|
||||
"AND s.purpose = 'auto_approver'"
|
||||
+ (" AND EXISTS (SELECT 1 FROM turn_usage u WHERE u.turn_id = t.turn_id AND u.persona_id = ?)"
|
||||
if persona_id else "") + " GROUP BY rkind",
|
||||
(start, end, persona_id) if persona_id else (start, end)).fetchall()
|
||||
return {r["rkind"] or "unknown": int(r["tok"] or 0) for r in rows}
|
||||
|
||||
def outside_calls_between(self, start: int, end: int) -> dict[str, int]:
|
||||
"""Model steps with no turn (proxy calls) in [start, end): in no pill, so outside the
|
||||
|
||||
@@ -101,9 +101,15 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = No
|
||||
steps = store.approver_steps(start, end, persona_id=persona_id)
|
||||
live = [s for s in steps if s["kind"] == "auto_approver" and not s["shadow"] and s["parent_step_id"] is None]
|
||||
shadow = [s for s in steps if s["kind"] == "auto_approver" and s["shadow"]]
|
||||
overridden = sum(1 for s in steps if s["kind"] == "tool" and any(
|
||||
overrides = [s for s in steps if s["kind"] == "tool" and any(
|
||||
x.get("stage") == "grant" and x.get("outcome") == "allow_anyway"
|
||||
for x in (s["attrs"].get("decision") or {}).get("path") or []))
|
||||
for x in (s["attrs"].get("decision") or {}).get("path") or [])]
|
||||
# T3-6: an override counts for the reviewer its turn ran with (the one whose block was
|
||||
# let through); the step itself names none, as no reviewer is asked again.
|
||||
overridden_by: dict[str, int] = {}
|
||||
for s in overrides:
|
||||
k = (s.get("turn_reviewer") or {}).get("kind") or "unknown"
|
||||
overridden_by[k] = overridden_by.get(k, 0) + 1
|
||||
|
||||
def counts(rows: list[dict]) -> dict[str, Any]:
|
||||
res = [r["attrs"].get("result") for r in rows]
|
||||
@@ -117,8 +123,10 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = No
|
||||
tokens = sum(g["total"] for g in store.usage_rollup(["purpose"], start, end,
|
||||
where={"persona_id": persona_id} if persona_id else None)
|
||||
if g["key"]["purpose"] == "auto_approver")
|
||||
return {**counts(live), "overridden": overridden, "tokens": tokens, "shadow_checks": len(shadow),
|
||||
"by_reviewer": [{"kind": k, **counts(v)} for k, v in sorted(by_kind.items())],
|
||||
tokens_by = store.approver_tokens_by_reviewer(start, end, persona_id=persona_id)
|
||||
return {**counts(live), "overridden": len(overrides), "tokens": tokens, "shadow_checks": len(shadow),
|
||||
"by_reviewer": [{"kind": k, **counts(v), "overridden": overridden_by.get(k, 0),
|
||||
"tokens": tokens_by.get(k, 0)} for k, v in sorted(by_kind.items())],
|
||||
"shown": bool(live)}
|
||||
|
||||
|
||||
|
||||
@@ -4733,7 +4733,9 @@ export interface ApproverStrip extends ApproverCounts {
|
||||
overridden: number;
|
||||
tokens: number;
|
||||
shadow_checks: number;
|
||||
by_reviewer: (ApproverCounts & { kind: string })[];
|
||||
// T3-6: each reviewer's own overrides and model tokens (live checks only; the strip's
|
||||
// `tokens` total has the shadow checks too)
|
||||
by_reviewer: (ApproverCounts & { kind: string; overridden?: number; tokens?: number })[];
|
||||
shown: boolean;
|
||||
}
|
||||
|
||||
|
||||
@@ -91,13 +91,37 @@ describe("ApproverStripView", () => {
|
||||
tokens: 3796, shadow_checks: 2, by_reviewer: [{ kind: "session_model", checks: 1, allowed: 1, asked: 0, blocked: 0, could_not_run: 0, median_ms: 1000 }], shown: true };
|
||||
render(<ApproverStripView ap={ap} t={t} />);
|
||||
const text = screen.getByTestId("usage-approver").textContent || "";
|
||||
expect(text).toContain("allowed 1");
|
||||
expect(text).not.toContain("%");
|
||||
expect(text).toContain("session model");
|
||||
expect(text).toContain("incl. 2 shadow checks");
|
||||
expect(screen.getByTestId("usage-approver-session_model").textContent).toContain("Session model");
|
||||
expect(screen.getByTestId("usage-approver-total").textContent).toContain("incl. 2 shadow checks");
|
||||
expect(screen.getByTestId("usage-approver-total").textContent).toContain("3.8k");
|
||||
expect(text).toContain("Auto-approve");
|
||||
expect(text).not.toContain("Safety reviewer");
|
||||
});
|
||||
|
||||
it("one row per reviewer, named in full, with its own overrides and tokens, and a Total row (T3-6)", () => {
|
||||
const row = (kind: string, checks: number, allowed: number, asked: number, blocked: number, overridden: number, tokens: number) =>
|
||||
({ kind, checks, allowed, asked, blocked, could_not_run: 0, median_ms: 1000, overridden, tokens });
|
||||
const ap = { checks: 98, allowed: 80, asked: 10, blocked: 8, could_not_run: 0, median_ms: 1000, overridden: 3,
|
||||
tokens: 457_000, shadow_checks: 2, shown: true,
|
||||
by_reviewer: [row("openai_decisions", 2, 1, 1, 0, 0, 0), row("session_model", 95, 79, 8, 8, 3, 450_000), row("typesafe_jev", 1, 0, 1, 0, 0, 0)] };
|
||||
render(<ApproverStripView ap={ap} t={t} />);
|
||||
const cells = (id: string) => [...screen.getByTestId(id).querySelectorAll("td")].map((c) => c.textContent);
|
||||
expect(cells("usage-approver-openai_decisions")[0]).toBe("Decision model · OpenAI Decisions");
|
||||
expect(cells("usage-approver-typesafe_jev")[0]).toBe("Decision model · Jev");
|
||||
// overridden is each reviewer's own, not the period's total on every line
|
||||
expect(cells("usage-approver-session_model")[6]).toBe("3");
|
||||
expect(cells("usage-approver-typesafe_jev")[6]).toBe("0");
|
||||
// 95 checks: shares; 2 checks: counts
|
||||
expect(cells("usage-approver-session_model")[2]).toBe("83%");
|
||||
expect(cells("usage-approver-openai_decisions")[2]).toBe("1");
|
||||
const total = cells("usage-approver-total");
|
||||
expect(total[0]).toContain("Total");
|
||||
expect(total[0]).toContain("incl. 2 shadow checks");
|
||||
expect(total[1]).toBe("98");
|
||||
expect(total[6]).toBe("3");
|
||||
expect(total[8]).toBe("457k");
|
||||
});
|
||||
});
|
||||
|
||||
describe("one coworker", () => {
|
||||
|
||||
@@ -322,36 +322,57 @@ export function coworkerTable(groups: RollupGroup[], names: Record<string, strin
|
||||
return [...rows.values()].sort((a, b) => b.total - a.total);
|
||||
}
|
||||
|
||||
/** T3-6: one row per reviewer, named in full, and a Total row; the point is comparing
|
||||
* reviewers, so they stay apart. Under 10 checks a cell is a count, from 10 a share (M3-R5). */
|
||||
export function ApproverStripView({ ap, t }: { ap: ApproverStrip; t: T }) {
|
||||
const lines = ap.by_reviewer.length ? ap.by_reviewer : [{ ...ap, kind: "" }];
|
||||
const rows = ap.by_reviewer.length ? ap.by_reviewer : [{ ...ap, kind: "unknown", tokens: undefined }];
|
||||
const part = (n: number, checks: number) => {
|
||||
const p = approverPart(n, checks);
|
||||
return p.share ? t("p3.ap.share", { n: p.value }) : String(p.value);
|
||||
};
|
||||
const cells = (l: { checks: number; allowed: number; asked: number; blocked: number; could_not_run: number; median_ms: number | null },
|
||||
overridden: number | undefined, tokens: number | undefined) => (
|
||||
<>
|
||||
<td className="n">{l.checks}</td>
|
||||
<td className="n c-ok">{part(l.allowed, l.checks)}</td>
|
||||
<td className="n c-warn">{part(l.asked, l.checks)}</td>
|
||||
<td className="n c-err">{part(l.blocked, l.checks)}</td>
|
||||
<td className="n">{l.could_not_run}</td>
|
||||
<td className="n">{overridden ?? "—"}</td>
|
||||
<td className="n">{formatDuration(l.median_ms)}</td>
|
||||
<td className="n">{tokens == null ? "—" : formatTokens(tokens)}</td>
|
||||
</>
|
||||
);
|
||||
const anyShare = rows.some((r) => r.checks >= 10) || ap.checks >= 10;
|
||||
return (
|
||||
<div className={CARD + " p3-card mt-3 flex flex-col gap-1.5"} data-testid="usage-approver">
|
||||
{lines.map((l) => {
|
||||
const part = (n: number, key: string) => {
|
||||
const p = approverPart(n, l.checks);
|
||||
return t(`p3.ap.${key}_${p.share ? "share" : "count"}`, { n: p.value });
|
||||
};
|
||||
return (
|
||||
<div key={l.kind} className="flex items-center gap-3 flex-wrap text-meta">
|
||||
<b className="p3-h">{t("p3.ap.title")}</b>
|
||||
<span className="text-muted">
|
||||
{t("p3.ap.checks", { count: l.checks })}
|
||||
{l.kind ? " · " + t(`p3.ap.kind.${l.kind}`, { defaultValue: l.kind }) : ""}
|
||||
</span>
|
||||
<span className="p3-chip ok">{part(l.allowed, "allowed")}</span>
|
||||
<span className="p3-chip warn">{part(l.asked, "asked")}</span>
|
||||
<span className="p3-chip err">{part(l.blocked, "blocked")}</span>
|
||||
<span className="text-muted">
|
||||
{t("p3.ap.could_not_run", { n: l.could_not_run })} · {t("p3.ap.overridden", { n: ap.overridden })} ·{" "}
|
||||
{t("p3.ap.median", { time: formatDuration(l.median_ms) })}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div className="text-meta text-muted">
|
||||
{t("p3.ap.tokens", { tokens: formatTokens(ap.tokens) })}
|
||||
{ap.shadow_checks > 0 ? " · " + t("p3.ap.incl_shadow", { count: ap.shadow_checks }) : ""}
|
||||
</div>
|
||||
<div className={CARD + " p3-card mt-3"} data-testid="usage-approver">
|
||||
<b className="p3-h">{t("p3.ap.title")}</b>
|
||||
<table className="p3-t p3-ap" data-testid="usage-approver-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>{t("p3.ap.col.reviewer")}</th>
|
||||
{(["checks", "allowed", "asked", "blocked", "could_not_run", "overridden", "median", "tokens"] as const).map((c) => (
|
||||
<th key={c} className="n">{t(`p3.ap.col.${c}`)}</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((l) => (
|
||||
<tr key={l.kind} data-testid={`usage-approver-${l.kind}`}>
|
||||
<td>{t(`p3.ap.name.${l.kind}`, { defaultValue: l.kind })}</td>
|
||||
{cells(l, l.overridden, l.tokens)}
|
||||
</tr>
|
||||
))}
|
||||
<tr className="p3-total" data-testid="usage-approver-total">
|
||||
<td>
|
||||
<b>{t("p3.ap.total")}</b>
|
||||
{ap.shadow_checks > 0 && <div className="text-faint">{t("p3.ap.incl_shadow", { count: ap.shadow_checks })}</div>}
|
||||
</td>
|
||||
{cells(ap, ap.overridden, ap.tokens)}
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
{anyShare && <div className="text-meta text-faint mt-1">{t("p3.ap.shares_note")}</div>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3968,23 +3968,26 @@
|
||||
},
|
||||
"ap": {
|
||||
"title": "Auto-approve",
|
||||
"checks_one": "{{count}} check",
|
||||
"checks_other": "{{count}} checks",
|
||||
"kind": {
|
||||
"session_model": "session model",
|
||||
"openai_decisions": "decision model",
|
||||
"typesafe_jev": "decision model"
|
||||
"col": {
|
||||
"reviewer": "Reviewer",
|
||||
"checks": "Checks",
|
||||
"allowed": "Allowed",
|
||||
"asked": "Asked you",
|
||||
"blocked": "Blocked",
|
||||
"could_not_run": "Could not run",
|
||||
"overridden": "Overridden",
|
||||
"median": "Median time",
|
||||
"tokens": "Tokens"
|
||||
},
|
||||
"allowed_count": "allowed {{n}}",
|
||||
"allowed_share": "allowed {{n}}%",
|
||||
"asked_count": "asked you {{n}}",
|
||||
"asked_share": "asked you {{n}}%",
|
||||
"blocked_count": "blocked {{n}}",
|
||||
"blocked_share": "blocked {{n}}%",
|
||||
"could_not_run": "could not run {{n}}",
|
||||
"overridden": "overridden {{n}}",
|
||||
"median": "{{time}} median",
|
||||
"tokens": "{{tokens}} tokens",
|
||||
"name": {
|
||||
"session_model": "Session model",
|
||||
"openai_decisions": "Decision model · OpenAI Decisions",
|
||||
"typesafe_jev": "Decision model · Jev",
|
||||
"unknown": "Unknown reviewer"
|
||||
},
|
||||
"total": "Total",
|
||||
"share": "{{n}}%",
|
||||
"shares_note": "From 10 checks, a row shows shares of its checks; under 10, counts.",
|
||||
"incl_shadow_one": "incl. {{count}} shadow check",
|
||||
"incl_shadow_other": "incl. {{count}} shadow checks"
|
||||
},
|
||||
|
||||
@@ -3908,22 +3908,26 @@
|
||||
},
|
||||
"ap": {
|
||||
"title": "自动审批",
|
||||
"checks_other": "{{count}} 次检查",
|
||||
"kind": {
|
||||
"session_model": "会话模型",
|
||||
"openai_decisions": "决策模型",
|
||||
"typesafe_jev": "决策模型"
|
||||
"col": {
|
||||
"reviewer": "审核者",
|
||||
"checks": "检查",
|
||||
"allowed": "允许",
|
||||
"asked": "询问你",
|
||||
"blocked": "阻止",
|
||||
"could_not_run": "无法运行",
|
||||
"overridden": "被你放行",
|
||||
"median": "中位用时",
|
||||
"tokens": "令牌"
|
||||
},
|
||||
"allowed_count": "允许 {{n}}",
|
||||
"allowed_share": "允许 {{n}}%",
|
||||
"asked_count": "询问你 {{n}}",
|
||||
"asked_share": "询问你 {{n}}%",
|
||||
"blocked_count": "阻止 {{n}}",
|
||||
"blocked_share": "阻止 {{n}}%",
|
||||
"could_not_run": "无法运行 {{n}}",
|
||||
"overridden": "被你放行 {{n}}",
|
||||
"median": "中位数 {{time}}",
|
||||
"tokens": "{{tokens}} 令牌",
|
||||
"name": {
|
||||
"session_model": "会话模型",
|
||||
"openai_decisions": "决策模型 · OpenAI Decisions",
|
||||
"typesafe_jev": "决策模型 · Jev",
|
||||
"unknown": "未知审核者"
|
||||
},
|
||||
"total": "合计",
|
||||
"share": "{{n}}%",
|
||||
"shares_note": "一行有 10 次及以上检查时显示占比,不足 10 次时显示次数。",
|
||||
"incl_shadow_other": "含 {{count}} 次影子检查"
|
||||
},
|
||||
"foot": {
|
||||
|
||||
@@ -2644,6 +2644,9 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3
|
||||
.p3-chip.ok { background:var(--ok-soft); color:var(--ok); }
|
||||
.p3-chip.warn { background:var(--warn-soft); color:var(--warn-ink); }
|
||||
.p3-chip.err { background:var(--danger-soft); color:var(--danger); }
|
||||
/* T3-6: the Auto-approve table, one row per reviewer and a Total row */
|
||||
.p3-ap td.c-ok { color:var(--ok); } .p3-ap td.c-warn { color:var(--warn-ink); } .p3-ap td.c-err { color:var(--danger); }
|
||||
.p3-ap tr.p3-total td { border-top:1px solid var(--line-strong, var(--line)); }
|
||||
.p3-callout { border-radius:9px; padding:8px 12px; font-size:var(--fs-meta); margin:8px 0; }
|
||||
.p3-callout.err { background:var(--danger-soft); color:var(--danger); }
|
||||
.p3-event { display:flex; align-items:center; gap:8px; width:100%; background:none; border:0; padding:3px 0; font-size:var(--fs-meta); color:var(--ink); cursor:pointer; }
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Phase 3 manual testing (5 Nov), the server side of rows T3-6 and T3-10: the Auto-approve table
|
||||
counts "overridden" per reviewer, and the Ratings card's numbers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from coworker.observability import RecordingStore, usage_api
|
||||
|
||||
# Turns an hour old: a store with the background writer sweeps on its first tick.
|
||||
RECENT = int(time.time() * 1000) - 3_600_000
|
||||
|
||||
|
||||
def _turn(tid, sid, seq, at, reviewer_kind=None):
|
||||
return {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "user", "started_at": at,
|
||||
"duration_ms": 1000, "status": "completed", "incomplete": False, "approval_mode": "auto_approve",
|
||||
"attended": True, "reviewer": {"kind": reviewer_kind, "active": True, "shadow": False} if reviewer_kind else None,
|
||||
"stamp": {"machine": {"kind": "this_machine", "id": None}}, "first_message_id": None,
|
||||
"last_message_id": None, "schema_version": 1, "wake": {"reason": "user"}}
|
||||
|
||||
|
||||
def _step(tid, sid, n, at, kind, attrs):
|
||||
return {"step_id": f"{tid}-s{n}", "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": n,
|
||||
"kind": kind, "name": kind, "started_at": at, "duration_ms": 100, "status": "ok", "call_id": None,
|
||||
"call_index": None, "message_id": None, "shadow": False, "after_turn": False, "attrs": attrs,
|
||||
"content": {}}
|
||||
|
||||
|
||||
def _check(result, kind):
|
||||
return {"result": result, "reviewer": {"kind": kind}}
|
||||
|
||||
|
||||
OVERRIDE = {"decision": {"path": [{"stage": "auto_approver", "outcome": "deny"}, {"stage": "grant", "outcome": "allow_anyway"}]}}
|
||||
|
||||
|
||||
def test_overridden_is_counted_per_reviewer(tmp_path):
|
||||
store = RecordingStore(tmp_path / "recordings.db", background=False)
|
||||
turns = [_turn("A", "s1", 1, RECENT, "session_model"), _turn("B", "s2", 1, RECENT + 10, "typesafe_jev"),
|
||||
_turn("C", "s2", 2, RECENT + 20, "typesafe_jev"), _turn("D", "s3", 1, RECENT + 30, "openai_decisions")]
|
||||
steps = [
|
||||
_step("A", "s1", 1, RECENT, "auto_approver", _check("allow", "session_model")),
|
||||
_step("A", "s1", 2, RECENT + 1, "auto_approver", _check("allow", "session_model")),
|
||||
_step("B", "s2", 1, RECENT + 10, "auto_approver", _check("deny", "typesafe_jev")),
|
||||
# the person let Jev's block through, twice
|
||||
_step("C", "s2", 1, RECENT + 20, "tool", OVERRIDE),
|
||||
_step("C", "s2", 2, RECENT + 21, "tool", OVERRIDE),
|
||||
_step("D", "s3", 1, RECENT + 30, "auto_approver", _check("unsure", "openai_decisions")),
|
||||
# the reviewers' own model calls: tokens per reviewer
|
||||
_step("A", "s1", 3, RECENT + 2, "model", {"purpose": "auto_approver", "model": "model-x", "usage": {
|
||||
"reported": True, "input_fresh": 300, "cache_read": 0, "cache_write": 0, "output": 20}}),
|
||||
_step("D", "s3", 2, RECENT + 31, "model", {"purpose": "auto_approver", "model": "model-y", "usage": {
|
||||
"reported": True, "input_fresh": 50, "cache_read": 0, "cache_write": 0, "output": 5}}),
|
||||
_step("A", "s1", 4, RECENT + 3, "model", {"purpose": "main", "model": "model-x", "usage": {
|
||||
"reported": True, "input_fresh": 9000, "cache_read": 0, "cache_write": 0, "output": 100}}),
|
||||
]
|
||||
for t in turns:
|
||||
store.write_turn(t)
|
||||
for s in steps:
|
||||
store.write_step(s)
|
||||
store.flush()
|
||||
got = usage_api.approver(store, start=RECENT - 1, end=RECENT + 1000)
|
||||
assert got["overridden"] == 2 # the total, as before
|
||||
rows = {r["kind"]: r for r in got["by_reviewer"]}
|
||||
assert set(rows) == {"session_model", "typesafe_jev", "openai_decisions"}
|
||||
assert rows["typesafe_jev"]["overridden"] == 2
|
||||
assert rows["session_model"]["overridden"] == 0 and rows["openai_decisions"]["overridden"] == 0
|
||||
assert rows["session_model"]["checks"] == 2 and rows["openai_decisions"]["asked"] == 1
|
||||
# each reviewer's own tokens; main work is not in them
|
||||
assert rows["session_model"]["tokens"] == 320 and rows["openai_decisions"]["tokens"] == 55
|
||||
assert rows["typesafe_jev"]["tokens"] == 0
|
||||
store.close()
|
||||
Reference in New Issue
Block a user