mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
Sessions saved before the model-prefix fix (PR #837) recorded one OpenAI model as both openai:model-x and model-x, so Usage showed it twice and split its tokens. Recordings are not rewritten; they are read through one rule: drop the openai: prefix, keep every other. The server's model_names.py canonical_model (and its SQL form for the ledger's model grouping) and the GUI's canonicalModel in modelNames.ts are the same, line for line, as #837's canonical_model and canonicalModel, so the two agree when both land. Applied to the Usage grouping by model, each turn's models (Heaviest turns, How it's been doing), the session usage by model and the Trace panel's model labels.
452 lines
26 KiB
Python
452 lines
26 KiB
Python
"""The Phase 3 read endpoints, as plain functions over the store (design §4).
|
|
|
|
Each takes the `RecordingStore` (and, where a screen needs names, a lookup the server passes)
|
|
and returns the JSON the endpoint serves. All of them read the `turn_usage` ledger, so their
|
|
totals add up to the pills (gate 1): the Usage page (OB-8), the coworker page's "How it's been
|
|
doing" (OB-15), TeamView's wake split and events (OB-17) and the Scheduled view's run health
|
|
(OB-18). Tokens only: no prices anywhere.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any, Callable, Iterable, Optional
|
|
|
|
from . import scopes
|
|
from . import signals as _signals
|
|
from .model_names import canonical_model
|
|
from .rollups import STEP_GROUPS, median
|
|
|
|
MIN_RUNS_TO_COMPARE = 10 # TD3-14
|
|
|
|
|
|
def _turn_groups(group: list[str]) -> bool:
|
|
return not (set(group) & STEP_GROUPS)
|
|
|
|
|
|
def rollup(store: Any, *, start: int, end: int, group: list[str], days: Optional[list] = None,
|
|
where: Optional[dict[str, Any]] = None) -> dict[str, Any]:
|
|
"""`GET /v1/observability/rollup`: the ledger grouped, the proxy line, how many turns of
|
|
the period are not checked yet, and how far back the ledger goes (`kept_from`).
|
|
Turn groups also get each group's median time without waiting (crash-open turns, with no
|
|
length, are left out)."""
|
|
groups = store.usage_rollup(group, start, end, days=days, where=where)
|
|
turns = store.usage_turns(start, end, where=where)
|
|
version = _signals.version()
|
|
not_checked = sum(1 for t in turns if t["warn"] is None or t.get("signals_version") not in (None, version))
|
|
if _turn_groups(group):
|
|
by_key: dict[tuple, list[int]] = {}
|
|
day_of = None
|
|
if "day" in group and days:
|
|
def day_of(ts: int) -> Optional[str]: # noqa: E306
|
|
return next((d[0] for d in days if d[1] <= ts < d[2]), None)
|
|
for t in turns:
|
|
if t["active_ms"] is None:
|
|
continue
|
|
key = []
|
|
for g in group:
|
|
if g == "all":
|
|
key.append("all")
|
|
elif g == "day":
|
|
key.append(day_of(t["started_at"]) if day_of else None)
|
|
elif g == "session":
|
|
key.append(t["session_id"])
|
|
elif g == "wake_reason":
|
|
key.append(t.get("wake_reason") or "unknown")
|
|
else:
|
|
key.append(t.get(g))
|
|
by_key.setdefault(tuple(key), []).append(t["active_ms"])
|
|
for gr in groups:
|
|
k = tuple(gr["key"][g] for g in group) if group else ("all",)
|
|
gr["median_active_ms"] = median(by_key.get(k, []))
|
|
return {"groups": groups, "no_turn": store.outside_calls_between(start, end),
|
|
"not_checked": not_checked, "kept_from": store.usage_kept_from(),
|
|
"turns": len(turns), "deleted_session_turns": sum(1 for t in turns if t["session_deleted"])}
|
|
|
|
|
|
def heaviest(store: Any, *, start: int, end: int, limit: int = 20,
|
|
title_of: Optional[Callable[[str], Optional[str]]] = None,
|
|
persona_id: Optional[str] = None) -> dict[str, Any]:
|
|
"""`GET /v1/observability/turns?order=tokens`: the heaviest turns of the period. Each names
|
|
its turn (`seq`, M3-R2), says when it was found open after a crash (M3-R3), and lists its
|
|
warning checks. A deleted session's turn has no title and no link."""
|
|
where = {"persona_id": persona_id} if persona_id else None
|
|
turns = sorted(store.usage_turns(start, end, where=where), key=lambda t: (-t["total"], t["started_at"]))[: max(1, min(limit, 100))]
|
|
seqs = store.turn_seqs([t["turn_id"] for t in turns]) if turns else {}
|
|
out = []
|
|
for t in turns:
|
|
title = None
|
|
if title_of is not None and not t["session_deleted"]:
|
|
try:
|
|
title = title_of(t["session_id"])
|
|
except Exception: # noqa: BLE001
|
|
title = None
|
|
out.append({"turn_id": t["turn_id"], "session_id": t["session_id"], "title": title,
|
|
"seq": seqs.get(t["turn_id"]), "session_deleted": bool(t["session_deleted"]),
|
|
"persona_id": t["persona_id"], "model_ids": t["models"], "total": t["total"],
|
|
"at_least": t["at_least"], "active_ms": t["active_ms"], "status": t["status"],
|
|
"found_open": t["status"] == "unknown" and t["duration_ms"] is None,
|
|
"warn": t["warn"], "checks": t["warn_checks"], "started_at": t["started_at"],
|
|
"run_id": t["run_id"], "task_id": t["task_id"]})
|
|
return {"turns": out}
|
|
|
|
|
|
ASKED = ("unsure", "context_changed", "could_not_run")
|
|
|
|
|
|
|
|
|
|
def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = None) -> dict[str, Any]:
|
|
"""`GET /v1/observability/approver`: the Auto-approve strip (§7.3). Live checks by result
|
|
(asked you includes could-not-run: the card falls back to asking), overridden tool
|
|
steps, the median check time, and the checks' tokens with shadow checks included, as
|
|
the pill counts them. One line per reviewer kind used in the period."""
|
|
steps = store.approver_steps(start, end, persona_id=persona_id)
|
|
live = [s for s in steps if s["kind"] == "auto_approver" and not s["shadow"] and s["parent_step_id"] is None]
|
|
shadow = [s for s in steps if s["kind"] == "auto_approver" and s["shadow"]]
|
|
overrides = [s for s in steps if s["kind"] == "tool" and any(
|
|
x.get("stage") == "grant" and x.get("outcome") == "allow_anyway"
|
|
for x in (s["attrs"].get("decision") or {}).get("path") or [])]
|
|
# T3-6: an override counts for the reviewer its turn ran with (the one whose block was
|
|
# let through); the step itself names none, as no reviewer is asked again.
|
|
overridden_by: dict[str, int] = {}
|
|
for s in overrides:
|
|
k = (s.get("turn_reviewer") or {}).get("kind") or "unknown"
|
|
overridden_by[k] = overridden_by.get(k, 0) + 1
|
|
|
|
def counts(rows: list[dict]) -> dict[str, Any]:
|
|
res = [r["attrs"].get("result") for r in rows]
|
|
return {"checks": len(rows), "allowed": res.count("allow"), "asked": sum(r in ASKED for r in res),
|
|
"blocked": res.count("deny"), "could_not_run": res.count("could_not_run"),
|
|
"median_ms": median([r["duration_ms"] for r in rows if r["duration_ms"] is not None])}
|
|
|
|
by_kind: dict[str, list] = {}
|
|
for s in live:
|
|
by_kind.setdefault((s["attrs"].get("reviewer") or {}).get("kind") or "unknown", []).append(s)
|
|
tokens = sum(g["total"] for g in store.usage_rollup(["purpose"], start, end,
|
|
where={"persona_id": persona_id} if persona_id else None)
|
|
if g["key"]["purpose"] == "auto_approver")
|
|
tokens_by = store.approver_tokens_by_reviewer(start, end, persona_id=persona_id)
|
|
return {**counts(live), "overridden": len(overrides), "tokens": tokens, "shadow_checks": len(shadow),
|
|
"by_reviewer": [{"kind": k, **counts(v), "overridden": overridden_by.get(k, 0),
|
|
"tokens": tokens_by.get(k, 0)} for k, v in sorted(by_kind.items())],
|
|
"shown": bool(live)}
|
|
|
|
|
|
RATED_DOWN_SHOWN = 50
|
|
|
|
|
|
def ratings(store: Any, *, start: int, end: int, persona_id: Optional[str] = None,
|
|
title_of: Optional[Callable[[str], Optional[str]]] = None) -> dict[str, Any]:
|
|
"""`GET /v1/observability/ratings`: the Usage page's Ratings card (T3-10, lean). The 👍 and
|
|
👎 of the turns that started in the period (the same turns the page counts), how often each
|
|
reason was picked, and the 👎 turns, newest first, with their note and how many of their
|
|
steps the auto-approver let through. No charts or trends: those stay with Langfuse."""
|
|
where = {"persona_id": persona_id} if persona_id else None
|
|
rated = [t for t in store.usage_turns(start, end, where=where) if t.get("rating") in ("up", "down")]
|
|
down = sorted((t for t in rated if t["rating"] == "down"), key=lambda t: (-(t["started_at"] or 0), t["turn_id"]))
|
|
feedback = store.feedback_for([t["turn_id"] for t in down])
|
|
reasons: dict[str, int] = {}
|
|
for f in feedback.values():
|
|
for r in f.get("reasons") or []:
|
|
reasons[r] = reasons.get(r, 0) + 1
|
|
shown = down[:RATED_DOWN_SHOWN]
|
|
seqs = store.turn_seqs([t["turn_id"] for t in shown]) if shown else {}
|
|
turns = []
|
|
for t in shown:
|
|
f = feedback.get(t["turn_id"]) or {}
|
|
title = None
|
|
if title_of is not None and not t["session_deleted"]:
|
|
try:
|
|
title = title_of(t["session_id"])
|
|
except Exception: # noqa: BLE001
|
|
title = None
|
|
turns.append({"turn_id": t["turn_id"], "session_id": t["session_id"], "title": title,
|
|
"seq": seqs.get(t["turn_id"]), "session_deleted": bool(t["session_deleted"]),
|
|
"persona_id": t["persona_id"], "started_at": t["started_at"],
|
|
"reasons": f.get("reasons") or [], "note": (f.get("content") or {}).get("note"),
|
|
"auto_approved": len(f.get("auto_approved") or [])})
|
|
return {"up": sum(1 for t in rated if t["rating"] == "up"), "down": len(down), "reasons": reasons,
|
|
"down_turns": turns, "more": max(0, len(down) - len(shown))}
|
|
|
|
|
|
# -- OB-15: how a coworker has been doing ------------------------------------------------------
|
|
|
|
|
|
def persona_doing(store: Any, persona_id: str, *, start: int, end: int, current_fp: Optional[str] = None,
|
|
min_runs: int = MIN_RUNS_TO_COMPARE) -> dict[str, Any]:
|
|
"""`GET /v1/personas/{id}/doing`: one row per fingerprint. A run is the turns of one
|
|
session under one fingerprint (TD3-12), so the runs add up to the coworker's share of the
|
|
Usage page. Fewer than `min_runs` runs: no medians, "too few runs to compare"."""
|
|
turns = store.usage_turns(start, end, where={"persona_id": persona_id})
|
|
kept_from = store.usage_kept_from()
|
|
models_of_turn = {t["turn_id"]: list(dict.fromkeys(canonical_model(m) for m in t.get("own_models", t["models"])))
|
|
for t in turns}
|
|
runs: dict[tuple, dict[str, Any]] = {}
|
|
for t in turns:
|
|
k = (t["persona_fp"], t["session_id"])
|
|
r = runs.setdefault(k, {"fp": t["persona_fp"], "turns": [], "automation": False})
|
|
r["turns"].append(t)
|
|
r["automation"] = r["automation"] or bool(t["run_id"])
|
|
rows: dict[Optional[str], dict[str, Any]] = {}
|
|
for (fp, _sid), r in runs.items():
|
|
row = rows.setdefault(fp, {"fp": fp, "runs": [], "first": None, "last": None})
|
|
row["runs"].append(r)
|
|
for t in r["turns"]:
|
|
row["first"] = t["started_at"] if row["first"] is None else min(row["first"], t["started_at"])
|
|
row["last"] = t["started_at"] if row["last"] is None else max(row["last"], t["started_at"])
|
|
versions = {v["fp"]: v for v in store.persona_versions(persona_id)}
|
|
out = []
|
|
for fp, row in sorted(rows.items(), key=lambda kv: -(kv[1]["last"] or 0)):
|
|
rr = row["runs"]
|
|
all_turns = [t for r in rr for t in r["turns"]]
|
|
checked = [t for t in all_turns if t["warn"] is not None]
|
|
models = sorted({m for t in all_turns for m in models_of_turn.get(t["turn_id"], [])})
|
|
enough = len(rr) >= min_runs
|
|
out.append({
|
|
"fp": fp, "current": fp is not None and fp == current_fp, "unknown_version": fp is None,
|
|
"first_seen": row["first"], "last_seen": row["last"],
|
|
# V3-10: when the first turn is the oldest the ledger keeps, it is "before" that day.
|
|
"before_kept": kept_from is not None and row["first"] is not None and row["first"] <= kept_from,
|
|
"runs": len(rr), "automation_runs": sum(1 for r in rr if r["automation"]),
|
|
"turns": len(all_turns), "checked": len(checked), "with_warn": sum(1 for t in checked if t["warn"]),
|
|
"not_checked": len(all_turns) - len(checked),
|
|
"median_tokens": median([sum(t["total"] for t in r["turns"]) for r in rr]) if enough else None,
|
|
"at_least": any(t["at_least"] for t in all_turns),
|
|
"approvals_per_run": (sum(t["approvals"] for t in all_turns) / len(rr)) if enough else None,
|
|
"up": sum(1 for t in all_turns if t["rating"] == "up"),
|
|
"down": sum(1 for t in all_turns if t["rating"] == "down"),
|
|
"too_few": not enough, "models": models,
|
|
"first_seen_version": (versions.get(fp) or {}).get("first_seen"),
|
|
})
|
|
# "What changed" between each version and the one before it (by first sighting).
|
|
ordered = [v for v in store.persona_versions(persona_id)]
|
|
changes = {}
|
|
for prev, cur in zip(ordered, ordered[1:]):
|
|
changes[cur["fp"]] = {"from": prev["fp"], **diff_versions(prev, cur, store)}
|
|
# a version whose files were not kept still gets its number, by when it was first seen here
|
|
fps = [v["fp"] for v in ordered]
|
|
fps += [r["fp"] for r in sorted(out, key=lambda r: r["first_seen"] or 0) if r["fp"] and r["fp"] not in fps]
|
|
number = {fp: i + 1 for i, fp in enumerate(fps)}
|
|
for row in out:
|
|
row["changed"] = changes.get(row["fp"])
|
|
# T3-15: "v1 · current": versions numbered oldest first, by first sighting
|
|
row["version_n"] = number.get(row["fp"])
|
|
# model line: "model unchanged", "model changed: a -> b", or several models
|
|
for i, row in enumerate(out):
|
|
older = out[i + 1] if i + 1 < len(out) else None
|
|
# T3-15: a first version has nothing to compare a model with (it says only that it is
|
|
# the first); the models are listed on their own line either way
|
|
if not row.get("changed"):
|
|
row["model_line"] = {"kind": "first", "models": row["models"]}
|
|
elif len(row["models"]) > 1:
|
|
row["model_line"] = {"kind": "several", "models": row["models"]}
|
|
elif older is not None and row["models"] and older["models"] and row["models"] != older["models"]:
|
|
row["model_line"] = {"kind": "changed", "from": older["models"][-1], "to": row["models"][0]}
|
|
else:
|
|
row["model_line"] = {"kind": "unchanged", "models": row["models"]}
|
|
return {"persona_id": persona_id, "start": start, "end": end, "kept_from": kept_from,
|
|
"min_runs": min_runs, "rows": out,
|
|
"total": sum(t["total"] for t in turns), "runs": sum(r["runs"] for r in out)}
|
|
|
|
|
|
def _skill_version(data: Optional[bytes]) -> Optional[str]:
|
|
if not data:
|
|
return None
|
|
for line in data.decode("utf-8", "replace").splitlines()[:40]:
|
|
s = line.strip()
|
|
if s.lower().startswith("version:"):
|
|
return s.split(":", 1)[1].strip().strip("\"'") or None
|
|
return None
|
|
|
|
|
|
def diff_versions(prev: dict[str, Any], cur: dict[str, Any], store: Any) -> dict[str, Any]:
|
|
"""What changed between two kept versions of a coworker, from their file lists: files
|
|
added, removed and edited (with the line change), and a skill's own `version:` when its
|
|
SKILL.md says one (§9.4)."""
|
|
a, b = prev["files"], cur["files"]
|
|
edits = []
|
|
for path in sorted(set(a) | set(b)):
|
|
if path in a and path in b and a[path]["sha256"] == b[path]["sha256"]:
|
|
continue
|
|
e: dict[str, Any] = {"path": path, "kind": "added" if path not in a else "removed" if path not in b else "edited"}
|
|
if e["kind"] == "edited":
|
|
e["lines"] = b[path]["lines"] - a[path]["lines"]
|
|
if path.endswith("SKILL.md"):
|
|
e["skill"] = path.split("/")[1] if path.count("/") >= 2 else path
|
|
e["from_version"] = _skill_version(store.persona_file(prev["persona_id"], prev["fp"], path)) if path in a else None
|
|
e["to_version"] = _skill_version(store.persona_file(cur["persona_id"], cur["fp"], path)) if path in b else None
|
|
edits.append(e)
|
|
return {"files": edits}
|
|
|
|
|
|
def version_diff(store: Any, persona_id: str, fp_from: str, fp_to: str) -> dict[str, Any]:
|
|
""""Show diff": both versions' text of each changed file (unified diff), local only."""
|
|
import difflib
|
|
|
|
vs = {v["fp"]: v for v in store.persona_versions(persona_id)}
|
|
if fp_from not in vs or fp_to not in vs:
|
|
return {"error": "version not kept"}
|
|
files = []
|
|
for e in diff_versions(vs[fp_from], vs[fp_to], store)["files"]:
|
|
old = store.persona_file(persona_id, fp_from, e["path"]) or b""
|
|
new = store.persona_file(persona_id, fp_to, e["path"]) or b""
|
|
lines = list(difflib.unified_diff(old.decode("utf-8", "replace").splitlines(),
|
|
new.decode("utf-8", "replace").splitlines(),
|
|
fromfile=f"{fp_from}/{e['path']}", tofile=f"{fp_to}/{e['path']}", lineterm=""))
|
|
files.append({**e, "diff": "\n".join(lines[:2000])})
|
|
return {"from": fp_from, "to": fp_to, "files": files}
|
|
|
|
|
|
# -- OB-17: team wake split and events ---------------------------------------------------------------
|
|
|
|
|
|
def team_wakes(store: Any, session_ids: Iterable[str], *, start: int = 0, end: int = 2**62) -> list[dict[str, Any]]:
|
|
"""TeamView's "By wake reason" (§11.5): wakes, tokens and cache-read share per reason, lead
|
|
and workers together, each split by role. Reads the ledger of every member session, so its
|
|
total equals the sum of their pills (gate 1)."""
|
|
sids = list(dict.fromkeys(session_ids))
|
|
out: dict[str, dict[str, Any]] = {}
|
|
for g in store.usage_rollup(["wake_reason", "team_role"], start, end, where={"session_id": sids}):
|
|
reason = g["key"]["wake_reason"]
|
|
r = out.setdefault(reason, {"reason": reason, "wakes": 0, "total": 0, "cache_read": 0, "input": 0,
|
|
"at_least": False, "by_role": {}})
|
|
r["wakes"] += g["turns"]
|
|
r["total"] += g["total"]
|
|
r["cache_read"] += g["cache_read"]
|
|
r["input"] += g["input_fresh"] + g["cache_read"] + g["cache_write"]
|
|
r["at_least"] = r["at_least"] or g["unreported"] > 0
|
|
role = g["key"]["team_role"] or "unknown"
|
|
b = r["by_role"].setdefault(role, {"wakes": 0, "total": 0})
|
|
b["wakes"] += g["turns"]
|
|
b["total"] += g["total"]
|
|
return sorted(out.values(), key=lambda r: (-r["total"], r["reason"]))
|
|
|
|
|
|
def team_events(store: Any, *, lead_session: str, worker_sessions: dict[str, str],
|
|
board_events: list[dict[str, Any]], links: list[tuple[int, int]], limit: int = 50,
|
|
turns: Optional[list[dict[str, Any]]] = None) -> list[dict[str, Any]]:
|
|
""""Team events that used to be silent" (§11.4), newest first: cap hits, long deferrals,
|
|
watchdog nudges, lead polling, the same approval asked twice and an item reviewed before
|
|
its prerequisite. Each names the session and turn whose Steps it opens. `turns`: the
|
|
members' turns when the caller has read them already (P3B-5)."""
|
|
sessions = {lead_session: "lead", **{sid: actor for actor, sid in worker_sessions.items()}}
|
|
rows = store.team_turns(list(sessions)) if turns is None else turns
|
|
events: list[dict[str, Any]] = []
|
|
for t in rows:
|
|
wake = t.get("wake") or {}
|
|
d = wake.get("deferred") or {}
|
|
who = sessions.get(t["session_id"], "?")
|
|
base = {"session_id": t["session_id"], "turn_id": t["turn_id"], "seq": t["seq"], "at": t["started_at"], "who": who}
|
|
if d.get("by") == "cap":
|
|
events.append({**base, "kind": "wake_capped", "severity": "warn", "deferred_ms": d.get("ms")})
|
|
elif d.get("by") in ("busy", "stopped") and (d.get("ms") or 0) >= 60_000:
|
|
events.append({**base, "kind": "deferred", "severity": "info", "by": d["by"], "deferred_ms": d.get("ms")})
|
|
if wake.get("reason") == "watchdog":
|
|
events.append({**base, "kind": "watchdog_nudged", "severity": "info", "item_ids": wake.get("item_ids") or []})
|
|
lead_turns = [t for t in rows if t["session_id"] == lead_session]
|
|
by_id = {t["turn_id"]: t for t in lead_turns}
|
|
for f in scopes.lead_polling([{"id": t["turn_id"], "wake": t.get("wake")} for t in lead_turns]):
|
|
t = by_id[f["to"]]
|
|
events.append({"session_id": lead_session, "turn_id": t["turn_id"], "seq": t["seq"], "at": t["started_at"],
|
|
"who": "lead", "kind": "lead_polling", "severity": "warn", "count": f["count"]})
|
|
cards = store.team_approval_cards(list(sessions))
|
|
for f, c in scopes.same_approval_repeats(cards): # the repeat card, as the Issues tab (P3B-4)
|
|
events.append({"session_id": f["again_by"], "turn_id": c.get("turn_id"), "seq": c.get("seq"),
|
|
"at": c.get("at"), "who": sessions.get(f["again_by"], "?"), "kind": "same_approval_twice",
|
|
"severity": "warn", "tool": f["tool"], "first_by": sessions.get(f["first_by"], f["first_by"]),
|
|
"again_by": sessions.get(f["again_by"], f["again_by"])})
|
|
transitions = [{"seq": e["seq"], "item": e.get("item_id"), "to": (e.get("payload") or {}).get("to"),
|
|
"at": e.get("ts")} for e in board_events if e.get("kind") == "item_transitioned"]
|
|
for f in scopes.reviewed_before_prereq(transitions, links):
|
|
e = next((x for x in transitions if x["seq"] == f["at_seq"]), {})
|
|
events.append({"kind": "reviewed_before_prereq", "severity": "warn", "item": f["item"],
|
|
"prerequisite": f["prerequisite"], "to": e.get("to"), "at": _ts_ms(e.get("at")),
|
|
"board_seq": f["at_seq"]})
|
|
for e in board_events:
|
|
if e.get("kind") == "wake_cap_reached":
|
|
p = e.get("payload") or {}
|
|
events.append({"kind": "wake_cap_reached", "severity": "warn", "at": _ts_ms(e.get("ts")),
|
|
"hour": p.get("hour"), "count": p.get("count"), "held": p.get("held") or [],
|
|
"board_seq": e.get("seq")})
|
|
events.sort(key=lambda e: -(e.get("at") or 0))
|
|
return events[:limit]
|
|
|
|
|
|
def _ts_ms(ts: Any) -> Optional[int]:
|
|
if ts is None:
|
|
return None
|
|
if isinstance(ts, (int, float)):
|
|
return int(ts * 1000) if ts < 10**11 else int(ts)
|
|
try:
|
|
from datetime import datetime
|
|
|
|
return int(datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp() * 1000)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
# -- OB-18: automation run health ---------------------------------------------------------------------
|
|
|
|
|
|
def automation_runs(store: Any, task_id: str, runs: list[dict[str, Any]], *, now_ms: int,
|
|
titles: Optional[dict[str, str]] = None) -> dict[str, Any]:
|
|
"""Each run of an automation gains `{turns, total, at_least, active_ms, warn, checks,
|
|
health}` from its own turns (`turn.automation`, TD3-23), and the automation gains the
|
|
`failed_in_a_row` banner (TD3-21). A run whose turn was never recorded reads tokens "—"
|
|
and its health from the run record only. Before 1.3, a run's own turns are the turns of
|
|
its session that started before the run finished (a follow-up typed later is left out)."""
|
|
titles = titles or {}
|
|
by_run: dict[str, list[dict]] = {}
|
|
for t in store.usage_turns(0, 2**62, where={"task_id": task_id}):
|
|
by_run.setdefault(t["run_id"], []).append(t)
|
|
sessions = [r.get("session_id") for r in runs if r.get("session_id")]
|
|
legacy: dict[str, list[dict]] = {}
|
|
for t in store.usage_turns(0, 2**62, where={"session_id": sessions}) if sessions else []:
|
|
if t["run_id"] is None:
|
|
legacy.setdefault(t["session_id"], []).append(t)
|
|
seqs = store.turn_seqs([t["turn_id"] for ts in list(by_run.values()) + list(legacy.values()) for t in ts])
|
|
out = []
|
|
for r in runs:
|
|
own = by_run.get(r.get("run_id")) or []
|
|
if not own:
|
|
started = int((r.get("started_at") or 0) * 1000)
|
|
finished = int(r["finished_at"] * 1000) if r.get("finished_at") else None
|
|
own = [t for t in legacy.get(r.get("session_id"), [])
|
|
if t["started_at"] >= started - 5000 and (finished is None or t["started_at"] <= finished)]
|
|
warn_checks = sorted({c for t in own for c in t["warn_checks"]})
|
|
last = own[-1] if own else None
|
|
summary = None if last is None else {
|
|
"status": next((t["status"] for t in own if t["status"] in ("error", "unknown", "max_iterations_exceeded")),
|
|
last["status"]),
|
|
"warn_checks": warn_checks,
|
|
"ended_at": (last["started_at"] + last["duration_ms"]) if last["duration_ms"] is not None else None}
|
|
health = scopes.run_health(r, summary, now_ms=now_ms)
|
|
did_not_finish = (r.get("status") == "running" and health == "failed")
|
|
out.append({**r, "turns": len(own), "total": sum(t["total"] for t in own) if own else None,
|
|
"at_least": any(t["at_least"] for t in own),
|
|
"active_ms": sum(t["active_ms"] or 0 for t in own) if own else None,
|
|
"duration_ms": sum(t["duration_ms"] or 0 for t in own) if own else (
|
|
int((r["finished_at"] - r["started_at"]) * 1000) if r.get("finished_at") else None),
|
|
"warn": sum(t["warn"] or 0 for t in own) if own else None,
|
|
"checks": warn_checks, "check_titles": [titles.get(c, c) for c in warn_checks],
|
|
"health": "did_not_finish" if did_not_finish else health,
|
|
"first_turn_id": own[0]["turn_id"] if own else None,
|
|
"first_turn_seq": seqs.get(own[0]["turn_id"]) if own else None})
|
|
ordered = sorted(out, key=lambda x: x.get("started_at") or 0)
|
|
series = [{"health": "failed" if x["health"] in ("failed", "did_not_finish") else x["health"],
|
|
"error": x.get("error") or (x["check_titles"][0] if x["check_titles"] else None)} for x in ordered]
|
|
banner = scopes.failed_in_a_row(series)
|
|
if banner is not None:
|
|
last_failed = next(x for x in reversed(ordered) if x["health"] in ("failed", "did_not_finish"))
|
|
banner = {**banner, "run_id": last_failed.get("run_id"), "session_id": last_failed.get("session_id"),
|
|
"turn_id": last_failed.get("first_turn_id")}
|
|
return {"runs": out, "failed_in_a_row": banner}
|
|
|
|
|
|
def json_or(text: Optional[str], default: Any) -> Any:
|
|
try:
|
|
return json.loads(text) if text else default
|
|
except ValueError:
|
|
return default
|