"""The one schema for a reviewer corpus row, and the checks every reviewer file passes. Four files hold one-action reviewer scenarios: `benign`, `dangerous`, `injection` (the original hand-written corpora, now kept under the generator as `legacy_reviewer_rows.py`) and `reviewer_actions` (the layered corpus). They used to differ in shape — a provenance STRING (the engine's rendered line) on one side, a provenance DICT (where the action came from) on the other, `planted` only on injection rows, `recommended_gate` only on layered rows — and the harness papered over it with special cases. That is how ten layered rows carried connector arguments that no longer existed for months: nothing checked them. This module is the single place that knows the shape. `normalize_reviewer_row` accepts every historical shape and returns the unified one; `validate_reviewer_row` enforces it and checks every tool name and every argument name against the live catalog. Unified row:: { "id": "benign-001-pytest", # stable; reports are joined on it "layer": "reviewer_action", "user_request": "run the tests", "setup": {"roots": [...], "remotes": [...], ...}, "history": ["earlier user message", ...], # chronological, current request excluded "reply": "", # an ask_user answer, lands in history tagged is_reply "action": {"tool": "run_shell", "arguments": {...}}, "action_context": null | {...}, # the source-labelled approval context the live # engine passes for delegated WORKER actions # (coworker_definition, user_approved_worker_guidance, # assignments_agent_authored_not_access_grants, # user_saved_rules, worker_session_unsourced_input, ...) "provenance": { "engine_line": "", # the fixed-vocabulary line the ENGINE renders # (agent-made file N steps ago); shown to the judge "action": "agent-derived", # where the action came from — taint metadata, "arguments": "user-request", # NEVER shown to the judge (production has no "user_authorized": "" # such line); for slicing reports by source }, "planted": null | {"source": "...", "text": "..."}, # injection rows: the attacker text "correct": "allow" | "ask" | "deny", "why": "...", "recommended_gate": "reviewer_eligible" | "human_only" | "hard_deny", "tags": [...], "holdout": false } """ from __future__ import annotations import functools import inspect import re import sys import tempfile from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parent.parent REVIEWER_LAYER = "reviewer_action" REVIEWER_FILES = ("benign", "dangerous", "injection", "reviewer_actions") REVIEW_LABELS = {"allow", "ask", "deny"} GATE_LABELS = {"allow_without_reviewer", "reviewer_eligible", "human_only", "hard_deny"} PROVENANCE_KEYS = ("engine_line", "action", "arguments", "user_authorized") # The canonical field order; `dump` writes rows in this order so diffs stay readable. FIELD_ORDER = ( "id", "layer", "user_request", "setup", "history", "reply", "action", "action_context", "provenance", "planted", "correct", "why", "recommended_gate", "tags", "holdout", ) # Present only when it has something to say, after every field above. `facts`: the engine's # own fact lines for this action (OPE-235 / OPE-237, coworker/review_facts.py), passed to the # reviewer exactly as the engine passes them. OPTIONAL_FIELDS = ("facts",) # Names production never had; a row naming one measures the reviewer's reaction to a tool # it can never see. STALE_ALIASES = {"send_email", "calendar_list_events", "gmail_delete", "gmail_forward"} CORE_TOOLS = { "read_file", "read_file_lines", "grep", "list_files", "write_file", "replace_in_file", "apply_patch", "apply_unified_diff", "run_shell", "shell_task_output", "shell_task_kill", "web_fetch", "web_search", "send_message", "send_file", "save_skill", "load_skill", "request_directory", "ask_user", "propose_plan", "create_scheduled_task", "list_scheduled_tasks", "update_scheduled_task", "delete_scheduled_task", "todo_write", } # `mcp__server__tool` names are runtime-defined: any server can ship any name, and some # rows deliberately use unfamiliar servers to test that the judge reads arguments, not # names. The family is legitimate by construction. MCP_SHAPE = re.compile(r"^mcp__[a-z0-9-]+__\w+$") class ValidationError(Exception): pass # -- the live catalog --------------------------------------------------------------- @functools.lru_cache(maxsize=1) def production_tools() -> set[str]: sys.path.insert(0, str(ROOT)) from coworker.connectors.tool_defs import TOOL_DEFS return CORE_TOOLS | {d.name for d in TOOL_DEFS} @functools.lru_cache(maxsize=1) def tool_signatures() -> dict[str, frozenset[str]]: """Parameter names of every connector tool, from the real callables. A row whose arguments name a parameter the tool does not have measures nothing — the judge is reading an argument production would never send.""" sys.path.insert(0, str(ROOT)) from coworker.connectors import email_tools, integration_tools from coworker.secrets import SecretStore with tempfile.TemporaryDirectory() as tmp: store = SecretStore(Path(tmp) / "s.json") fns = {t.__name__: t for t in integration_tools.make_integration_tools(store)} fns.update({t.__name__: t for t in email_tools.make_email_tools(store)}) return {name: frozenset(inspect.signature(fn).parameters) for name, fn in fns.items()} # -- normalisation ------------------------------------------------------------------ def normalize_provenance(raw: Any) -> dict[str, str]: """Every historical shape → the unified dict. * str → the engine's rendered line (legacy `dangerous` rows). * dict → taint metadata (layered rows), or an already-unified dict. * None / "" → nothing known. """ out = {k: "" for k in PROVENANCE_KEYS} if isinstance(raw, str): out["engine_line"] = raw elif isinstance(raw, dict): for k in PROVENANCE_KEYS: v = raw.get(k, "") out[k] = "" if v is None else str(v) elif raw is not None: raise ValidationError(f"provenance must be a string, object, or null; got {type(raw).__name__}") return out def normalize_reviewer_row(d: dict[str, Any]) -> dict[str, Any]: """Accept any shape a reviewer row has ever had; return the unified shape, keys in `FIELD_ORDER`. Pure — never mutates the input.""" planted = d.get("planted") context = d.get("action_context") row = { "id": d.get("id", ""), "layer": REVIEWER_LAYER, "user_request": d.get("user_request", ""), "setup": dict(d.get("setup") or {}), "history": [str(h) for h in (d.get("history") or [])], "reply": str(d.get("reply") or ""), "action": dict(d.get("action") or {}), "action_context": dict(context) if isinstance(context, dict) else None, "provenance": normalize_provenance(d.get("provenance")), "planted": dict(planted) if isinstance(planted, dict) else None, "correct": d.get("correct", ""), "why": d.get("why", ""), "recommended_gate": d.get("recommended_gate") or "reviewer_eligible", "tags": sorted(set(str(t) for t in (d.get("tags") or []))), "holdout": bool(d.get("holdout", False)), } out = {k: row[k] for k in FIELD_ORDER} facts = d.get("facts") if facts: out["facts"] = [str(f) for f in facts] return out def provenance_line(row: dict[str, Any]) -> str: """What the judge is shown: the engine's rendered line, or nothing.""" return str((row.get("provenance") or {}).get("engine_line") or "") def provenance_taint(row: dict[str, Any]) -> dict[str, str] | None: """What the judge is NOT shown: where the action and arguments came from. None when the row says nothing about it.""" prov = row.get("provenance") or {} taint = {k: str(prov.get(k) or "") for k in ("action", "arguments", "user_authorized")} return taint if any(taint.values()) else None # -- validation --------------------------------------------------------------------- def validate_reviewer_row( row: dict[str, Any], where: str, *, tools: set[str] | None = None, signatures: dict[str, frozenset[str]] | None = None, ) -> None: """Raise `ValidationError` on the first defect. Checks shape, labels, tool-name parity with the live catalog, and argument-name parity with the real connector callables.""" tools = production_tools() if tools is None else tools signatures = tool_signatures() if signatures is None else signatures missing = [k for k in FIELD_ORDER if k not in row] if missing: raise ValidationError(f"{where}: missing fields: {', '.join(missing)}") extra = sorted(set(row) - set(FIELD_ORDER) - set(OPTIONAL_FIELDS)) if extra: raise ValidationError(f"{where}: unknown fields: {', '.join(extra)}") if row["layer"] != REVIEWER_LAYER: raise ValidationError(f"{where}: layer must be {REVIEWER_LAYER!r}") if not isinstance(row["id"], str) or not row["id"]: raise ValidationError(f"{where}: id must be a non-empty string") if not isinstance(row["user_request"], str) or not row["user_request"].strip(): raise ValidationError(f"{where}: user_request must be a non-empty string") if not isinstance(row["setup"], dict): raise ValidationError(f"{where}: setup must be an object") if not isinstance(row["history"], list) or not all(isinstance(h, str) for h in row["history"]): raise ValidationError(f"{where}: history must be a list of strings") if not isinstance(row["reply"], str): raise ValidationError(f"{where}: reply must be a string") if row["correct"] not in REVIEW_LABELS: raise ValidationError(f"{where}: invalid correct label {row['correct']!r}") if row["recommended_gate"] not in GATE_LABELS: raise ValidationError(f"{where}: invalid recommended_gate {row['recommended_gate']!r}") if not isinstance(row["why"], str) or not row["why"].strip(): raise ValidationError(f"{where}: why must be a non-empty string") if not isinstance(row["tags"], list) or not row["tags"]: raise ValidationError(f"{where}: tags must be a non-empty list") if not isinstance(row["holdout"], bool): raise ValidationError(f"{where}: holdout must be a boolean") if "facts" in row: from coworker.review_facts import FACT_VOCABULARY if not isinstance(row["facts"], list) or not row["facts"] or any(f not in FACT_VOCABULARY for f in row["facts"]): raise ValidationError(f"{where}: facts must be a non-empty list of lines the engine can produce") prov = row["provenance"] if not isinstance(prov, dict) or set(prov) != set(PROVENANCE_KEYS): raise ValidationError(f"{where}: provenance must have exactly the keys {PROVENANCE_KEYS}") if not all(isinstance(v, str) for v in prov.values()): raise ValidationError(f"{where}: provenance values must be strings") if "provenance" in row["tags"] and not (prov["engine_line"] or prov["action"] or prov["arguments"]): raise ValidationError(f"{where}: tagged provenance but carries none") if row["action_context"] is not None and not isinstance(row["action_context"], dict): raise ValidationError(f"{where}: action_context must be null or an object") planted = row["planted"] if planted is not None: if not isinstance(planted, dict) or not planted.get("source") or not planted.get("text"): raise ValidationError(f"{where}: planted must be null or {{source, text}}") if "injection" in row["tags"] and row["id"].startswith("inject-") and planted is None: raise ValidationError(f"{where}: injection row needs a planted source") action = row["action"] if not isinstance(action, dict) or not action.get("tool"): raise ValidationError(f"{where}: action must be {{tool, arguments}}") tool = str(action["tool"]) args = action.get("arguments", {}) if not isinstance(args, dict): raise ValidationError(f"{where}: action.arguments must be an object") if tool in STALE_ALIASES: raise ValidationError(f"{where}: stale/non-production alias {tool!r}") if not MCP_SHAPE.match(tool) and tool not in tools and "unknown-tool" not in row["tags"]: raise ValidationError(f"{where}: unknown production tool {tool!r}") params = signatures.get(tool) if params is not None: unknown = sorted(set(args) - params) if unknown: raise ValidationError( f"{where}: {tool} has no parameter(s) {unknown}; it takes {sorted(params)}" ) def validate_reviewer_rows(name: str, rows: list[dict[str, Any]]) -> None: tools = production_tools() signatures = tool_signatures() for index, row in enumerate(rows, 1): validate_reviewer_row(row, f"{name}:{index}", tools=tools, signatures=signatures)