"""Validate every Auto-Approve corpus under tests/corpora. Checks syntax, schema, IDs, labels, holdout splits, coverage tags, tool-name parity with the production connector catalog and, for the one-action reviewer files, argument-name parity with the real connector callables (scripts/corpus_schema.py). Exits non-zero on any defect. """ from __future__ import annotations import json import sys from collections import Counter from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parent.parent CORPUS_DIR = ROOT / "tests" / "corpora" sys.path.insert(0, str(ROOT)) from scripts.corpus_schema import ( # noqa: E402 GATE_LABELS, REVIEW_LABELS, STALE_ALIASES, ValidationError, production_tools, validate_reviewer_rows, ) # name → (layer, minimum rows). The reviewer files share one schema; the gate and sequence # files have their own, checked below. FILES = { "benign.jsonl": ("reviewer_action", 40), "dangerous.jsonl": ("reviewer_action", 30), "injection.jsonl": ("reviewer_action", 20), "reviewer_actions.jsonl": ("reviewer_action", 120), "permission_gate.jsonl": ("permission_gate", 120), "action_sequences.jsonl": ("action_sequence", 60), } MODES = {"discuss", "plan", "interactive", "custom", "auto-approve", "bypass-approvals"} # Coverage tags each file must span. The legacy reviewer files predate the tag vocabulary # and are not held to it; the layered file is. REQUIRED_TAGS = { "permission_gate.jsonl": {"exec", "outside-root", "credentials", "environment", "self-protection", "egress", "persistence", "privilege", "browser", "mcp", "connector", "persistent-authority"}, "reviewer_actions.jsonl": {"exec", "egress", "connector", "browser", "transformed-injection", "explicit-danger", "wrong-destination", "production-tool", "persistent-authority", "judge-injection", "judge-injection-control", "multi-turn", "reply-context", "worker-context", "long-context"}, "action_sequences.jsonl": {"injection", "cross-connector", "benign-control", "persistence", "browser", "exfiltration", "automation"}, } def load_jsonl(path: Path) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for line_no, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): if not raw.strip(): continue try: row = json.loads(raw) except json.JSONDecodeError as exc: raise ValidationError(f"{path.name}:{line_no}: invalid JSON: {exc}") from exc if not isinstance(row, dict): raise ValidationError(f"{path.name}:{line_no}: row must be an object") rows.append(row) return rows def require(row: dict[str, Any], fields: set[str], where: str) -> None: missing = sorted(k for k in fields if k not in row) if missing: raise ValidationError(f"{where}: missing fields: {', '.join(missing)}") def action_tools(row: dict[str, Any]) -> list[str]: if row.get("layer") == "action_sequence": actions = row.get("actions") if not isinstance(actions, list) or not actions: raise ValidationError(f"{row.get('id')}: actions must be a non-empty list") return [str(a.get("tool", "")) for a in actions if isinstance(a, dict)] action = row.get("action") if not isinstance(action, dict): raise ValidationError(f"{row.get('id')}: action must be an object") return [str(action.get("tool", ""))] def validate_rows(name: str, rows: list[dict[str, Any]], tools: set[str]) -> None: expected_layer, minimum = FILES[name] if len(rows) < minimum: raise ValidationError(f"{name}: expected at least {minimum} rows, found {len(rows)}") holdouts = sum(bool(r.get("holdout")) for r in rows) if holdouts == 0 or holdouts == len(rows): raise ValidationError(f"{name}: requires both holdout and non-holdout rows") labels: Counter[str] = Counter() tags: Counter[str] = Counter() for index, row in enumerate(rows, 1): where = f"{name}:{index}" require(row, {"id", "layer", "user_request", "setup", "tags", "holdout"}, where) if row["layer"] != expected_layer: raise ValidationError(f"{where}: expected layer {expected_layer!r}") if not isinstance(row["id"], str) or not row["id"]: raise ValidationError(f"{where}: id must be a non-empty string") if not isinstance(row["tags"], list) or not row["tags"]: raise ValidationError(f"{where}: tags must be a non-empty list") tags.update(str(t) for t in row["tags"]) if expected_layer == "permission_gate": require(row, {"mode", "action", "expected_current", "expected_secure", "why"}, where) if row["mode"] not in MODES: raise ValidationError(f"{where}: invalid mode {row['mode']!r}") for field in ("expected_current", "expected_secure"): if row[field] not in GATE_LABELS: raise ValidationError(f"{where}: invalid {field} {row[field]!r}") if row["expected_current"] != row["expected_secure"]: if row.get("known_gap") is not True or not row.get("failure_point"): raise ValidationError(f"{where}: differing expectations require known_gap and failure_point") labels.update([row["expected_current"], row["expected_secure"]]) elif expected_layer == "reviewer_action": # Shape, labels, tool and argument parity: all in validate_reviewer_rows below. labels.update([row.get("correct", "")]) else: require(row, {"observations", "actions", "correct", "why"}, where) if row["correct"] not in REVIEW_LABELS: raise ValidationError(f"{where}: invalid correct label {row['correct']!r}") if not isinstance(row["observations"], list): raise ValidationError(f"{where}: observations must be a list") labels.update([row["correct"]]) if expected_layer != "reviewer_action": for tool in action_tools(row): if not tool: raise ValidationError(f"{where}: action has no tool name") if tool in STALE_ALIASES: raise ValidationError(f"{where}: stale/non-production alias {tool!r}") if tool not in tools and "unknown-tool" not in row["tags"]: raise ValidationError(f"{where}: unknown production tool {tool!r}") if expected_layer == "reviewer_action": validate_reviewer_rows(name, rows) if expected_layer == "permission_gate": missing_labels = GATE_LABELS - set(labels) elif name == "reviewer_actions.jsonl" or expected_layer == "action_sequence": missing_labels = REVIEW_LABELS - set(labels) else: missing_labels = set() # a legacy file is one key by design (benign = all allow) if missing_labels: raise ValidationError(f"{name}: missing labels {sorted(missing_labels)}") missing_tags = REQUIRED_TAGS.get(name, set()) - set(tags) if missing_tags: raise ValidationError(f"{name}: missing required coverage tags {sorted(missing_tags)}") def validate_all() -> dict[str, Any]: tools = production_tools() seen: set[str] = set() summary: dict[str, Any] = {} for name in FILES: rows = load_jsonl(CORPUS_DIR / name) validate_rows(name, rows, tools) for row in rows: rid = row["id"] if rid in seen: raise ValidationError(f"duplicate id across corpora: {rid}") seen.add(rid) label_field = "expected_secure" if row["layer"] == "permission_gate" else "correct" summary[name] = { "rows": len(rows), "holdout": sum(bool(r.get("holdout")) for r in rows), "labels": dict(Counter(str(r[label_field]) for r in rows)), "tools": len(set(t for r in rows for t in action_tools(r))), "tags": len(set(str(tag) for r in rows for tag in r["tags"])), } summary["total"] = sum(v["rows"] for v in summary.values()) return summary def main() -> int: try: summary = validate_all() except (OSError, ValidationError) as exc: print(f"INVALID: {exc}", file=sys.stderr) return 1 print(json.dumps(summary, indent=2, sort_keys=True)) print("VALID") return 0 if __name__ == "__main__": raise SystemExit(main())