mirror of
https://github.com/andrewyng/openworker.git
synced 2026-09-03 04:49:26 +00:00
Devika Verma
d976e82a88
Freezes today's evaluate() verdict across 26 (mode, tool, args, grants) situations, so any later permission change shows up as a row-diff. Four rows are marked BASELINE-WRONG / BASELINE-ANNOYING on purpose: they record known gaps (shell auto with no sandbox, find -delete and find -exec auto-allowed via a find prefix, git status && git diff rejected for the operator, web_fetch never gating in any mode). The PRs that fix these flip their rows here as the visible proof. Design of record: ocw-context/docs/reviewed-auto-mode.md Parts 3 and 7.
91 lines
3.3 KiB
Python
91 lines
3.3 KiB
Python
"""Golden decision table — freezes the PermissionEngine's verdict for a fixed set of
|
|
(mode, tool, arguments, grants) situations.
|
|
|
|
The point of this test is regression detection: any change to `permissions.py` /
|
|
`risk.py` shows up as a diff in exactly the rows it was meant to change. Rows whose
|
|
`note` starts with BASELINE-WRONG or BASELINE-ANNOYING record *today's* behaviour on
|
|
purpose — they document known gaps (see `ocw-context/docs/reviewed-auto-mode.md` Part 3),
|
|
and a PR that fixes one flips its row here as the visible proof.
|
|
|
|
The matrix lives in `tests/corpora/decision_matrix.csv`. Each row builds a fresh
|
|
`PermissionEngine`; relative paths resolve against a per-row temp workspace.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from coworker.permissions import Mode, PermissionEngine
|
|
|
|
_MATRIX = Path(__file__).parent / "corpora" / "decision_matrix.csv"
|
|
|
|
|
|
def _meta(kind: str):
|
|
"""The `meta` column → a stand-in aisuite ToolMetadata (only the fields classify()
|
|
and evaluate() read). Empty → None (built-ins classify by name)."""
|
|
kind = (kind or "").strip()
|
|
if not kind:
|
|
return None
|
|
if kind == "external":
|
|
return SimpleNamespace(requires_approval=True, category="connector")
|
|
if kind == "read":
|
|
return SimpleNamespace(requires_approval=False, category="connector")
|
|
raise ValueError(f"unknown meta kind: {kind!r}")
|
|
|
|
|
|
def _pipes(value: str) -> list[str]:
|
|
return [v for v in (value or "").split("|") if v]
|
|
|
|
|
|
def _verdict(decision) -> str:
|
|
if decision.allowed and not decision.needs_user:
|
|
return "allow"
|
|
if decision.needs_user:
|
|
return "ask"
|
|
return "deny"
|
|
|
|
|
|
def _rows() -> list[dict]:
|
|
with open(_MATRIX, newline="", encoding="utf-8") as fh:
|
|
return list(csv.DictReader(fh))
|
|
|
|
|
|
def _build_engine(row: dict, workspace: Path) -> PermissionEngine:
|
|
kwargs: dict = {
|
|
"workspace_root": workspace,
|
|
"mode": Mode(row["mode"]),
|
|
"allowed_commands": _pipes(row.get("allowed_commands", "")),
|
|
"auto_allow_tools": set(_pipes(row.get("auto_allow", ""))),
|
|
}
|
|
eng = PermissionEngine(**kwargs)
|
|
for tool in _pipes(row.get("session_tools", "")):
|
|
eng.allow_tool_for_session(tool)
|
|
for cmd in _pipes(row.get("session_commands", "")):
|
|
eng.allow_command_for_session(cmd)
|
|
standing = (row.get("standing") or "").strip()
|
|
if standing:
|
|
tool, _, target = standing.partition(" ")
|
|
eng.task_rules.setdefault(tool, set()).add(target.strip())
|
|
# allowed_domains: applied only if the engine supports it (arrives with PR1).
|
|
domains = _pipes(row.get("allowed_domains", ""))
|
|
if domains and hasattr(eng, "allowed_domains"):
|
|
eng.allowed_domains = list(domains)
|
|
return eng
|
|
|
|
|
|
@pytest.mark.parametrize("row", _rows(), ids=lambda r: r["id"])
|
|
def test_decision_matrix(row, tmp_path):
|
|
eng = _build_engine(row, tmp_path)
|
|
args = json.loads(row["args"]) if row.get("args") else {}
|
|
decision = eng.evaluate(row["tool"], args, _meta(row.get("meta", "")))
|
|
got = _verdict(decision)
|
|
assert got == row["expected"], (
|
|
f"{row['id']}: expected {row['expected']}, got {got} "
|
|
f"(reason: {decision.reason!r}) — note: {row.get('note', '')}"
|
|
)
|