"""Replay recorded agent runs and count the calls that newly need a person because of the "files that run later" floor (OPE-231), compared with an earlier revision. Reads the runs only. Two formats are understood: * OpenWorker transcripts: `messages.json` files (OpenAI-style tool calls), given as a glob; * SWE-smith rows: a JSON file with `{"rows": [{"row": {"messages": ...}}]}`, whose tools are `str_replace_editor` and `bash`. python scripts/measure_protected_paths.py --before origin/main \ --openworker "runs/**/messages.json" --swe-smith rows.json Shell commands are read as bash (both sources ran on Linux). A file-tool write counts when the earlier revision did not protect its path and this one does; a shell command counts when this revision asks a person for it (the earlier one had no shell check). """ from __future__ import annotations import argparse import collections import glob import importlib.util import json import re import shlex import subprocess import sys import tempfile from pathlib import Path REPO = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO)) from coworker import permissions as now # noqa: E402 WRITE_TOOLS = ("write_file", "replace_in_file", "apply_patch", "apply_unified_diff") EDITOR_WRITES = ("create", "str_replace", "insert", "undo_edit") SHELL_TOOLS = ("run_shell", "bash", "execute_bash") def load_before(rev: str): src = subprocess.run( ["git", "-C", str(REPO), "show", f"{rev}:coworker/permissions.py"], capture_output=True, text=True, encoding="utf-8", check=True, ).stdout path = Path(tempfile.mkdtemp()) / "permissions_before.py" path.write_text(src, encoding="utf-8") spec = importlib.util.spec_from_file_location("coworker.permissions_before", path) module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def marker_of(text: str, markers) -> str | None: for m in markers: if now._names_protected(text, (m,)): return m return None def tool_calls(msgs): for m in msgs: if m.get("role") != "assistant": continue for c in m.get("tool_calls") or []: fn = c.get("function") or {} args = fn.get("arguments") if isinstance(args, str): try: args = json.loads(args or "{}") except ValueError: args = {} yield fn.get("name", ""), args if isinstance(args, dict) else {} def write_targets(name: str, args: dict) -> list[str]: if name in WRITE_TOOLS: return now.write_paths(name, args)[0] if name == "str_replace_editor" and args.get("command") in EDITOR_WRITES: return [str(args.get("path", ""))] return [] class Tally: def __init__(self, label: str, before, engine): self.label, self.before, self.engine = label, before, engine self.runs = self.writes = self.shells = 0 self.hits = collections.Counter() self.hit_runs = collections.defaultdict(set) self.examples = collections.defaultdict(list) self.reads = collections.Counter() def hit(self, run: str, key: tuple[str, str], example: str) -> None: self.hits[key] += 1 self.hit_runs[key[0]].add(run) if len(self.examples[key]) < 3: self.examples[key].append(example[:160]) def replay(self, run: str, msgs) -> None: self.runs += 1 for name, args in tool_calls(msgs): for p in filter(None, write_targets(name, args)): self.writes += 1 cand = Path(p) if now._is_protected_in_project(cand) and not self.before._is_protected_in_project(cand): marker = marker_of(p, now._PROTECTED_IN_PROJECT) or "(where it lands on disk)" self.hit(run, ("file tool", marker), f"{name}: {p}") if name not in SHELL_TOOLS: continue cmd = str(args.get("command", "") or "") if not cmd.strip(): continue self.shells += 1 text = cmd.replace("\\", "/") words = [w for w in now._WORD_BREAK.split(re.sub(r"[\"'`]", "", text)) if w] if self.engine._shell_changes_protected(cmd): argvs = [] for part in now._split_commands(text): try: argvs.append(shlex.split(part)) except ValueError: argvs.append(part.split()) if any(now._changes_git_config(a) for a in argvs): marker = "git config (a setting that is not a plain one)" else: found = [marker_of(w, now._shell_markers()) for w in words if not w.startswith("-")] marker = next((m for m in found if m), "(where it lands on disk)") self.hit(run, ("shell", marker), cmd) elif self.engine._only_reads(cmd): found = [marker_of(w, now._PROTECTED_IN_PROJECT) for w in words] if any(found): self.reads[next(m for m in found if m)] += 1 def report(self) -> None: print(f"\n== {self.label}: {self.runs} runs, {self.writes} file-tool writes, " f"{self.shells} shell commands") if not self.hits: print(" nothing newly needs a person") for channel in ("file tool", "shell"): keys = [k for k in self.hits if k[0] == channel] if not keys: continue total = sum(self.hits[k] for k in keys) print(f" {channel}: {total} calls in {len(self.hit_runs[channel])} runs") for key in sorted(keys, key=lambda k: -self.hits[k]): print(f" {self.hits[key]:5d} {key[1]}") for ex in self.examples[key]: print(f" e.g. {ex!r}") if self.reads: print(f" reads naming a protected path, still handled as reads: {sum(self.reads.values())}") for marker, n in self.reads.most_common(6): print(f" {n:5d} {marker}") def main() -> None: ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--before", default="origin/main", help="revision to compare with") ap.add_argument("--openworker", help="glob of OpenWorker messages.json files") ap.add_argument("--swe-smith", help="SWE-smith rows JSON file") a = ap.parse_args() before = load_before(a.before) engine = now.PermissionEngine( workspace_root=Path(tempfile.mkdtemp()), mode=now.Mode.AUTO_APPROVE, shell_kind="bash" ) print(f"compared with {a.before}") if a.swe_smith: t = Tally("SWE-smith", before, engine) data = json.load(open(a.swe_smith, encoding="utf-8")) for i, row in enumerate(data["rows"]): r = row.get("row", row) msgs = json.loads(r["messages"]) if isinstance(r["messages"], str) else r["messages"] t.replay(f"swe-{i}", msgs) t.report() if a.openworker: t = Tally("OpenWorker", before, engine) for p in sorted(glob.glob(a.openworker, recursive=True)): try: msgs = json.load(open(p, encoding="utf-8")) except ValueError: continue t.replay(p, msgs) t.report() if __name__ == "__main__": main()