mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
* Shell commands that name a protected in-project path need a person, unless the read-only check accepts the command (also each part of a `;` / `&&` / `||` sequence, with `cd` and output-discarding redirects allowed). Paths are matched as written and from the shell's current folder, with wildcards expanded. Running a protected file is not a change to it. * `git config` commands that change a setting need a person, except plain ones that only label commits or pick a default (user.name, user.email, init.defaultBranch, core.autocrlf, pull.rebase, color.ui). Reads are unaffected. * Protected list gains git's own settings (.git/config, .git/info/attributes, .git/modules/, a .git file), conftest.py at any depth, more CI files (azure-pipelines.yml, .travis.yml, Jenkinsfile, bitbucket-pipelines.yml, .buildkite/), .vscode/launch.json, and .npmrc / .yarnrc / .yarnrc.yml. * File-tool writes are also checked against where the path really lands on disk, read the way the operating system reads it; a location that can't be worked out counts as protected. * Corpora: gate-133..gate-141 added; danger-022 (a global git config change) moved out of the reviewer corpus because the gate now sends it to a person. * scripts/measure_protected_paths.py replays recorded runs and counts what newly needs a person.
189 lines
7.4 KiB
Python
189 lines
7.4 KiB
Python
"""Replay recorded agent runs and count the calls that newly need a person because of the
|
|
"files that run later" floor (OPE-231), compared with an earlier revision.
|
|
|
|
Reads the runs only. Two formats are understood:
|
|
|
|
* OpenWorker transcripts: `messages.json` files (OpenAI-style tool calls), given as a glob;
|
|
* SWE-smith rows: a JSON file with `{"rows": [{"row": {"messages": ...}}]}`, whose tools
|
|
are `str_replace_editor` and `bash`.
|
|
|
|
python scripts/measure_protected_paths.py --before origin/main \
|
|
--openworker "runs/**/messages.json" --swe-smith rows.json
|
|
|
|
Shell commands are read as bash (both sources ran on Linux). A file-tool write counts when
|
|
the earlier revision did not protect its path and this one does; a shell command counts
|
|
when this revision asks a person for it (the earlier one had no shell check).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import collections
|
|
import glob
|
|
import importlib.util
|
|
import json
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(REPO))
|
|
|
|
from coworker import permissions as now # noqa: E402
|
|
|
|
WRITE_TOOLS = ("write_file", "replace_in_file", "apply_patch", "apply_unified_diff")
|
|
EDITOR_WRITES = ("create", "str_replace", "insert", "undo_edit")
|
|
SHELL_TOOLS = ("run_shell", "bash", "execute_bash")
|
|
|
|
|
|
def load_before(rev: str):
|
|
src = subprocess.run(
|
|
["git", "-C", str(REPO), "show", f"{rev}:coworker/permissions.py"],
|
|
capture_output=True, text=True, encoding="utf-8", check=True,
|
|
).stdout
|
|
path = Path(tempfile.mkdtemp()) / "permissions_before.py"
|
|
path.write_text(src, encoding="utf-8")
|
|
spec = importlib.util.spec_from_file_location("coworker.permissions_before", path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def marker_of(text: str, markers) -> str | None:
|
|
for m in markers:
|
|
if now._names_protected(text, (m,)):
|
|
return m
|
|
return None
|
|
|
|
|
|
def tool_calls(msgs):
|
|
for m in msgs:
|
|
if m.get("role") != "assistant":
|
|
continue
|
|
for c in m.get("tool_calls") or []:
|
|
fn = c.get("function") or {}
|
|
args = fn.get("arguments")
|
|
if isinstance(args, str):
|
|
try:
|
|
args = json.loads(args or "{}")
|
|
except ValueError:
|
|
args = {}
|
|
yield fn.get("name", ""), args if isinstance(args, dict) else {}
|
|
|
|
|
|
def write_targets(name: str, args: dict) -> list[str]:
|
|
if name in WRITE_TOOLS:
|
|
return now.write_paths(name, args)[0]
|
|
if name == "str_replace_editor" and args.get("command") in EDITOR_WRITES:
|
|
return [str(args.get("path", ""))]
|
|
return []
|
|
|
|
|
|
class Tally:
|
|
def __init__(self, label: str, before, engine):
|
|
self.label, self.before, self.engine = label, before, engine
|
|
self.runs = self.writes = self.shells = 0
|
|
self.hits = collections.Counter()
|
|
self.hit_runs = collections.defaultdict(set)
|
|
self.examples = collections.defaultdict(list)
|
|
self.reads = collections.Counter()
|
|
|
|
def hit(self, run: str, key: tuple[str, str], example: str) -> None:
|
|
self.hits[key] += 1
|
|
self.hit_runs[key[0]].add(run)
|
|
if len(self.examples[key]) < 3:
|
|
self.examples[key].append(example[:160])
|
|
|
|
def replay(self, run: str, msgs) -> None:
|
|
self.runs += 1
|
|
for name, args in tool_calls(msgs):
|
|
for p in filter(None, write_targets(name, args)):
|
|
self.writes += 1
|
|
cand = Path(p)
|
|
if now._is_protected_in_project(cand) and not self.before._is_protected_in_project(cand):
|
|
marker = marker_of(p, now._PROTECTED_IN_PROJECT) or "(where it lands on disk)"
|
|
self.hit(run, ("file tool", marker), f"{name}: {p}")
|
|
if name not in SHELL_TOOLS:
|
|
continue
|
|
cmd = str(args.get("command", "") or "")
|
|
if not cmd.strip():
|
|
continue
|
|
self.shells += 1
|
|
text = cmd.replace("\\", "/")
|
|
words = [w for w in now._WORD_BREAK.split(re.sub(r"[\"'`]", "", text)) if w]
|
|
if self.engine._shell_changes_protected(cmd):
|
|
argvs = []
|
|
for part in now._split_commands(text):
|
|
try:
|
|
argvs.append(shlex.split(part))
|
|
except ValueError:
|
|
argvs.append(part.split())
|
|
if any(now._changes_git_config(a) for a in argvs):
|
|
marker = "git config (a setting that is not a plain one)"
|
|
else:
|
|
found = [marker_of(w, now._shell_markers()) for w in words if not w.startswith("-")]
|
|
marker = next((m for m in found if m), "(where it lands on disk)")
|
|
self.hit(run, ("shell", marker), cmd)
|
|
elif self.engine._only_reads(cmd):
|
|
found = [marker_of(w, now._PROTECTED_IN_PROJECT) for w in words]
|
|
if any(found):
|
|
self.reads[next(m for m in found if m)] += 1
|
|
|
|
def report(self) -> None:
|
|
print(f"\n== {self.label}: {self.runs} runs, {self.writes} file-tool writes, "
|
|
f"{self.shells} shell commands")
|
|
if not self.hits:
|
|
print(" nothing newly needs a person")
|
|
for channel in ("file tool", "shell"):
|
|
keys = [k for k in self.hits if k[0] == channel]
|
|
if not keys:
|
|
continue
|
|
total = sum(self.hits[k] for k in keys)
|
|
print(f" {channel}: {total} calls in {len(self.hit_runs[channel])} runs")
|
|
for key in sorted(keys, key=lambda k: -self.hits[k]):
|
|
print(f" {self.hits[key]:5d} {key[1]}")
|
|
for ex in self.examples[key]:
|
|
print(f" e.g. {ex!r}")
|
|
if self.reads:
|
|
print(f" reads naming a protected path, still handled as reads: {sum(self.reads.values())}")
|
|
for marker, n in self.reads.most_common(6):
|
|
print(f" {n:5d} {marker}")
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("--before", default="origin/main", help="revision to compare with")
|
|
ap.add_argument("--openworker", help="glob of OpenWorker messages.json files")
|
|
ap.add_argument("--swe-smith", help="SWE-smith rows JSON file")
|
|
a = ap.parse_args()
|
|
before = load_before(a.before)
|
|
engine = now.PermissionEngine(
|
|
workspace_root=Path(tempfile.mkdtemp()), mode=now.Mode.AUTO_APPROVE, shell_kind="bash"
|
|
)
|
|
print(f"compared with {a.before}")
|
|
if a.swe_smith:
|
|
t = Tally("SWE-smith", before, engine)
|
|
data = json.load(open(a.swe_smith, encoding="utf-8"))
|
|
for i, row in enumerate(data["rows"]):
|
|
r = row.get("row", row)
|
|
msgs = json.loads(r["messages"]) if isinstance(r["messages"], str) else r["messages"]
|
|
t.replay(f"swe-{i}", msgs)
|
|
t.report()
|
|
if a.openworker:
|
|
t = Tally("OpenWorker", before, engine)
|
|
for p in sorted(glob.glob(a.openworker, recursive=True)):
|
|
try:
|
|
msgs = json.load(open(p, encoding="utf-8"))
|
|
except ValueError:
|
|
continue
|
|
t.replay(p, msgs)
|
|
t.report()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|