mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
Destination fact: - A bare name ending in a file-like top-level domain (main.py, README.md, run.sh, model.ai) is a file name, not a site. It counts as a site only when written as one: in a URL or email address, after `www.`, or followed by a path or port. - Naming a shared hosting platform (herokuapp.com, github.io, vercel.app, s3.amazonaws.com, pages.dev, netlify.app, ...) names only itself, never the sites hosted under it. Naming the exact site still names it. The Public Suffix List's private section is vendored in coworker/_psl_private.py, generated by scripts/refresh_public_suffixes.py (source, version and licence recorded there). - Connector targets match only in their own form: `#channel` or the full channel id, `owner/repo` or `owner/repo#12`, the full `platform:id` handle, an exact email address or phone number; a bare id only when long enough to be an id. Outside-content fact: - The tool result that brought outside content in is marked on its message. The line is shown while such a result is still in what the agent is sent: a summary written from it keeps the line, the no-summary trim that drops it clears the line. - Spilled tool output and the compacted transcript count as outside files when they hold outside content. - Files that arrive through the shell count: git clone / gh repo clone / submodule add, wget, curl -o/-O in every flag spelling, pip download, Start-BitsTransfer, and archives from outside unpacked with tar, unzip, 7z, Expand-Archive or gunzip. Reading such a file, searching a folder that holds one, or running a command inside one counts. A folder that already existed is never marked as a whole, only the names that appear in it during the call, so the user's own files stay the user's. Fact sentences and the reviewer instructions are unchanged.
94 lines
3.2 KiB
Python
94 lines
3.2 KiB
Python
"""Regenerate coworker/_psl_private.py from the Public Suffix List (OPE-235).
|
|
|
|
The reviewer's "destination named" fact needs to know which domains are shared hosting
|
|
platforms (`github.io`, `herokuapp.com`, ...), where naming the platform names no site on
|
|
it. Those platforms are the PRIVATE section of the Public Suffix List. No PSL library is in
|
|
OpenWorker's dependency tree, so the section is vendored as a generated module.
|
|
|
|
python scripts/refresh_public_suffixes.py # download the current list
|
|
python scripts/refresh_public_suffixes.py list.dat # or use a copy you already have
|
|
|
|
Then run `pytest tests/test_review_facts.py` and commit the regenerated module. The list
|
|
changes weekly; refreshing every few months is plenty for this use.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
SOURCE = "https://publicsuffix.org/list/public_suffix_list.dat"
|
|
OUT = Path(__file__).resolve().parent.parent / "coworker" / "_psl_private.py"
|
|
|
|
|
|
def _ascii(rule: str) -> str:
|
|
prefix = "!" if rule.startswith("!") else ""
|
|
body = rule[len(prefix):]
|
|
labels = []
|
|
for label in body.split("."):
|
|
if label == "*" or label.isascii():
|
|
labels.append(label.lower())
|
|
else:
|
|
labels.append(label.encode("idna").decode("ascii"))
|
|
return prefix + ".".join(labels)
|
|
|
|
|
|
def private_rules(text: str) -> tuple[list[str], str, str]:
|
|
version = (re.search(r"^// VERSION: (\S+)", text, re.M) or [None, ""])[1]
|
|
commit = (re.search(r"^// COMMIT: (\S+)", text, re.M) or [None, ""])[1]
|
|
start = text.index("// ===BEGIN PRIVATE DOMAINS===")
|
|
end = text.index("// ===END PRIVATE DOMAINS===")
|
|
rules = []
|
|
for line in text[start:end].splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("//"):
|
|
continue
|
|
try:
|
|
rules.append(_ascii(line.split()[0]))
|
|
except UnicodeError:
|
|
continue # a label Python's IDNA codec cannot encode: no ASCII host matches it
|
|
return sorted(set(rules)), version, commit
|
|
|
|
|
|
def render(rules: list[str], version: str, commit: str) -> str:
|
|
body = "\n".join(rules)
|
|
return f'''"""The PRIVATE section of the Public Suffix List: shared hosting platforms (OPE-235).
|
|
|
|
GENERATED by scripts/refresh_public_suffixes.py; do not edit by hand. To refresh, run
|
|
`python scripts/refresh_public_suffixes.py` and commit the result.
|
|
|
|
Source: {SOURCE}
|
|
Licence: Mozilla Public License 2.0 (https://mozilla.org/MPL/2.0/).
|
|
Rules follow the PSL format: `*.x` is a wildcard, `!x` an exception. Non-ASCII rules
|
|
are stored in their ASCII (punycode) form.
|
|
"""
|
|
|
|
SOURCE = "{SOURCE}"
|
|
VERSION = "{version}"
|
|
COMMIT = "{commit}"
|
|
|
|
RULES = frozenset(
|
|
"""
|
|
{body}
|
|
""".split()
|
|
)
|
|
'''
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
if len(argv) > 1:
|
|
text = Path(argv[1]).read_text(encoding="utf-8")
|
|
else:
|
|
with urllib.request.urlopen(SOURCE, timeout=60) as resp: # noqa: S310 - fixed https URL
|
|
text = resp.read().decode("utf-8")
|
|
rules, version, commit = private_rules(text)
|
|
OUT.write_text(render(rules, version, commit), encoding="utf-8", newline="\n")
|
|
print(f"wrote {len(rules)} rules (PSL {version}) to {OUT}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main(sys.argv))
|