"""Regenerate coworker/_psl_private.py from the Public Suffix List (OPE-235). The reviewer's "destination named" fact needs to know which domains are shared hosting platforms (`github.io`, `herokuapp.com`, ...), where naming the platform names no site on it. Those platforms are the PRIVATE section of the Public Suffix List. No PSL library is in OpenWorker's dependency tree, so the section is vendored as a generated module. python scripts/refresh_public_suffixes.py # download the current list python scripts/refresh_public_suffixes.py list.dat # or use a copy you already have Then run `pytest tests/test_review_facts.py` and commit the regenerated module. The list changes weekly; refreshing every few months is plenty for this use. """ from __future__ import annotations import re import sys import urllib.request from pathlib import Path SOURCE = "https://publicsuffix.org/list/public_suffix_list.dat" OUT = Path(__file__).resolve().parent.parent / "coworker" / "_psl_private.py" def _ascii(rule: str) -> str: prefix = "!" if rule.startswith("!") else "" body = rule[len(prefix):] labels = [] for label in body.split("."): if label == "*" or label.isascii(): labels.append(label.lower()) else: labels.append(label.encode("idna").decode("ascii")) return prefix + ".".join(labels) def private_rules(text: str) -> tuple[list[str], str, str]: version = (re.search(r"^// VERSION: (\S+)", text, re.M) or [None, ""])[1] commit = (re.search(r"^// COMMIT: (\S+)", text, re.M) or [None, ""])[1] start = text.index("// ===BEGIN PRIVATE DOMAINS===") end = text.index("// ===END PRIVATE DOMAINS===") rules = [] for line in text[start:end].splitlines(): line = line.strip() if not line or line.startswith("//"): continue try: rules.append(_ascii(line.split()[0])) except UnicodeError: continue # a label Python's IDNA codec cannot encode: no ASCII host matches it return sorted(set(rules)), version, commit def render(rules: list[str], version: str, commit: str) -> str: body = "\n".join(rules) return f'''"""The PRIVATE section of the Public Suffix List: shared hosting platforms (OPE-235). GENERATED by scripts/refresh_public_suffixes.py; do not edit by hand. To refresh, run `python scripts/refresh_public_suffixes.py` and commit the result. Source: {SOURCE} Licence: Mozilla Public License 2.0 (https://mozilla.org/MPL/2.0/). Rules follow the PSL format: `*.x` is a wildcard, `!x` an exception. Non-ASCII rules are stored in their ASCII (punycode) form. """ SOURCE = "{SOURCE}" VERSION = "{version}" COMMIT = "{commit}" RULES = frozenset( """ {body} """.split() ) ''' def main(argv: list[str]) -> int: if len(argv) > 1: text = Path(argv[1]).read_text(encoding="utf-8") else: with urllib.request.urlopen(SOURCE, timeout=60) as resp: # noqa: S310 - fixed https URL text = resp.read().decode("utf-8") rules, version, commit = private_rules(text) OUT.write_text(render(rules, version, commit), encoding="utf-8", newline="\n") print(f"wrote {len(rules)} rules (PSL {version}) to {OUT}") return 0 if __name__ == "__main__": raise SystemExit(main(sys.argv))