Files
openworker/tests/test_model_catalog.py
T
Devika Verma 9d472d1840 feat(models): the model matrix is a data file (OPE-252)
The 95 hand-written rows of `MATRIX` move from matrix.py to
coworker/providers/model_catalog.json: per provider and model, the label,
tools / vision / pdf / parallel_tool_calls, the context window, and an optional
output limit. Each provider block carries a `note` naming the vendor page or
listing its rows were read from and when (the comments matrix.py used to hold).
A row a maintainer sized on purpose (a window pinned to a particular host, a
capability unverified over this backend) says so with `pinned` and a reason in
its note; 13 rows are pinned.

matrix.py loads the file into the same `MATRIX` dict at import; `entry_for`,
`model_labels`, `model_context_windows` and `models_for_provider` are unchanged,
so no consumer changes. The signed-in OpenRouter provider stays a load-time
mirror and is never listed. `OVERRIDES` (empty) beats the file, before and after
the mirror. `reload()` refills the dict in place under a lock, the helpers read
a snapshot, and a file that fails validation leaves MATRIX untouched.

The loader refuses anything it cannot read as a whole: wrong schema, unknown or
missing fields, a non-boolean capability, a context that is not a positive
whole number, an OpenAI id with a prefix, a mirrored provider listed by hand,
a pin on a field that cannot be pinned. The file is in canonical form
(json.dumps indent 2, one trailing newline) so a catalog pull request diffs
only the rows that changed.

Keeping the file current is a maintainer job, not a runtime one: the app never
fetches a catalog from anyone. `serial`, `generated` and `last_checked` are
informational to the app and move when the maintainers change or check a fact.

Round trip against the hand-written dict: the same 104 ids (95 rows plus the
9 openrouter-account mirrors) in the same order, with the same labels,
capabilities and windows.

Tests: tests/test_model_catalog.py (file contract, package data, the four
helpers against the file, the mirror row for row, capabilities_for, reload in
place, a refused file leaves MATRIX, OVERRIDES on both twins, four reader
threads over 120 reloads, 34 refused shapes, the optional fields accepted);
test_matrix_current_models reuses matrix.full_id. The 251 existing
matrix-reading tests pass unchanged.
2026-11-17 18:38:30 +05:30

456 lines
18 KiB
Python

"""OPE-252: the model matrix is a data file, `coworker/providers/model_catalog.json`.
`matrix.py` loads it into MATRIX at import; the four helpers every consumer reads
(`entry_for`, `model_labels`, `model_context_windows`, `models_for_provider`) are unchanged.
These tests pin the file's contract: what it must contain, what the loader refuses, that the
loader fills the same dict object in place (so `from ..matrix import MATRIX` keeps working
after a reload), that OVERRIDES beat the file, that the signed-in OpenRouter provider is a
mirror and never a block of its own, and that the file is in canonical form so a catalog
pull request diff shows only real changes.
"""
from __future__ import annotations
import copy
import importlib.resources
import json
import re
from pathlib import Path
import pytest
from coworker.providers.base import ModelCapabilities
from coworker.providers.capabilities import capabilities_for
from coworker.providers.matrix import (
CATALOG_PATH,
MATRIX,
CatalogError,
ModelEntry,
catalog_meta,
entry_for,
full_id,
load_catalog,
model_context_windows,
model_labels,
models_for_provider,
parse_catalog,
reload,
split_id,
)
from coworker.providers import matrix
from coworker.providers.registry import DESCRIPTORS
REPO = Path(__file__).resolve().parents[1]
def _doc() -> dict:
return json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
def _row(**over) -> dict:
row = {
"label": "Model · Vendor",
"tools": True,
"vision": False,
"pdf": False,
"parallel_tool_calls": True,
"context": 128_000,
}
row.update(over)
return row
def _minimal() -> dict:
"""The smallest valid catalog: an OpenAI row (stored bare) and an OpenRouter row (which
the loader mirrors to openrouter-account)."""
return {
"schema": 1,
"serial": "2026.11.07.1",
"generated": "2026-11-07",
"providers": {
"openai": {"models": {"gpt-x": _row(label="GPT X · OpenAI", vision=True, pdf=True, context=400_000)}},
"openrouter": {"note": "test rows", "models": {"lab/m": _row(label="M · via OpenRouter", context=None)}},
},
}
@pytest.fixture
def restore_matrix():
"""Whatever a test loads, the shipped catalog is back afterwards."""
yield
matrix.OVERRIDES.clear()
reload()
# -- the shipped file -----------------------------------------------------------------------
def test_the_shipped_catalog_is_what_matrix_serves():
meta, rows = load_catalog(CATALOG_PATH)
assert meta["schema"] == 1
assert re.fullmatch(r"\d{4}\.\d{2}\.\d{2}\.\d+", meta["serial"]), meta["serial"]
assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", meta["generated"]), meta["generated"]
mirrored = {f"openrouter-account:{m[len('openrouter:'):]}" for m in rows if m.startswith("openrouter:")}
assert set(MATRIX) == set(rows) | mirrored
assert catalog_meta()["serial"] == meta["serial"]
assert catalog_meta()["path"] == str(CATALOG_PATH)
def test_every_provider_in_the_file_is_a_registered_cloud_provider():
known = {d.name for d in DESCRIPTORS if d.kind != "local"}
unknown = sorted(set(_doc()["providers"]) - known)
assert not unknown, f"catalog providers the registry does not know: {unknown}"
assert "openrouter-account" not in _doc()["providers"] # mirrored, never listed
def test_the_file_is_in_canonical_form():
"""json.dumps(indent=2, ensure_ascii=False) plus one trailing newline, so a catalog pull
request diffs only the rows that changed, never whitespace."""
doc = _doc()
expected = json.dumps(doc, indent=2, ensure_ascii=False) + "\n"
assert CATALOG_PATH.read_text(encoding="utf-8") == expected
def test_every_row_is_tool_capable_and_carries_a_label():
for mid, entry in MATRIX.items():
assert entry.caps.tools, mid
assert entry.caps.streaming, mid
assert entry.label.strip(), mid
def test_pinned_rows_say_why():
"""A pin tells the maintainers' check to report, never propose; the reason must be in
the file."""
for provider, block in _doc()["providers"].items():
for model, row in block["models"].items():
if row.get("pinned"):
assert row.get("note", "").strip(), f"{provider}/{model} is pinned without a note"
def test_every_provider_block_names_its_source():
for provider, block in _doc()["providers"].items():
assert block.get("note", "").strip(), f"{provider} has no provenance note"
def test_the_catalog_ships_inside_the_package():
assert importlib.resources.files("coworker.providers").joinpath("model_catalog.json").is_file()
pyproject = (REPO / "pyproject.toml").read_text(encoding="utf-8")
assert '"providers/*.json"' in pyproject
# -- migration gate (self-retiring) --------------------------------------------------------
SNAPSHOT = Path(__file__).with_name("fixtures") / "model_catalog_snapshot_2026-11-07.json"
MIGRATION_SERIAL = "2026.11.07.1"
def test_the_migration_to_the_data_file_changed_no_fact():
"""The snapshot is MATRIX exactly as the hand-written matrix.py served it on 2026-11-07:
104 rows (95 in the file plus the openrouter-account mirror) with every id, label,
capability and window. While the file still carries the migration serial, the loaded
matrix must equal it. The first weekly sync bumps the serial and retires this gate: from
then on the file is meant to change, and the snapshot is history, not a contract."""
if catalog_meta()["serial"] != MIGRATION_SERIAL:
pytest.skip("the catalog has moved past the migration serial; this gate has done its job")
snapshot = json.loads(SNAPSHOT.read_text(encoding="utf-8"))
assert len(snapshot) == 104
assert set(MATRIX) == set(snapshot)
for mid, was in snapshot.items():
entry = MATRIX[mid]
now = {
"label": entry.label,
"tools": entry.caps.tools,
"vision": entry.caps.vision,
"pdf": entry.caps.pdf,
"parallel_tool_calls": entry.caps.parallel_tool_calls,
"streaming": entry.caps.streaming,
"context": entry.context_window,
}
assert now == was, mid
# -- helpers -------------------------------------------------------------------------------
def test_helpers_agree_with_the_file():
doc = _doc()
labels = model_labels()
windows = model_context_windows()
for provider, block in doc["providers"].items():
bare = list(block["models"])
assert models_for_provider(provider) == bare, provider
for model, row in block["models"].items():
mid = full_id(provider, model)
assert labels[mid] == row["label"]
assert entry_for(mid).context_window == row["context"]
assert (mid in windows) == bool(row["context"])
if row["context"]:
assert windows[mid] == row["context"]
caps = entry_for(mid).caps
assert (caps.vision, caps.pdf, caps.parallel_tool_calls) == (
row["vision"], row["pdf"], row["parallel_tool_calls"]
)
def test_openai_rows_are_bare_and_everyone_else_is_prefixed():
assert all(":" not in m for m in models_for_provider("openai"))
assert "gpt-5.5" in MATRIX and "openai:gpt-5.5" not in MATRIX
assert all(mid.startswith("anthropic:") for mid in MATRIX if "claude" in mid and mid.startswith("anthropic"))
def test_openrouter_account_mirrors_openrouter_row_for_row():
assert models_for_provider("openrouter-account") == models_for_provider("openrouter")
for model in models_for_provider("openrouter"):
assert MATRIX[f"openrouter-account:{model}"] == MATRIX[f"openrouter:{model}"]
@pytest.mark.parametrize(
"provider,model,mid",
[
("openai", "gpt-5.5", "gpt-5.5"),
("anthropic", "claude-fable-5-1", "anthropic:claude-fable-5-1"),
("bedrock", "claude/anthropic.claude-sonnet-4-6-v1:0", "bedrock:claude/anthropic.claude-sonnet-4-6-v1:0"),
],
)
def test_full_id_and_split_id_round_trip(provider, model, mid):
assert full_id(provider, model) == mid
assert split_id(mid) == (provider, model)
def test_capabilities_probe_reads_the_file_then_falls_back_to_heuristics():
assert capabilities_for("anthropic:claude-fable-5-1").pdf is True
assert capabilities_for("groq:openai/gpt-oss-120b").parallel_tool_calls is False
assert capabilities_for("anthropic:claude-unlisted-9").tools is True # heuristic path
def test_unverified_windows_stay_null_and_hidden_from_the_meter():
assert entry_for("minimax:MiniMax-M2.5").context_window is None
assert "minimax:MiniMax-M2.5" not in model_context_windows()
# -- reload / overrides --------------------------------------------------------------------
def test_reload_fills_the_same_dict_object_in_place(tmp_path, restore_matrix):
before = id(MATRIX)
path = tmp_path / "catalog.json"
path.write_text(json.dumps(_minimal()), encoding="utf-8")
meta = reload(path)
assert id(MATRIX) == before
assert set(MATRIX) == {"gpt-x", "openrouter:lab/m", "openrouter-account:lab/m"}
assert meta["serial"] == "2026.11.07.1" and meta["path"] == str(path)
assert models_for_provider("openai") == ["gpt-x"]
assert model_context_windows() == {"gpt-x": 400_000}
reload()
assert "anthropic:claude-fable-5-1" in MATRIX
def test_a_file_that_fails_validation_leaves_matrix_untouched(tmp_path, restore_matrix):
snapshot = dict(MATRIX)
bad = _minimal()
bad["providers"]["openai"]["models"]["gpt-x"]["context"] = -1
path = tmp_path / "bad.json"
path.write_text(json.dumps(bad), encoding="utf-8")
with pytest.raises(CatalogError):
reload(path)
assert MATRIX == snapshot
assert catalog_meta()["path"] == str(CATALOG_PATH)
def test_a_missing_or_unparsable_file_is_a_catalog_error(tmp_path):
with pytest.raises(CatalogError):
load_catalog(tmp_path / "nope.json")
broken = tmp_path / "broken.json"
broken.write_text("{not json", encoding="utf-8")
with pytest.raises(CatalogError):
load_catalog(broken)
def test_overrides_beat_the_file(restore_matrix):
matrix.OVERRIDES["gpt-5.5"] = ModelEntry("GPT-5.5 (corrected)", ModelCapabilities(), 7)
matrix.OVERRIDES["extra:model"] = ModelEntry("Extra")
reload()
assert MATRIX["gpt-5.5"].context_window == 7
assert model_labels()["gpt-5.5"] == "GPT-5.5 (corrected)"
assert "model" in models_for_provider("extra")
def test_an_override_on_an_openrouter_row_reaches_the_signed_in_twin(restore_matrix):
matrix.OVERRIDES["openrouter:z-ai/glm-5.3"] = ModelEntry("GLM 5.3 (corrected)", ModelCapabilities(), 131_072)
reload()
assert MATRIX["openrouter-account:z-ai/glm-5.3"].context_window == 131_072
assert MATRIX["openrouter-account:z-ai/glm-5.3"].label == "GLM 5.3 (corrected)"
def test_an_override_written_for_the_twin_itself_still_wins(restore_matrix):
matrix.OVERRIDES["openrouter:z-ai/glm-5.3"] = ModelEntry("key users", ModelCapabilities(), 1)
matrix.OVERRIDES["openrouter-account:z-ai/glm-5.3"] = ModelEntry("account users", ModelCapabilities(), 2)
reload()
assert MATRIX["openrouter:z-ai/glm-5.3"].context_window == 1
assert MATRIX["openrouter-account:z-ai/glm-5.3"].context_window == 2
def test_reload_does_not_race_readers(tmp_path, restore_matrix):
"""A daily catalog refresh (PR 3) reloads on one thread while requests read on others:
no 'dictionary changed size during iteration', and no reader ever sees an empty matrix."""
import threading
path = tmp_path / "catalog.json"
path.write_text(json.dumps(_minimal()), encoding="utf-8")
stop = threading.Event()
problems: list = []
def reader():
while not stop.is_set():
try:
labels = model_labels()
windows = model_context_windows()
ids = models_for_provider("openai")
if not labels or not ids or not windows:
problems.append("empty read")
except RuntimeError as exc: # pragma: no cover - the failure this test guards
problems.append(repr(exc))
threads = [threading.Thread(target=reader) for _ in range(4)]
for t in threads:
t.start()
try:
for _ in range(60):
reload(path)
reload()
finally:
stop.set()
for t in threads:
t.join()
assert not problems, problems[:3]
def test_snapshot_is_a_copy_that_a_reload_cannot_change(tmp_path, restore_matrix):
before = matrix.snapshot()
path = tmp_path / "catalog.json"
path.write_text(json.dumps(_minimal()), encoding="utf-8")
reload(path)
assert "anthropic:claude-fable-5-1" in before and "anthropic:claude-fable-5-1" not in MATRIX
assert before is not MATRIX
# -- validation ----------------------------------------------------------------------------
def _mutate(path: list, value) -> dict:
"""A minimal catalog with one value replaced (`...DELETE` removes the key)."""
doc = _minimal()
node = doc
for key in path[:-1]:
node = node[key]
if value == "...DELETE":
del node[path[-1]]
else:
node[path[-1]] = value
return doc
ROW = ["providers", "openai", "models", "gpt-x"]
INVALID = [
("not an object", [], "the catalog must be a JSON object"),
("unknown top-level key", ["extra"], "unknown top-level"),
("schema 2", ["schema"], "schema 2"),
("schema missing", ["schema"], "schema None"),
("empty serial", ["serial"], "serial must be"),
("generated missing", ["generated"], "generated must be"),
("last_checked not a string", ["last_checked"], "last_checked must be"),
("providers empty", ["providers"], "providers must be"),
("providers not an object", ["providers"], "providers must be"),
("provider name with colon", ["providers", "bad:name"], "bad provider name"),
("openrouter-account listed", ["providers", "openrouter-account"], "mirrored from openrouter"),
("provider block not an object", ["providers", "openai"], "must be an object"),
("provider unknown key", ["providers", "openai", "extra"], "unknown field"),
("provider note not a string", ["providers", "openai", "note"], "note must be a string"),
("models empty", ["providers", "openai", "models"], "models must be"),
("model id with spaces", ["providers", "openai", "models", " gpt "], "bad model id"),
("model id empty", ["providers", "openai", "models", ""], "bad model id"),
("openai id with colon", ["providers", "openai", "models", "openai:gpt"], "stored bare"),
("row not an object", ROW, "must be an object"),
("row unknown field", ROW + ["speed"], "unknown field"),
("row missing field", ROW + ["vision"], "missing field"),
("label empty", ROW + ["label"], "label must be"),
("label not a string", ROW + ["label"], "label must be"),
("tools not a bool", ROW + ["tools"], "tools must be true or false"),
("vision not a bool", ROW + ["vision"], "vision must be true or false"),
("context zero", ROW + ["context"], "context must be a positive"),
("context negative", ROW + ["context"], "context must be a positive"),
("context float", ROW + ["context"], "context must be a positive"),
("context bool", ROW + ["context"], "context must be a positive"),
("context string", ROW + ["context"], "context must be a positive"),
("output zero", ROW + ["output"], "output must be a positive"),
("note not a string", ROW + ["note"], "note must be a string"),
("pinned not a list", ROW + ["pinned"], "pinned must list"),
("pinned unknown field", ROW + ["pinned"], "pinned must list"),
("pinned duplicate", ROW + ["pinned"], "pinned must list"),
]
VALUES = {
"not an object": [],
"unknown top-level key": 1,
"schema 2": 2,
"schema missing": "...DELETE",
"empty serial": " ",
"generated missing": "...DELETE",
"last_checked not a string": 20261107,
"providers empty": {},
"providers not an object": [],
"provider name with colon": {"models": {"m": _row()}},
"openrouter-account listed": {"models": {"m": _row()}},
"provider block not an object": [],
"provider unknown key": 1,
"provider note not a string": 1,
"models empty": {},
"model id with spaces": _row(),
"model id empty": _row(),
"openai id with colon": _row(),
"row not an object": "x",
"row unknown field": 1,
"row missing field": "...DELETE",
"label empty": " ",
"label not a string": 3,
"tools not a bool": "yes",
"vision not a bool": 1,
"context zero": 0,
"context negative": -5,
"context float": 1.5,
"context bool": True,
"context string": "128k",
"output zero": 0,
"note not a string": [],
"pinned not a list": "context",
"pinned unknown field": ["label"],
"pinned duplicate": ["context", "context"],
}
@pytest.mark.parametrize("name,path,message", INVALID, ids=[i[0] for i in INVALID])
def test_the_loader_refuses(name, path, message):
doc = VALUES[name] if not path else _mutate(path, VALUES[name])
with pytest.raises(CatalogError, match=re.escape(message)):
parse_catalog(doc)
def test_the_loader_accepts_the_optional_fields():
doc = _minimal()
doc["last_checked"] = "2026-11-07"
row = doc["providers"]["openai"]["models"]["gpt-x"]
row.update({"output": 128_000, "note": "why", "pinned": ["context", "pdf"]})
rows = parse_catalog(doc)
assert rows["gpt-x"].context_window == 400_000
assert rows["openrouter:lab/m"].context_window is None # null is "not verified"
def test_parse_does_not_apply_mirrors_or_overrides():
rows = parse_catalog(_minimal())
assert "openrouter-account:lab/m" not in rows
def test_the_minimal_catalog_is_valid_and_the_mutations_start_from_it():
"""Guards the test fixture itself: every INVALID case must be a single change away."""
parse_catalog(_minimal())
assert copy.deepcopy(_minimal()) == _minimal()