Files
openworker/tests/test_matrix_current_models.py
T
Devika Verma 9d472d1840 feat(models): the model matrix is a data file (OPE-252)
The 95 hand-written rows of `MATRIX` move from matrix.py to
coworker/providers/model_catalog.json: per provider and model, the label,
tools / vision / pdf / parallel_tool_calls, the context window, and an optional
output limit. Each provider block carries a `note` naming the vendor page or
listing its rows were read from and when (the comments matrix.py used to hold).
A row a maintainer sized on purpose (a window pinned to a particular host, a
capability unverified over this backend) says so with `pinned` and a reason in
its note; 13 rows are pinned.

matrix.py loads the file into the same `MATRIX` dict at import; `entry_for`,
`model_labels`, `model_context_windows` and `models_for_provider` are unchanged,
so no consumer changes. The signed-in OpenRouter provider stays a load-time
mirror and is never listed. `OVERRIDES` (empty) beats the file, before and after
the mirror. `reload()` refills the dict in place under a lock, the helpers read
a snapshot, and a file that fails validation leaves MATRIX untouched.

The loader refuses anything it cannot read as a whole: wrong schema, unknown or
missing fields, a non-boolean capability, a context that is not a positive
whole number, an OpenAI id with a prefix, a mirrored provider listed by hand,
a pin on a field that cannot be pinned. The file is in canonical form
(json.dumps indent 2, one trailing newline) so a catalog pull request diffs
only the rows that changed.

Keeping the file current is a maintainer job, not a runtime one: the app never
fetches a catalog from anyone. `serial`, `generated` and `last_checked` are
informational to the app and move when the maintainers change or check a fact.

Round trip against the hand-written dict: the same 104 ids (95 rows plus the
9 openrouter-account mirrors) in the same order, with the same labels,
capabilities and windows.

Tests: tests/test_model_catalog.py (file contract, package data, the four
helpers against the file, the mirror row for row, capabilities_for, reload in
place, a refused file leaves MATRIX, OVERRIDES on both twins, four reader
threads over 120 reloads, 34 refused shapes, the optional fields accepted);
test_matrix_current_models reuses matrix.full_id. The 251 existing
matrix-reading tests pass unchanged.
2026-11-17 18:38:30 +05:30

187 lines
7.8 KiB
Python

"""OPE-170 problem 1: the model matrix knows the current models' context windows.
A model missing from the matrix falls back to `DEFAULT_CONTEXT_WINDOW` (128,000), so
auto-compaction fires at 102,400 tokens — half or a tenth of what the newest models have
(observed on Claude Sonnet 5: two compactions in one 33-minute run, each rebuilding the
prompt cache). These tests pin the current Anthropic rows, keep every model the product
recommends or has evaluated in the table, and make the fallback visible in the log.
"""
from __future__ import annotations
import asyncio
import logging
import re
from pathlib import Path
import aisuite as ai
import pytest
from coworker.compaction import DEFAULT_CONTEXT_WINDOW, trigger_tokens
from coworker.engine import TurnEngine
from coworker.permissions import PermissionEngine
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient
from coworker.providers.matrix import MATRIX, full_id, model_context_windows
from coworker.providers.registry import DESCRIPTORS
from coworker.tools import ToolRegistry
REPO = Path(__file__).resolve().parents[1]
# -- rows ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"model,window",
[
# platform.claude.com/docs/en/models/overview, read 2026-09-08: 1M is the default
# and only context size on the Claude 5 generation; Haiku 4.5 stays at 200K.
("anthropic:claude-fable-5-1", 1_000_000),
("anthropic:claude-opus-5", 1_000_000),
("anthropic:claude-sonnet-5", 1_000_000),
("anthropic:claude-fable-5", 1_000_000),
("anthropic:claude-haiku-4-5", 200_000),
# Together /v1/models `context_length`, read 2026-09-08 (exact per model).
("together:moonshotai/Kimi-K3", 1_048_576),
("openrouter:moonshotai/kimi-k3", 1_048_576), # openrouter.ai models API, 2026-09-09
("together:zai-org/GLM-5.2", 1_048_575),
("together:deepseek-ai/DeepSeek-V4-Pro", 512_000),
# Vendor pages read 2026-09-08 (docs.z.ai "1M"; openrouter 1,048,576;
# fireworks "1040k").
("zai:glm-5.2", 1_000_000),
("openrouter:z-ai/glm-5.2", 1_048_576),
("fireworks:accounts/fireworks/models/glm-5p2", 1_040_000),
# openrouter.ai models API, read 2026-10-01 (OPE-215). Ultra: the endpoint's own
# context_length (202,800), not the model-level 262,144 the catalog shows.
("openrouter:nvidia/nemotron-3-ultra-550b-a55b", 202_800),
("openrouter:nvidia/nemotron-3.5-lightning", 262_144),
# openrouter.ai endpoints API, read 2026-10-01 (OPE-215): the Z.AI host's window.
("openrouter:z-ai/glm-5.3", 1_048_576),
],
)
def test_current_models_have_verified_windows(model, window):
assert model_context_windows()[model] == window
def test_claude_5_rows_are_vision_capable_and_labelled():
for mid in ("anthropic:claude-fable-5-1", "anthropic:claude-opus-5", "anthropic:claude-sonnet-5"):
entry = MATRIX[mid]
assert entry.caps.tools and entry.caps.vision and entry.caps.pdf, mid
assert "Anthropic" in entry.label, mid
_matrix_key = full_id # OpenAI rows are stored bare (bare ids route to the OpenAI default).
# Recommended models whose vendor spec has not been re-checked: the matrix keeps their
# window at None on purpose (the GUI meter hides rather than show a made-up number) and
# they compact on the 128k guess. Verify the spec, fill the row, and shrink this set.
_KNOWN_UNVERIFIED_RECOMMENDATIONS = {"minimax:MiniMax-M2.5", "meta:muse-spark-1.1"}
def test_every_recommended_model_is_in_the_matrix():
"""The Settings pane suggests each provider's `recommended_model`; every one must be a
curated row (local servers excepted: Ollama and LM Studio recommend a model the user
downloads, named as that app names it, not a vendor catalog id)."""
missing = [
_matrix_key(d.name, d.recommended_model)
for d in DESCRIPTORS
if d.kind != "local" and d.recommended_model
and _matrix_key(d.name, d.recommended_model) not in MATRIX
]
assert not missing, f"recommended models without a matrix row: {missing}"
def test_recommended_models_without_a_window_are_exactly_the_known_set():
"""A recommendation without a window compacts on the 128k guess. New ones must not
slip in silently; verified ones must leave the allowlist."""
windows = model_context_windows()
without = {
_matrix_key(d.name, d.recommended_model)
for d in DESCRIPTORS
if d.kind != "local" and d.recommended_model
and _matrix_key(d.name, d.recommended_model) not in windows
}
assert without == _KNOWN_UNVERIFIED_RECOMMENDATIONS
def test_every_model_in_the_reviewer_eval_reports_has_a_window():
"""`reports/reviewer-eval-*.md` are the models the product has been evaluated on."""
reports = sorted((REPO / "reports").glob("reviewer-eval-*.md"))
if not reports:
pytest.skip("no reviewer-eval reports in this checkout")
pattern = re.compile(r"\b(anthropic|openai|together|gemini|zai|deepseek|fireworks|openrouter):([A-Za-z0-9._/-]+)")
ids = set()
for path in reports:
for provider, model in pattern.findall(path.read_text(encoding="utf-8")):
ids.add(_matrix_key(provider, model))
assert ids, "no model ids found in the reports"
windows = model_context_windows()
missing = sorted(m for m in ids if m not in windows)
assert not missing, f"evaluated models without a matrix window: {missing}"
# -- trigger arithmetic (documents what the rows buy) -------------------------------
def test_trigger_for_a_1m_model_is_the_cap_not_the_128k_guess():
assert trigger_tokens(None) == int(0.8 * DEFAULT_CONTEXT_WINDOW) # 102,400
assert trigger_tokens(1_000_000) == 250_000
assert trigger_tokens(200_000) == 160_000
# -- fallback is visible ------------------------------------------------------------
class _OneTurn(ProviderClient):
def complete(self, *, model, messages, tools=None, **settings):
return AssistantTurn(text="done", finish_reason="stop")
def capabilities(self, model):
return ModelCapabilities()
def _engine(tmp_path, model):
registry = ToolRegistry()
registry.register_all(ai.toolkits.files(root=str(tmp_path), allow_write=True))
return TurnEngine(
provider=_OneTurn(),
registry=registry,
permissions=PermissionEngine(workspace_root=tmp_path),
model=model,
)
def _run(engine):
async def _collect():
return [ev async for ev in engine.run("hello")]
return asyncio.run(_collect())
def test_unlisted_model_logs_the_128k_fallback_once(tmp_path, caplog):
engine = _engine(tmp_path, "anthropic:claude-unlisted-9")
with caplog.at_level(logging.WARNING, logger="coworker.engine"):
_run(engine)
_run(engine) # second turn: no repeat
hits = [r for r in caplog.records if "context window" in r.getMessage()]
assert len(hits) == 1, [r.getMessage() for r in caplog.records]
msg = hits[0].getMessage()
assert "anthropic:claude-unlisted-9" in msg and str(DEFAULT_CONTEXT_WINDOW) in msg
def test_listed_model_does_not_warn(tmp_path, caplog):
engine = _engine(tmp_path, "anthropic:claude-sonnet-5")
with caplog.at_level(logging.WARNING, logger="coworker.engine"):
_run(engine)
assert not [r for r in caplog.records if "context window" in r.getMessage()]
def test_compaction_settings_override_silences_the_warning(tmp_path, caplog):
"""A caller that supplies its own context_window (the GUI's per-model settings) has
made the choice explicitly; nothing to warn about."""
engine = _engine(tmp_path, "anthropic:claude-unlisted-9")
engine.compaction_settings = lambda: {"context_window": 400_000}
with caplog.at_level(logging.WARNING, logger="coworker.engine"):
_run(engine)
assert not [r for r in caplog.records if "context window" in r.getMessage()]