openworker/tests/test_matrix_current_models.py

"""OPE-170 problem 1: the model matrix knows the current models' context windows.

A model missing from the matrix falls back to `DEFAULT_CONTEXT_WINDOW` (128,000), so
auto-compaction fires at 102,400 tokens — half or a tenth of what the newest models have
(observed on Claude Sonnet 5: two compactions in one 33-minute run, each rebuilding the
prompt cache). These tests pin the current Anthropic rows, keep every model the product
recommends or has evaluated in the table, and make the fallback visible in the log.
"""

from __future__ import annotations

import asyncio
import logging
import re
from pathlib import Path

import aisuite as ai
import pytest
from coworker.compaction import DEFAULT_CONTEXT_WINDOW, trigger_tokens
from coworker.engine import TurnEngine
from coworker.permissions import PermissionEngine
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient
from coworker.providers.matrix import MATRIX, model_context_windows
from coworker.providers.registry import DESCRIPTORS
from coworker.tools import ToolRegistry

REPO = Path(__file__).resolve().parents[1]


# -- rows ---------------------------------------------------------------------------


@pytest.mark.parametrize(
    "model,window",
    [
        # platform.claude.com/docs/en/models/overview, read 2026-09-08: 1M is the default
        # and only context size on the Claude 5 generation; Haiku 4.5 stays at 200K.
        ("anthropic:claude-fable-5-1", 1_000_000),
        ("anthropic:claude-opus-5", 1_000_000),
        ("anthropic:claude-sonnet-5", 1_000_000),
        ("anthropic:claude-fable-5", 1_000_000),
        ("anthropic:claude-haiku-4-5", 200_000),
        # Together /v1/models `context_length`, read 2026-09-08 (exact per model).
        ("together:moonshotai/Kimi-K3", 1_048_576),
        ("openrouter:moonshotai/kimi-k3", 1_048_576),  # openrouter.ai models API, 2026-09-09
        ("together:zai-org/GLM-5.2", 1_048_575),
        ("together:deepseek-ai/DeepSeek-V4-Pro", 512_000),
        # Vendor pages read 2026-09-08 (docs.z.ai "1M"; openrouter 1,048,576;
        # fireworks "1040k").
        ("zai:glm-5.2", 1_000_000),
        ("openrouter:z-ai/glm-5.2", 1_048_576),
        ("fireworks:accounts/fireworks/models/glm-5p2", 1_040_000),
    ],
)
def test_current_models_have_verified_windows(model, window):
    assert model_context_windows()[model] == window


def test_claude_5_rows_are_vision_capable_and_labelled():
    for mid in ("anthropic:claude-fable-5-1", "anthropic:claude-opus-5", "anthropic:claude-sonnet-5"):
        entry = MATRIX[mid]
        assert entry.caps.tools and entry.caps.vision and entry.caps.pdf, mid
        assert "Anthropic" in entry.label, mid


def _matrix_key(provider: str, model: str) -> str:
    # OpenAI rows are stored bare (bare ids route to the OpenAI default).
    return model if provider == "openai" else f"{provider}:{model}"


# Recommended models whose vendor spec has not been re-checked: the matrix keeps their
# window at None on purpose (the GUI meter hides rather than show a made-up number) and
# they compact on the 128k guess. Verify the spec, fill the row, and shrink this set.
_KNOWN_UNVERIFIED_RECOMMENDATIONS = {"minimax:MiniMax-M2.5", "meta:muse-spark-1.1"}


def test_every_recommended_model_is_in_the_matrix():
    """The Settings pane suggests each provider's `recommended_model`; every one must be a
    curated row (Ollama excepted: a user-local model name, not a vendor catalog id)."""
    missing = [
        _matrix_key(d.name, d.recommended_model)
        for d in DESCRIPTORS
        if d.name != "ollama" and d.recommended_model
        and _matrix_key(d.name, d.recommended_model) not in MATRIX
    ]
    assert not missing, f"recommended models without a matrix row: {missing}"


def test_recommended_models_without_a_window_are_exactly_the_known_set():
    """A recommendation without a window compacts on the 128k guess. New ones must not
    slip in silently; verified ones must leave the allowlist."""
    windows = model_context_windows()
    without = {
        _matrix_key(d.name, d.recommended_model)
        for d in DESCRIPTORS
        if d.name != "ollama" and d.recommended_model
        and _matrix_key(d.name, d.recommended_model) not in windows
    }
    assert without == _KNOWN_UNVERIFIED_RECOMMENDATIONS


def test_every_model_in_the_reviewer_eval_reports_has_a_window():
    """`reports/reviewer-eval-*.md` are the models the product has been evaluated on."""
    reports = sorted((REPO / "reports").glob("reviewer-eval-*.md"))
    if not reports:
        pytest.skip("no reviewer-eval reports in this checkout")
    pattern = re.compile(r"\b(anthropic|openai|together|gemini|zai|deepseek|fireworks|openrouter):([A-Za-z0-9._/-]+)")
    ids = set()
    for path in reports:
        for provider, model in pattern.findall(path.read_text(encoding="utf-8")):
            ids.add(_matrix_key(provider, model))
    assert ids, "no model ids found in the reports"
    windows = model_context_windows()
    missing = sorted(m for m in ids if m not in windows)
    assert not missing, f"evaluated models without a matrix window: {missing}"


# -- trigger arithmetic (documents what the rows buy) -------------------------------


def test_trigger_for_a_1m_model_is_the_cap_not_the_128k_guess():
    assert trigger_tokens(None) == int(0.8 * DEFAULT_CONTEXT_WINDOW)  # 102,400
    assert trigger_tokens(1_000_000) == 250_000
    assert trigger_tokens(200_000) == 160_000


# -- fallback is visible ------------------------------------------------------------


class _OneTurn(ProviderClient):
    def complete(self, *, model, messages, tools=None, **settings):
        return AssistantTurn(text="done", finish_reason="stop")

    def capabilities(self, model):
        return ModelCapabilities()


def _engine(tmp_path, model):
    registry = ToolRegistry()
    registry.register_all(ai.toolkits.files(root=str(tmp_path), allow_write=True))
    return TurnEngine(
        provider=_OneTurn(),
        registry=registry,
        permissions=PermissionEngine(workspace_root=tmp_path),
        model=model,
    )


def _run(engine):
    async def _collect():
        return [ev async for ev in engine.run("hello")]

    return asyncio.run(_collect())


def test_unlisted_model_logs_the_128k_fallback_once(tmp_path, caplog):
    engine = _engine(tmp_path, "anthropic:claude-unlisted-9")
    with caplog.at_level(logging.WARNING, logger="coworker.engine"):
        _run(engine)
        _run(engine)  # second turn: no repeat
    hits = [r for r in caplog.records if "context window" in r.getMessage()]
    assert len(hits) == 1, [r.getMessage() for r in caplog.records]
    msg = hits[0].getMessage()
    assert "anthropic:claude-unlisted-9" in msg and str(DEFAULT_CONTEXT_WINDOW) in msg


def test_listed_model_does_not_warn(tmp_path, caplog):
    engine = _engine(tmp_path, "anthropic:claude-sonnet-5")
    with caplog.at_level(logging.WARNING, logger="coworker.engine"):
        _run(engine)
    assert not [r for r in caplog.records if "context window" in r.getMessage()]


def test_compaction_settings_override_silences_the_warning(tmp_path, caplog):
    """A caller that supplies its own context_window (the GUI's per-model settings) has
    made the choice explicitly; nothing to warn about."""
    engine = _engine(tmp_path, "anthropic:claude-unlisted-9")
    engine.compaction_settings = lambda: {"context_window": 400_000}
    with caplog.at_level(logging.WARNING, logger="coworker.engine"):
        _run(engine)
    assert not [r for r in caplog.records if "context window" in r.getMessage()]