mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-04 05:17:09 +00:00
Rohit C Prasad
833bb291d2
A per-model table says whether a model has a thinking switch and which effort levels; the composer shows a grey effort slider and a brain switch only when the model has them. llama.cpp and vLLM get thinking as enable_thinking; the provider JSON tables ship in the package.
277 lines
14 KiB
Python
277 lines
14 KiB
Python
"""Per-model settings (coworker/model_config.py) and the recommended-settings table
|
|
(coworker/providers/recommended.py): what the user saves wins, the maker's
|
|
recommendation fills the rest, and both reach the provider call, the Ollama transport,
|
|
and compaction."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from types import SimpleNamespace
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from coworker import model_config
|
|
from coworker.engine import TurnEngine
|
|
from coworker.providers import local_machine
|
|
from coworker.providers.ollama_context import OllamaContextTransport, to_native_chat
|
|
from coworker.providers.recommended import recommendation_for
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _state(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
|
|
|
|
|
|
# -- the table ----------------------------------------------------------------------------
|
|
|
|
|
|
def test_the_table_matches_a_model_under_any_provider_prefix():
|
|
for model in (
|
|
"ollama:nemotron-3.5-lightning:30b",
|
|
"ollama:nemotron-3.5-lightning:30b-mlx",
|
|
"vllm:nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4",
|
|
):
|
|
rec = recommendation_for(model)
|
|
assert rec is not None and rec.name == "NVIDIA Nemotron 3.5 Lightning"
|
|
assert rec.sampling == {"temperature": 1.0, "top_p": 0.95}
|
|
assert rec.thinking_available and rec.thinking_default
|
|
assert rec.max_output_tokens == 16000 and rec.context_for_agents == 65536
|
|
assert rec.source.startswith("https://huggingface.co/nvidia/")
|
|
assert recommendation_for("ollama:llama3.1:8b") is None
|
|
assert recommendation_for("gpt-5.6-sol") is None
|
|
|
|
|
|
def test_every_table_entry_is_complete_and_sourced():
|
|
from coworker.providers.recommended import _TABLE
|
|
|
|
raw = json.loads(_TABLE.read_text())
|
|
assert raw["version"] == 1
|
|
for item in raw["models"]:
|
|
assert item["match"] and item["name"] and item["source"].startswith("https://"), item
|
|
assert isinstance(item["context_max"], int) and item["context_max"] > 0
|
|
assert set(item["thinking"]) == {"available", "default"}
|
|
|
|
|
|
# -- the store ------------------------------------------------------------------------------
|
|
|
|
|
|
def test_saved_settings_are_checked_merged_and_kept(tmp_path):
|
|
record, error = model_config.set("ollama:qwen3-coder:30b", {"context_size": 65536, "thinking": False})
|
|
assert error is None and record == {"context_size": 65536, "thinking": False}
|
|
record, error = model_config.set("ollama:qwen3-coder:30b", {"temperature": 0.2, "context_size": None})
|
|
assert error is None and record == {"thinking": False, "temperature": 0.2}
|
|
assert (tmp_path / "state" / "model_config.json").is_file()
|
|
assert model_config.get("ollama:qwen3-coder:30b") == {"thinking": False, "temperature": 0.2}
|
|
for bad in ({"context_size": 0}, {"context_size": "big"}, {"temperature": 3}, {"top_p": 0},
|
|
{"compaction_threshold_pct": 0.99}, {"thinking": "yes"}, {"nope": 1}):
|
|
_, error = model_config.set("m", bad)
|
|
assert error, bad
|
|
model_config.remove("ollama:qwen3-coder:30b")
|
|
assert model_config.get("ollama:qwen3-coder:30b") == {}
|
|
|
|
|
|
def test_only_one_model_is_the_default():
|
|
model_config.set("a", {"default": True})
|
|
model_config.set("b", {"default": True})
|
|
assert model_config.default_model() == "b"
|
|
assert model_config.get("a") == {}
|
|
|
|
|
|
def test_effective_settings_say_where_each_value_came_from():
|
|
model_config.set("ollama:nemotron-3.5-lightning:30b", {"max_output_tokens": 8000})
|
|
eff = model_config.effective("ollama:nemotron-3.5-lightning:30b")
|
|
assert eff["max_output_tokens"] == {"value": 8000, "from": "user"}
|
|
assert eff["temperature"] == {"value": 1.0, "from": "recommended"}
|
|
assert eff["thinking"] == {"value": True, "from": "recommended"}
|
|
assert eff["recommendation"]["name"] == "NVIDIA Nemotron 3.5 Lightning"
|
|
assert "temperature" not in model_config.effective("ollama:llama3.1:8b")
|
|
|
|
|
|
# -- reaching the provider call -------------------------------------------------------------
|
|
|
|
|
|
def test_provider_settings_come_from_the_user_then_the_recommendation():
|
|
assert model_config.model_settings_for("ollama:llama3.1:8b") == {}
|
|
assert model_config.model_settings_for("ollama:nemotron-3.5-lightning:30b") == {
|
|
"max_tokens": 16000, "temperature": 1.0, "top_p": 0.95, "extra_body": {"think": True},
|
|
}
|
|
model_config.set("ollama:nemotron-3.5-lightning:30b", {"thinking": False, "temperature": 0.3})
|
|
assert model_config.model_settings_for("ollama:nemotron-3.5-lightning:30b") == {
|
|
"max_tokens": 16000, "temperature": 0.3, "top_p": 0.95, "extra_body": {"think": False},
|
|
}
|
|
# An effort level for a model with levels; no thinking switch rides along.
|
|
model_config.set("gpt-5.6-sol", {"reasoning_effort": "high", "max_output_tokens": 32000})
|
|
assert model_config.model_settings_for("gpt-5.6-sol") == {"max_tokens": 32000, "reasoning_effort": "high"}
|
|
|
|
|
|
def test_the_thinking_switch_reaches_ollamas_native_call():
|
|
body = {"model": "m", "messages": [], "think": False, "temperature": 0.3, "top_p": 0.9, "max_tokens": 500}
|
|
native = to_native_chat(body, 32768)
|
|
assert native["think"] is False
|
|
assert native["options"] == {"num_ctx": 32768, "num_predict": 500, "temperature": 0.3, "top_p": 0.9}
|
|
|
|
|
|
class _Inner(httpx.BaseTransport):
|
|
def __init__(self):
|
|
self.requests = []
|
|
|
|
def handle_request(self, request):
|
|
self.requests.append(request)
|
|
if request.url.path.endswith("/api/show"):
|
|
return httpx.Response(200, json={"model_info": {"general.architecture": "qwen3moe", "qwen3moe.context_length": 262144}})
|
|
return httpx.Response(200, json={"model": "qwen3-coder:30b", "message": {"role": "assistant", "content": "ok"}, "done": True, "done_reason": "stop"})
|
|
|
|
|
|
def test_a_saved_context_size_wins_over_the_machine_rule():
|
|
inner = _Inner()
|
|
transport = OllamaContextTransport(inner=inner, memory_bytes=8 * local_machine.GB)
|
|
req = lambda: httpx.Request("POST", "http://127.0.0.1:11434/v1/chat/completions", json={"model": "qwen3-coder:30b", "messages": []})
|
|
transport.handle_request(req())
|
|
sent = [json.loads(r.content) for r in inner.requests if r.url.path == "/api/chat"]
|
|
assert sent[0]["options"]["num_ctx"] == 16384 # the 8 GB tier
|
|
model_config.set("ollama:qwen3-coder:30b", {"context_size": 65536})
|
|
transport.handle_request(req())
|
|
sent = [json.loads(r.content) for r in inner.requests if r.url.path == "/api/chat"]
|
|
assert sent[1]["options"]["num_ctx"] == 65536
|
|
|
|
|
|
def test_compaction_uses_the_saved_threshold_and_window():
|
|
engine = SimpleNamespace(model="acme:unlisted-model", compaction_settings=None, _warned_context_fallback=False)
|
|
before = TurnEngine._compaction_config(engine)
|
|
assert before["context_window"] is None
|
|
model_config.set("acme:unlisted-model", {"context_size": 100_000, "compaction_threshold_pct": 0.6})
|
|
cfg = TurnEngine._compaction_config(engine)
|
|
assert cfg["context_window"] == 100_000 and cfg["threshold_pct"] == 0.6
|
|
|
|
|
|
# -- the server's view ----------------------------------------------------------------------
|
|
|
|
|
|
def test_manager_saves_reads_and_removes_model_settings(tmp_path, monkeypatch):
|
|
from coworker.server.manager import SessionManager
|
|
|
|
mgr = SessionManager(data_dir=tmp_path)
|
|
out = mgr.set_model_config("ollama:qwen3-coder:30b", {"context_size": 32768, "default": True})
|
|
assert out["ok"] and out["context_size"] == {"value": 32768, "from": "user"}
|
|
assert mgr.model == "ollama:qwen3-coder:30b" # default: true is the Make default button
|
|
assert mgr.get_settings()["model_config"]["ollama:qwen3-coder:30b"]["context_size"] == 32768
|
|
assert mgr.get_model_config("ollama:qwen3-coder:30b")["model"] == "ollama:qwen3-coder:30b"
|
|
assert mgr.set_model_config("x", {"temperature": 9}) == {"ok": False, "error": "temperature must be between 0 and 2"}
|
|
assert mgr.remove_model_config("ollama:qwen3-coder:30b") == {"ok": True}
|
|
assert mgr.get_model_config()["models"] == {}
|
|
|
|
|
|
# -- the machine and the models on it -------------------------------------------------------
|
|
|
|
|
|
def test_fit_and_system_facts(monkeypatch):
|
|
GB = local_machine.GB
|
|
assert local_machine.fit_for(25 * GB, 96 * GB) == "runs_well"
|
|
assert local_machine.fit_for(70 * GB, 96 * GB) == "tight"
|
|
assert local_machine.fit_for(87 * GB, 96 * GB) == "too_large"
|
|
assert local_machine.fit_for(None, 96 * GB) == "unknown"
|
|
facts = local_machine.system_facts()
|
|
for key in ("processor", "graphics", "kind", "memory_bytes", "storage_free_bytes", "runs_well_up_to_bytes"):
|
|
assert key in facts
|
|
assert facts["kind"] in ("apple_silicon", "nvidia", "dgx_spark", "jetson", "cpu")
|
|
|
|
|
|
def test_ollama_model_facts_read_tools_thinking_and_context(monkeypatch):
|
|
from coworker.providers import ollama_facts
|
|
|
|
tags = {"models": [
|
|
{"name": "nemotron-3.5-lightning:30b", "size": 25 * local_machine.GB, "details": {"parameter_size": "30.5B", "quantization_level": "Q4_K_M"}},
|
|
{"name": "gemma3:12b", "size": 8 * local_machine.GB, "details": {}},
|
|
{"name": "nemotron-3-super:cloud", "size": 0, "remote_model": "nemotron-3-super:cloud", "remote_host": "https://ollama.com"},
|
|
]}
|
|
shows = {
|
|
"nemotron-3.5-lightning:30b": {"capabilities": ["completion", "tools", "thinking"], "model_info": {"general.architecture": "nemotron", "nemotron.context_length": 1048576}},
|
|
"gemma3:12b": {"capabilities": ["completion", "vision"], "model_info": {"general.architecture": "gemma3", "gemma3.context_length": 131072}},
|
|
"nemotron-3-super:cloud": {"capabilities": ["completion", "tools", "thinking"], "model_info": {}},
|
|
}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path == "/api/tags":
|
|
return httpx.Response(200, json=tags)
|
|
name = json.loads(request.content)["model"]
|
|
return httpx.Response(200, json=shows[name])
|
|
|
|
fake = httpx.Client(transport=httpx.MockTransport(handler))
|
|
monkeypatch.setattr(httpx, "get", lambda url, timeout=None: fake.get(url))
|
|
monkeypatch.setattr(httpx, "post", lambda url, json=None, timeout=None: fake.post(url, json=json))
|
|
monkeypatch.setattr(local_machine, "model_memory_bytes", lambda: 32 * local_machine.GB)
|
|
ollama_facts.forget_all()
|
|
rows = {r["name"]: r for r in ollama_facts.model_facts("http://localhost:11434/v1", fresh=True)}
|
|
lightning = rows["nemotron-3.5-lightning:30b"]
|
|
assert lightning["tools"] and lightning["thinking"] and not lightning["vision"]
|
|
assert lightning["context_max"] == 1048576 and lightning["context"] == 65536 # the 32 GB tier
|
|
assert lightning["fit"] == "tight" and lightning["recommendation"] == "NVIDIA Nemotron 3.5 Lightning" # 25 of 32 GB
|
|
gemma = rows["gemma3:12b"]
|
|
assert not gemma["tools"] and gemma["vision"] and gemma["fit"] == "runs_well"
|
|
cloud = rows["nemotron-3-super:cloud"]
|
|
assert cloud["remote"] and cloud["fit"] == "cloud"
|
|
# A saved context size shows as the user's.
|
|
model_config.set("ollama:nemotron-3.5-lightning:30b", {"context_size": 32768})
|
|
row = {r["name"]: r for r in ollama_facts.model_facts("http://localhost:11434/v1", fresh=True)}["nemotron-3.5-lightning:30b"]
|
|
assert row["context"] == 32768 and row["context_from"] == "user"
|
|
|
|
|
|
def test_only_tool_capable_ollama_models_join_the_picker(tmp_path, monkeypatch):
|
|
from coworker.providers import ollama_facts
|
|
from coworker.server.manager import SessionManager
|
|
|
|
monkeypatch.setattr(ollama_facts, "model_facts", lambda base=None, fresh=False: [
|
|
{"model": "ollama:qwen3-coder:30b", "tools": True},
|
|
{"model": "ollama:gemma3:12b", "tools": False},
|
|
{"model": "ollama:old-server-model", "tools": None}, # an older Ollama does not say
|
|
])
|
|
mgr = SessionManager(data_dir=tmp_path)
|
|
assert mgr._ollama_models() == ["ollama:qwen3-coder:30b", "ollama:old-server-model"]
|
|
|
|
|
|
def test_a_session_can_switch_thinking_for_itself(tmp_path):
|
|
from types import SimpleNamespace
|
|
|
|
from coworker.server.manager import SessionManager
|
|
|
|
mgr = SessionManager(data_dir=tmp_path)
|
|
engine = SimpleNamespace(model="ollama:nemotron-3.5-lightning:30b", model_settings={"max_tokens": 16000, "extra_body": {"think": True}})
|
|
mgr._engines["s1"] = engine
|
|
got = mgr.session_model_settings("s1")
|
|
assert (got["thinking"], got["reasoning_effort"]) == (None, None) # at the model's default
|
|
assert got["controls"]["thinking"] == {"support": "supported", "default": True}
|
|
out = mgr.set_session_model_settings("s1", {"thinking": False})
|
|
assert out["ok"] and out["thinking"] is False
|
|
assert engine.model_settings["extra_body"] == {"think": False}
|
|
assert engine.model_settings["max_tokens"] == 16000 # the rest stays
|
|
# Nemotron has no effort levels: refused, nothing changes.
|
|
assert mgr.set_session_model_settings("s1", {"reasoning_effort": "high"})["ok"] is False
|
|
# Back to the model's own setting.
|
|
out = mgr.set_session_model_settings("s1", {"thinking": None})
|
|
assert out["thinking"] is None and engine.model_settings["extra_body"] == {"think": True}
|
|
assert mgr.set_session_model_settings("nope", {"thinking": True}) == {"ok": False, "error": "session not running"}
|
|
assert mgr.set_session_model_settings("s1", {"thinking": "yes"})["ok"] is False
|
|
|
|
|
|
def test_a_session_can_set_reasoning_effort_within_the_model_levels(tmp_path):
|
|
from types import SimpleNamespace
|
|
|
|
from coworker.server.manager import SessionManager
|
|
|
|
mgr = SessionManager(data_dir=tmp_path)
|
|
engine = SimpleNamespace(model="anthropic:claude-fable-5-1", model_settings={})
|
|
mgr._engines["s1"] = engine
|
|
controls = mgr.session_model_settings("s1")["controls"]
|
|
assert controls["reasoning"] == {"support": "supported", "levels": ["low", "medium", "high", "xhigh", "max"], "default": "high"}
|
|
assert controls["thinking"] == {"support": "not_supported"}
|
|
out = mgr.set_session_model_settings("s1", {"reasoning_effort": "low"})
|
|
assert out["ok"] and out["reasoning_effort"] == "low" and engine.model_settings["reasoning_effort"] == "low"
|
|
assert mgr.set_session_model_settings("s1", {"reasoning_effort": "none"})["ok"] is False # no Off
|
|
assert mgr.set_session_model_settings("s1", {"thinking": False})["ok"] is False # no switch
|
|
out = mgr.set_session_model_settings("s1", {"reasoning_effort": None})
|
|
assert out["reasoning_effort"] is None and "reasoning_effort" not in engine.model_settings
|
|
# A level saved for the model becomes its default, and Reset goes back to it.
|
|
model_config.set("anthropic:claude-fable-5-1", {"reasoning_effort": "max"})
|
|
assert mgr.model_controls("anthropic:claude-fable-5-1")["reasoning"]["default"] == "max"
|