Files
openworker/tests/test_model_errors.py
Devika Verma 7b9630ddc2 feat(models): the picker shows only ticked models a key can use; a free check sets refused ones aside; chat errors point to another model (OPE-252)
- Picker: a catalog row starts ticked only when the provider's list for the key carries it (by id or alias); models the user ticked or added stay, unticked ones never show. Providers with no list (Bedrock, Vertex, Ark agent plan) and the time before the first list keep the catalog rows. The default model always stays.
- Free check: with the daily list fetch, and in parallel with it, Gemini, Anthropic and MiniMax count tokens (no generation, no charge) for this provider's picker models; a model the provider refuses (a 400/403/404 that names it and says it is not found, no longer available, unknown, retired...) is set aside on the page with the provider's own sentence and leaves the picker. Ticking a model runs the same check. 'Use anyway' overrides it. A failed or unclear check never hides a model. Seen live 2026-11-10: gemini-2.5-flash and gemini-2.5-pro (no longer available to new users), MiniMax-M2.5 and M2.1 (no longer available).
- Chat errors: friendly_model_error also recognises a withdrawn model, a model without tool support, a plan with no quota for the model (limit: 0), a busy provider (high demand, overloaded), and any provider's 404 that names the model as not found; each sentence ends by pointing to another model. Nothing is remembered from a failed turn.
2026-11-17 18:38:31 +05:30

190 lines
9.0 KiB
Python

"""New-flagship rollout (2026-07-14): GPT-5.6 Sol/Terra/Luna + Claude Fable 5 in the
matrix, both families' flagships as defaults, and friendly errors when an account can't
use them (GPT-5.6 rolls out per-organization; quota/credits can run out on any model).
"""
from coworker.config import Config
from coworker.providers.errors import friendly_model_error
from coworker.providers.matrix import MATRIX, models_for_provider
from coworker.providers.registry import get_descriptor
def test_new_flagships_in_matrix_with_labels():
for mid, label in {
"gpt-5.6-sol": "GPT-5.6 Sol · OpenAI",
"gpt-5.6-terra": "GPT-5.6 Terra · OpenAI",
"gpt-5.6-luna": "GPT-5.6 Luna · OpenAI",
"anthropic:claude-fable-5": "Claude Fable 5 · Anthropic",
}.items():
assert MATRIX[mid].label == label
assert MATRIX[mid].caps.tools and MATRIX[mid].caps.vision
assert "gpt-5.6-sol" in models_for_provider("openai")
assert "claude-fable-5" in models_for_provider("anthropic")
def test_flagships_are_the_defaults():
assert Config().model == "gpt-5.6-sol"
assert get_descriptor("openai").recommended_model == "gpt-5.6-sol"
assert get_descriptor("anthropic").recommended_model == "claude-fable-5"
# -- friendly access/quota errors --------------------------------------------------------
def test_no_access_errors_are_translated():
# OpenAI's 404/403 body for a model the org can't use yet
exc = RuntimeError(
"Error code: 404 - {'error': {'code': 'model_not_found', 'message': "
"'The model `gpt-5.6-sol` does not exist or you do not have access to it.'}}"
)
msg = friendly_model_error("gpt-5.6-sol", exc)
assert msg and "doesn't have access to gpt-5.6-sol" in msg
# Anthropic's 404 body is type not_found_error + "model: <id>"
exc = RuntimeError(
"Error code: 404 - {'type': 'error', 'error': {'type': 'not_found_error', "
"'message': 'model: claude-fable-5'}}"
)
msg = friendly_model_error("anthropic:claude-fable-5", exc)
assert msg and "doesn't have access to anthropic:claude-fable-5" in msg
def test_quota_errors_are_translated():
exc = RuntimeError(
"Error code: 429 - {'error': {'code': 'insufficient_quota', 'message': "
"'You exceeded your current quota, please check your plan and billing details.'}}"
)
msg = friendly_model_error("gpt-5.6-sol", exc)
assert msg and "out of quota for gpt-5.6-sol" in msg
exc = RuntimeError(
"Error code: 400 - {'type': 'error', 'error': {'type': 'invalid_request_error', "
"'message': 'Your credit balance is too low to access the Anthropic API.'}}"
)
msg = friendly_model_error("anthropic:claude-fable-5", exc)
assert msg and "out of quota" in msg
def test_withdrawn_no_tools_plan_and_busy_errors_say_to_pick_another_model():
"""Real bodies from Google and MiniMax, 2026-11-10. Each sentence ends by pointing the
user at another model; nothing is remembered from it."""
google_gone = RuntimeError("404 NOT_FOUND. " + '{"error": {"code": 404, "message": "This model models/gemini-2.5-flash is no longer available to new users. Please update your code to use models/gemini-3.8-flash for the latest features and improvements.", "status": "NOT_FOUND"}}')
msg = friendly_model_error("gemini:gemini-2.5-flash", google_gone)
assert msg and "no longer available to your account" in msg and "Pick a different model" in msg
minimax_gone = RuntimeError("Error code: 404 - " + '{"type":"error","error":{"type":"not_found_error","message":"The model \\"MiniMax-M2.5\\" is no longer available. (2069)"}}')
assert "no longer available" in friendly_model_error("minimax:MiniMax-M2.5", minimax_gone)
no_tools = RuntimeError("400 INVALID_ARGUMENT. {'error': {'code': 400, 'message': 'Function calling is not enabled for models/antigravity-preview-latest'}}")
assert "can't use tools" in friendly_model_error("gemini:antigravity-preview-latest", no_tools)
free_tier = RuntimeError("429 RESOURCE_EXHAUSTED. " + '{"error": {"code": 429, "message": "You exceeded your current quota, please check your plan and billing details. * Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: computer-use-preview", "status": "RESOURCE_EXHAUSTED"}}')
assert "plan doesn't include" in friendly_model_error("gemini:gemini-2.5-computer-use-preview-10-2025", free_tier)
busy = RuntimeError("503 UNAVAILABLE. " + '{"error": {"code": 503, "message": "This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.", "status": "UNAVAILABLE"}}')
msg = friendly_model_error("gemini:gemini-3.8-flash", busy)
assert msg and "busy at the provider" in msg and "pick a different model" in msg
# any provider: a 404 that names the model and says it is not found
other = RuntimeError("Error code: 404 - {'error': {'message': 'Model new-vendor-x1 not found'}}")
assert "isn't available to your account" in friendly_model_error("newvendor:new-vendor-x1", other)
def test_unrelated_errors_pass_through_raw():
# a plain rate-limit (429 without a quota code) must NOT be dressed up
assert (
friendly_model_error(
"gpt-5.6-sol",
RuntimeError("Error code: 429 - rate_limit_exceeded, retry after 2s"),
)
is None
)
# a 404 from a wrong base_url isn't an access problem
assert (
friendly_model_error(
"gpt-5.6-sol", RuntimeError("Error code: 404 - no route /v2/chat")
)
is None
)
assert (
friendly_model_error("gpt-5.6-sol", RuntimeError("connection reset by peer"))
is None
)
def test_os_permission_error_is_not_a_model_access_error():
"""An unreadable FILE is not a billing problem.
Every OS PermissionError stringifies as "[Errno 13] Permission denied: <path>". The
marker list used to contain "permission denied" with a space, so any such error was
reported as "your account doesn't have access to <model>", sending debugging after a
provider outage that never happened. Unrecognized errors must return None so the
caller surfaces the raw message.
"""
exc = PermissionError(13, "Permission denied", "/src/.coworker/skills")
assert friendly_model_error("openrouter:moonshotai/kimi-k3", exc) is None
# The genuine provider marker (Anthropic's underscored error type) still translates.
real = RuntimeError("Error code: 403 - {'type': 'permission_error', 'message': '...'}")
msg = friendly_model_error("anthropic:claude-fable-5", real)
assert msg and "doesn't have access to anthropic:claude-fable-5" in msg
# -- friendly authentication errors ------------------------------------------------------
def test_qwen_auth_error_mentions_endpoint_mismatch():
# DashScope's 401 InvalidApiKey — the exact shape the OpenAI SDK surfaces when a
# China-portal key hits the international endpoint (or vice versa).
exc = RuntimeError(
"Error code: 401 - {'code': 'InvalidApiKey', 'message': 'Invalid API-key "
"provided.'}"
)
msg = friendly_model_error("qwen:qwen3-max", exc)
assert msg is not None
assert "qwen3-max" in msg
assert "dashscope.aliyuncs.com" in msg
assert "dashscope-intl.aliyuncs.com" in msg
assert "endpoint" in msg.lower()
def test_qwen_auth_error_works_with_bare_model_name():
# A bare "qwen3-max" (no provider prefix) should still be recognized as Qwen.
exc = RuntimeError(
"Error code: 401 - {'code': 'InvalidApiKey', 'message': 'Invalid API-key "
"provided.'}"
)
msg = friendly_model_error("qwen3-max", exc)
assert msg is not None
assert "qwen3-max" in msg
def test_generic_auth_error_for_non_qwen_provider():
# DeepSeek uses "invalid_api_key" — should get the generic key-check message,
# not the Qwen-specific endpoint guidance.
exc = RuntimeError(
"Error code: 401 - {'error': {'code': 'invalid_api_key', 'message': "
"'Incorrect API key provided.'}}"
)
msg = friendly_model_error("deepseek:deepseek-v4-pro", exc)
assert msg is not None
assert "deepseek-v4-pro" in msg
assert "Settings" in msg
# Must NOT mention DashScope endpoints for a non-Qwen provider
assert "dashscope" not in msg.lower()
def test_permission_error_is_not_treated_as_auth_failure():
# A 403 permission_error is about model access, not key validity — it must be
# caught by _NO_ACCESS, not by the auth-failure path.
exc = RuntimeError(
"Error code: 403 - {'error': {'type': 'permission_error', 'message': "
"'You do not have access to this model.'}}"
)
msg = friendly_model_error("qwen:qwen3-max", exc)
assert msg is not None
assert "doesn't have access" in msg
assert "endpoint" not in msg.lower()
def test_unrelated_401_passes_through_raw():
# A 401 that doesn't carry an invalid-key marker should NOT be dressed up.
assert (
friendly_model_error(
"qwen:qwen3-max",
RuntimeError("Error code: 401 - token expired, refresh required"),
)
is None
)