diff --git a/coworker/engine.py b/coworker/engine.py index bbfd75ce..5b002c83 100644 --- a/coworker/engine.py +++ b/coworker/engine.py @@ -896,6 +896,10 @@ def _assistant_message(turn: AssistantTurn) -> dict[str, Any]: "content": turn.text or "", "ts": time.time(), } + if turn.extras: + # Provider-private sidecars (e.g. `_gemini` thought signatures) persist with the + # message; the owning provider reattaches them, the rest strip them (base.py). + message.update(turn.extras) if turn.tool_calls: message["tool_calls"] = [ { diff --git a/coworker/providers/base.py b/coworker/providers/base.py index 032bed58..aed82282 100644 --- a/coworker/providers/base.py +++ b/coworker/providers/base.py @@ -29,6 +29,11 @@ class AssistantTurn: tool_calls: list[ToolCall] = field(default_factory=list) finish_reason: Optional[str] = None raw: Any = field(default=None, repr=False, compare=False) + # Provider-private sidecars to persist on the canonical assistant message + # (underscore-prefixed keys, e.g. `_gemini` thought signatures). Contract: the + # owning provider consumes its own key when converting history; every other + # provider must strip or ignore foreign underscore keys before its wire call. + extras: dict[str, Any] = field(default_factory=dict) @property def has_tool_calls(self) -> bool: diff --git a/coworker/providers/gemini_provider.py b/coworker/providers/gemini_provider.py index 89cdf9fd..04772a6c 100644 --- a/coworker/providers/gemini_provider.py +++ b/coworker/providers/gemini_provider.py @@ -10,10 +10,16 @@ must absorb: back by name (an id→name map built from the assistant turns during conversion). - Tool parameter schemas are an OpenAPI 3.0 subset: unsupported JSON Schema keys (`additionalProperties`, `$schema`, …) must be stripped or the API rejects the request. +- Gemini 3 thought signatures: response parts carry `thought_signature` (bytes) that MUST + be echoed back on the same parts in later requests — tool loops break without them. They + ride the canonical assistant message as the `_gemini` sidecar (base64 strings; the SDK's + `val_json_bytes="base64"` decodes them on send) and are reattached here. Parts flagged + `thought` are reasoning summaries, never answer text. """ from __future__ import annotations +import base64 import json import re from typing import Any, Optional @@ -194,22 +200,28 @@ def convert_messages( if parts: converted.append({"role": "user", "parts": parts}) elif role == "assistant": + sidecar = message.get("_gemini") or {} + call_sigs = sidecar.get("call_sigs") or [] parts = [] text = message.get("content") if isinstance(text, str) and text: - parts.append({"text": text}) - for call in message.get("tool_calls") or []: + part: dict[str, Any] = {"text": text} + if sidecar.get("text_sig"): + part["thought_signature"] = sidecar["text_sig"] + parts.append(part) + for i, call in enumerate(message.get("tool_calls") or []): function = call.get("function") or {} name = function.get("name") or "" call_names[call.get("id") or ""] = name - parts.append( - { - "function_call": { - "name": name, - "args": _parse_args(function.get("arguments")), - } + part = { + "function_call": { + "name": name, + "args": _parse_args(function.get("arguments")), } - ) + } + if i < len(call_sigs) and call_sigs[i]: + part["thought_signature"] = call_sigs[i] + parts.append(part) if parts: converted.append({"role": "model", "parts": parts}) elif role == "tool": @@ -278,21 +290,45 @@ def convert_tools(tools: Optional[list[dict[str, Any]]]) -> list[dict[str, Any]] return [{"function_declarations": declarations}] if declarations else [] -def _parse_candidate(response: Any) -> tuple[list[str], list[ToolCall], Optional[str]]: - """Pull text parts, function calls (with synthesized ids), and the finish reason out of a - GenerateContentResponse (or one streamed chunk of it).""" +def _sig_str(part: Any) -> Optional[str]: + """A part's thought signature as a base64 string (jsonl-safe; the SDK's base64 bytes + validation turns it back into the original bytes on send).""" + sig = getattr(part, "thought_signature", None) + if not sig: + return None + if isinstance(sig, (bytes, bytearray)): + return base64.b64encode(bytes(sig)).decode("ascii") + return str(sig) + + +def _signature_extras( + text_sig: Optional[str], call_sigs: list[Optional[str]] +) -> dict[str, Any]: + """Captured signatures → the `_gemini` assistant-message sidecar (empty when none).""" + if not text_sig and not any(call_sigs): + return {} + return {"_gemini": {"text_sig": text_sig, "call_sigs": call_sigs}} + + +def _parse_candidate( + response: Any, +) -> tuple[list[str], list[ToolCall], Optional[str], tuple[Optional[str], list[Optional[str]]]]: + """Pull answer text parts, function calls (with synthesized ids), the finish reason, and + thought signatures out of a GenerateContentResponse (or one streamed chunk of it). + Parts flagged `thought` are reasoning summaries — their signature is kept, their text is + NOT answer text.""" texts: list[str] = [] calls: list[ToolCall] = [] finish = None + text_sig: Optional[str] = None + call_sigs: list[Optional[str]] = [] candidates = getattr(response, "candidates", None) or [] if not candidates: - return texts, calls, finish + return texts, calls, finish, (text_sig, call_sigs) candidate = candidates[0] content = getattr(candidate, "content", None) for part in getattr(content, "parts", None) or []: - text = getattr(part, "text", None) - if text: - texts.append(text) + sig = _sig_str(part) function_call = getattr(part, "function_call", None) if function_call is not None: calls.append( @@ -302,10 +338,19 @@ def _parse_candidate(response: Any) -> tuple[list[str], list[ToolCall], Optional arguments=dict(getattr(function_call, "args", None) or {}), ) ) + call_sigs.append(sig) + continue + if sig: + text_sig = sig + if getattr(part, "thought", False): + continue + text = getattr(part, "text", None) + if text: + texts.append(text) raw_finish = getattr(candidate, "finish_reason", None) if raw_finish is not None: finish = getattr(raw_finish, "name", None) or str(raw_finish) - return texts, calls, finish + return texts, calls, finish, (text_sig, call_sigs) def _map_finish(finish: Optional[str], has_calls: bool) -> Optional[str]: @@ -384,7 +429,7 @@ class GeminiProvider(ProviderClient): model=model, messages=messages, tools=tools, settings=settings ) response = self._ensure_client().models.generate_content(**kwargs) - texts, calls, finish = _parse_candidate(response) + texts, calls, finish, (text_sig, call_sigs) = _parse_candidate(response) tool_calls = [ ToolCall(id=f"call_{i}", name=c.name, arguments=c.arguments) for i, c in enumerate(calls) @@ -394,6 +439,7 @@ class GeminiProvider(ProviderClient): tool_calls=tool_calls, finish_reason=_map_finish(finish, bool(tool_calls)), raw=response, + extras=_signature_extras(text_sig, call_sigs), ) def capabilities(self, model: str) -> ModelCapabilities: @@ -415,15 +461,22 @@ class GeminiProvider(ProviderClient): text_parts: list[str] = [] calls: list[ToolCall] = [] finish = None + text_sig: Optional[str] = None + call_sigs: list[Optional[str]] = [] # Unlike Anthropic, function_call parts arrive whole (args are a complete dict per # part), so there is no JSON accumulation — just collect parts across chunks. for chunk in client.models.generate_content_stream(**kwargs): - texts, chunk_calls, chunk_finish = _parse_candidate(chunk) + texts, chunk_calls, chunk_finish, (chunk_sig, chunk_call_sigs) = ( + _parse_candidate(chunk) + ) for text in texts: text_parts.append(text) yield StreamChunk(text_delta=text) calls.extend(chunk_calls) + call_sigs.extend(chunk_call_sigs) + if chunk_sig: + text_sig = chunk_sig if chunk_finish: finish = chunk_finish @@ -436,5 +489,6 @@ class GeminiProvider(ProviderClient): text="".join(text_parts) or None, tool_calls=tool_calls, finish_reason=_map_finish(finish, bool(tool_calls)), + extras=_signature_extras(text_sig, call_sigs), ) ) diff --git a/coworker/providers/matrix.py b/coworker/providers/matrix.py index eb0f0c65..2c234412 100644 --- a/coworker/providers/matrix.py +++ b/coworker/providers/matrix.py @@ -62,6 +62,12 @@ MATRIX: dict[str, ModelEntry] = { "anthropic:claude-haiku-4-5": ModelEntry( "Claude Haiku 4.5 · Anthropic", _AGENTIC_VISION ), + # Gemini 3 (thought signatures required in tool loops — carried via the `_gemini` + # message sidecar, see gemini_provider.py; ids from the vendor catalog 2026-07-22). + "gemini:gemini-3.1-pro-preview": ModelEntry( + "Gemini 3.1 Pro · Google", _AGENTIC_VISION + ), + "gemini:gemini-3.6-flash": ModelEntry("Gemini 3.6 Flash · Google", _AGENTIC_VISION), "gemini:gemini-2.5-pro": ModelEntry("Gemini 2.5 Pro · Google", _AGENTIC_VISION), "gemini:gemini-2.5-flash": ModelEntry("Gemini 2.5 Flash · Google", _AGENTIC_VISION), # -- direct OpenAI-compatible vendors ---------------------------------------- diff --git a/coworker/providers/openai_provider.py b/coworker/providers/openai_provider.py index 06d42269..45b0b6fe 100644 --- a/coworker/providers/openai_provider.py +++ b/coworker/providers/openai_provider.py @@ -51,6 +51,20 @@ def _pin_reasoning_effort(kwargs: dict[str, Any]) -> None: kwargs.setdefault("reasoning_effort", "none") +def _strip_foreign_sidecars(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Drop provider-private message sidecars (underscore-prefixed keys, e.g. `_gemini` + thought signatures — see providers/base.py): they belong to other providers, and the + OpenAI wire (and its compat servers) rejects unknown message fields.""" + return [ + ( + {k: v for k, v in m.items() if not k.startswith("_")} + if any(k.startswith("_") for k in m) + else m + ) + for m in messages + ] + + _MAX_TOKENS_ERROR = "'max_tokens' is not supported" @@ -122,7 +136,11 @@ class OpenAIProvider(ProviderClient): tools: Optional[list[dict[str, Any]]] = None, **settings: Any, ) -> AssistantTurn: - kwargs: dict[str, Any] = {"model": model, "messages": messages, **settings} + kwargs: dict[str, Any] = { + "model": model, + "messages": _strip_foreign_sidecars(messages), + **settings, + } if tools: kwargs["tools"] = tools _pin_reasoning_effort(kwargs) @@ -162,7 +180,7 @@ class OpenAIProvider(ProviderClient): ): kwargs: dict[str, Any] = { "model": model, - "messages": messages, + "messages": _strip_foreign_sidecars(messages), "stream": True, **settings, } diff --git a/coworker/providers/registry.py b/coworker/providers/registry.py index 0ced46e2..05897277 100644 --- a/coworker/providers/registry.py +++ b/coworker/providers/registry.py @@ -237,7 +237,7 @@ DESCRIPTORS: list[ProviderDescriptor] = [ ), ], build=_build_gemini, - recommended_model="gemini-2.5-flash", + recommended_model="gemini-3.6-flash", env_key="GEMINI_API_KEY", ), # OpenAI-compatible vendors, listed as first-class providers so users don't need to know the diff --git a/tests/test_engine.py b/tests/test_engine.py index a4134ed8..98f5537c 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -362,3 +362,22 @@ def test_outbound_keeps_pdf_for_native_models(tmp_path): } engine.messages.append(message) assert engine._outbound_messages()[-1]["content"][1]["type"] == "file" + + +def test_provider_extras_persist_on_message_and_survive_outbound(tmp_path): + """A turn's provider-private sidecar (`extras`, e.g. Gemini thought signatures) rides + the persisted assistant message and is NOT stripped by _outbound_messages — the owning + provider needs it back; foreign providers strip it themselves.""" + turn = AssistantTurn( + text="ok", + finish_reason="stop", + extras={"_gemini": {"text_sig": "c2ln", "call_sigs": []}}, + ) + engine, _ = _engine(tmp_path, [turn]) + _collect(engine, "hi") + + persisted = engine.messages[-1] + assert persisted["_gemini"] == {"text_sig": "c2ln", "call_sigs": []} + outbound = engine._outbound_messages()[-1] + assert outbound["_gemini"] == {"text_sig": "c2ln", "call_sigs": []} + assert "ts" not in outbound # display sidecars still stripped diff --git a/tests/test_gemini_provider.py b/tests/test_gemini_provider.py index 7d49153e..4759d628 100644 --- a/tests/test_gemini_provider.py +++ b/tests/test_gemini_provider.py @@ -501,3 +501,106 @@ def test_convert_pdf_file_part_to_inline_data(): "inline_data": {"mime_type": "application/pdf", "data": "JVBERi0="} } assert parts[2] == {"text": "[unsupported file attachment]"} + +# -- Gemini 3 thought signatures (2026-07 roadmap item 2) --------------------------- + + +def _sig_call_part(name, args, sig): + return SimpleNamespace( + text=None, + function_call=SimpleNamespace(name=name, args=args), + thought_signature=sig, + ) + + +def _thought_part(text, sig=None): + return SimpleNamespace( + text=text, function_call=None, thought=True, thought_signature=sig + ) + + +def test_complete_captures_signatures_and_filters_thought_parts(): + response = _response( + [ + _thought_part("secret reasoning", sig=b"tsig"), + SimpleNamespace(text="the answer", function_call=None, thought_signature=None), + _sig_call_part("run_shell", {"command": "ls"}, b"csig"), + ] + ) + provider = GeminiProvider(client=_FakeClient(response=response)) + turn = provider.complete(model="m", messages=[{"role": "user", "content": "x"}]) + + assert turn.text == "the answer" # thought text never leaks into the answer + assert turn.tool_calls[0].name == "run_shell" + import base64 + + sidecar = turn.extras["_gemini"] + assert sidecar["text_sig"] == base64.b64encode(b"tsig").decode() + assert sidecar["call_sigs"] == [base64.b64encode(b"csig").decode()] + + +def test_complete_without_signatures_has_no_extras(): + response = _response([_text_part("plain")]) + provider = GeminiProvider(client=_FakeClient(response=response)) + turn = provider.complete(model="m", messages=[{"role": "user", "content": "x"}]) + assert turn.extras == {} + + +def test_stream_accumulates_signatures_across_chunks(): + chunks = [ + _response([_text_part("hi ")], finish_reason=None), + _response([_sig_call_part("t1", {}, b"s1")], finish_reason=None), + _response([_sig_call_part("t2", {}, None)], finish_reason="STOP"), + ] + provider = GeminiProvider(client=_FakeClient(chunks=chunks)) + final = list(provider.stream(model="m", messages=[{"role": "user", "content": "x"}]))[-1].turn + import base64 + + assert [c.name for c in final.tool_calls] == ["t1", "t2"] + assert final.extras["_gemini"]["call_sigs"] == [ + base64.b64encode(b"s1").decode(), + None, + ] + + +def test_convert_reattaches_signatures_to_parts(): + system, contents = convert_messages( + [ + {"role": "user", "content": "do it"}, + { + "role": "assistant", + "content": "on it", + "_gemini": {"text_sig": "dHNpZw==", "call_sigs": [None, "Y3NpZw=="]}, + "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}, + {"id": "call_1", "type": "function", "function": {"name": "b", "arguments": "{}"}}, + ], + }, + ] + ) + parts = contents[-1]["parts"] + assert parts[0] == {"text": "on it", "thought_signature": "dHNpZw=="} + assert "thought_signature" not in parts[1] # first call had no signature + assert parts[2]["function_call"]["name"] == "b" + assert parts[2]["thought_signature"] == "Y3NpZw==" + + +def test_convert_signature_parts_validate_as_sdk_types(): + """The dict parts we emit (base64-string signatures) must round-trip through the real + SDK Part model — its base64 bytes-validation is what decodes them on send.""" + types_mod = pytest.importorskip("google.genai.types") + _, contents = convert_messages( + [ + {"role": "user", "content": "go"}, + { + "role": "assistant", + "content": "", + "_gemini": {"text_sig": None, "call_sigs": ["c2ln"]}, + "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}, + ], + }, + ] + ) + part = types_mod.Part.model_validate(contents[-1]["parts"][0]) + assert part.thought_signature == b"sig" diff --git a/tests/test_providers.py b/tests/test_providers.py index 88423362..94281f11 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -415,3 +415,19 @@ def test_reseller_descriptors_and_matrix_stay_in_lockstep(): # full ids in the matrix must round-trip: prefix + bare == matrix key base = next(f for f in d.fields if f.key == "base_url") assert base.default.startswith("https://") + + +def test_foreign_sidecars_stripped_from_outbound_messages(): + """Provider-private sidecars (`_gemini` thought signatures et al) must never reach the + OpenAI wire — it and its compat servers reject unknown message fields.""" + client = _FakeClient(_response(content="ok")) + provider = OpenAIProvider(client=client) + provider.complete( + model="gpt-5.5", + messages=[ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "prev", "_gemini": {"call_sigs": ["x"]}}, + ], + ) + sent = client.chat.completions.calls[0]["messages"] + assert sent[1] == {"role": "assistant", "content": "prev"}