mirror of
https://github.com/andrewyng/openworker.git
synced 2026-09-03 04:49:26 +00:00
Add Gemini 3 models with thought-signature support
Signatures ride the assistant message as a _gemini sidecar and are reattached in tool loops. Thought-flagged parts are filtered from answer text; foreign sidecars stripped on the OpenAI wire. Live-verified both Gemini 3 models end-to-end; without the echo they 400 on every tool loop.
This commit is contained in:
@@ -896,6 +896,10 @@ def _assistant_message(turn: AssistantTurn) -> dict[str, Any]:
|
||||
"content": turn.text or "",
|
||||
"ts": time.time(),
|
||||
}
|
||||
if turn.extras:
|
||||
# Provider-private sidecars (e.g. `_gemini` thought signatures) persist with the
|
||||
# message; the owning provider reattaches them, the rest strip them (base.py).
|
||||
message.update(turn.extras)
|
||||
if turn.tool_calls:
|
||||
message["tool_calls"] = [
|
||||
{
|
||||
|
||||
@@ -29,6 +29,11 @@ class AssistantTurn:
|
||||
tool_calls: list[ToolCall] = field(default_factory=list)
|
||||
finish_reason: Optional[str] = None
|
||||
raw: Any = field(default=None, repr=False, compare=False)
|
||||
# Provider-private sidecars to persist on the canonical assistant message
|
||||
# (underscore-prefixed keys, e.g. `_gemini` thought signatures). Contract: the
|
||||
# owning provider consumes its own key when converting history; every other
|
||||
# provider must strip or ignore foreign underscore keys before its wire call.
|
||||
extras: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def has_tool_calls(self) -> bool:
|
||||
|
||||
@@ -10,10 +10,16 @@ must absorb:
|
||||
back by name (an id→name map built from the assistant turns during conversion).
|
||||
- Tool parameter schemas are an OpenAPI 3.0 subset: unsupported JSON Schema keys
|
||||
(`additionalProperties`, `$schema`, …) must be stripped or the API rejects the request.
|
||||
- Gemini 3 thought signatures: response parts carry `thought_signature` (bytes) that MUST
|
||||
be echoed back on the same parts in later requests — tool loops break without them. They
|
||||
ride the canonical assistant message as the `_gemini` sidecar (base64 strings; the SDK's
|
||||
`val_json_bytes="base64"` decodes them on send) and are reattached here. Parts flagged
|
||||
`thought` are reasoning summaries, never answer text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Optional
|
||||
@@ -194,22 +200,28 @@ def convert_messages(
|
||||
if parts:
|
||||
converted.append({"role": "user", "parts": parts})
|
||||
elif role == "assistant":
|
||||
sidecar = message.get("_gemini") or {}
|
||||
call_sigs = sidecar.get("call_sigs") or []
|
||||
parts = []
|
||||
text = message.get("content")
|
||||
if isinstance(text, str) and text:
|
||||
parts.append({"text": text})
|
||||
for call in message.get("tool_calls") or []:
|
||||
part: dict[str, Any] = {"text": text}
|
||||
if sidecar.get("text_sig"):
|
||||
part["thought_signature"] = sidecar["text_sig"]
|
||||
parts.append(part)
|
||||
for i, call in enumerate(message.get("tool_calls") or []):
|
||||
function = call.get("function") or {}
|
||||
name = function.get("name") or ""
|
||||
call_names[call.get("id") or ""] = name
|
||||
parts.append(
|
||||
{
|
||||
"function_call": {
|
||||
"name": name,
|
||||
"args": _parse_args(function.get("arguments")),
|
||||
}
|
||||
part = {
|
||||
"function_call": {
|
||||
"name": name,
|
||||
"args": _parse_args(function.get("arguments")),
|
||||
}
|
||||
)
|
||||
}
|
||||
if i < len(call_sigs) and call_sigs[i]:
|
||||
part["thought_signature"] = call_sigs[i]
|
||||
parts.append(part)
|
||||
if parts:
|
||||
converted.append({"role": "model", "parts": parts})
|
||||
elif role == "tool":
|
||||
@@ -278,21 +290,45 @@ def convert_tools(tools: Optional[list[dict[str, Any]]]) -> list[dict[str, Any]]
|
||||
return [{"function_declarations": declarations}] if declarations else []
|
||||
|
||||
|
||||
def _parse_candidate(response: Any) -> tuple[list[str], list[ToolCall], Optional[str]]:
|
||||
"""Pull text parts, function calls (with synthesized ids), and the finish reason out of a
|
||||
GenerateContentResponse (or one streamed chunk of it)."""
|
||||
def _sig_str(part: Any) -> Optional[str]:
|
||||
"""A part's thought signature as a base64 string (jsonl-safe; the SDK's base64 bytes
|
||||
validation turns it back into the original bytes on send)."""
|
||||
sig = getattr(part, "thought_signature", None)
|
||||
if not sig:
|
||||
return None
|
||||
if isinstance(sig, (bytes, bytearray)):
|
||||
return base64.b64encode(bytes(sig)).decode("ascii")
|
||||
return str(sig)
|
||||
|
||||
|
||||
def _signature_extras(
|
||||
text_sig: Optional[str], call_sigs: list[Optional[str]]
|
||||
) -> dict[str, Any]:
|
||||
"""Captured signatures → the `_gemini` assistant-message sidecar (empty when none)."""
|
||||
if not text_sig and not any(call_sigs):
|
||||
return {}
|
||||
return {"_gemini": {"text_sig": text_sig, "call_sigs": call_sigs}}
|
||||
|
||||
|
||||
def _parse_candidate(
|
||||
response: Any,
|
||||
) -> tuple[list[str], list[ToolCall], Optional[str], tuple[Optional[str], list[Optional[str]]]]:
|
||||
"""Pull answer text parts, function calls (with synthesized ids), the finish reason, and
|
||||
thought signatures out of a GenerateContentResponse (or one streamed chunk of it).
|
||||
Parts flagged `thought` are reasoning summaries — their signature is kept, their text is
|
||||
NOT answer text."""
|
||||
texts: list[str] = []
|
||||
calls: list[ToolCall] = []
|
||||
finish = None
|
||||
text_sig: Optional[str] = None
|
||||
call_sigs: list[Optional[str]] = []
|
||||
candidates = getattr(response, "candidates", None) or []
|
||||
if not candidates:
|
||||
return texts, calls, finish
|
||||
return texts, calls, finish, (text_sig, call_sigs)
|
||||
candidate = candidates[0]
|
||||
content = getattr(candidate, "content", None)
|
||||
for part in getattr(content, "parts", None) or []:
|
||||
text = getattr(part, "text", None)
|
||||
if text:
|
||||
texts.append(text)
|
||||
sig = _sig_str(part)
|
||||
function_call = getattr(part, "function_call", None)
|
||||
if function_call is not None:
|
||||
calls.append(
|
||||
@@ -302,10 +338,19 @@ def _parse_candidate(response: Any) -> tuple[list[str], list[ToolCall], Optional
|
||||
arguments=dict(getattr(function_call, "args", None) or {}),
|
||||
)
|
||||
)
|
||||
call_sigs.append(sig)
|
||||
continue
|
||||
if sig:
|
||||
text_sig = sig
|
||||
if getattr(part, "thought", False):
|
||||
continue
|
||||
text = getattr(part, "text", None)
|
||||
if text:
|
||||
texts.append(text)
|
||||
raw_finish = getattr(candidate, "finish_reason", None)
|
||||
if raw_finish is not None:
|
||||
finish = getattr(raw_finish, "name", None) or str(raw_finish)
|
||||
return texts, calls, finish
|
||||
return texts, calls, finish, (text_sig, call_sigs)
|
||||
|
||||
|
||||
def _map_finish(finish: Optional[str], has_calls: bool) -> Optional[str]:
|
||||
@@ -384,7 +429,7 @@ class GeminiProvider(ProviderClient):
|
||||
model=model, messages=messages, tools=tools, settings=settings
|
||||
)
|
||||
response = self._ensure_client().models.generate_content(**kwargs)
|
||||
texts, calls, finish = _parse_candidate(response)
|
||||
texts, calls, finish, (text_sig, call_sigs) = _parse_candidate(response)
|
||||
tool_calls = [
|
||||
ToolCall(id=f"call_{i}", name=c.name, arguments=c.arguments)
|
||||
for i, c in enumerate(calls)
|
||||
@@ -394,6 +439,7 @@ class GeminiProvider(ProviderClient):
|
||||
tool_calls=tool_calls,
|
||||
finish_reason=_map_finish(finish, bool(tool_calls)),
|
||||
raw=response,
|
||||
extras=_signature_extras(text_sig, call_sigs),
|
||||
)
|
||||
|
||||
def capabilities(self, model: str) -> ModelCapabilities:
|
||||
@@ -415,15 +461,22 @@ class GeminiProvider(ProviderClient):
|
||||
text_parts: list[str] = []
|
||||
calls: list[ToolCall] = []
|
||||
finish = None
|
||||
text_sig: Optional[str] = None
|
||||
call_sigs: list[Optional[str]] = []
|
||||
|
||||
# Unlike Anthropic, function_call parts arrive whole (args are a complete dict per
|
||||
# part), so there is no JSON accumulation — just collect parts across chunks.
|
||||
for chunk in client.models.generate_content_stream(**kwargs):
|
||||
texts, chunk_calls, chunk_finish = _parse_candidate(chunk)
|
||||
texts, chunk_calls, chunk_finish, (chunk_sig, chunk_call_sigs) = (
|
||||
_parse_candidate(chunk)
|
||||
)
|
||||
for text in texts:
|
||||
text_parts.append(text)
|
||||
yield StreamChunk(text_delta=text)
|
||||
calls.extend(chunk_calls)
|
||||
call_sigs.extend(chunk_call_sigs)
|
||||
if chunk_sig:
|
||||
text_sig = chunk_sig
|
||||
if chunk_finish:
|
||||
finish = chunk_finish
|
||||
|
||||
@@ -436,5 +489,6 @@ class GeminiProvider(ProviderClient):
|
||||
text="".join(text_parts) or None,
|
||||
tool_calls=tool_calls,
|
||||
finish_reason=_map_finish(finish, bool(tool_calls)),
|
||||
extras=_signature_extras(text_sig, call_sigs),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -62,6 +62,12 @@ MATRIX: dict[str, ModelEntry] = {
|
||||
"anthropic:claude-haiku-4-5": ModelEntry(
|
||||
"Claude Haiku 4.5 · Anthropic", _AGENTIC_VISION
|
||||
),
|
||||
# Gemini 3 (thought signatures required in tool loops — carried via the `_gemini`
|
||||
# message sidecar, see gemini_provider.py; ids from the vendor catalog 2026-07-22).
|
||||
"gemini:gemini-3.1-pro-preview": ModelEntry(
|
||||
"Gemini 3.1 Pro · Google", _AGENTIC_VISION
|
||||
),
|
||||
"gemini:gemini-3.6-flash": ModelEntry("Gemini 3.6 Flash · Google", _AGENTIC_VISION),
|
||||
"gemini:gemini-2.5-pro": ModelEntry("Gemini 2.5 Pro · Google", _AGENTIC_VISION),
|
||||
"gemini:gemini-2.5-flash": ModelEntry("Gemini 2.5 Flash · Google", _AGENTIC_VISION),
|
||||
# -- direct OpenAI-compatible vendors ----------------------------------------
|
||||
|
||||
@@ -51,6 +51,20 @@ def _pin_reasoning_effort(kwargs: dict[str, Any]) -> None:
|
||||
kwargs.setdefault("reasoning_effort", "none")
|
||||
|
||||
|
||||
def _strip_foreign_sidecars(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Drop provider-private message sidecars (underscore-prefixed keys, e.g. `_gemini`
|
||||
thought signatures — see providers/base.py): they belong to other providers, and the
|
||||
OpenAI wire (and its compat servers) rejects unknown message fields."""
|
||||
return [
|
||||
(
|
||||
{k: v for k, v in m.items() if not k.startswith("_")}
|
||||
if any(k.startswith("_") for k in m)
|
||||
else m
|
||||
)
|
||||
for m in messages
|
||||
]
|
||||
|
||||
|
||||
_MAX_TOKENS_ERROR = "'max_tokens' is not supported"
|
||||
|
||||
|
||||
@@ -122,7 +136,11 @@ class OpenAIProvider(ProviderClient):
|
||||
tools: Optional[list[dict[str, Any]]] = None,
|
||||
**settings: Any,
|
||||
) -> AssistantTurn:
|
||||
kwargs: dict[str, Any] = {"model": model, "messages": messages, **settings}
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": _strip_foreign_sidecars(messages),
|
||||
**settings,
|
||||
}
|
||||
if tools:
|
||||
kwargs["tools"] = tools
|
||||
_pin_reasoning_effort(kwargs)
|
||||
@@ -162,7 +180,7 @@ class OpenAIProvider(ProviderClient):
|
||||
):
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"messages": _strip_foreign_sidecars(messages),
|
||||
"stream": True,
|
||||
**settings,
|
||||
}
|
||||
|
||||
@@ -237,7 +237,7 @@ DESCRIPTORS: list[ProviderDescriptor] = [
|
||||
),
|
||||
],
|
||||
build=_build_gemini,
|
||||
recommended_model="gemini-2.5-flash",
|
||||
recommended_model="gemini-3.6-flash",
|
||||
env_key="GEMINI_API_KEY",
|
||||
),
|
||||
# OpenAI-compatible vendors, listed as first-class providers so users don't need to know the
|
||||
|
||||
@@ -362,3 +362,22 @@ def test_outbound_keeps_pdf_for_native_models(tmp_path):
|
||||
}
|
||||
engine.messages.append(message)
|
||||
assert engine._outbound_messages()[-1]["content"][1]["type"] == "file"
|
||||
|
||||
|
||||
def test_provider_extras_persist_on_message_and_survive_outbound(tmp_path):
|
||||
"""A turn's provider-private sidecar (`extras`, e.g. Gemini thought signatures) rides
|
||||
the persisted assistant message and is NOT stripped by _outbound_messages — the owning
|
||||
provider needs it back; foreign providers strip it themselves."""
|
||||
turn = AssistantTurn(
|
||||
text="ok",
|
||||
finish_reason="stop",
|
||||
extras={"_gemini": {"text_sig": "c2ln", "call_sigs": []}},
|
||||
)
|
||||
engine, _ = _engine(tmp_path, [turn])
|
||||
_collect(engine, "hi")
|
||||
|
||||
persisted = engine.messages[-1]
|
||||
assert persisted["_gemini"] == {"text_sig": "c2ln", "call_sigs": []}
|
||||
outbound = engine._outbound_messages()[-1]
|
||||
assert outbound["_gemini"] == {"text_sig": "c2ln", "call_sigs": []}
|
||||
assert "ts" not in outbound # display sidecars still stripped
|
||||
|
||||
@@ -501,3 +501,106 @@ def test_convert_pdf_file_part_to_inline_data():
|
||||
"inline_data": {"mime_type": "application/pdf", "data": "JVBERi0="}
|
||||
}
|
||||
assert parts[2] == {"text": "[unsupported file attachment]"}
|
||||
|
||||
# -- Gemini 3 thought signatures (2026-07 roadmap item 2) ---------------------------
|
||||
|
||||
|
||||
def _sig_call_part(name, args, sig):
|
||||
return SimpleNamespace(
|
||||
text=None,
|
||||
function_call=SimpleNamespace(name=name, args=args),
|
||||
thought_signature=sig,
|
||||
)
|
||||
|
||||
|
||||
def _thought_part(text, sig=None):
|
||||
return SimpleNamespace(
|
||||
text=text, function_call=None, thought=True, thought_signature=sig
|
||||
)
|
||||
|
||||
|
||||
def test_complete_captures_signatures_and_filters_thought_parts():
|
||||
response = _response(
|
||||
[
|
||||
_thought_part("secret reasoning", sig=b"tsig"),
|
||||
SimpleNamespace(text="the answer", function_call=None, thought_signature=None),
|
||||
_sig_call_part("run_shell", {"command": "ls"}, b"csig"),
|
||||
]
|
||||
)
|
||||
provider = GeminiProvider(client=_FakeClient(response=response))
|
||||
turn = provider.complete(model="m", messages=[{"role": "user", "content": "x"}])
|
||||
|
||||
assert turn.text == "the answer" # thought text never leaks into the answer
|
||||
assert turn.tool_calls[0].name == "run_shell"
|
||||
import base64
|
||||
|
||||
sidecar = turn.extras["_gemini"]
|
||||
assert sidecar["text_sig"] == base64.b64encode(b"tsig").decode()
|
||||
assert sidecar["call_sigs"] == [base64.b64encode(b"csig").decode()]
|
||||
|
||||
|
||||
def test_complete_without_signatures_has_no_extras():
|
||||
response = _response([_text_part("plain")])
|
||||
provider = GeminiProvider(client=_FakeClient(response=response))
|
||||
turn = provider.complete(model="m", messages=[{"role": "user", "content": "x"}])
|
||||
assert turn.extras == {}
|
||||
|
||||
|
||||
def test_stream_accumulates_signatures_across_chunks():
|
||||
chunks = [
|
||||
_response([_text_part("hi ")], finish_reason=None),
|
||||
_response([_sig_call_part("t1", {}, b"s1")], finish_reason=None),
|
||||
_response([_sig_call_part("t2", {}, None)], finish_reason="STOP"),
|
||||
]
|
||||
provider = GeminiProvider(client=_FakeClient(chunks=chunks))
|
||||
final = list(provider.stream(model="m", messages=[{"role": "user", "content": "x"}]))[-1].turn
|
||||
import base64
|
||||
|
||||
assert [c.name for c in final.tool_calls] == ["t1", "t2"]
|
||||
assert final.extras["_gemini"]["call_sigs"] == [
|
||||
base64.b64encode(b"s1").decode(),
|
||||
None,
|
||||
]
|
||||
|
||||
|
||||
def test_convert_reattaches_signatures_to_parts():
|
||||
system, contents = convert_messages(
|
||||
[
|
||||
{"role": "user", "content": "do it"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "on it",
|
||||
"_gemini": {"text_sig": "dHNpZw==", "call_sigs": [None, "Y3NpZw=="]},
|
||||
"tool_calls": [
|
||||
{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}},
|
||||
{"id": "call_1", "type": "function", "function": {"name": "b", "arguments": "{}"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
)
|
||||
parts = contents[-1]["parts"]
|
||||
assert parts[0] == {"text": "on it", "thought_signature": "dHNpZw=="}
|
||||
assert "thought_signature" not in parts[1] # first call had no signature
|
||||
assert parts[2]["function_call"]["name"] == "b"
|
||||
assert parts[2]["thought_signature"] == "Y3NpZw=="
|
||||
|
||||
|
||||
def test_convert_signature_parts_validate_as_sdk_types():
|
||||
"""The dict parts we emit (base64-string signatures) must round-trip through the real
|
||||
SDK Part model — its base64 bytes-validation is what decodes them on send."""
|
||||
types_mod = pytest.importorskip("google.genai.types")
|
||||
_, contents = convert_messages(
|
||||
[
|
||||
{"role": "user", "content": "go"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"_gemini": {"text_sig": None, "call_sigs": ["c2ln"]},
|
||||
"tool_calls": [
|
||||
{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
)
|
||||
part = types_mod.Part.model_validate(contents[-1]["parts"][0])
|
||||
assert part.thought_signature == b"sig"
|
||||
|
||||
@@ -415,3 +415,19 @@ def test_reseller_descriptors_and_matrix_stay_in_lockstep():
|
||||
# full ids in the matrix must round-trip: prefix + bare == matrix key
|
||||
base = next(f for f in d.fields if f.key == "base_url")
|
||||
assert base.default.startswith("https://")
|
||||
|
||||
|
||||
def test_foreign_sidecars_stripped_from_outbound_messages():
|
||||
"""Provider-private sidecars (`_gemini` thought signatures et al) must never reach the
|
||||
OpenAI wire — it and its compat servers reject unknown message fields."""
|
||||
client = _FakeClient(_response(content="ok"))
|
||||
provider = OpenAIProvider(client=client)
|
||||
provider.complete(
|
||||
model="gpt-5.5",
|
||||
messages=[
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "prev", "_gemini": {"call_sigs": ["x"]}},
|
||||
],
|
||||
)
|
||||
sent = client.chat.completions.calls[0]["messages"]
|
||||
assert sent[1] == {"role": "assistant", "content": "prev"}
|
||||
|
||||
Reference in New Issue
Block a user