From a990f6872fd45f09de2ef9e96d76e167223fed19 Mon Sep 17 00:00:00 2001 From: Devika Verma Date: Mon, 17 Aug 2026 22:57:18 +0530 Subject: [PATCH] OPE-101: extract token usage in the OpenAI Responses provider Any model routed through /v1/responses (gpt-5.6+ with tools, i.e. the default gpt-5.6-sol) reported 0 tokens for every call: both AssistantTurn constructions omitted usage while every sibling adapter populated TokenUsage. First consumer to notice was the Auto-Approve reviewer metering; the 2026-08-13 eval report printed 'Tokens: 0 in / 0 out' across ~40 live calls. _usage_from maps the Responses shape like the Chat Completions adapter: fresh input = input_tokens - cached_tokens (input_tokens is INCLUSIVE of the cached share), cached share -> cache_read, output_tokens as-is (reasoning already included). Defensive reads: compat servers may omit input_tokens_details; a missing usage object stays None, never a fake zero. Wired in _parse_response, which both complete() and the stream's terminal response.completed event flow through - one extraction, both paths. Tests: cache split, partial/missing degradation, stream terminal event, stream-without-terminal stays None. Verified live against gpt-5.6-sol: call 1 'in=1820 cache_read=0', call 2 'in=3 cache_read=1817' - the 0-token era ends and the cache split reports correctly. --- coworker/providers/openai_responses.py | 20 ++++++++++ tests/test_openai_responses.py | 55 ++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/coworker/providers/openai_responses.py b/coworker/providers/openai_responses.py index 566c620b..8c673841 100644 --- a/coworker/providers/openai_responses.py +++ b/coworker/providers/openai_responses.py @@ -40,6 +40,7 @@ from .base import ( ModelCapabilities, ProviderClient, StreamChunk, + TokenUsage, ToolCall, ) from .capabilities import capabilities_for @@ -233,6 +234,24 @@ def _sidecar_extras(items: list[dict[str, Any]]) -> dict[str, Any]: return {} +def _usage_from(usage: Any) -> Optional[TokenUsage]: + """Responses-API usage → normalized counts (OPE-101). `input_tokens` INCLUDES the + cached share, so fresh input = input_tokens − cached_tokens (the same convention as + the Chat Completions and Anthropic adapters); `output_tokens` already includes + reasoning tokens (billed as output). Defensive reads throughout — compat/older + servers may omit `input_tokens_details`.""" + if usage is None: + return None + prompt = int(getattr(usage, "input_tokens", 0) or 0) + details = getattr(usage, "input_tokens_details", None) + cached = int(getattr(details, "cached_tokens", 0) or 0) + return TokenUsage( + input=max(prompt - cached, 0), + output=int(getattr(usage, "output_tokens", 0) or 0), + cache_read=cached, + ) + + def _parse_response(response: Any) -> AssistantTurn: """One Responses result → an AssistantTurn (+ `_openai` extras).""" items = [_dump(item) for item in getattr(response, "output", None) or []] @@ -278,6 +297,7 @@ def _parse_response(response: Any) -> AssistantTurn: raw=response, reasoning="".join(summaries) or None, extras=_sidecar_extras(items), + usage=_usage_from(getattr(response, "usage", None)), ) diff --git a/tests/test_openai_responses.py b/tests/test_openai_responses.py index 9e871d57..952232bc 100644 --- a/tests/test_openai_responses.py +++ b/tests/test_openai_responses.py @@ -273,6 +273,39 @@ def test_complete_text_turn_and_request_shape(): assert fake.kwargs["reasoning"] == {"summary": "auto"} +def test_complete_extracts_usage_with_cache_split(): + # OPE-101: the Responses API reports `input_tokens` INCLUSIVE of the cached share; + # normalized like every other adapter — fresh input = input − cached, cache_read + # carries the cached share. Before this, the field was dropped entirely and every + # Responses-routed model metered as 0 tokens. + resp = _response([_message_item("hello")]) + resp.usage = SimpleNamespace( + input_tokens=1500, + output_tokens=80, + input_tokens_details=SimpleNamespace(cached_tokens=1400), + ) + provider = OpenAIResponsesProvider(client=_FakeClient(response=resp)) + turn = provider.complete(model="m", messages=[{"role": "user", "content": "hi"}]) + assert turn.usage is not None + assert (turn.usage.input, turn.usage.output, turn.usage.cache_read) == (100, 80, 1400) + + +def test_complete_usage_degrades_on_partial_or_missing_fields(): + # Compat/older servers may omit `input_tokens_details` or the whole usage object — + # never a crash, and absence stays None (not a fake zero-usage). + resp = _response([_message_item("x")]) + resp.usage = SimpleNamespace(input_tokens=500, output_tokens=20) # no details + provider = OpenAIResponsesProvider(client=_FakeClient(response=resp)) + turn = provider.complete(model="m", messages=[{"role": "user", "content": "hi"}]) + assert (turn.usage.input, turn.usage.output, turn.usage.cache_read) == (500, 20, 0) + + bare = _response([_message_item("y")]) # SimpleNamespace without a usage attr at all + turn2 = OpenAIResponsesProvider(client=_FakeClient(response=bare)).complete( + model="m", messages=[{"role": "user", "content": "hi"}] + ) + assert turn2.usage is None + + def test_complete_parses_function_calls_with_call_ids(): fake = _FakeClient( response=_response( @@ -498,6 +531,28 @@ def test_stream_without_terminal_event_keeps_accumulated_text(): provider.stream(model="m", messages=[{"role": "user", "content": "x"}]) )[-1].turn assert turn.text == "partial" and turn.finish_reason is None + assert turn.usage is None # nothing terminal arrived — no usage to invent + + +def test_stream_terminal_event_carries_usage(): + # OPE-101 streaming path: usage rides the terminal `response.completed` event's full + # response object, which the stream parses whole — same extraction as complete(). + final = _response([_message_item("done")]) + final.usage = SimpleNamespace( + input_tokens=1430, + output_tokens=65, + input_tokens_details=SimpleNamespace(cached_tokens=1408), + ) + events = [ + SimpleNamespace(type="response.output_text.delta", delta="done"), + SimpleNamespace(type="response.completed", response=final), + ] + provider = OpenAIResponsesProvider(client=_FakeClient(events=events)) + turn = list( + provider.stream(model="m", messages=[{"role": "user", "content": "x"}]) + )[-1].turn + assert turn.usage is not None + assert (turn.usage.input, turn.usage.output, turn.usage.cache_read) == (22, 65, 1408) def test_stream_requests_stream_flag():