diff --git a/.gitignore b/.gitignore index dc328dd6c8..ff24609488 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea +.review .vscode .zed .history @@ -20,7 +21,7 @@ tiktoken_cache .gocache .gomodcache/ .cache -plans +.plans .claude .cursor @@ -37,7 +38,7 @@ skills-lock.json # Local-only live probes and scratch test workspaces. .local-tests/ -service/relayconvert/chat_responses_live_local_test.go +relaykit/relayconvert/chat_responses_live_local_test.go service/openaicompat/chat_responses_live_local_test.go go.work go.work.sum diff --git a/controller/channel-test.go b/controller/channel-test.go index 895099a106..fca5264bb9 100644 --- a/controller/channel-test.go +++ b/controller/channel-test.go @@ -259,6 +259,13 @@ func testChannel(ctx context.Context, channel *model.Channel, testUserID int, te newAPIError: types.NewError(err, types.ErrorCodeChannelModelMappedError), } } + if err = helper.ApplyReasoningModelSuffix(info, request); err != nil { + return testResult{ + context: c, + localErr: err, + newAPIError: types.NewErrorWithStatusCode(err, types.ErrorCodeConvertRequestFailed, http.StatusBadRequest, types.ErrOptionWithSkipRetry()), + } + } testModel = info.UpstreamModelName // 更新请求中的模型名称 @@ -943,7 +950,7 @@ func testChannelForHealthCheck(ctx context.Context, channel *model.Channel, test } if allowDisable && isChannelEnabled && shouldBanChannel && channel.GetAutoBan() { - processChannelError(result.context, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError) + processChannelError(result.context, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError, nil) summary.Disabled++ } diff --git a/controller/relay.go b/controller/relay.go index a678888d93..099b3fb70d 100644 --- a/controller/relay.go +++ b/controller/relay.go @@ -238,7 +238,7 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { newAPIError = service.NormalizeViolationFeeError(newAPIError) relayInfo.LastError = newAPIError - processChannelError(c, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(c, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError) + processChannelError(c, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(c, constant.ContextKeyChannelKey), channel.GetAutoBan()), newAPIError, relayInfo) if !shouldRetry(c, newAPIError, common.RetryTimes-retryParam.GetRetry()) { break @@ -257,6 +257,38 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) { } } +// CountClaudeTokens implements Anthropic's token-counting utility endpoint. +// It deliberately skips upstream generation and billing; callers use this +// endpoint to size prompts before creating a Message. +func CountClaudeTokens(c *gin.Context) { + request, err := helper.GetAndValidateClaudeRequest(c) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "type": "error", + "error": gin.H{ + "type": "invalid_request_error", + "message": common.MessageWithRequestId(err.Error(), c.GetString(common.RequestIdKey)), + }, + }) + return + } + + info := relaycommon.GenRelayInfoClaude(c, request) + inputTokens, err := service.CountRequestToken(c, request.GetTokenCountMeta(), info) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "type": "error", + "error": gin.H{ + "type": "api_error", + "message": common.MessageWithRequestId(err.Error(), c.GetString(common.RequestIdKey)), + }, + }) + return + } + + c.JSON(http.StatusOK, gin.H{"input_tokens": inputTokens}) +} + var upgrader = websocket.Upgrader{ Subprotocols: []string{"realtime"}, // WS 握手支持的协议，如果有使用 Sec-WebSocket-Protocol，则必须在此声明对应的 Protocol TODO add other protocol CheckOrigin: func(r *http.Request) bool { @@ -362,7 +394,7 @@ func shouldRetry(c *gin.Context, openaiErr *types.NewAPIError, retryTimes int) b return operation_setting.ShouldRetryByStatusCode(code) } -func processChannelError(c *gin.Context, channelError types.ChannelError, err *types.NewAPIError) { +func processChannelError(c *gin.Context, channelError types.ChannelError, err *types.NewAPIError, relayInfo *relaycommon.RelayInfo) { logger.LogError(c, fmt.Sprintf("channel error (channel #%d, status code: %d): %s", channelError.ChannelId, err.StatusCode, common.LocalLogPreview(err.Error()))) // 不要使用context获取渠道信息，异步处理时可能会出现渠道信息不一致的情况 // do not use context to get channel info, there may be inconsistent channel info when processing asynchronously @@ -392,6 +424,14 @@ func processChannelError(c *gin.Context, channelError types.ChannelError, err *t other["channel_type"] = c.GetInt("channel_type") adminInfo := make(map[string]interface{}) adminInfo["use_channel"] = c.GetStringSlice("use_channel") + if relayInfo != nil { + if diagnostics := relayInfo.ConversionDiagnostics(); len(diagnostics) > 0 { + adminInfo["conversion_diagnostics"] = diagnostics + } + if relayInfo.ConversionDiagnosticsTruncated() { + adminInfo["conversion_diagnostics_truncated"] = true + } + } isMultiKey := common.GetContextKeyBool(c, constant.ContextKeyChannelIsMultiKey) if isMultiKey { adminInfo["is_multi_key"] = true @@ -655,7 +695,8 @@ func executeTaskSubmissionWith( processChannelError(c, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(c, constant.ContextKeyChannelKey), channel.GetAutoBan()), - types.NewOpenAIError(taskErr.Error, types.ErrorCodeBadResponseStatusCode, taskErr.StatusCode)) + types.NewOpenAIError(taskErr.Error, types.ErrorCodeBadResponseStatusCode, taskErr.StatusCode), + relayInfo) } willRetry := shouldRetryTaskRelay(c, channel.Id, taskErr, common.RetryTimes-retryParam.GetRetry()) diff --git a/controller/relay_count_tokens_test.go b/controller/relay_count_tokens_test.go new file mode 100644 index 0000000000..2889bb76a7 --- /dev/null +++ b/controller/relay_count_tokens_test.go @@ -0,0 +1,73 @@ +package controller + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCountClaudeTokensReturnsInputTokensWhenRelayCountingDisabled(t *testing.T) { + gin.SetMode(gin.TestMode) + originalCountToken := constant.CountToken + constant.CountToken = false + t.Cleanup(func() { + constant.CountToken = originalCountToken + }) + + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest( + http.MethodPost, + "/v1/messages/count_tokens?beta=true", + strings.NewReader(`{ + "model":"gemini-3.6-flash", + "messages":[{"role":"user","content":"count this prompt"}], + "tools":[{"name":"lookup","description":"Look up a value","input_schema":{"type":"object","properties":{"query":{"type":"string"}}}}] + }`), + ) + ctx.Request.Header.Set("Content-Type", "application/json") + common.SetContextKey(ctx, constant.ContextKeyOriginalModel, "gemini-3.6-flash") + + CountClaudeTokens(ctx) + + require.Equal(t, http.StatusOK, recorder.Code) + var response struct { + InputTokens int `json:"input_tokens"` + } + require.NoError(t, common.Unmarshal(recorder.Body.Bytes(), &response)) + assert.Positive(t, response.InputTokens) +} + +func TestCountClaudeTokensRejectsMissingMessages(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest( + http.MethodPost, + "/v1/messages/count_tokens", + strings.NewReader(`{"model":"gemini-3.6-flash"}`), + ) + ctx.Request.Header.Set("Content-Type", "application/json") + + CountClaudeTokens(ctx) + + require.Equal(t, http.StatusBadRequest, recorder.Code) + var response struct { + Type string `json:"type"` + Error struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error"` + } + require.NoError(t, common.Unmarshal(recorder.Body.Bytes(), &response)) + assert.Equal(t, "error", response.Type) + assert.Equal(t, "invalid_request_error", response.Error.Type) + assert.Contains(t, response.Error.Message, "messages") +} diff --git a/model/channel.go b/model/channel.go index 705c852b7a..268f190e70 100644 --- a/model/channel.go +++ b/model/channel.go @@ -989,6 +989,9 @@ func (channel *Channel) ValidateSettings() error { return err } } + if err := channelOtherSettings.ValidateToolLossPolicy(); err != nil { + return err + } if channel.Type == constant.ChannelTypeAdvancedCustom { if channelOtherSettings.AdvancedCustom == nil { return fmt.Errorf("advanced_custom is required") diff --git a/relay/channel/aws/adaptor.go b/relay/channel/aws/adaptor.go index 480aea3993..92519a1dee 100644 --- a/relay/channel/aws/adaptor.go +++ b/relay/channel/aws/adaptor.go @@ -39,6 +39,10 @@ func (a *Adaptor) ConvertGeminiRequest(*gin.Context, *relaycommon.RelayInfo, *dt } func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayInfo, request *dto.ClaudeRequest) (any, error) { + claudeAdaptor := claude.Adaptor{} + if _, err := claudeAdaptor.ConvertClaudeRequest(c, info, request); err != nil { + return nil, err + } for i, message := range request.Messages { updated := false if !message.IsStringContent() { diff --git a/relay/channel/aws/relay_aws_test.go b/relay/channel/aws/relay_aws_test.go index 22d8373873..5f92566984 100644 --- a/relay/channel/aws/relay_aws_test.go +++ b/relay/channel/aws/relay_aws_test.go @@ -357,7 +357,7 @@ func TestAwsStreamHandlerUsesFinalUpstreamUsage(t *testing.T) { assert.Contains(t, recorder.Body.String(), "[DONE]") } -func TestAwsStreamHandlerStopsAtClientCancellationAndKeepsPartialBillingUsage(t *testing.T) { +func TestAwsStreamHandlerStopsAtClientCancellation(t *testing.T) { originalRelayTimeout := common.RelayTimeout common.RelayTimeout = 0 t.Cleanup(func() { @@ -439,12 +439,6 @@ func TestAwsStreamHandlerStopsAtClientCancellationAndKeepsPartialBillingUsage(t require.ErrorIs(t, upstreamContext.Err(), context.Canceled) require.Nil(t, result.err) require.NotNil(t, result.usage) - require.NotNil(t, result.usage.BillingUsage) - require.NotNil(t, result.usage.BillingUsage.ClaudeUsage) - assert.Equal(t, dto.BillingUsageSourceClaudeMessages, result.usage.BillingUsage.Source) - assert.Equal(t, dto.BillingUsageSemanticAnthropic, result.usage.BillingUsage.Semantic) - assert.Equal(t, 100, result.usage.BillingUsage.ClaudeUsage.InputTokens) - assert.Equal(t, 1, result.usage.BillingUsage.ClaudeUsage.OutputTokens) assert.Equal(t, bodyLengthBeforeCancel, responseWriter.Body.Len()) assert.NotContains(t, responseWriter.Body.String(), "[DONE]") diff --git a/relay/channel/claude/adaptor.go b/relay/channel/claude/adaptor.go index bbd711ff2c..8d3583f09a 100644 --- a/relay/channel/claude/adaptor.go +++ b/relay/channel/claude/adaptor.go @@ -12,6 +12,7 @@ import ( "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert" "github.com/QuantumNous/new-api/relaykit/types" + "github.com/QuantumNous/new-api/service" "github.com/QuantumNous/new-api/setting/model_setting" "github.com/gin-gonic/gin" @@ -26,6 +27,22 @@ func (a *Adaptor) ConvertGeminiRequest(*gin.Context, *relaycommon.RelayInfo, *dt } func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayInfo, request *dto.ClaudeRequest) (any, error) { + if request.MaxTokens != nil && *request.MaxTokens == 0 { + request.MaxTokens = nil + } + if err := relayconvert.ApplyClaudeThinkingModel(request, info); err != nil { + return nil, err + } + if request.MaxTokens == nil { + defaultMaxTokens := uint(model_setting.GetClaudeSettings().GetDefaultMaxTokens(request.Model)) + request.MaxTokens = &defaultMaxTokens + } + // ApplyClaudeThinkingModel no longer rewrites request.Model. Do not write + // a still-suffixed name back over the entry-normalized UpstreamModelName + // (AWS/Vertex look up getAwsModelID / claudeModelMap from that field). + if info.UpstreamModelName == "" { + info.UpstreamModelName = request.Model + } return request, nil } @@ -96,7 +113,7 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn if request == nil { return nil, errors.New("request is nil") } - result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatClaude, request) + result, err := service.ConvertRequest(c, info, types.RelayFormatClaude, request) if err != nil { return nil, err } @@ -113,8 +130,15 @@ func (a *Adaptor) ConvertEmbeddingRequest(c *gin.Context, info *relaycommon.Rela } func (a *Adaptor) ConvertOpenAIResponsesRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.OpenAIResponsesRequest) (any, error) { - // TODO implement me - return nil, errors.New("not implemented") + result, err := service.ConvertRequest(c, info, types.RelayFormatClaude, &request) + if err != nil { + return nil, err + } + claudeRequest, ok := result.Value.(*dto.ClaudeRequest) + if !ok { + return nil, fmt.Errorf("expected Anthropic Messages request, got %T", result.Value) + } + return claudeRequest, nil } func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (any, error) { @@ -123,6 +147,9 @@ func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, request func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (usage any, err *types.NewAPIError) { info.FinalRequestRelayFormat = types.RelayFormatClaude + if info.RelayFormat == types.RelayFormatOpenAIResponses && info.IsStream { + return ClaudeResponsesStreamHandler(c, resp, info) + } if info.IsStream { return ClaudeStreamHandler(c, resp, info) } else { diff --git a/relay/channel/claude/adaptor_test.go b/relay/channel/claude/adaptor_test.go new file mode 100644 index 0000000000..01c035638e --- /dev/null +++ b/relay/channel/claude/adaptor_test.go @@ -0,0 +1,90 @@ +package claude + +import ( + "net/http/httptest" + "testing" + + "github.com/QuantumNous/new-api/common" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relay/helper" + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/setting/model_setting" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertClaudeRequestTreatsZeroMaxTokensAsUnset(t *testing.T) { + zero := uint(0) + req := &dto.ClaudeRequest{ + Model: "claude-sonnet-4-5", + MaxTokens: &zero, + Messages: []dto.ClaudeMessage{ + {Role: "user", Content: "hello"}, + }, + } + info := &relaycommon.RelayInfo{ + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "claude-sonnet-4-5", + }, + } + + out, err := (&Adaptor{}).ConvertClaudeRequest(nil, info, req) + require.NoError(t, err) + converted, ok := out.(*dto.ClaudeRequest) + require.True(t, ok) + require.NotNil(t, converted.MaxTokens) + assert.Equal(t, uint(model_setting.GetClaudeSettings().GetDefaultMaxTokens(req.Model)), *converted.MaxTokens) +} + +func TestConvertClaudeRequestZeroMaxTokensStillRaisesThinkingBudget(t *testing.T) { + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + + zero := uint(0) + original := &dto.ClaudeRequest{ + Model: "claude-3-7-sonnet-thinking", + MaxTokens: &zero, + Messages: []dto.ClaudeMessage{ + {Role: "user", Content: "hello"}, + }, + } + info := &relaycommon.RelayInfo{ + OriginModelName: "claude-3-7-sonnet-thinking", + Request: original, + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "claude-3-7-sonnet-thinking", + }, + } + outbound, err := common.DeepCopy(original) + require.NoError(t, err) + require.NoError(t, helper.ModelMappedHelper(c, info, outbound)) + require.NoError(t, helper.ApplyReasoningModelSuffix(info, outbound)) + + out, err := (&Adaptor{}).ConvertClaudeRequest(nil, info, outbound) + require.NoError(t, err) + converted, ok := out.(*dto.ClaudeRequest) + require.True(t, ok) + assert.Equal(t, "claude-3-7-sonnet", converted.Model) + require.NotNil(t, converted.Thinking) + require.NotNil(t, converted.MaxTokens) + assert.Greater(t, *converted.MaxTokens, uint(1024)) +} + +func TestConvertClaudeRequestDoesNotOverwriteTrimmedUpstreamModelName(t *testing.T) { + req := &dto.ClaudeRequest{ + Model: "claude-3-7-sonnet-thinking", + Messages: []dto.ClaudeMessage{ + {Role: "user", Content: "hello"}, + }, + } + info := &relaycommon.RelayInfo{ + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "claude-3-7-sonnet", + }, + } + + _, err := (&Adaptor{}).ConvertClaudeRequest(nil, info, req) + require.NoError(t, err) + assert.Equal(t, "claude-3-7-sonnet", info.UpstreamModelName) +} diff --git a/relay/channel/claude/relay-claude.go b/relay/channel/claude/relay-claude.go index 2f424b32ab..511c7eb79a 100644 --- a/relay/channel/claude/relay-claude.go +++ b/relay/channel/claude/relay-claude.go @@ -1,6 +1,7 @@ package claude import ( + "fmt" "io" "net/http" "strings" @@ -19,6 +20,8 @@ import ( "github.com/gin-gonic/gin" ) +const claudeToChatStreamStateKey = "relaykit.claude_to_chat_stream_state" + func stopReasonClaude2OpenAI(reason string) string { return relayconvert.StopReasonClaudeToOpenAI(reason) } @@ -117,7 +120,14 @@ func HandleStreamResponseData(c *gin.Context, info *relaycommon.RelayInfo, claud countClaudeStreamBillableTools(c, info, &claudeResponse) helper.ClaudeChunkData(c, claudeResponse, data) } else if info.RelayFormat == types.RelayFormatOpenAI { - response := StreamResponseClaude2OpenAI(&claudeResponse) + state, err := claudeToChatStreamState(c) + if err != nil { + return types.NewError(err, types.ErrorCodeBadResponseBody) + } + response, err := state.ConvertChunk(&claudeResponse) + if err != nil { + return types.NewError(err, types.ErrorCodeBadResponseBody) + } if !FormatClaudeResponseInfo(&claudeResponse, response, claudeInfo) { return nil @@ -125,6 +135,9 @@ func HandleStreamResponseData(c *gin.Context, info *relaycommon.RelayInfo, claud countClaudeStreamBillableTools(c, info, &claudeResponse) + if response == nil { + return nil + } err = helper.ObjectData(c, response) if err != nil { logger.LogError(c, "send_stream_response_failed: "+err.Error()) @@ -133,6 +146,20 @@ func HandleStreamResponseData(c *gin.Context, info *relaycommon.RelayInfo, claud return nil } +func claudeToChatStreamState(c *gin.Context) (*relayconvert.ClaudeToChatStreamState, error) { + if value, ok := c.Get(claudeToChatStreamStateKey); ok { + state, ok := value.(*relayconvert.ClaudeToChatStreamState) + if !ok || state == nil { + return nil, fmt.Errorf("invalid Claude-to-Chat stream state %T", value) + } + return state, nil + } + + state := relayconvert.NewClaudeToChatStreamState() + c.Set(claudeToChatStreamStateKey, state) + return state, nil +} + func countClaudeStreamBillableTools(c *gin.Context, info *relaycommon.RelayInfo, claudeResponse *dto.ClaudeResponse) { if claudeResponse == nil { return @@ -172,9 +199,7 @@ func HandleStreamFinalResponse(c *gin.Context, info *relaycommon.RelayInfo, clau if claudeInfo.Usage != nil { claudeInfo.Usage.UsageSemantic = "anthropic" } - if claudeInfo.Usage != nil && claudeInfo.Usage.BillingUsage == nil { - claudeInfo.Usage.BillingUsage = dto.NewClaudeMessagesBillingUsage(buildMessageDeltaPatchUsage(nil, claudeInfo)) - } + relayconvert.FinalizeClaudeStreamBillingUsage(claudeInfo) if info.RelayFormat == types.RelayFormatClaude { // @@ -232,7 +257,10 @@ func HandleClaudeResponseData(c *gin.Context, info *relaycommon.RelayInfo, claud claudeInfo.Usage.CompletionTokens = claudeResponse.Usage.OutputTokens claudeInfo.Usage.TotalTokens = claudeResponse.Usage.InputTokens + claudeResponse.Usage.OutputTokens claudeInfo.Usage.UsageSemantic = "anthropic" - claudeInfo.Usage.BillingUsage = dto.NewClaudeMessagesBillingUsage(claudeResponse.Usage) + claudeInfo.Usage.BillingUsage = dto.CloneBillingUsage(claudeResponse.Usage.BillingUsage) + if claudeInfo.Usage.BillingUsage == nil { + claudeInfo.Usage.BillingUsage = dto.NewClaudeMessagesBillingUsage(claudeResponse.Usage) + } claudeInfo.Usage.PromptTokensDetails.CachedTokens = claudeResponse.Usage.CacheReadInputTokens claudeInfo.Usage.PromptTokensDetails.CachedCreationTokens = claudeResponse.Usage.CacheCreationInputTokens claudeInfo.Usage.ClaudeCacheCreation5mTokens = claudeResponse.Usage.GetCacheCreation5mTokens() @@ -247,6 +275,22 @@ func HandleClaudeResponseData(c *gin.Context, info *relaycommon.RelayInfo, claud if err != nil { return types.NewError(err, types.ErrorCodeBadResponseBody) } + case types.RelayFormatOpenAIResponses: + convertResult, err := service.ConvertResponse(c, info, types.RelayFormatOpenAIResponses, &claudeResponse) + if err != nil { + return types.NewError(err, types.ErrorCodeBadResponseBody) + } + responsesResponse, ok := convertResult.Value.(*dto.OpenAIResponsesResponse) + if !ok { + return types.NewError(fmt.Errorf("expected OpenAI Responses response, got %T", convertResult.Value), types.ErrorCodeBadResponseBody) + } + if responseID := helper.GetResponseID(c); responseID != "" { + responsesResponse.ID = responseID + } + responseData, err = common.Marshal(responsesResponse) + if err != nil { + return types.NewError(err, types.ErrorCodeBadResponseBody) + } case types.RelayFormatClaude: responseData = data } diff --git a/relay/channel/claude/relay_claude_test.go b/relay/channel/claude/relay_claude_test.go index 3975658fa4..703d78a370 100644 --- a/relay/channel/claude/relay_claude_test.go +++ b/relay/channel/claude/relay_claude_test.go @@ -1,12 +1,16 @@ package claude import ( + "net/http/httptest" "strings" "testing" + "github.com/QuantumNous/new-api/common" relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relay/helper" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert" + "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -323,8 +327,27 @@ func TestBuildOpenAIStyleUsageFromClaudeUsageDefaultsAggregateCacheCreationTo5m( require.Equal(t, 0, openAIUsage.ClaudeCacheCreation1hTokens) } +func applyOpenAIChatReasoningThroughHandlerOrder(t *testing.T, original dto.GeneralOpenAIRequest) (*dto.GeneralOpenAIRequest, *relaycommon.RelayInfo) { + t.Helper() + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + + info := &relaycommon.RelayInfo{ + OriginModelName: original.Model, + Request: &original, + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: original.Model, + }, + } + outbound, err := common.DeepCopy(&original) + require.NoError(t, err) + require.NoError(t, helper.ModelMappedHelper(c, info, outbound)) + require.NoError(t, helper.ApplyReasoningModelSuffix(info, outbound)) + return outbound, info +} + func TestOpenAIChatRequestToClaudeMessages_ClaudeOpus48HighUsesAdaptiveThinking(t *testing.T) { - request := dto.GeneralOpenAIRequest{ + original := dto.GeneralOpenAIRequest{ Model: "claude-opus-4-8-high", Temperature: commonPointer(0.7), TopP: commonPointer(0.9), @@ -337,7 +360,8 @@ func TestOpenAIChatRequestToClaudeMessages_ClaudeOpus48HighUsesAdaptiveThinking( }, } - claudeRequest, err := relayconvert.OpenAIChatRequestToClaudeMessages(nil, &relaycommon.RelayInfo{}, request) + outbound, info := applyOpenAIChatReasoningThroughHandlerOrder(t, original) + claudeRequest, err := relayconvert.OpenAIChatRequestToClaudeMessages(nil, info, *outbound) require.NoError(t, err) require.Equal(t, "claude-opus-4-8", claudeRequest.Model) require.NotNil(t, claudeRequest.Thinking) @@ -350,7 +374,7 @@ func TestOpenAIChatRequestToClaudeMessages_ClaudeOpus48HighUsesAdaptiveThinking( } func TestOpenAIChatRequestToClaudeMessages_ClaudeOpus48ThinkingUsesAdaptiveHighEffort(t *testing.T) { - request := dto.GeneralOpenAIRequest{ + original := dto.GeneralOpenAIRequest{ Model: "claude-opus-4-8-thinking", Temperature: commonPointer(0.7), TopP: commonPointer(0.9), @@ -363,7 +387,8 @@ func TestOpenAIChatRequestToClaudeMessages_ClaudeOpus48ThinkingUsesAdaptiveHighE }, } - claudeRequest, err := relayconvert.OpenAIChatRequestToClaudeMessages(nil, &relaycommon.RelayInfo{}, request) + outbound, info := applyOpenAIChatReasoningThroughHandlerOrder(t, original) + claudeRequest, err := relayconvert.OpenAIChatRequestToClaudeMessages(nil, info, *outbound) require.NoError(t, err) require.Equal(t, "claude-opus-4-8", claudeRequest.Model) require.NotNil(t, claudeRequest.Thinking) diff --git a/relay/channel/claude/relay_responses.go b/relay/channel/claude/relay_responses.go new file mode 100644 index 0000000000..1e3697ae9d --- /dev/null +++ b/relay/channel/claude/relay_responses.go @@ -0,0 +1,180 @@ +package claude + +import ( + "fmt" + "net/http" + "strings" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/logger" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relay/helper" + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/relayconvert" + "github.com/QuantumNous/new-api/relaykit/types" + "github.com/QuantumNous/new-api/service" + "github.com/gin-gonic/gin" +) + +func ClaudeResponsesStreamHandler(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (*dto.Usage, *types.NewAPIError) { + responseID := helper.GetResponseID(c) + created := common.GetTimestamp() + state, err := relayconvert.NewResponseStreamState(types.RelayFormatClaude, types.RelayFormatOpenAIResponses, relayconvert.ResponseStreamOptions{ + ID: responseID, + Model: info.UpstreamModelName, + Created: created, + EmitSequenceNumber: true, + }) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) + } + hostedBridge := relayconvert.NewClaudeHostedStreamBridge() + + claudeInfo := &ClaudeResponseInfo{ + ResponseId: responseID, + Created: created, + Model: info.UpstreamModelName, + ResponseText: strings.Builder{}, + Usage: &dto.Usage{}, + } + var streamErr *types.NewAPIError + // streamFailed means a Responses-native terminal error was sent successfully. + // In that case the scanner stops without a transport error and the partial + // upstream usage remains billable. + streamFailed := false + + sendResponsesEvent := func(eventType string, payload dto.ResponsesStreamResponse) bool { + payload.Type = eventType + data, err := common.Marshal(payload) + if err != nil { + streamErr = types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError) + return false + } + if err := helper.ResponseChunkData(c, dto.ResponsesStreamResponse{Type: eventType}, string(data)); err != nil { + streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) + return false + } + return true + } + sendResult := func(result relayconvert.ResponseResult) bool { + event, ok := result.Value.(relayconvert.ChatToResponsesStreamEvent) + if !ok { + streamErr = types.NewOpenAIError( + fmt.Errorf("expected OpenAI Responses stream event, got %T", result.Value), + types.ErrorCodeBadResponse, + http.StatusInternalServerError, + ) + return false + } + return sendResponsesEvent(event.Type, event.Payload) + } + failResponsesStream := func(err error) bool { + failureResults, handled := state.FailResponsesStream("server_error", err.Error(), "") + if !handled { + return false + } + for _, result := range failureResults { + if !sendResult(result) { + return true + } + } + streamFailed = true + return true + } + + helper.StreamScannerHandler(c, resp, info, func(data string, sr *helper.StreamResult) { + var claudeResponse dto.ClaudeResponse + if err := common.UnmarshalJsonStr(data, &claudeResponse); err != nil { + logger.LogError(c, "failed to unmarshal Claude stream event: "+err.Error()) + if failResponsesStream(err) { + // A nil streamErr here is intentional: the protocol-level failure + // event was delivered, so only the scanner needs to stop. + sr.Stop(streamErr) + return + } + streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + sr.Stop(streamErr) + return + } + if claudeError := claudeResponse.GetClaudeError(); claudeError != nil && claudeError.Type != "" { + if failResponsesStream(fmt.Errorf("%s", claudeError.Message)) { + sr.Stop(streamErr) + return + } + streamErr = types.WithClaudeError(*claudeError, http.StatusInternalServerError) + sr.Stop(streamErr) + return + } + + if claudeResponse.StopReason != "" { + maybeMarkClaudeRefusal(c, claudeResponse.StopReason) + } + if claudeResponse.Delta != nil && claudeResponse.Delta.StopReason != nil { + maybeMarkClaudeRefusal(c, *claudeResponse.Delta.StopReason) + } + if claudeResponse.Type == "message_start" && claudeResponse.Message != nil { + info.UpstreamModelName = claudeResponse.Message.Model + } + FormatClaudeResponseInfo(&claudeResponse, nil, claudeInfo) + countClaudeStreamBillableTools(c, info, &claudeResponse) + hostedEvents, consumed, err := hostedBridge.Convert(&claudeResponse, state) + if err != nil { + if failResponsesStream(err) { + sr.Stop(streamErr) + return + } + streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) + sr.Stop(streamErr) + return + } + for _, event := range hostedEvents { + if !sendResponsesEvent(event.Type, event.Payload) { + sr.Stop(streamErr) + return + } + } + if consumed { + return + } + + results, err := service.ConvertStreamResponseChunk(c, info, state, &claudeResponse) + if err != nil { + if failResponsesStream(err) { + sr.Stop(streamErr) + return + } + streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) + sr.Stop(streamErr) + return + } + for _, result := range results { + if !sendResult(result) { + sr.Stop(streamErr) + return + } + } + }) + if streamErr != nil { + return nil, streamErr + } + if streamFailed { + return claudeInfo.Usage, nil + } + + HandleStreamFinalResponse(c, info, claudeInfo) + openAIUsage := buildOpenAIStyleUsageFromClaudeUsage(claudeInfo.Usage) + state.SetUsage(&openAIUsage) + finalResults, err := service.FinalizeStreamResponse(c, info, state) + if err != nil { + if failResponsesStream(err) { + return claudeInfo.Usage, streamErr + } + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) + } + for _, result := range finalResults { + if !sendResult(result) { + return nil, streamErr + } + } + return claudeInfo.Usage, nil +} diff --git a/relay/channel/gemini/adaptor.go b/relay/channel/gemini/adaptor.go index c96ecca6fb..073e501526 100644 --- a/relay/channel/gemini/adaptor.go +++ b/relay/channel/gemini/adaptor.go @@ -13,8 +13,8 @@ import ( "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert" "github.com/QuantumNous/new-api/relaykit/types" + "github.com/QuantumNous/new-api/service" "github.com/QuantumNous/new-api/setting/model_setting" - "github.com/QuantumNous/new-api/setting/reasoning" "github.com/gin-gonic/gin" "github.com/samber/lo" @@ -24,6 +24,9 @@ type Adaptor struct { } func (a *Adaptor) ConvertGeminiRequest(c *gin.Context, info *relaycommon.RelayInfo, request *dto.GeminiChatRequest) (any, error) { + if err := relayconvert.ApplyGeminiThinkingConfigChecked(request, info); err != nil { + return nil, err + } if len(request.Contents) > 0 { for i, content := range request.Contents { if i == 0 { @@ -44,7 +47,7 @@ func (a *Adaptor) ConvertGeminiRequest(c *gin.Context, info *relaycommon.RelayIn } func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayInfo, req *dto.ClaudeRequest) (any, error) { - result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatGemini, req) + result, err := service.ConvertRequest(c, info, types.RelayFormatGemini, req) if err != nil { return nil, err } @@ -132,21 +135,6 @@ func (a *Adaptor) Init(info *relaycommon.RelayInfo) { func (a *Adaptor) GetRequestURL(info *relaycommon.RelayInfo) (string, error) { - if model_setting.GetGeminiSettings().ThinkingAdapterEnabled && - !model_setting.ShouldPreserveThinkingSuffix(info.OriginModelName) { - // 新增逻辑：处理 -thinking- 格式 - if strings.Contains(info.UpstreamModelName, "-thinking-") { - parts := strings.Split(info.UpstreamModelName, "-thinking-") - info.UpstreamModelName = parts[0] - } else if strings.HasSuffix(info.UpstreamModelName, "-thinking") { // 旧的适配 - info.UpstreamModelName = strings.TrimSuffix(info.UpstreamModelName, "-thinking") - } else if strings.HasSuffix(info.UpstreamModelName, "-nothinking") { - info.UpstreamModelName = strings.TrimSuffix(info.UpstreamModelName, "-nothinking") - } else if baseModel, level, ok := reasoning.TrimEffortSuffix(info.UpstreamModelName); ok && level != "" { - info.UpstreamModelName = baseModel - } - } - version := model_setting.GetGeminiVersionSetting(info.UpstreamModelName) if strings.HasPrefix(info.UpstreamModelName, "imagen") { @@ -183,7 +171,7 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn if request == nil { return nil, errors.New("request is nil") } - result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatGemini, request) + result, err := service.ConvertRequest(c, info, types.RelayFormatGemini, request) if err != nil { return nil, err } @@ -239,7 +227,7 @@ func (a *Adaptor) ConvertEmbeddingRequest(c *gin.Context, info *relaycommon.Rela } func (a *Adaptor) ConvertOpenAIResponsesRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.OpenAIResponsesRequest) (any, error) { - result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatGemini, &request) + result, err := service.ConvertRequest(c, info, types.RelayFormatGemini, &request) if err != nil { return nil, err } diff --git a/relay/channel/gemini/relay-gemini-native.go b/relay/channel/gemini/relay-gemini-native.go index 74742d666b..c6c913383b 100644 --- a/relay/channel/gemini/relay-gemini-native.go +++ b/relay/channel/gemini/relay-gemini-native.go @@ -34,6 +34,7 @@ func GeminiTextGenerationHandler(c *gin.Context, info *relaycommon.RelayInfo, re if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } + countGeminiBillableFunctionCalls(info, &geminiResponse) if len(geminiResponse.Candidates) == 0 && geminiResponse.PromptFeedback != nil && geminiResponse.PromptFeedback.BlockReason != nil { common.SetContextKey(c, constant.ContextKeyAdminRejectReason, fmt.Sprintf("gemini_block_reason=%s", *geminiResponse.PromptFeedback.BlockReason)) diff --git a/relay/channel/gemini/relay-gemini.go b/relay/channel/gemini/relay-gemini.go index 84acea73c5..e437f1277b 100644 --- a/relay/channel/gemini/relay-gemini.go +++ b/relay/channel/gemini/relay-gemini.go @@ -55,10 +55,13 @@ func patchGeminiZeroCompletionUsage(c *gin.Context, info *relaycommon.RelayInfo, usage.CompletionTokens = imageCount * 1400 } usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens - // Overwrite the metadata-derived billing usage: effectiveBillingUsage prefers - // BillingUsage during settlement, so keeping the prompt-only metadata there - // would still bill zero completion tokens. - usage.BillingUsage = dto.NewEstimatedGeminiChatBillingUsage(usage) + // Settlement prefers BillingUsage, so fill the missing completion in the + // original upstream dialect without discarding cache or modality details. + if usage.BillingUsage != nil { + usage.BillingUsage = dto.CloneBillingUsageWithEstimatedCompletion(usage.BillingUsage, usage.CompletionTokens) + } else { + usage.BillingUsage = dto.NewEstimatedGeminiChatBillingUsage(usage) + } } func geminiResponseUsageText(response *dto.GeminiChatResponse) string { @@ -88,6 +91,23 @@ func markGeminiGoogleSearchCall(c *gin.Context, response *dto.GeminiChatResponse } } +func countGeminiBillableFunctionCalls(info *relaycommon.RelayInfo, response *dto.GeminiChatResponse) { + if info == nil || response == nil { + return + } + for _, candidate := range response.Candidates { + for _, part := range candidate.Content.Parts { + if part.FunctionCall == nil { + continue + } + if part.FunctionCall.WillContinue != nil && *part.FunctionCall.WillContinue { + continue + } + info.CountBillableToolCall(dto.BuildInCallFunctionCall, part.FunctionCall.FunctionName) + } + } +} + func buildUsageFromGeminiResponse(c *gin.Context, info *relaycommon.RelayInfo, response *dto.GeminiChatResponse) dto.Usage { metadata := response.GetUsageMetadata() if dto.HasGeminiUsageMetadataTokens(metadata) { @@ -148,12 +168,15 @@ func geminiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http var usage = &dto.Usage{} var imageCount int var hasBillableUsageMetadata bool + var streamErr error + var accumulatedUsageMetadata *dto.GeminiUsageMetadata responseText := strings.Builder{} helper.StreamScannerHandler(c, resp, info, func(data string, sr *helper.StreamResult) { var geminiResponse dto.GeminiChatResponse if err := common.UnmarshalJsonStr(data, &geminiResponse); err != nil { - sr.Stop(fmt.Errorf("unmarshal: %w", err)) + streamErr = fmt.Errorf("unmarshal Gemini stream response: %w", err) + sr.Stop(streamErr) return } @@ -162,6 +185,7 @@ func geminiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http } markGeminiGoogleSearchCall(c, &geminiResponse) + countGeminiBillableFunctionCalls(info, &geminiResponse) // 统计图片数量 for _, candidate := range geminiResponse.Candidates { @@ -177,13 +201,19 @@ func geminiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http // 更新使用量统计 if metadata := geminiResponse.GetUsageMetadata(); dto.HasGeminiUsageMetadataTokens(metadata) { - mappedUsage := buildUsageFromGeminiMetadata(metadata, info.GetEstimatePromptTokens()) + accumulatedUsageMetadata = dto.MergeGeminiUsageMetadataNonZero(accumulatedUsageMetadata, metadata) + mappedUsage := buildUsageFromGeminiMetadata(accumulatedUsageMetadata, info.GetEstimatePromptTokens()) *usage = mappedUsage hasBillableUsageMetadata = true } if !callback(data, &geminiResponse) { - sr.Stop(fmt.Errorf("gemini callback stopped")) + if isGeminiDownstreamStop(c, info) { + sr.Stop(nil) + return + } + streamErr = errors.New("Gemini stream callback stopped") + sr.Stop(streamErr) } }) @@ -203,9 +233,24 @@ func geminiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http patchGeminiZeroCompletionUsage(c, info, usage, responseText.String(), imageCount) } + if streamErr != nil { + return usage, types.NewOpenAIError(streamErr, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + } + if info.StreamStatus != nil && !info.StreamStatus.IsNormalEnd() { + logger.LogWarn(c, fmt.Sprintf("Gemini stream ended unexpectedly: %s", info.StreamStatus.Summary())) + } + return usage, nil } +func isGeminiDownstreamStop(c *gin.Context, info *relaycommon.RelayInfo) bool { + if c != nil && c.Request != nil && c.Request.Context().Err() != nil { + return true + } + return info != nil && info.StreamStatus != nil && + info.StreamStatus.EndReason == relaycommon.StreamEndReasonClientGone +} + func GeminiChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) { id := helper.GetResponseID(c) createAt := common.GetTimestamp() @@ -323,6 +368,7 @@ func GeminiChatHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.R return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } markGeminiGoogleSearchCall(c, &geminiResponse) + countGeminiBillableFunctionCalls(info, &geminiResponse) if len(geminiResponse.Candidates) == 0 { usage := buildUsageFromGeminiResponse(c, info, &geminiResponse) @@ -371,7 +417,7 @@ func GeminiChatHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.R return nil, types.NewError(err, types.ErrorCodeBadResponseBody) } case types.RelayFormatClaude: - convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatClaude, fullTextResponse) + convertResult, err := service.ConvertResponse(c, info, types.RelayFormatClaude, fullTextResponse) if err != nil { return nil, types.NewError(err, types.ErrorCodeBadResponseBody) } diff --git a/relay/channel/gemini/relay_responses.go b/relay/channel/gemini/relay_responses.go index 7b3d746b27..924fd04d04 100644 --- a/relay/channel/gemini/relay_responses.go +++ b/relay/channel/gemini/relay_responses.go @@ -32,6 +32,7 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } markGeminiGoogleSearchCall(c, &geminiResponse) + countGeminiBillableFunctionCalls(info, &geminiResponse) if len(geminiResponse.Candidates) == 0 { usage := buildUsageFromGeminiResponse(c, info, &geminiResponse) if geminiResponse.PromptFeedback != nil && geminiResponse.PromptFeedback.BlockReason != nil { @@ -50,15 +51,9 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h ) } - chatResp := responseGeminiChat2OpenAI(c, &geminiResponse) - chatResp.Model = info.UpstreamModelName - if responseID := helper.GetResponseID(c); responseID != "" { - chatResp.Id = responseID - } usage := buildUsageFromGeminiResponse(c, info, &geminiResponse) - chatResp.Usage = usage - convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatOpenAIResponses, chatResp) + convertResult, err := service.ConvertResponse(c, info, types.RelayFormatOpenAIResponses, &geminiResponse) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } @@ -66,10 +61,11 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h if !ok { return nil, types.NewOpenAIError(fmt.Errorf("expected OpenAI responses response, got %T", convertResult.Value), types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } - responsesUsage := convertResult.Usage - if responsesUsage == nil || responsesUsage.TotalTokens == 0 { - responsesResp.Usage = relayconvert.UsageFromChatUsage(&usage) + if responseID := helper.GetResponseID(c); responseID != "" { + responsesResp.ID = responseID } + responsesResp.Model = info.UpstreamModelName + responsesResp.Usage = relayconvert.UsageFromChatUsage(&usage) responseBody, err = common.Marshal(responsesResp) if err != nil { @@ -82,17 +78,16 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) { responseID := helper.GetResponseID(c) created := common.GetTimestamp() - state, err := relayconvert.NewResponseStreamState(types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses, relayconvert.ResponseStreamOptions{ - ID: responseID, - Model: info.UpstreamModelName, - Created: created, + state, err := relayconvert.NewResponseStreamState(types.RelayFormatGemini, types.RelayFormatOpenAIResponses, relayconvert.ResponseStreamOptions{ + ID: responseID, + Model: info.UpstreamModelName, + Created: created, + EmitSequenceNumber: true, }) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) } - finishReason := constant.FinishReasonStop - toolCallIndexByChoice := make(map[int]map[string]int) - nextToolCallIndexByChoice := make(map[int]int) + hostedBridge := relayconvert.NewGeminiHostedStreamBridge() var streamErr *types.NewAPIError sendEvent := func(event relayconvert.ChatToResponsesStreamEvent) bool { @@ -101,12 +96,37 @@ func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, r streamErr = types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError) return false } - helper.ResponseChunkData(c, dto.ResponsesStreamResponse{Type: event.Type}, string(data)) + if err := helper.ResponseChunkData(c, dto.ResponsesStreamResponse{Type: event.Type}, string(data)); err != nil { + if info.StreamStatus != nil { + info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonClientGone, err) + } + return false + } return true } - sendChunk := func(chunk *dto.ChatCompletionsStreamResponse) bool { - results, err := relayconvert.ConvertStreamResponseChunk(c, info, state, chunk) + failResponsesStream := func(err error) bool { + failureResults, handled := state.FailResponsesStream("server_error", err.Error(), "") + if !handled { + return false + } + for _, result := range failureResults { + event, ok := result.Value.(relayconvert.ChatToResponsesStreamEvent) + if !ok { + streamErr = types.NewOpenAIError(fmt.Errorf("expected OAI responses stream event, got %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError) + return true + } + if !sendEvent(event) { + return true + } + } + return true + } + sendChunk := func(chunk *dto.GeminiChatResponse) bool { + results, err := service.ConvertStreamResponseChunk(c, info, state, chunk) if err != nil { + if failResponsesStream(err) { + return false + } streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) return false } @@ -123,58 +143,46 @@ func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, r return true } - usage, streamAPIError := geminiStreamHandler(c, info, resp, func(data string, geminiResponse *dto.GeminiChatResponse) bool { - response, isStop := streamResponseGeminiChat2OpenAI(geminiResponse) - response.Id = responseID - response.Created = created - response.Model = info.UpstreamModelName - - if response.IsToolCall() { - finishReason = constant.FinishReasonToolCalls - } - for choiceIdx := range response.Choices { - choiceKey := response.Choices[choiceIdx].Index - for toolIdx := range response.Choices[choiceIdx].Delta.ToolCalls { - tool := &response.Choices[choiceIdx].Delta.ToolCalls[toolIdx] - if tool.ID == "" { - continue - } - indexByID := toolCallIndexByChoice[choiceKey] - if indexByID == nil { - indexByID = make(map[string]int) - toolCallIndexByChoice[choiceKey] = indexByID - } - if idx, ok := indexByID[tool.ID]; ok { - tool.SetIndex(idx) - continue - } - idx := nextToolCallIndexByChoice[choiceKey] - nextToolCallIndexByChoice[choiceKey] = idx + 1 - indexByID[tool.ID] = idx - tool.SetIndex(idx) - } - } - - if !sendChunk(response) { - return false - } - if isStop { - return sendChunk(helper.GenerateStopResponse(responseID, created, info.UpstreamModelName, finishReason)) - } - return true + usage, streamAPIError := geminiStreamHandler(c, info, resp, func(_ string, geminiResponse *dto.GeminiChatResponse) bool { + hostedBridge.Observe(geminiResponse) + return sendChunk(geminiResponse) }) if streamAPIError != nil { + if failResponsesStream(streamAPIError) && streamErr == nil { + return usage, nil + } return usage, streamAPIError } + if info.StreamStatus != nil && !info.StreamStatus.IsNormalEnd() { + if info.StreamStatus.EndReason != relaycommon.StreamEndReasonClientGone { + failResponsesStream(fmt.Errorf("gemini stream ended unexpectedly: %s", info.StreamStatus.Summary())) + } + return usage, nil + } if streamErr != nil { return nil, streamErr } + hostedEvents, err := hostedBridge.Finalize(state) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) + } + for _, event := range hostedEvents { + if !sendEvent(event) { + if streamErr != nil { + return usage, streamErr + } + return usage, nil + } + } if usage != nil { state.SetUsage(usage) } - finalResults, err := relayconvert.FinalizeStreamResponse(c, info, state) + finalResults, err := service.FinalizeStreamResponse(c, info, state) if err != nil { + if failResponsesStream(err) { + return usage, streamErr + } return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) } for _, result := range finalResults { @@ -183,7 +191,10 @@ func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, r return nil, types.NewOpenAIError(fmt.Errorf("expected OAI responses stream event, got %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError) } if !sendEvent(event) { - return nil, streamErr + if streamErr != nil { + return usage, streamErr + } + return usage, nil } } return usage, nil diff --git a/relay/channel/newapi/adaptor.go b/relay/channel/newapi/adaptor.go index 979d5c2769..63c3ccfac2 100644 --- a/relay/channel/newapi/adaptor.go +++ b/relay/channel/newapi/adaptor.go @@ -75,14 +75,14 @@ func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayIn if request == nil { return nil, errors.New("request is nil") } - return request, nil + return a.claudeAdaptor.ConvertClaudeRequest(c, info, request) } func (a *Adaptor) ConvertGeminiRequest(c *gin.Context, info *relaycommon.RelayInfo, request *dto.GeminiChatRequest) (any, error) { if request == nil { return nil, errors.New("request is nil") } - return request, nil + return a.geminiAdaptor.ConvertGeminiRequest(c, info, request) } func (a *Adaptor) ConvertImageRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.ImageRequest) (any, error) { diff --git a/relay/channel/openai/adaptor.go b/relay/channel/openai/adaptor.go index 64ae3102b3..dd64149b20 100644 --- a/relay/channel/openai/adaptor.go +++ b/relay/channel/openai/adaptor.go @@ -19,14 +19,15 @@ import ( "github.com/QuantumNous/new-api/relay/channel" "github.com/QuantumNous/new-api/relay/channel/ai360" "github.com/QuantumNous/new-api/relay/channel/lingyiwanwu" + "github.com/QuantumNous/new-api/relay/channel/openrouter" "github.com/QuantumNous/new-api/relaykit/dto" //"github.com/QuantumNous/new-api/relay/channel/minimax" - "github.com/QuantumNous/new-api/relay/channel/openrouter" "github.com/QuantumNous/new-api/relay/channel/xinference" relaycommon "github.com/QuantumNous/new-api/relay/common" "github.com/QuantumNous/new-api/relay/common_handler" relayconstant "github.com/QuantumNous/new-api/relay/constant" + kitreasoning "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/service" "github.com/QuantumNous/new-api/setting/model_setting" @@ -249,80 +250,122 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn request.StreamOptions = nil } if info.ChannelType == constant.ChannelTypeOpenRouter { + initialIntent, err := kitreasoning.FromOpenAIChat(request) + if err != nil { + return nil, kitreasoning.AsClientError(err) + } + if request.THINKING != nil && strings.HasPrefix(info.UpstreamModelName, "anthropic") { + var thinking dto.Thinking + if err := common.Unmarshal(request.THINKING, &thinking); err != nil { + return nil, fmt.Errorf("error Unmarshal thinking: %w", err) + } + legacyIntent, err := kitreasoning.FromClaude(&dto.ClaudeRequest{Thinking: &thinking}) + if err != nil { + return nil, kitreasoning.AsClientError(err) + } + initialIntent, err = kitreasoning.MergeExplicit(initialIntent, legacyIntent, request.Model) + if err != nil { + return nil, kitreasoning.AsClientError(err) + } + request.THINKING = nil + } if len(request.Usage) == 0 { request.Usage = json.RawMessage(`{"include":true}`) } // 适配 OpenRouter 的 thinking 后缀 - if !model_setting.ShouldPreserveThinkingSuffix(info.OriginModelName) && - strings.HasSuffix(info.UpstreamModelName, "-thinking") { + preserveSuffix := model_setting.ShouldPreserveThinkingSuffix(info.OriginModelName) || model_setting.ShouldPreserveThinkingSuffix(info.UpstreamModelName) + mergeEffortSuffix := func(modelName string) error { + rawEffort, _ := reasoning.ParseOpenAIReasoningEffortFromModelSuffix(modelName) + if rawEffort == "" { + return nil + } + effort, err := kitreasoning.ParseEffort(rawEffort) + if err != nil { + return err + } + mode := kitreasoning.ModeEnabled + if effort == kitreasoning.EffortNone { + mode = kitreasoning.ModeDisabled + } + initialIntent, err = kitreasoning.MergeExplicitAndSuffix(initialIntent, kitreasoning.Intent{Mode: mode, Effort: effort, Source: kitreasoning.SourceSuffix}, modelName) + return err + } + if !preserveSuffix { + if err := mergeEffortSuffix(info.UpstreamModelName); err != nil { + return nil, kitreasoning.AsClientError(err) + } + if _, baseModel := reasoning.ParseOpenAIReasoningEffortFromModelSuffix(info.UpstreamModelName); baseModel != info.UpstreamModelName { + info.UpstreamModelName = baseModel + request.Model = baseModel + } + if info.OriginModelName != info.UpstreamModelName { + if err := mergeEffortSuffix(info.OriginModelName); err != nil { + return nil, kitreasoning.AsClientError(err) + } + } + } + if !preserveSuffix && strings.HasSuffix(info.UpstreamModelName, "-thinking") { + initialIntent, err = kitreasoning.MergeExplicitAndSuffix( + initialIntent, + kitreasoning.Intent{Mode: kitreasoning.ModeEnabled}, + info.UpstreamModelName, + ) + if err != nil { + return nil, kitreasoning.AsClientError(err) + } info.UpstreamModelName = strings.TrimSuffix(info.UpstreamModelName, "-thinking") request.Model = info.UpstreamModelName - if len(request.Reasoning) == 0 { - reasoning := map[string]any{ - "enabled": true, - } - if request.ReasoningEffort != "" && request.ReasoningEffort != "none" { - reasoning["effort"] = request.ReasoningEffort - } - marshal, err := common.Marshal(reasoning) - if err != nil { - return nil, fmt.Errorf("error marshalling reasoning: %w", err) - } - request.Reasoning = marshal - } - // 清空多余的ReasoningEffort - request.ReasoningEffort = "" - } else { - if len(request.Reasoning) == 0 { - // 适配 OpenAI 的 ReasoningEffort 格式 - if request.ReasoningEffort != "" { - reasoning := map[string]any{ - "enabled": true, - } - if request.ReasoningEffort != "none" { - reasoning["effort"] = request.ReasoningEffort - marshal, err := common.Marshal(reasoning) - if err != nil { - return nil, fmt.Errorf("error marshalling reasoning: %w", err) - } - request.Reasoning = marshal - } - } - } - request.ReasoningEffort = "" } - - // https://docs.anthropic.com/en/api/openai-sdk#extended-thinking-support - // 没有做排除3.5Haiku等，要出问题再加吧，最佳兼容性（不是 - if request.THINKING != nil && strings.HasPrefix(info.UpstreamModelName, "anthropic") { - var thinking dto.Thinking // Claude标准Thinking格式 - if err := json.Unmarshal(request.THINKING, &thinking); err != nil { - return nil, fmt.Errorf("error Unmarshal thinking: %w", err) + if !preserveSuffix && info.OriginModelName != info.UpstreamModelName && strings.HasSuffix(info.OriginModelName, "-thinking") { + initialIntent, err = kitreasoning.MergeExplicitAndSuffix( + initialIntent, + kitreasoning.Intent{Mode: kitreasoning.ModeEnabled}, + info.OriginModelName, + ) + if err != nil { + return nil, kitreasoning.AsClientError(err) } - - // 只有当 thinking.Type 是 "enabled" 时才处理 - if thinking.Type == "enabled" { - // 检查 BudgetTokens 是否为 nil - if thinking.BudgetTokens == nil { - return nil, fmt.Errorf("BudgetTokens is nil when thinking is enabled") - } - - reasoning := openrouter.RequestReasoning{ - Enabled: true, - MaxTokens: *thinking.BudgetTokens, - } - - marshal, err := common.Marshal(reasoning) - if err != nil { - return nil, fmt.Errorf("error marshalling reasoning: %w", err) - } - - request.Reasoning = marshal - } - - // 清空 THINKING - request.THINKING = nil } + if !initialIntent.IsEmpty() { + reasoningConfig := make(map[string]any) + if len(request.Reasoning) > 0 { + if err := common.Unmarshal(request.Reasoning, &reasoningConfig); err != nil { + return nil, fmt.Errorf("error unmarshalling reasoning: %w", err) + } + if reasoningConfig == nil { + reasoningConfig = make(map[string]any) + } + } + disabled := initialIntent.Mode == kitreasoning.ModeDisabled || initialIntent.Effort == kitreasoning.EffortNone + if initialIntent.HasStrength() { + reasoningConfig["enabled"] = !disabled + if disabled { + delete(reasoningConfig, "effort") + delete(reasoningConfig, "max_tokens") + } + } + if !disabled && initialIntent.BudgetTokens != nil { + reasoningConfig["max_tokens"] = *initialIntent.BudgetTokens + delete(reasoningConfig, "effort") + } else if !disabled && initialIntent.Effort != "" && initialIntent.Effort != kitreasoning.EffortNone { + reasoningConfig["effort"] = string(initialIntent.Effort) + delete(reasoningConfig, "max_tokens") + } + if initialIntent.IncludeThoughts != nil { + reasoningConfig["exclude"] = !*initialIntent.IncludeThoughts + } + marshal, err := common.Marshal(reasoningConfig) + if err != nil { + return nil, fmt.Errorf("error marshalling reasoning: %w", err) + } + request.Reasoning = marshal + } + request.ReasoningEffort = "" + effectiveEffort := kitreasoning.EffectiveEffort(initialIntent) + if initialIntent.BudgetTokens != nil { + effectiveEffort = kitreasoning.EffortFromBudget(*initialIntent.BudgetTokens) + } + info.SetReasoningEffort(string(effectiveEffort)) } isOModel := dto.IsOpenAIReasoningOModel(info.UpstreamModelName) @@ -344,16 +387,6 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn request.LogProbs = nil } - // 转换模型推理力度后缀 - effort, originModel := reasoning.ParseOpenAIReasoningEffortFromModelSuffix(info.UpstreamModelName) - if effort != "" { - request.ReasoningEffort = effort - info.UpstreamModelName = originModel - request.Model = originModel - } - - info.SetReasoningEffort(request.ReasoningEffort) - // o系列模型developer适配（o1-mini除外） if !strings.HasPrefix(info.UpstreamModelName, "o1-mini") && !strings.HasPrefix(info.UpstreamModelName, "o1-preview") { //修改第一个Message的内容，将system改为developer @@ -363,6 +396,53 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn } } + if info.ChannelType != constant.ChannelTypeOpenRouter { + preserveSuffix := model_setting.ShouldPreserveThinkingSuffix(info.OriginModelName) || model_setting.ShouldPreserveThinkingSuffix(info.UpstreamModelName) + effort, baseModel := reasoning.ParseOpenAIReasoningEffortFromModelSuffix(info.UpstreamModelName) + if preserveSuffix { + effort = "" + } + currentIntent, err := kitreasoning.FromOpenAIChat(request) + if err != nil { + return nil, kitreasoning.AsClientError(err) + } + mergeSuffix := func(modelName, rawEffort string) error { + if rawEffort == "" { + return nil + } + suffixEffort, err := kitreasoning.ParseEffort(rawEffort) + if err != nil { + return err + } + mode := kitreasoning.ModeEnabled + if suffixEffort == kitreasoning.EffortNone { + mode = kitreasoning.ModeDisabled + } + currentIntent, err = kitreasoning.MergeExplicitAndSuffix(currentIntent, kitreasoning.Intent{Mode: mode, Effort: suffixEffort, Source: kitreasoning.SourceSuffix}, modelName) + return err + } + if err := mergeSuffix(info.UpstreamModelName, effort); err != nil { + return nil, kitreasoning.AsClientError(err) + } + if !preserveSuffix && info.OriginModelName != info.UpstreamModelName { + originEffort, _ := reasoning.ParseOpenAIReasoningEffortFromModelSuffix(info.OriginModelName) + if err := mergeSuffix(info.OriginModelName, originEffort); err != nil { + return nil, kitreasoning.AsClientError(err) + } + } + if effort != "" { + info.UpstreamModelName = baseModel + request.Model = baseModel + } + if canonicalEffort := kitreasoning.OpenAIEffort(kitreasoning.EffectiveEffort(currentIntent)); canonicalEffort != "" { + request.ReasoningEffort = string(canonicalEffort) + info.SetReasoningEffort(string(canonicalEffort)) + } + if info.ChannelType == constant.ChannelTypeOpenAI || info.ChannelType == constant.ChannelTypeAzure { + request.Reasoning = nil + } + } + return request, nil } @@ -604,18 +684,52 @@ func detectImageMimeType(filename string) string { func (a *Adaptor) ConvertOpenAIResponsesRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.OpenAIResponsesRequest) (any, error) { // 转换模型推理力度后缀 effort, originModel := reasoning.ParseOpenAIReasoningEffortFromModelSuffix(request.Model) - if effort != "" { - if request.Reasoning == nil { - request.Reasoning = &dto.Reasoning{ - Effort: effort, - } - } else { - request.Reasoning.Effort = effort - } - request.Model = originModel + preserveSuffix := model_setting.ShouldPreserveThinkingSuffix(request.Model) || (info != nil && model_setting.ShouldPreserveThinkingSuffix(info.OriginModelName)) + if preserveSuffix { + effort = "" } - if info != nil && request.Reasoning != nil && request.Reasoning.Effort != "" { - info.SetReasoningEffort(request.Reasoning.Effort) + currentIntent, err := kitreasoning.FromOpenAIResponses(&request) + if err != nil { + return nil, kitreasoning.AsClientError(err) + } + mergeSuffix := func(modelName, rawEffort string) error { + if rawEffort == "" { + return nil + } + suffixEffort, err := kitreasoning.ParseEffort(rawEffort) + if err != nil { + return err + } + mode := kitreasoning.ModeEnabled + if suffixEffort == kitreasoning.EffortNone { + mode = kitreasoning.ModeDisabled + } + currentIntent, err = kitreasoning.MergeExplicitAndSuffix(currentIntent, kitreasoning.Intent{Mode: mode, Effort: suffixEffort, Source: kitreasoning.SourceSuffix}, modelName) + return err + } + if err := mergeSuffix(request.Model, effort); err != nil { + return nil, kitreasoning.AsClientError(err) + } + if !preserveSuffix && info != nil && info.OriginModelName != request.Model { + originEffort, _ := reasoning.ParseOpenAIReasoningEffortFromModelSuffix(info.OriginModelName) + if err := mergeSuffix(info.OriginModelName, originEffort); err != nil { + return nil, kitreasoning.AsClientError(err) + } + } + if effort != "" { + request.Model = originModel + if info != nil { + info.UpstreamModelName = originModel + } + } + if canonicalEffort := kitreasoning.OpenAIEffort(kitreasoning.EffectiveEffort(currentIntent)); canonicalEffort != "" { + if request.Reasoning == nil { + request.Reasoning = &dto.Reasoning{} + } + request.Reasoning.Effort = string(canonicalEffort) + if info != nil { + info.SetReasoningEffort(string(canonicalEffort)) + } } return request, nil } diff --git a/relay/channel/openai/chat_via_responses.go b/relay/channel/openai/chat_via_responses.go index 25caeb5854..5a45585dcc 100644 --- a/relay/channel/openai/chat_via_responses.go +++ b/relay/channel/openai/chat_via_responses.go @@ -41,33 +41,10 @@ func OaiResponsesToChatHandler(c *gin.Context, info *relaycommon.RelayInfo, resp return nil, types.WithOpenAIError(*oaiError, resp.StatusCode) } - chatResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatOpenAI, &responsesResp) + responseValue, usage, err := convertResponsesResponseForClient(c, info, &responsesResp) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } - chatResp, ok := chatResult.Value.(*dto.OpenAITextResponse) - if !ok { - return nil, types.NewOpenAIError(fmt.Errorf("expected OpenAI chat response, got %T", chatResult.Value), types.ErrorCodeBadResponseBody, http.StatusInternalServerError) - } - if chatID := helper.GetResponseID(c); chatID != "" { - chatResp.Id = chatID - } - usage := chatResult.Usage - - if usage == nil || usage.TotalTokens == 0 { - text := service.ExtractOutputTextFromResponses(&responsesResp) - usage = service.ResponseText2Usage(c, text, info.UpstreamModelName, info.GetEstimatePromptTokens()) - chatResp.Usage = *usage - } - - responseValue := any(chatResp) - if info.RelayFormat != types.RelayFormatOpenAI { - targetResult, err := relayconvert.ConvertResponse(c, info, info.RelayFormat, chatResp) - if err != nil { - return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) - } - responseValue = targetResult.Value - } responseBody, err := common.Marshal(responseValue) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError) @@ -150,32 +127,10 @@ func OaiResponsesToChatBufferedStreamHandler(c *gin.Context, info *relaycommon.R } accumulator.SupplementResponseOutput(finalResponse) - chatResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatOpenAI, finalResponse) + responseValue, usage, err := convertResponsesResponseForClient(c, info, finalResponse) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } - chatResp, ok := chatResult.Value.(*dto.OpenAITextResponse) - if !ok { - return nil, types.NewOpenAIError(fmt.Errorf("expected OpenAI chat response, got %T", chatResult.Value), types.ErrorCodeBadResponseBody, http.StatusInternalServerError) - } - if chatID := helper.GetResponseID(c); chatID != "" { - chatResp.Id = chatID - } - usage := chatResult.Usage - if usage == nil || usage.TotalTokens == 0 { - text := service.ExtractOutputTextFromResponses(finalResponse) - usage = service.ResponseText2Usage(c, text, info.UpstreamModelName, info.GetEstimatePromptTokens()) - chatResp.Usage = *usage - } - - responseValue := any(chatResp) - if info.RelayFormat != types.RelayFormatOpenAI { - targetResult, err := relayconvert.ConvertResponse(c, info, info.RelayFormat, chatResp) - if err != nil { - return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) - } - responseValue = targetResult.Value - } responseBody, err := common.Marshal(responseValue) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError) @@ -185,6 +140,28 @@ func OaiResponsesToChatBufferedStreamHandler(c *gin.Context, info *relaycommon.R return usage, nil } +func convertResponsesResponseForClient(c *gin.Context, info *relaycommon.RelayInfo, response *dto.OpenAIResponsesResponse) (any, *dto.Usage, error) { + if responseID := helper.GetResponseID(c); responseID != "" { + response.ID = responseID + } + + usage := relayconvert.UsageFromResponsesUsage(response.Usage) + if usage == nil || usage.TotalTokens == 0 { + text := service.ExtractOutputTextFromResponses(response) + usage = service.ResponseText2Usage(c, text, info.UpstreamModelName, info.GetEstimatePromptTokens()) + response.Usage = relayconvert.UsageFromChatUsage(usage) + } + + result, err := service.ConvertResponse(c, info, info.RelayFormat, response) + if err != nil { + return nil, nil, err + } + if result.Usage != nil && result.Usage.TotalTokens != 0 { + usage = result.Usage + } + return result.Value, usage, nil +} + func OaiResponsesToChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) { if resp == nil || resp.Body == nil { return nil, types.NewOpenAIError(fmt.Errorf("invalid response"), types.ErrorCodeBadResponse, http.StatusInternalServerError) @@ -293,7 +270,7 @@ func OaiResponsesToChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo return } - results, err := relayconvert.ConvertStreamResponseChunk(c, info, state, &streamResp) + results, err := service.ConvertStreamResponseChunk(c, info, state, &streamResp) if err != nil { streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) sr.Stop(streamErr) @@ -320,7 +297,7 @@ func OaiResponsesToChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo if info.RelayFormat == types.RelayFormatClaude && info.ClaudeConvertInfo != nil { info.ClaudeConvertInfo.Usage = usage } - finalResults, err := relayconvert.FinalizeStreamResponse(c, info, state) + finalResults, err := service.FinalizeStreamResponse(c, info, state) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) } diff --git a/relay/channel/openai/chat_via_responses_test.go b/relay/channel/openai/chat_via_responses_test.go index df83b1d616..27ee71c9ab 100644 --- a/relay/channel/openai/chat_via_responses_test.go +++ b/relay/channel/openai/chat_via_responses_test.go @@ -10,6 +10,7 @@ import ( "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/constant" relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/types" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" @@ -171,6 +172,50 @@ func TestOaiResponsesToChatBufferedStreamHandlerReturnsJSONFromSSE(t *testing.T) require.Contains(t, got, `"finish_reason":"tool_calls"`) } +func TestOaiResponsesToChatBufferedStreamHandlerPreservesInterleavedClaudeContent(t *testing.T) { + oldMode := gin.Mode() + gin.SetMode(gin.TestMode) + t.Cleanup(func() { gin.SetMode(oldMode) }) + + body := strings.Join([]string{ + `data: {"type":"response.output_item.added","output_index":0,"item":{"type":"reasoning","id":"rs_1","summary":[]}}`, + `data: {"type":"response.reasoning_summary_text.delta","output_index":0,"item_id":"rs_1","delta":"**Planning file inspection**"}`, + `data: {"type":"response.output_item.added","output_index":1,"item":{"type":"message","id":"msg_1","role":"assistant","content":[]}}`, + `data: {"type":"response.output_text.delta","output_index":1,"item_id":"msg_1","delta":"I’ll inspect the starter repository."}`, + `data: {"type":"response.output_item.added","output_index":2,"item":{"type":"reasoning","id":"rs_2","summary":[]}}`, + `data: {"type":"response.reasoning_summary_text.delta","output_index":2,"item_id":"rs_2","delta":"**Clarifying environment task requirements**"}`, + `data: {"type":"response.output_item.added","output_index":3,"item":{"type":"message","id":"msg_2","role":"assistant","content":[]}}`, + `data: {"type":"response.output_text.delta","output_index":3,"item_id":"msg_2","delta":"What would you like me to build?"}`, + `data: {"type":"response.done","response":{"id":"resp_1","model":"gpt-test","status":"completed","usage":{"input_tokens":1,"output_tokens":2,"total_tokens":3}}}`, + `data: [DONE]`, + ``, + }, "\n") + + c, recorder, resp, info := newResponsesChatTestContext(t, body, false) + info.RelayFormat = types.RelayFormatClaude + + usage, apiErr := OaiResponsesToChatBufferedStreamHandler(c, info, resp) + require.Nil(t, apiErr) + require.NotNil(t, usage) + assert.Equal(t, 3, usage.TotalTokens) + + var claudeResponse dto.ClaudeResponse + require.NoError(t, common.Unmarshal(recorder.Body.Bytes(), &claudeResponse)) + require.Len(t, claudeResponse.Content, 4) + assert.Equal(t, []string{"thinking", "text", "thinking", "text"}, []string{ + claudeResponse.Content[0].Type, + claudeResponse.Content[1].Type, + claudeResponse.Content[2].Type, + claudeResponse.Content[3].Type, + }) + require.NotNil(t, claudeResponse.Content[0].Thinking) + require.NotNil(t, claudeResponse.Content[2].Thinking) + assert.Equal(t, "**Planning file inspection**", *claudeResponse.Content[0].Thinking) + assert.Equal(t, "I’ll inspect the starter repository.", claudeResponse.Content[1].GetText()) + assert.Equal(t, "**Clarifying environment task requirements**", *claudeResponse.Content[2].Thinking) + assert.Equal(t, "What would you like me to build?", claudeResponse.Content[3].GetText()) +} + func TestOaiChatToResponsesStreamHandlerConvertsSSEOrderAndUsage(t *testing.T) { oldMode := gin.Mode() gin.SetMode(gin.TestMode) diff --git a/relay/channel/openai/helper.go b/relay/channel/openai/helper.go index 666235ff56..3999ba560f 100644 --- a/relay/channel/openai/helper.go +++ b/relay/channel/openai/helper.go @@ -19,16 +19,20 @@ import ( "github.com/gin-gonic/gin" ) +const chatToGeminiStreamStateKey = "relaykit.chat_to_gemini_stream_state" + // 辅助函数 func HandleStreamFormat(c *gin.Context, info *relaycommon.RelayInfo, data string, forceFormat bool, thinkToContent bool) error { - info.SendResponseCount++ - switch info.RelayFormat { case types.RelayFormatOpenAI: + info.SendResponseCount++ return sendStreamData(c, info, data, forceFormat, thinkToContent) case types.RelayFormatClaude: + info.SendResponseCount++ return handleClaudeFormat(c, data, info) case types.RelayFormatGemini: + // The stateful relaykit path owns its chunk counter so multi-hop and + // direct conversions observe the same stream state semantics. return handleGeminiFormat(c, data, info) } return nil @@ -41,9 +45,9 @@ func handleClaudeFormat(c *gin.Context, data string, info *relaycommon.RelayInfo } if streamResponse.Usage != nil { - info.ClaudeConvertInfo.Usage = streamResponse.Usage + info.EnsureClaudeConvertInfo().Usage = streamResponse.Usage } - result, err := relayconvert.ConvertStreamResponse(c, info, types.RelayFormatClaude, &streamResponse) + result, err := service.ConvertStreamResponse(c, info, types.RelayFormatClaude, &streamResponse) if err != nil { return err } @@ -64,29 +68,55 @@ func handleGeminiFormat(c *gin.Context, data string, info *relaycommon.RelayInfo return err } - result, err := relayconvert.ConvertStreamResponse(c, info, types.RelayFormatGemini, &streamResponse) + state, err := chatToGeminiStreamState(c, &streamResponse) if err != nil { return err } - geminiResponse, ok := result.Value.(*dto.GeminiChatResponse) - if !ok { - return fmt.Errorf("expected Gemini stream response, got %T", result.Value) - } - - // 如果返回 nil，表示没有实际内容，跳过发送 - if geminiResponse == nil { - return nil - } - - geminiResponseStr, err := common.Marshal(geminiResponse) + results, err := service.ConvertStreamResponseChunk(c, info, state, &streamResponse) if err != nil { - logger.LogError(c, "failed to marshal gemini response: "+err.Error()) return err } + return sendGeminiStreamResults(c, results) +} - // send gemini format response - c.Render(-1, common.CustomEvent{Data: "data: " + string(geminiResponseStr)}) - _ = helper.FlushWriter(c) +func chatToGeminiStreamState(c *gin.Context, streamResponse *dto.ChatCompletionsStreamResponse) (*relayconvert.ResponseStreamState, error) { + if value, ok := c.Get(chatToGeminiStreamStateKey); ok { + state, ok := value.(*relayconvert.ResponseStreamState) + if !ok || state == nil { + return nil, fmt.Errorf("invalid Chat-to-Gemini stream state %T", value) + } + return state, nil + } + + state, err := relayconvert.NewResponseStreamState(types.RelayFormatOpenAI, types.RelayFormatGemini, relayconvert.ResponseStreamOptions{ + ID: streamResponse.Id, + Model: streamResponse.Model, + Created: streamResponse.Created, + }) + if err != nil { + return nil, err + } + c.Set(chatToGeminiStreamStateKey, state) + return state, nil +} + +func sendGeminiStreamResults(c *gin.Context, results []relayconvert.ResponseResult) error { + for _, result := range results { + geminiResponse, ok := result.Value.(*dto.GeminiChatResponse) + if !ok { + return fmt.Errorf("expected Gemini stream response, got %T", result.Value) + } + if geminiResponse == nil { + continue + } + data, err := common.Marshal(geminiResponse) + if err != nil { + logger.LogError(c, "failed to marshal gemini response: "+err.Error()) + return err + } + c.Render(-1, common.CustomEvent{Data: "data: " + string(data)}) + _ = helper.FlushWriter(c) + } return nil } @@ -148,7 +178,7 @@ func handleLastResponse(lastStreamData string, responseId *string, createAt *int if service.ValidUsage(lastStreamResponse.Usage) { *containStreamUsage = true - *usage = lastStreamResponse.Usage + *usage = dto.MergeUsageNonZero(*usage, lastStreamResponse.Usage) if !info.ShouldIncludeUsage { *shouldSendLastResp = lo.SomeBy(lastStreamResponse.Choices, func(choice dto.ChatCompletionsStreamResponseChoice) bool { return choice.Delta.GetContentString() != "" || choice.Delta.GetReasoningContent() != "" @@ -181,7 +211,7 @@ func HandleFinalResponse(c *gin.Context, info *relaycommon.RelayInfo, lastStream info.ClaudeConvertInfo.Usage = usage - result, err := relayconvert.ConvertStreamResponse(c, info, types.RelayFormatClaude, &streamResponse) + result, err := service.ConvertStreamResponse(c, info, types.RelayFormatClaude, &streamResponse) if err != nil { common.SysLog("error converting Claude stream response: " + err.Error()) return @@ -203,36 +233,31 @@ func HandleFinalResponse(c *gin.Context, info *relaycommon.RelayInfo, lastStream return } - // 这里处理的是 openai 最后一个流响应，其 delta 为空，有 finish_reason 字段 - // 因此相比较于 google 官方的流响应，由 openai 转换而来会多一个 parts 为空，finishReason 为 STOP 的响应 - // 而包含最后一段文本输出的响应（倒数第二个）的 finishReason 为 null - // 暂不知是否有程序会不兼容。 - - result, err := relayconvert.ConvertStreamResponse(c, info, types.RelayFormatGemini, &streamResponse) + state, err := chatToGeminiStreamState(c, &streamResponse) if err != nil { - common.SysLog("error converting Gemini stream response: " + err.Error()) - return - } - geminiResponse, ok := result.Value.(*dto.GeminiChatResponse) - if !ok { - common.SysLog(fmt.Sprintf("expected Gemini stream response, got %T", result.Value)) + common.SysLog("error creating Gemini stream state: " + err.Error()) return } + state.SetUsage(usage) - // openai 流响应开头的空数据 - if geminiResponse == nil { - return - } - - geminiResponseStr, err := common.Marshal(geminiResponse) + results, err := service.ConvertStreamResponseChunk(c, info, state, &streamResponse) if err != nil { - common.SysLog("error marshalling gemini response: " + err.Error()) + common.SysLog("error converting final Gemini stream response: " + err.Error()) + return + } + if err := sendGeminiStreamResults(c, results); err != nil { + common.SysLog("error sending final Gemini stream response: " + err.Error()) return } - // 发送最终的 Gemini 响应 - c.Render(-1, common.CustomEvent{Data: "data: " + string(geminiResponseStr)}) - _ = helper.FlushWriter(c) + results, err = service.FinalizeStreamResponse(c, info, state) + if err != nil { + common.SysLog("error finalizing Gemini stream response: " + err.Error()) + return + } + if err := sendGeminiStreamResults(c, results); err != nil { + common.SysLog("error sending finalized Gemini stream response: " + err.Error()) + } } } diff --git a/relay/channel/openai/relay-openai.go b/relay/channel/openai/relay-openai.go index 9a0619eb27..285e342f82 100644 --- a/relay/channel/openai/relay-openai.go +++ b/relay/channel/openai/relay-openai.go @@ -13,7 +13,6 @@ import ( relaycommon "github.com/QuantumNous/new-api/relay/common" "github.com/QuantumNous/new-api/relay/helper" "github.com/QuantumNous/new-api/relaykit/dto" - "github.com/QuantumNous/new-api/relaykit/relayconvert" "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/service" @@ -118,13 +117,10 @@ func OaiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Re var toolCount int var usage = &dto.Usage{} var lastStreamData string - var secondLastStreamData string // 存储倒数第二个stream data，用于音频模型 + var secondLastStreamData string // 保留倒数第二个stream data；部分兼容网关把完整usage放在倒数第二个事件 seenStreamToolCalls := make(map[string]struct{}) var streamFunctionCallNames []string - // 检查是否为音频模型 - isAudioModel := strings.Contains(strings.ToLower(model), "audio") - helper.StreamScannerHandler(c, resp, info, func(data string, sr *helper.StreamResult) { if lastStreamData != "" { if err := HandleStreamFormat(c, info, lastStreamData, info.ChannelSetting.ForceFormat, info.ChannelSetting.ThinkingToContent); err != nil { @@ -133,8 +129,7 @@ func OaiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Re } } if len(data) > 0 { - // 对音频模型，保存倒数第二个stream data - if isAudioModel && lastStreamData != "" { + if lastStreamData != "" { secondLastStreamData = lastStreamData } @@ -147,24 +142,6 @@ func OaiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Re } }) - // 对音频模型，从倒数第二个stream data中提取usage信息 - if isAudioModel && secondLastStreamData != "" { - var streamResp struct { - Usage *dto.Usage `json:"usage"` - } - err := common.Unmarshal([]byte(secondLastStreamData), &streamResp) - if err == nil && streamResp.Usage != nil && service.ValidUsage(streamResp.Usage) { - usage = streamResp.Usage - containStreamUsage = true - - if common.DebugEnabled { - logger.LogDebug(c, "Audio model usage extracted from second last SSE: PromptTokens=%d, CompletionTokens=%d, TotalTokens=%d, InputTokens=%d, OutputTokens=%d", - usage.PromptTokens, usage.CompletionTokens, usage.TotalTokens, - usage.InputTokens, usage.OutputTokens) - } - } - } - // 处理最后的响应 shouldSendLastResp := true if err := handleLastResponse(lastStreamData, &responseId, &createAt, &systemFingerprint, &model, &usage, @@ -172,6 +149,29 @@ func OaiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Re logger.LogError(c, fmt.Sprintf("error handling last response: %s, lastStreamData: [%s]", err.Error(), lastStreamData)) } + // 部分兼容网关把完整的累计usage附在倒数第二个事件上，随后发送一个空的最后事件。 + // 仅当最后一个事件没有有效usage时，回退到倒数第二个事件的完整快照。 + usageFrame := lastStreamData + if !containStreamUsage && secondLastStreamData != "" { + var streamResp struct { + Usage *dto.Usage `json:"usage"` + } + err := common.Unmarshal([]byte(secondLastStreamData), &streamResp) + if err == nil && streamResp.Usage != nil && + streamResp.Usage.PromptTokens > 0 && + (streamResp.Usage.CompletionTokens > 0 || streamResp.Usage.TotalTokens > 0) { + usage = dto.MergeUsageNonZero(usage, streamResp.Usage) + containStreamUsage = true + usageFrame = secondLastStreamData + + if common.DebugEnabled { + logger.LogDebug(c, "usage extracted from second last SSE: PromptTokens=%d, CompletionTokens=%d, TotalTokens=%d, InputTokens=%d, OutputTokens=%d", + usage.PromptTokens, usage.CompletionTokens, usage.TotalTokens, + usage.InputTokens, usage.OutputTokens) + } + } + } + if info.RelayFormat == types.RelayFormatOpenAI { if shouldSendLastResp { _ = sendStreamData(c, info, lastStreamData, info.ChannelSetting.ForceFormat, info.ChannelSetting.ThinkingToContent) @@ -183,7 +183,7 @@ func OaiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Re usage.CompletionTokens += toolCount * 7 } - applyUsagePostProcessing(info, usage, common.StringToByteSlice(lastStreamData)) + applyUsagePostProcessing(info, usage, common.StringToByteSlice(usageFrame)) for _, name := range streamFunctionCallNames { info.CountBillableToolCall(dto.BuildInCallFunctionCall, name) @@ -201,7 +201,7 @@ func collectStreamFunctionCallNames(data string, seen map[string]struct{}, names } for _, choice := range streamResponse.Choices { for i, tc := range choice.Delta.ToolCalls { - name := tc.Function.Name + name := strings.TrimSpace(tc.Function.Name) if name == "" { continue } @@ -209,11 +209,30 @@ func collectStreamFunctionCallNames(data string, seen map[string]struct{}, names if tc.Index != nil { toolIdx = *tc.Index } - key := fmt.Sprintf("%d-%d", choice.Index, toolIdx) - if _, ok := seen[key]; ok { - continue + fallbackKey := fmt.Sprintf("index\x00%d\x00%d\x00%s", choice.Index, toolIdx, name) + activeKey := fmt.Sprintf("active\x00%d\x00%d\x00%s", choice.Index, toolIdx, name) + callID := strings.TrimSpace(tc.ID) + if callID != "" { + idKey := fmt.Sprintf("id\x00%d\x00%s", choice.Index, callID) + if _, ok := seen[idKey]; ok { + continue + } + seen[idKey] = struct{}{} + seen[activeKey] = struct{}{} + if _, delayedID := seen[fallbackKey]; delayedID { + delete(seen, fallbackKey) + continue + } + } else { + if _, ok := seen[fallbackKey]; ok { + continue + } + if _, ok := seen[activeKey]; ok { + continue + } + seen[fallbackKey] = struct{}{} + seen[activeKey] = struct{}{} } - seen[key] = struct{}{} *names = append(*names, name) } } @@ -280,11 +299,12 @@ func OpenaiHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respo completionTokens += ctkm } } - simpleResponse.Usage = dto.Usage{ + fallbackUsage := &dto.Usage{ PromptTokens: info.GetEstimatePromptTokens(), CompletionTokens: completionTokens, TotalTokens: info.GetEstimatePromptTokens() + completionTokens, } + simpleResponse.Usage = *fallbackUsage usageModified = true } @@ -310,7 +330,7 @@ func OpenaiHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respo break } case types.RelayFormatClaude: - convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatClaude, &simpleResponse) + convertResult, err := service.ConvertResponse(c, info, types.RelayFormatClaude, &simpleResponse) if err != nil { return nil, types.NewError(err, types.ErrorCodeBadResponseBody) } @@ -320,7 +340,7 @@ func OpenaiHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respo } responseBody = claudeRespStr case types.RelayFormatGemini: - convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatGemini, &simpleResponse) + convertResult, err := service.ConvertResponse(c, info, types.RelayFormatGemini, &simpleResponse) if err != nil { return nil, types.NewError(err, types.ErrorCodeBadResponseBody) } diff --git a/relay/channel/openai/relay_responses.go b/relay/channel/openai/relay_responses.go index ceca1af3b3..93b2599ded 100644 --- a/relay/channel/openai/relay_responses.go +++ b/relay/channel/openai/relay_responses.go @@ -11,6 +11,7 @@ import ( relaycommon "github.com/QuantumNous/new-api/relay/common" "github.com/QuantumNous/new-api/relay/helper" "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/relayconvert" "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/service" @@ -38,16 +39,7 @@ func OaiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http service.IOCopyBytesGracefully(c, resp, responseBody) // compute usage - usage := dto.Usage{} - if responsesResponse.Usage != nil { - usage.PromptTokens = responsesResponse.Usage.InputTokens - usage.CompletionTokens = responsesResponse.Usage.OutputTokens - usage.TotalTokens = responsesResponse.Usage.TotalTokens - if responsesResponse.Usage.InputTokensDetails != nil { - usage.PromptTokensDetails.CachedTokens = responsesResponse.Usage.InputTokensDetails.CachedTokens - usage.PromptTokensDetails.CacheWriteTokens = responsesResponse.Usage.InputTokensDetails.CacheWriteTokens - } - } + usage := relayconvert.NormalizeResponsesUsage(responsesResponse.Usage) // Count actual tool invocations from Output (not tool declarations). for _, output := range responsesResponse.Output { switch output.Type { @@ -69,7 +61,7 @@ func OaiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http } imageCounter.Commit(info) - return &usage, nil + return usage, nil } func OaiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) { @@ -99,19 +91,8 @@ func OaiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp case "response.completed", "response.done": if streamResponse.Response != nil { if streamResponse.Response.Usage != nil { - if streamResponse.Response.Usage.InputTokens != 0 { - usage.PromptTokens = streamResponse.Response.Usage.InputTokens - } - if streamResponse.Response.Usage.OutputTokens != 0 { - usage.CompletionTokens = streamResponse.Response.Usage.OutputTokens - } - if streamResponse.Response.Usage.TotalTokens != 0 { - usage.TotalTokens = streamResponse.Response.Usage.TotalTokens - } - if streamResponse.Response.Usage.InputTokensDetails != nil { - usage.PromptTokensDetails.CachedTokens = streamResponse.Response.Usage.InputTokensDetails.CachedTokens - usage.PromptTokensDetails.CacheWriteTokens = streamResponse.Response.Usage.InputTokensDetails.CacheWriteTokens - } + incomingUsage := relayconvert.NormalizeResponsesUsage(streamResponse.Response.Usage) + usage = dto.MergeUsageNonZero(usage, incomingUsage) } if !imageCommitted { if relaycommon.IsNonBillableResponsesStatus(streamResponse.Response.Status) { @@ -173,6 +154,9 @@ func OaiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp } usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens + if usage.BillingUsage != nil { + usage.BillingUsage = dto.CloneBillingUsageWithEstimatedCompletion(usage.BillingUsage, usage.CompletionTokens) + } return usage, nil } diff --git a/relay/channel/openai/responses_via_chat.go b/relay/channel/openai/responses_via_chat.go index 53b9d33cbc..6487708328 100644 --- a/relay/channel/openai/responses_via_chat.go +++ b/relay/channel/openai/responses_via_chat.go @@ -38,7 +38,7 @@ func OaiChatToResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp if responseID := helper.GetResponseID(c); responseID != "" { chatResp.Id = responseID } - convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatOpenAIResponses, &chatResp) + convertResult, err := service.ConvertResponse(c, info, types.RelayFormatOpenAIResponses, &chatResp) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) } @@ -70,8 +70,9 @@ func OaiChatToResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo responseID := helper.GetResponseID(c) state, err := relayconvert.NewResponseStreamState(types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses, relayconvert.ResponseStreamOptions{ - ID: responseID, - Model: info.UpstreamModelName, + ID: responseID, + Model: info.UpstreamModelName, + EmitSequenceNumber: true, }) if err != nil { return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) @@ -84,7 +85,27 @@ func OaiChatToResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo streamErr = types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError) return false } - helper.ResponseChunkData(c, dto.ResponsesStreamResponse{Type: event.Type}, string(data)) + if err := helper.ResponseChunkData(c, dto.ResponsesStreamResponse{Type: event.Type}, string(data)); err != nil { + streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) + return false + } + return true + } + failResponsesStream := func(err error) bool { + failureResults, handled := state.FailResponsesStream("server_error", err.Error(), "") + if !handled { + return false + } + for _, result := range failureResults { + event, ok := result.Value.(relayconvert.ChatToResponsesStreamEvent) + if !ok { + streamErr = types.NewOpenAIError(fmt.Errorf("expected OAI responses stream event, got %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError) + return true + } + if !sendEvent(event) { + return true + } + } return true } @@ -97,6 +118,10 @@ func OaiChatToResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo var errorResp dto.OpenAITextResponse if err := common.UnmarshalJsonStr(data, &errorResp); err == nil { if oaiError := errorResp.GetOpenAIError(); oaiError != nil && oaiError.Type != "" { + if failResponsesStream(fmt.Errorf("%s", oaiError.Message)) { + sr.Stop(streamErr) + return + } streamErr = types.WithOpenAIError(*oaiError, resp.StatusCode) sr.Stop(streamErr) return @@ -106,12 +131,21 @@ func OaiChatToResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo var chunk dto.ChatCompletionsStreamResponse if err := common.UnmarshalJsonStr(data, &chunk); err != nil { logger.LogError(c, "failed to unmarshal chat stream response: "+err.Error()) - sr.Error(err) + if failResponsesStream(err) { + sr.Stop(streamErr) + return + } + streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError) + sr.Stop(streamErr) return } - results, err := relayconvert.ConvertStreamResponseChunk(c, info, state, &chunk) + results, err := service.ConvertStreamResponseChunk(c, info, state, &chunk) if err != nil { + if failResponsesStream(err) { + sr.Stop(streamErr) + return + } streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) sr.Stop(streamErr) return @@ -140,8 +174,11 @@ func OaiChatToResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo state.SetUsage(usage) } - finalResults, err := relayconvert.FinalizeStreamResponse(c, info, state) + finalResults, err := service.FinalizeStreamResponse(c, info, state) if err != nil { + if failResponsesStream(err) { + return usage, streamErr + } return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError) } for _, result := range finalResults { diff --git a/relay/channel/sub2api/adaptor_test.go b/relay/channel/sub2api/adaptor_test.go index 4ce3912a7b..7c1f1c33f0 100644 --- a/relay/channel/sub2api/adaptor_test.go +++ b/relay/channel/sub2api/adaptor_test.go @@ -1,11 +1,13 @@ package sub2api import ( + "encoding/json" "testing" "github.com/QuantumNous/new-api/constant" relaycommon "github.com/QuantumNous/new-api/relay/common" relayconstant "github.com/QuantumNous/new-api/relay/constant" + "github.com/QuantumNous/new-api/relaykit/dto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -44,3 +46,40 @@ func TestAdaptorInheritsNewAPIResponsesCompactSupport(t *testing.T) { assert.Equal(t, "sub2api", adaptor.GetChannelName()) assert.Empty(t, adaptor.GetModelList()) } + +func TestConvertClaudeRequestPreservesAdaptiveThinkingForCompatibleModel(t *testing.T) { + adaptor := &Adaptor{} + maxTokens := uint(8192) + temperature := 0.2 + topP := 0.99 + request := &dto.ClaudeRequest{ + Model: "gpt-5.6-sol", + MaxTokens: &maxTokens, + Temperature: &temperature, + TopP: &topP, + Thinking: &dto.Thinking{Type: "adaptive", Display: "summarized"}, + OutputConfig: json.RawMessage(`{"effort":"xhigh","provider_option":true}`), + Messages: []dto.ClaudeMessage{ + {Role: "user", Content: "hello"}, + }, + } + info := &relaycommon.RelayInfo{ + OriginModelName: "gpt-5.6-sol", + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelType: constant.ChannelTypeSub2API, + }, + } + + converted, err := adaptor.ConvertClaudeRequest(nil, info, request) + + require.NoError(t, err) + assert.Same(t, request, converted) + require.NotNil(t, request.Thinking) + assert.Equal(t, "adaptive", request.Thinking.Type) + assert.Equal(t, "summarized", request.Thinking.Display) + assert.JSONEq(t, `{"effort":"xhigh","provider_option":true}`, string(request.OutputConfig)) + assert.Same(t, &temperature, request.Temperature) + assert.Same(t, &topP, request.TopP) + assert.Equal(t, "xhigh", info.ReasoningEffort) + assert.Equal(t, "gpt-5.6-sol", info.UpstreamModelName) +} diff --git a/relay/channel/vertex/adaptor.go b/relay/channel/vertex/adaptor.go index c60d75d29f..5e306318bb 100644 --- a/relay/channel/vertex/adaptor.go +++ b/relay/channel/vertex/adaptor.go @@ -18,7 +18,6 @@ import ( "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/service" "github.com/QuantumNous/new-api/setting/model_setting" - "github.com/QuantumNous/new-api/setting/reasoning" "github.com/gin-gonic/gin" "github.com/samber/lo" @@ -56,15 +55,16 @@ type Adaptor struct { } func (a *Adaptor) ConvertGeminiRequest(c *gin.Context, info *relaycommon.RelayInfo, request *dto.GeminiChatRequest) (any, error) { - // Vertex AI does not support functionResponse.id; keep it stripped here for consistency. + // Vertex AI's generateContent schema does not expose the Gemini API's + // function-call identity fields. Strip both sides at this provider boundary. if model_setting.GetGeminiSettings().RemoveFunctionResponseIdEnabled { - removeFunctionResponseID(request) + removeFunctionCallIDs(request) } geminiAdaptor := gemini.Adaptor{} return geminiAdaptor.ConvertGeminiRequest(c, info, request) } -func removeFunctionResponseID(request *dto.GeminiChatRequest) { +func removeFunctionCallIDs(request *dto.GeminiChatRequest) { if request == nil { return } @@ -76,10 +76,10 @@ func removeFunctionResponseID(request *dto.GeminiChatRequest) { } for j := range request.Contents[i].Parts { part := &request.Contents[i].Parts[j] - if part.FunctionResponse == nil { - continue + if part.FunctionCall != nil { + part.FunctionCall.ID = "" } - if len(part.FunctionResponse.ID) > 0 { + if part.FunctionResponse != nil && len(part.FunctionResponse.ID) > 0 { part.FunctionResponse.ID = nil } } @@ -88,12 +88,16 @@ func removeFunctionResponseID(request *dto.GeminiChatRequest) { if len(request.Requests) > 0 { for i := range request.Requests { - removeFunctionResponseID(&request.Requests[i]) + removeFunctionCallIDs(&request.Requests[i]) } } } func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayInfo, request *dto.ClaudeRequest) (any, error) { + claudeAdaptor := claude.Adaptor{} + if _, err := claudeAdaptor.ConvertClaudeRequest(c, info, request); err != nil { + return nil, err + } if v, ok := claudeModelMap[info.UpstreamModelName]; ok { c.Set("request_model", v) } else { @@ -170,21 +174,6 @@ func (a *Adaptor) getRequestUrl(info *relaycommon.RelayInfo, modelName, suffix s func (a *Adaptor) GetRequestURL(info *relaycommon.RelayInfo) (string, error) { suffix := "" if a.RequestMode == RequestModeGemini { - if model_setting.GetGeminiSettings().ThinkingAdapterEnabled && - !model_setting.ShouldPreserveThinkingSuffix(info.OriginModelName) { - // 新增逻辑：处理 -thinking- 格式 - if strings.Contains(info.UpstreamModelName, "-thinking-") { - parts := strings.Split(info.UpstreamModelName, "-thinking-") - info.UpstreamModelName = parts[0] - } else if strings.HasSuffix(info.UpstreamModelName, "-thinking") { // 旧的适配 - info.UpstreamModelName = strings.TrimSuffix(info.UpstreamModelName, "-thinking") - } else if strings.HasSuffix(info.UpstreamModelName, "-nothinking") { - info.UpstreamModelName = strings.TrimSuffix(info.UpstreamModelName, "-nothinking") - } else if baseModel, level, ok := reasoning.TrimEffortSuffix(info.UpstreamModelName); ok && level != "" { - info.UpstreamModelName = baseModel - } - } - if info.IsStream { suffix = "streamGenerateContent?alt=sse" } else { @@ -310,6 +299,9 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn if !ok { return nil, fmt.Errorf("expected Gemini generateContent request, got %T", result.Value) } + if model_setting.GetGeminiSettings().RemoveFunctionResponseIdEnabled { + removeFunctionCallIDs(geminiRequest) + } c.Set("request_model", request.Model) return geminiRequest, nil } else if a.RequestMode == RequestModeOpenSource { diff --git a/relay/channel/zhipu_4v/adaptor.go b/relay/channel/zhipu_4v/adaptor.go index 9153d39f35..d27232e2d1 100644 --- a/relay/channel/zhipu_4v/adaptor.go +++ b/relay/channel/zhipu_4v/adaptor.go @@ -28,7 +28,8 @@ func (a *Adaptor) ConvertGeminiRequest(*gin.Context, *relaycommon.RelayInfo, *dt } func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayInfo, req *dto.ClaudeRequest) (any, error) { - return req, nil + claudeAdaptor := claude.Adaptor{} + return claudeAdaptor.ConvertClaudeRequest(c, info, req) } func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.AudioRequest) (io.Reader, error) { diff --git a/relay/chat_completions_via_responses.go b/relay/chat_completions_via_responses.go index b8a6fc8758..202c04137c 100644 --- a/relay/chat_completions_via_responses.go +++ b/relay/chat_completions_via_responses.go @@ -70,30 +70,35 @@ func applySystemPromptIfNeeded(c *gin.Context, info *relaycommon.RelayInfo, requ } } -func chatCompletionsViaResponses(c *gin.Context, info *relaycommon.RelayInfo, adaptor channel.Adaptor, request *dto.GeneralOpenAIRequest) (*dto.Usage, *types.NewAPIError) { - chatJSON, err := common.Marshal(request) - if err != nil { - return nil, types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) - } - - chatJSON, err = relaycommon.RemoveDisabledFields(chatJSON, info.ChannelOtherSettings, info.ChannelSetting.PassThroughBodyEnabled) - if err != nil { - return nil, types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) - } - - if len(info.ParamOverride) > 0 { - chatJSON, err = relaycommon.ApplyParamOverrideWithRelayInfo(chatJSON, info) +func textRequestViaResponses(c *gin.Context, info *relaycommon.RelayInfo, adaptor channel.Adaptor, request any) (*dto.Usage, *types.NewAPIError) { + paramOverrideApplied := false + if chatRequest, ok := request.(*dto.GeneralOpenAIRequest); ok { + chatJSON, err := common.Marshal(chatRequest) if err != nil { - return nil, newAPIErrorFromParamOverride(err) + return nil, types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) } + + chatJSON, err = relaycommon.RemoveDisabledFields(chatJSON, info.ChannelOtherSettings, info.ChannelSetting.PassThroughBodyEnabled) + if err != nil { + return nil, types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) + } + + if len(info.ParamOverride) > 0 { + chatJSON, err = relaycommon.ApplyParamOverrideWithRelayInfo(chatJSON, info) + if err != nil { + return nil, newAPIErrorFromParamOverride(err) + } + paramOverrideApplied = true + } + + var overriddenChatReq dto.GeneralOpenAIRequest + if err := common.Unmarshal(chatJSON, &overriddenChatReq); err != nil { + return nil, types.NewError(err, types.ErrorCodeChannelParamOverrideInvalid, types.ErrOptionWithSkipRetry()) + } + request = &overriddenChatReq } - var overriddenChatReq dto.GeneralOpenAIRequest - if err := common.Unmarshal(chatJSON, &overriddenChatReq); err != nil { - return nil, types.NewError(err, types.ErrorCodeChannelParamOverrideInvalid, types.ErrOptionWithSkipRetry()) - } - - result, err := service.ConvertRequestVia(c, info, &overriddenChatReq, types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses) + result, err := service.ConvertRequest(c, info, types.RelayFormatOpenAIResponses, request) if err != nil { return nil, types.NewErrorWithStatusCode(err, types.ErrorCodeInvalidRequest, http.StatusBadRequest, types.ErrOptionWithSkipRetry()) } @@ -101,7 +106,10 @@ func chatCompletionsViaResponses(c *gin.Context, info *relaycommon.RelayInfo, ad if !ok { return nil, types.NewError(fmt.Errorf("expected OpenAI responses request, got %T", result.Value), types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) } + return relayResponsesRequest(c, info, adaptor, responsesReq, paramOverrideApplied) +} +func relayResponsesRequest(c *gin.Context, info *relaycommon.RelayInfo, adaptor channel.Adaptor, responsesReq *dto.OpenAIResponsesRequest, paramOverrideApplied bool) (*dto.Usage, *types.NewAPIError) { savedRelayMode := info.RelayMode savedRequestURLPath := info.RequestURLPath defer func() { @@ -114,7 +122,7 @@ func chatCompletionsViaResponses(c *gin.Context, info *relaycommon.RelayInfo, ad convertedRequest, err := adaptor.ConvertOpenAIResponsesRequest(c, info, *responsesReq) if err != nil { - return nil, types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) + return nil, newConvertRequestFailedError(c, info, err) } relaycommon.AppendRequestConversionFromRequest(info, convertedRequest) @@ -127,6 +135,12 @@ func chatCompletionsViaResponses(c *gin.Context, info *relaycommon.RelayInfo, ad if err != nil { return nil, types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) } + if !paramOverrideApplied && len(info.ParamOverride) > 0 { + jsonData, err = relaycommon.ApplyParamOverrideWithRelayInfo(jsonData, info) + if err != nil { + return nil, newAPIErrorFromParamOverride(err) + } + } body, closer, err := relaycommon.NewOutboundJSONBody(jsonData) if err != nil { diff --git a/relay/chat_completions_via_responses_test.go b/relay/chat_completions_via_responses_test.go index 1858787444..5b1067ee2e 100644 --- a/relay/chat_completions_via_responses_test.go +++ b/relay/chat_completions_via_responses_test.go @@ -1,11 +1,21 @@ package relay import ( + "io" "math" + "net/http" + "net/http/httptest" "testing" + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + openaichannel "github.com/QuantumNous/new-api/relay/channel/openai" relaycommon "github.com/QuantumNous/new-api/relay/common" - "github.com/QuantumNous/new-api/types" + relayconstant "github.com/QuantumNous/new-api/relay/constant" + "github.com/QuantumNous/new-api/relaykit/dto" + relaytypes "github.com/QuantumNous/new-api/relaykit/types" + hosttypes "github.com/QuantumNous/new-api/types" + "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -31,7 +41,7 @@ func TestIsResponsesEventStreamContentType(t *testing.T) { func TestRecalcQuotaFromRatiosIgnoresInvalidMultipliers(t *testing.T) { info := &relaycommon.RelayInfo{ - PriceData: types.PriceData{ + PriceData: hosttypes.PriceData{ Quota: 100, }, } @@ -52,7 +62,7 @@ func TestRecalcQuotaFromRatiosIgnoresInvalidMultipliers(t *testing.T) { func TestRecalcQuotaFromRatiosRejectsAllInvalidAdjustedRatios(t *testing.T) { info := &relaycommon.RelayInfo{ - PriceData: types.PriceData{ + PriceData: hosttypes.PriceData{ Quota: 100, }, } @@ -69,3 +79,77 @@ func TestRecalcQuotaFromRatiosRejectsAllInvalidAdjustedRatios(t *testing.T) { assert.Equal(t, 0, quota) assert.True(t, info.PriceData.HasOtherRatio("duration")) } + +func TestTextRequestViaResponsesConvertsClaudeDirectly(t *testing.T) { + type capturedRequest struct { + path string + body []byte + } + captured := make(chan capturedRequest, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + captured <- capturedRequest{path: r.URL.Path, body: body} + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id":"resp_1", + "object":"response", + "status":"completed", + "model":"gpt-5.6-sol", + "output":[{"type":"message","id":"msg_1","role":"assistant","content":[{"type":"output_text","text":"ok"}]}], + "usage":{"input_tokens":3,"output_tokens":2,"total_tokens":5} + }`)) + })) + defer server.Close() + + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil) + c.Request.Header.Set("Content-Type", "application/json") + + info := &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeChatCompletions, + RelayFormat: relaytypes.RelayFormatClaude, + OriginModelName: "gpt-5.6-sol", + RequestConversionChain: []relaytypes.RelayFormat{relaytypes.RelayFormatClaude}, + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelType: constant.ChannelTypeOpenAI, + ChannelBaseUrl: server.URL, + ApiKey: "test-key", + UpstreamModelName: "gpt-5.6-sol", + }, + } + adaptor := &openaichannel.Adaptor{} + adaptor.Init(info) + request := &dto.ClaudeRequest{ + Model: "gpt-5.6-sol", + Thinking: &dto.Thinking{Type: "adaptive", Display: "summarized"}, + Messages: []dto.ClaudeMessage{{Role: "user", Content: "hello"}}, + } + + usage, apiErr := textRequestViaResponses(c, info, adaptor, request) + + require.Nil(t, apiErr) + require.NotNil(t, usage) + assert.Equal(t, 5, usage.TotalTokens) + assert.Equal(t, []relaytypes.RelayFormat{relaytypes.RelayFormatClaude, relaytypes.RelayFormatOpenAIResponses}, info.RequestConversionChain) + + upstream := <-captured + assert.Equal(t, "/v1/responses", upstream.path) + var upstreamBody map[string]any + require.NoError(t, common.Unmarshal(upstream.body, &upstreamBody)) + assert.NotContains(t, upstreamBody, "messages") + reasoning, ok := upstreamBody["reasoning"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "high", reasoning["effort"]) + assert.Equal(t, "detailed", reasoning["summary"]) + + var response dto.ClaudeResponse + require.NoError(t, common.Unmarshal(recorder.Body.Bytes(), &response)) + require.Len(t, response.Content, 1) + assert.Equal(t, "ok", response.Content[0].GetText()) +} diff --git a/relay/claude_handler.go b/relay/claude_handler.go index ff7854469d..1dfbc3b657 100644 --- a/relay/claude_handler.go +++ b/relay/claude_handler.go @@ -1,7 +1,6 @@ package relay import ( - "encoding/json" "fmt" "io" "net/http" @@ -16,7 +15,6 @@ import ( "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/service" "github.com/QuantumNous/new-api/setting/model_setting" - "github.com/QuantumNous/new-api/setting/reasoning" "github.com/gin-gonic/gin" ) @@ -40,6 +38,9 @@ func ClaudeHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *typ if err != nil { return types.NewError(err, types.ErrorCodeChannelModelMappedError, types.ErrOptionWithSkipRetry()) } + if err = helper.ApplyReasoningModelSuffix(info, request); err != nil { + return newConvertRequestFailedError(c, info, err) + } adaptor := GetAdaptor(info.ApiType) if adaptor == nil { @@ -47,71 +48,6 @@ func ClaudeHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *typ } adaptor.Init(info) - if request.MaxTokens == nil || *request.MaxTokens == 0 { - defaultMaxTokens := uint(model_setting.GetClaudeSettings().GetDefaultMaxTokens(request.Model)) - request.MaxTokens = &defaultMaxTokens - } - - if baseModel, effortLevel, ok := reasoning.TrimEffortSuffix(request.Model); ok && effortLevel != "" && - (strings.HasPrefix(request.Model, "claude-opus-4-6") || - strings.HasPrefix(request.Model, "claude-opus-4-7") || - strings.HasPrefix(request.Model, "claude-opus-4-8")) { - request.Model = baseModel - request.Thinking = &dto.Thinking{ - Type: "adaptive", - } - request.OutputConfig = json.RawMessage(fmt.Sprintf(`{"effort":"%s"}`, effortLevel)) - if strings.HasPrefix(request.Model, "claude-opus-4-7") || - strings.HasPrefix(request.Model, "claude-opus-4-8") { - // Opus 4.7/4.8 reject non-default temperature/top_p/top_k with 400 - // and defaults display to "omitted"; restore the 4.6 visible summary. - request.Thinking.Display = "summarized" - request.Temperature = nil - request.TopP = nil - request.TopK = nil - } else { - request.Temperature = common.GetPointer[float64](1.0) - } - info.UpstreamModelName = request.Model - } else if model_setting.GetClaudeSettings().ThinkingAdapterEnabled && - strings.HasSuffix(request.Model, "-thinking") { - if request.Thinking == nil { - baseModel := strings.TrimSuffix(request.Model, "-thinking") - if strings.HasPrefix(baseModel, "claude-opus-4-7") || - strings.HasPrefix(baseModel, "claude-opus-4-8") { - // Opus 4.7/4.8 reject thinking.type="enabled"; use adaptive at high effort. - request.Thinking = &dto.Thinking{Type: "adaptive", Display: "summarized"} - request.OutputConfig = json.RawMessage(`{"effort":"high"}`) - request.Temperature = nil - request.TopP = nil - request.TopK = nil - } else { - // 因为BudgetTokens 必须大于1024 - if request.MaxTokens == nil || *request.MaxTokens < 1280 { - request.MaxTokens = common.GetPointer[uint](1280) - } - - // BudgetTokens 为 max_tokens 的 80% - request.Thinking = &dto.Thinking{ - Type: "enabled", - BudgetTokens: common.GetPointer[int](int(float64(*request.MaxTokens) * model_setting.GetClaudeSettings().ThinkingAdapterBudgetTokensPercentage)), - } - // TODO: 临时处理 - // https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#important-considerations-when-using-extended-thinking - request.Temperature = common.GetPointer[float64](1.0) - } - } - if !model_setting.ShouldPreserveThinkingSuffix(info.OriginModelName) { - request.Model = strings.TrimSuffix(request.Model, "-thinking") - } - info.UpstreamModelName = request.Model - } - if !model_setting.GetGlobalSettings().PassThroughRequestEnabled && !info.ChannelSetting.PassThroughBodyEnabled { - if effort := request.GetEfforts(); effort != "" { - info.SetReasoningEffort(effort) - } - } - if info.ChannelSetting.SystemPrompt != "" { if request.System == nil { request.SetStringSystem(info.ChannelSetting.SystemPrompt) @@ -140,16 +76,7 @@ func ClaudeHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *typ if !model_setting.GetGlobalSettings().PassThroughRequestEnabled && !info.ChannelSetting.PassThroughBodyEnabled && service.ShouldChatCompletionsUseResponsesGlobal(info.ChannelId, info.ChannelType, info.OriginModelName) { - result, convErr := service.ConvertRequest(c, info, types.RelayFormatOpenAI, request) - if convErr != nil { - return types.NewError(convErr, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) - } - openAIRequest, ok := result.Value.(*dto.GeneralOpenAIRequest) - if !ok { - return types.NewError(fmt.Errorf("expected OpenAI chat completions request, got %T", result.Value), types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) - } - - usage, newApiErr := chatCompletionsViaResponses(c, info, adaptor, openAIRequest) + usage, newApiErr := textRequestViaResponses(c, info, adaptor, request) if newApiErr != nil { return newApiErr } @@ -168,7 +95,7 @@ func ClaudeHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *typ } else { convertedRequest, err := adaptor.ConvertClaudeRequest(c, info, request) if err != nil { - return types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) + return newConvertRequestFailedError(c, info, err) } relaycommon.AppendRequestConversionFromRequest(info, convertedRequest) jsonData, err := common.Marshal(convertedRequest) diff --git a/relay/common/conversion_diagnostics.go b/relay/common/conversion_diagnostics.go new file mode 100644 index 0000000000..bc1e1cd24e --- /dev/null +++ b/relay/common/conversion_diagnostics.go @@ -0,0 +1,71 @@ +package common + +import ( + "context" + "fmt" + + "github.com/QuantumNous/new-api/logger" + "github.com/QuantumNous/new-api/relaykit/types" + "github.com/gin-gonic/gin" +) + +const maxConversionDiagnostics = 32 + +type conversionDiagnosticKey struct { + code string + path string + severity types.ConversionDiagnosticSeverity + from types.RelayFormat + to types.RelayFormat +} + +// RecordConversionDiagnostics retains conversion losses for the consume log +// and emits one request-correlated warning per distinct diagnostic. The cap +// prevents malformed streams from growing request state without bound. +func (info *RelayInfo) RecordConversionDiagnostics(ctx context.Context, diagnostics []types.ConversionDiagnostic) { + if info == nil || len(diagnostics) == 0 { + return + } + if ginCtx, ok := ctx.(*gin.Context); ok && ginCtx == nil { + ctx = nil + } + if info.conversionDiagnosticKeys == nil { + info.conversionDiagnosticKeys = make(map[conversionDiagnosticKey]struct{}) + } + for _, diagnostic := range diagnostics { + key := conversionDiagnosticKey{ + code: diagnostic.Code, + path: diagnostic.Path, + severity: diagnostic.Severity, + from: diagnostic.From, + to: diagnostic.To, + } + if _, exists := info.conversionDiagnosticKeys[key]; exists { + continue + } + if len(info.conversionDiagnostics) >= maxConversionDiagnostics { + if !info.conversionDiagnosticsTruncated { + info.conversionDiagnosticsTruncated = true + logger.LogWarn(ctx, fmt.Sprintf("conversion diagnostics truncated after %d distinct entries", maxConversionDiagnostics)) + } + continue + } + info.conversionDiagnosticKeys[key] = struct{}{} + info.conversionDiagnostics = append(info.conversionDiagnostics, diagnostic) + logger.LogWarn(ctx, fmt.Sprintf( + "conversion diagnostic: code=%q severity=%q from=%q to=%q path=%q message=%q", + diagnostic.Code, diagnostic.Severity, diagnostic.From, diagnostic.To, diagnostic.Path, diagnostic.Message, + )) + } +} + +func (info *RelayInfo) ConversionDiagnostics() []types.ConversionDiagnostic { + if info == nil || len(info.conversionDiagnostics) == 0 { + return nil + } + return append([]types.ConversionDiagnostic(nil), info.conversionDiagnostics...) +} + +func (info *RelayInfo) ConversionDiagnosticsTruncated() bool { + return info != nil && info.conversionDiagnosticsTruncated +} diff --git a/relay/common/override.go b/relay/common/override.go index b1a7d17744..477af24677 100644 --- a/relay/common/override.go +++ b/relay/common/override.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/QuantumNous/new-api/common" + kitreasoning "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" "github.com/QuantumNous/new-api/relaykit/types" "github.com/samber/lo" "github.com/tidwall/gjson" @@ -224,22 +225,73 @@ func syncReasoningEffortAfterParamOverride(info *RelayInfo, before, after []byte } func extractReasoningEffortFromJSON(format types.RelayFormat, data []byte) (string, bool) { - var paths []string switch format { case types.RelayFormatOpenAI: - paths = []string{"reasoning_effort", "reasoning.effort"} + if effort, exists := firstStringValue(data, "reasoning_effort"); exists && effort != "" { + return effort, true + } + if enabled := gjson.GetBytes(data, "reasoning.enabled"); enabled.Exists() { + if enabled.Type != gjson.True && enabled.Type != gjson.False { + return "", true + } + if !enabled.Bool() { + return string(kitreasoning.EffortNone), true + } + if effort, exists := firstStringValue(data, "reasoning.effort"); exists && effort != "" { + return effort, true + } + if budget := gjson.GetBytes(data, "reasoning.max_tokens"); budget.Exists() { + return reasoningEffortFromBudgetValue(budget) + } + return string(kitreasoning.EffortHigh), true + } + if effort, exists := firstStringValue(data, "reasoning.effort"); exists && effort != "" { + return effort, true + } + if budget := gjson.GetBytes(data, "reasoning.max_tokens"); budget.Exists() { + return reasoningEffortFromBudgetValue(budget) + } + return "", false case types.RelayFormatOpenAIResponses: - paths = []string{"reasoning.effort"} + return firstStringValue(data, "reasoning.effort") case types.RelayFormatClaude: - paths = []string{"output_config.effort"} + if effort, exists := firstStringValue(data, "output_config.effort"); exists && effort != "" { + return effort, true + } + thinkingType, hasThinkingType := firstStringValue(data, "thinking.type") + if thinkingType == "disabled" { + return string(kitreasoning.EffortNone), true + } + if budget := gjson.GetBytes(data, "thinking.budget_tokens"); budget.Exists() { + return reasoningEffortFromBudgetValue(budget) + } + if thinkingType == "enabled" || thinkingType == "adaptive" { + return string(kitreasoning.EffortHigh), true + } + return "", hasThinkingType case types.RelayFormatGemini: - paths = []string{ + level, hasLevel := firstStringValue(data, "generationConfig.thinkingConfig.thinkingLevel", "generation_config.thinking_config.thinking_level", + ) + if level != "" { + return level, true } + for _, path := range []string{ + "generationConfig.thinkingConfig.thinkingBudget", + "generation_config.thinking_config.thinking_budget", + } { + if budget := gjson.GetBytes(data, path); budget.Exists() { + return reasoningEffortFromBudgetValue(budget) + } + } + return "", hasLevel default: return "", false } +} + +func firstStringValue(data []byte, paths ...string) (string, bool) { for _, path := range paths { value := gjson.GetBytes(data, path) if !value.Exists() { @@ -253,6 +305,25 @@ func extractReasoningEffortFromJSON(format types.RelayFormat, data []byte) (stri return "", false } +func reasoningEffortFromBudgetValue(value gjson.Result) (string, bool) { + if value.Type != gjson.Number { + return "", true + } + budget := value.Float() + switch { + case budget == 0: + return string(kitreasoning.EffortNone), true + case budget < 0: + return string(kitreasoning.EffortHigh), true + case budget <= 1024: + return string(kitreasoning.EffortLow), true + case budget <= 8192: + return string(kitreasoning.EffortMedium), true + default: + return string(kitreasoning.EffortHigh), true + } +} + func shouldEnableParamOverrideAudit(paramOverride map[string]interface{}) bool { if common.DebugEnabled { return true diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index 6154cfc790..727476b1ec 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -14,6 +14,7 @@ import ( relayconstant "github.com/QuantumNous/new-api/relay/constant" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + kitreasoning "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/setting/model_setting" hosttypes "github.com/QuantumNous/new-api/types" @@ -98,25 +99,34 @@ type RelayInfo struct { UsePrice bool RelayMode int OriginModelName string - RequestURLPath string - RequestHeaders map[string]string - ShouldIncludeUsage bool - DisablePing bool // 是否禁止向下游发送自定义 Ping - ClientWs *websocket.Conn - TargetWs *websocket.Conn - InputAudioFormat string - OutputAudioFormat string - RealtimeTools []dto.RealTimeTool - IsFirstRequest bool - AudioUsage bool - ReasoningEffort string - UserSetting dto.UserSetting - UserEmail string - UserQuota int - RelayFormat types.RelayFormat - SendResponseCount int - ReceivedResponseCount int - FinalPreConsumedQuota int // 最终预消耗的配额 + + // BillingModelName is the pricing identity for this request. It is kept + // separate from OriginModelName and UpstreamModelName so virtual pricing + // aliases never participate in channel selection or upstream routing. + BillingModelName string + + RequestURLPath string + RequestHeaders map[string]string + ShouldIncludeUsage bool + DisablePing bool // 是否禁止向下游发送自定义 Ping + ClientWs *websocket.Conn + TargetWs *websocket.Conn + InputAudioFormat string + OutputAudioFormat string + RealtimeTools []dto.RealTimeTool + IsFirstRequest bool + AudioUsage bool + ReasoningEffort string + // ReasoningConversion is the suffix-derived reasoning intent attached + // after model mapping. Converters read it via ReasoningState(). + ReasoningConversion *dto.ReasoningConversionState + UserSetting dto.UserSetting + UserEmail string + UserQuota int + RelayFormat types.RelayFormat + SendResponseCount int + ReceivedResponseCount int + FinalPreConsumedQuota int // 最终预消耗的配额 // ForcePreConsume 为 true 时禁用 BillingSession 的信任额度旁路， // 强制预扣全额。用于异步任务（视频/音乐生成等），因为请求返回后任务仍在运行， // 必须在提交前锁定全额。 @@ -176,6 +186,10 @@ type RelayInfo struct { // convOptions caches the converter settings snapshot (see ConvOptions). convOptions *convmeta.Options + conversionDiagnostics []types.ConversionDiagnostic + conversionDiagnosticKeys map[conversionDiagnosticKey]struct{} + conversionDiagnosticsTruncated bool + ThinkingContentInfo TokenCountMeta *ClaudeConvertInfo @@ -186,6 +200,9 @@ type RelayInfo struct { } func (info *RelayInfo) InitChannelMeta(c *gin.Context) { + info.FinalRequestRelayFormat = "" + info.RequestConversionChain = nil + info.InitRequestConversionChain() channelType := common.GetContextKeyInt(c, constant.ContextKeyChannelType) paramOverride := common.GetContextKeyStringMap(c, constant.ContextKeyChannelParamOverride) headerOverride := common.GetContextKeyStringMap(c, constant.ContextKeyChannelHeaderOverride) @@ -236,8 +253,10 @@ func (info *RelayInfo) InitChannelMeta(c *gin.Context) { info.convOptions = nil if model_setting.GetGlobalSettings().PassThroughRequestEnabled || channelMeta.ChannelSetting.PassThroughBodyEnabled { info.ReasoningEffort = "" + info.ReasoningConversion = nil } else { info.ReasoningEffort = reasoningEffortFromRequest(info.Request) + info.ReasoningConversion = nil } // reset some fields based on channel meta @@ -261,6 +280,9 @@ func (info *RelayInfo) ToString() string { fmt.Fprintf(b, "IsPlayground: %t, ", info.IsPlayground) fmt.Fprintf(b, "RequestURLPath: %q, ", info.RequestURLPath) fmt.Fprintf(b, "OriginModelName: %q, ", info.OriginModelName) + if info.BillingModelName != "" && info.BillingModelName != info.OriginModelName { + fmt.Fprintf(b, "BillingModelName: %q, ", info.BillingModelName) + } fmt.Fprintf(b, "EstimatePromptTokens: %d, ", info.estimatePromptTokens) fmt.Fprintf(b, "ShouldIncludeUsage: %t, ", info.ShouldIncludeUsage) fmt.Fprintf(b, "DisablePing: %t, ", info.DisablePing) @@ -464,7 +486,10 @@ func reasoningEffortFromRequest(request dto.Request) string { } case *dto.GeminiChatRequest: if req != nil && req.GenerationConfig.ThinkingConfig != nil { - effort = req.GenerationConfig.ThinkingConfig.ThinkingLevel + intent, err := kitreasoning.FromGemini(req) + if err == nil { + effort = string(kitreasoning.EffectiveEffort(intent)) + } } } return strings.TrimSpace(effort) @@ -739,6 +764,18 @@ func (info *RelayInfo) GetOriginModelName() string { return info.OriginModelName } +// GetBillingModelName returns the effective pricing identity without changing +// either the client-visible model or the model sent to the selected channel. +func (info *RelayInfo) GetBillingModelName() string { + if info == nil { + return "" + } + if info.BillingModelName != "" { + return info.BillingModelName + } + return info.OriginModelName +} + func (info *RelayInfo) GetUpstreamModelName() string { if info == nil || info.ChannelMeta == nil { return "" @@ -780,6 +817,13 @@ func (info *RelayInfo) SetReasoningEffort(effort string) { info.ReasoningEffort = strings.TrimSpace(effort) } +func (info *RelayInfo) ReasoningState() *dto.ReasoningConversionState { + if info == nil { + return nil + } + return info.ReasoningConversion +} + func (info *RelayInfo) EnsureClaudeConvertInfo() *convmeta.ClaudeConvertInfo { if info == nil { return &convmeta.ClaudeConvertInfo{ @@ -832,8 +876,12 @@ func (info *RelayInfo) ConvOptions() *convmeta.Options { }, OpenRouterDialect: info != nil && info.GetChannelType() == constant.ChannelTypeOpenRouter, PreserveThinkingSuffix: model_setting.ShouldPreserveThinkingSuffix, + PreserveEffortTail: model_setting.ShouldPreserveEffortTail, } if info != nil { + if info.ChannelMeta != nil { + options.ToolLossPolicy = types.ConversionLossPolicy(info.ChannelOtherSettings.ToolLossPolicy) + } info.convOptions = options } return options diff --git a/relay/common/relay_info_test.go b/relay/common/relay_info_test.go index 42a0f8567b..5142c5414b 100644 --- a/relay/common/relay_info_test.go +++ b/relay/common/relay_info_test.go @@ -56,6 +56,7 @@ func TestRelayInfoMetaTypedNilReceiver(t *testing.T) { assert.Zero(t, meta.GetChannelType()) assert.False(t, meta.GetIsStream()) assert.Empty(t, meta.GetReasoningEffort()) + assert.Nil(t, meta.ReasoningState()) assert.Zero(t, meta.GetEstimatePromptTokens()) assert.Zero(t, meta.GetSendResponseCount()) @@ -81,6 +82,7 @@ func TestRelayInfoMetaTypedNilReceiver(t *testing.T) { assert.NotNil(t, firstOptions.Gemini.SupportsImagine) assert.NotNil(t, firstOptions.Gemini.SafetySetting) assert.NotNil(t, firstOptions.PreserveThinkingSuffix) + assert.NotNil(t, firstOptions.PreserveEffortTail) } func TestGenRelayInfoCapturesRequestReasoningEffort(t *testing.T) { diff --git a/relay/common/tool_usage.go b/relay/common/tool_usage.go index e9bed6250f..499953fa29 100644 --- a/relay/common/tool_usage.go +++ b/relay/common/tool_usage.go @@ -46,7 +46,7 @@ func (info *RelayInfo) CountBillableToolCall(itemType string, functionName strin if _, reserved := reservedBillableToolNames[functionName]; reserved { return } - if operation_setting.GetToolPriceForModel(functionName, info.OriginModelName) <= 0 { + if operation_setting.GetToolPriceForModel(functionName, info.GetBillingModelName()) <= 0 { return } info.incrementBillableToolCall(functionName) diff --git a/relay/compatible_handler.go b/relay/compatible_handler.go index 8edb3362b6..ba816c4624 100644 --- a/relay/compatible_handler.go +++ b/relay/compatible_handler.go @@ -43,6 +43,9 @@ func TextHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *types if err != nil { return types.NewError(err, types.ErrorCodeChannelModelMappedError, types.ErrOptionWithSkipRetry()) } + if err = helper.ApplyReasoningModelSuffix(info, request); err != nil { + return newConvertRequestFailedError(c, info, err) + } includeUsage := true // 判断用户是否需要返回使用情况 @@ -76,7 +79,7 @@ func TextHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *types !info.ChannelSetting.PassThroughBodyEnabled && service.ShouldChatCompletionsUseResponsesGlobal(info.ChannelId, info.ChannelType, info.OriginModelName) { applySystemPromptIfNeeded(c, info, request) - usage, newApiErr := chatCompletionsViaResponses(c, info, adaptor, request) + usage, newApiErr := textRequestViaResponses(c, info, adaptor, request) if newApiErr != nil { return newApiErr } @@ -108,7 +111,7 @@ func TextHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *types } else { convertedRequest, err := adaptor.ConvertOpenAIRequest(c, info, request) if err != nil { - return types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) + return newConvertRequestFailedError(c, info, err) } relaycommon.AppendRequestConversionFromRequest(info, convertedRequest) diff --git a/relay/convert_request_error.go b/relay/convert_request_error.go new file mode 100644 index 0000000000..31468d90c3 --- /dev/null +++ b/relay/convert_request_error.go @@ -0,0 +1,23 @@ +package relay + +import ( + "errors" + "net/http" + + relaycommon "github.com/QuantumNous/new-api/relay/common" + kitreasoning "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" + "github.com/QuantumNous/new-api/relaykit/types" + "github.com/gin-gonic/gin" +) + +func newConvertRequestFailedError(c *gin.Context, info *relaycommon.RelayInfo, err error) *types.NewAPIError { + var loss *types.ConversionLossError + if errors.As(err, &loss) { + info.RecordConversionDiagnostics(c, loss.Diagnostics) + return types.NewErrorWithStatusCode(err, types.ErrorCodeConvertRequestFailed, http.StatusBadRequest, types.ErrOptionWithSkipRetry()) + } + if kitreasoning.IsClientError(err) { + return types.NewErrorWithStatusCode(err, types.ErrorCodeConvertRequestFailed, http.StatusBadRequest, types.ErrOptionWithSkipRetry()) + } + return types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) +} diff --git a/relay/convert_request_error_test.go b/relay/convert_request_error_test.go new file mode 100644 index 0000000000..c3caba2cf9 --- /dev/null +++ b/relay/convert_request_error_test.go @@ -0,0 +1,72 @@ +package relay + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/QuantumNous/new-api/common" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/types" + "github.com/QuantumNous/new-api/service" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOptInSafeToolLossRejectedAsBadRequestWithAdminDiagnostics(t *testing.T) { + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + + info := &relaycommon.RelayInfo{ + OriginModelName: "gpt-4o", + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "gpt-4o", + ChannelOtherSettings: dto.ChannelOtherSettings{ + ToolLossPolicy: string(types.ConversionLossPolicySafe), + }, + }, + } + + tools, err := common.Marshal([]map[string]any{{"codeExecution": map[string]any{}}}) + require.NoError(t, err) + req := &dto.GeminiChatRequest{ + Contents: []dto.GeminiChatContent{ + {Role: "user", Parts: []dto.GeminiPart{{Text: "run this"}}}, + }, + Tools: tools, + } + + result, convErr := service.ConvertRequest(c, info, types.RelayFormatOpenAI, req) + require.Error(t, convErr) + var loss *types.ConversionLossError + require.ErrorAs(t, convErr, &loss) + require.NotEmpty(t, loss.Diagnostics) + require.NotNil(t, result) + + apiErr := newConvertRequestFailedError(c, info, convErr) + require.NotNil(t, apiErr) + assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode) + assert.Equal(t, types.ErrorCodeConvertRequestFailed, apiErr.GetErrorCode()) + assert.True(t, types.IsSkipRetryError(apiErr)) + + diagnostics := info.ConversionDiagnostics() + require.NotEmpty(t, diagnostics) + assert.True(t, hasHostDiagnosticCode(diagnostics, "unsupported_hosted_tool")) + + other := service.GenerateTextOtherInfo(c, info, 1, 1, 1, 0, 0, 0, 1) + adminInfo, ok := other["admin_info"].(map[string]interface{}) + require.True(t, ok) + require.Contains(t, adminInfo, "conversion_diagnostics") +} + +func hasHostDiagnosticCode(diagnostics []types.ConversionDiagnostic, code string) bool { + for _, diagnostic := range diagnostics { + if diagnostic.Code == code { + return true + } + } + return false +} diff --git a/relay/gemini_handler.go b/relay/gemini_handler.go index 57010d87c3..ffa6e996be 100644 --- a/relay/gemini_handler.go +++ b/relay/gemini_handler.go @@ -12,7 +12,6 @@ import ( relaycommon "github.com/QuantumNous/new-api/relay/common" "github.com/QuantumNous/new-api/relay/helper" "github.com/QuantumNous/new-api/relaykit/dto" - "github.com/QuantumNous/new-api/relaykit/relayconvert" "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/service" "github.com/QuantumNous/new-api/setting/model_setting" @@ -20,37 +19,6 @@ import ( "github.com/gin-gonic/gin" ) -func isNoThinkingRequest(req *dto.GeminiChatRequest) bool { - if req.GenerationConfig.ThinkingConfig != nil && req.GenerationConfig.ThinkingConfig.ThinkingBudget != nil { - configBudget := req.GenerationConfig.ThinkingConfig.ThinkingBudget - if configBudget != nil && *configBudget == 0 { - // 如果思考预算为 0，则认为是非思考请求 - return true - } - } - return false -} - -func trimModelThinking(modelName string) string { - // 去除模型名称中的 -nothinking 后缀 - if strings.HasSuffix(modelName, "-nothinking") { - return strings.TrimSuffix(modelName, "-nothinking") - } - // 去除模型名称中的 -thinking 后缀 - if strings.HasSuffix(modelName, "-thinking") { - return strings.TrimSuffix(modelName, "-thinking") - } - - // 去除模型名称中的 -thinking-number - if strings.Contains(modelName, "-thinking-") { - parts := strings.Split(modelName, "-thinking-") - if len(parts) > 1 { - return parts[0] + "-thinking" - } - } - return modelName -} - func GeminiHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *types.NewAPIError) { info.InitChannelMeta(c) @@ -69,23 +37,8 @@ func GeminiHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *typ if err != nil { return types.NewError(err, types.ErrorCodeChannelModelMappedError, types.ErrOptionWithSkipRetry()) } - - if model_setting.GetGeminiSettings().ThinkingAdapterEnabled { - if isNoThinkingRequest(request) { - // check is thinking - if !strings.Contains(info.OriginModelName, "-nothinking") { - // try to get no thinking model price - noThinkingModelName := info.OriginModelName + "-nothinking" - containPrice := helper.HasModelBillingConfig(noThinkingModelName) - if containPrice { - info.OriginModelName = noThinkingModelName - info.UpstreamModelName = noThinkingModelName - } - } - } - if request.GenerationConfig.ThinkingConfig == nil { - relayconvert.ApplyGeminiThinkingConfig(request, info) - } + if err = helper.ApplyReasoningModelSuffix(info, request); err != nil { + return newConvertRequestFailedError(c, info, err) } adaptor := GetAdaptor(info.ApiType) @@ -146,7 +99,7 @@ func GeminiHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError *typ // 使用 ConvertGeminiRequest 转换请求格式 convertedRequest, err := adaptor.ConvertGeminiRequest(c, info, request) if err != nil { - return types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) + return newConvertRequestFailedError(c, info, err) } relaycommon.AppendRequestConversionFromRequest(info, convertedRequest) jsonData, err := common.Marshal(convertedRequest) @@ -245,6 +198,9 @@ func GeminiEmbeddingHandler(c *gin.Context, info *relaycommon.RelayInfo) (newAPI if err != nil { return types.NewError(err, types.ErrorCodeChannelModelMappedError, types.ErrOptionWithSkipRetry()) } + if err = helper.ApplyReasoningModelSuffix(info, req); err != nil { + return newConvertRequestFailedError(c, info, err) + } req.SetModelName("models/" + info.UpstreamModelName) diff --git a/relay/helper/price.go b/relay/helper/price.go index b9ae819bf5..1db88d816f 100644 --- a/relay/helper/price.go +++ b/relay/helper/price.go @@ -71,13 +71,14 @@ func HandleGroupRatio(ctx *gin.Context, relayInfo *relaycommon.RelayInfo) hostty } func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens int, meta *types.TokenCountMeta) (hosttypes.PriceData, error) { - modelPrice, usePrice := ratio_setting.GetModelPrice(info.OriginModelName, false) + billingModelName := info.GetBillingModelName() + modelPrice, usePrice := ratio_setting.GetModelPrice(billingModelName, false) groupRatioInfo := HandleGroupRatio(c, info) // Check if this model uses tiered_expr billing - if billing_setting.GetBillingMode(info.OriginModelName) == billing_setting.BillingModeTieredExpr { - return modelPriceHelperTiered(c, info, promptTokens, meta, groupRatioInfo) + if billing_setting.GetBillingMode(billingModelName) == billing_setting.BillingModeTieredExpr { + return modelPriceHelperTiered(c, info, billingModelName, promptTokens, meta, groupRatioInfo) } var preConsumedQuota int @@ -98,7 +99,7 @@ func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens } var success bool var matchName string - modelRatio, success, matchName = ratio_setting.GetModelRatio(info.OriginModelName) + modelRatio, success, matchName = ratio_setting.GetModelRatio(billingModelName) if !success { acceptUnsetRatio := false if info.UserSetting.AcceptUnsetRatioModel { @@ -108,15 +109,15 @@ func ModelPriceHelper(c *gin.Context, info *relaycommon.RelayInfo, promptTokens return hosttypes.PriceData{}, modelPriceNotConfiguredError(matchName, info.UserId) } } - completionRatio = ratio_setting.GetCompletionRatio(info.OriginModelName) - cacheRatio, _ = ratio_setting.GetCacheRatio(info.OriginModelName) - cacheCreationRatio, _ = ratio_setting.GetCreateCacheRatio(info.OriginModelName) + completionRatio = ratio_setting.GetCompletionRatio(billingModelName) + cacheRatio, _ = ratio_setting.GetCacheRatio(billingModelName) + cacheCreationRatio, _ = ratio_setting.GetCreateCacheRatio(billingModelName) cacheCreationRatio5m = cacheCreationRatio // 固定1h和5min缓存写入价格的比例 cacheCreationRatio1h = cacheCreationRatio * claudeCacheCreation1hMultiplier - imageRatio, _ = ratio_setting.GetImageRatio(info.OriginModelName) - audioRatio = ratio_setting.GetAudioRatio(info.OriginModelName) - audioCompletionRatio = ratio_setting.GetAudioCompletionRatio(info.OriginModelName) + imageRatio, _ = ratio_setting.GetImageRatio(billingModelName) + audioRatio = ratio_setting.GetAudioRatio(billingModelName) + audioCompletionRatio = ratio_setting.GetAudioCompletionRatio(billingModelName) ratio := modelRatio * groupRatioInfo.GroupRatio quota, err := common.QuotaFromFloatStrict(float64(preConsumedTokens) * ratio) if err != nil { @@ -266,10 +267,10 @@ func HasModelBillingConfig(modelName string) bool { return ok && strings.TrimSpace(expr) != "" } -func modelPriceHelperTiered(c *gin.Context, info *relaycommon.RelayInfo, promptTokens int, meta *types.TokenCountMeta, groupRatioInfo hosttypes.GroupRatioInfo) (hosttypes.PriceData, error) { - exprStr, ok := billing_setting.GetBillingExpr(info.OriginModelName) +func modelPriceHelperTiered(c *gin.Context, info *relaycommon.RelayInfo, billingModelName string, promptTokens int, meta *types.TokenCountMeta, groupRatioInfo hosttypes.GroupRatioInfo) (hosttypes.PriceData, error) { + exprStr, ok := billing_setting.GetBillingExpr(billingModelName) if !ok { - return hosttypes.PriceData{}, fmt.Errorf("model %s is configured as tiered_expr but has no billing expression", info.OriginModelName) + return hosttypes.PriceData{}, fmt.Errorf("model %s is configured as tiered_expr but has no billing expression", billingModelName) } estimatedCompletionTokens := meta.MaxTokens @@ -288,7 +289,7 @@ func modelPriceHelperTiered(c *gin.Context, info *relaycommon.RelayInfo, promptT Len: float64(promptTokens), }, requestInput) if err != nil { - return hosttypes.PriceData{}, fmt.Errorf("model %s tiered expr run failed: %w", info.OriginModelName, err) + return hosttypes.PriceData{}, fmt.Errorf("model %s tiered expr run failed: %w", billingModelName, err) } // Expression coefficients are $/1M tokens prices; convert to quota the same way per-call billing does. @@ -309,7 +310,7 @@ func modelPriceHelperTiered(c *gin.Context, info *relaycommon.RelayInfo, promptT exprHash := billingexpr.ExprHashString(exprStr) snapshot := &billingexpr.BillingSnapshot{ BillingMode: billing_setting.BillingModeTieredExpr, - ModelName: info.OriginModelName, + ModelName: billingModelName, ExprString: exprStr, ExprHash: exprHash, GroupRatio: groupRatioInfo.GroupRatio, @@ -330,7 +331,7 @@ func modelPriceHelperTiered(c *gin.Context, info *relaycommon.RelayInfo, promptT QuotaToPreConsume: preConsumedQuota, } - logger.LogDebug(c, "model_price_helper_tiered result: model=%s preConsume=%d quotaBeforeGroup=%.2f groupRatio=%.2f tier=%s", info.OriginModelName, preConsumedQuota, quotaBeforeGroup, groupRatioInfo.GroupRatio, trace.MatchedTier) + logger.LogDebug(c, "model_price_helper_tiered result: model=%s preConsume=%d quotaBeforeGroup=%.2f groupRatio=%.2f tier=%s", billingModelName, preConsumedQuota, quotaBeforeGroup, groupRatioInfo.GroupRatio, trace.MatchedTier) info.PriceData = priceData return priceData, nil diff --git a/relay/helper/price_test.go b/relay/helper/price_test.go index ca38b54829..75f825f016 100644 --- a/relay/helper/price_test.go +++ b/relay/helper/price_test.go @@ -8,11 +8,15 @@ import ( "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/pkg/billingexpr" relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/setting/billing_setting" "github.com/QuantumNous/new-api/setting/config" + "github.com/QuantumNous/new-api/setting/model_setting" + "github.com/QuantumNous/new-api/setting/operation_setting" "github.com/QuantumNous/new-api/setting/ratio_setting" "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -272,3 +276,97 @@ func TestModelPriceHelperRequestBillingRatiosOnlyApplyToFixedPrice(t *testing.T) require.Equal(t, common.QuotaClampOverflow, clamp.Kind) require.Nil(t, info.Billing) } + +// Pricing at controller/relay.go runs before ApplyReasoningModelSuffix. +// Identity is GetBillingModelName() → OriginModelName (the suffixed client +// name), matching main's info.OriginModelName lookup. Wildcard entries such +// as gemini-2.5-flash-thinking-* depend on that unstripped origin form. +func TestModelPriceHelperUsesSuffixedOriginLikeMain(t *testing.T) { + gin.SetMode(gin.TestMode) + + savedRatios := ratio_setting.ModelRatio2JSONString() + t.Cleanup(func() { + require.NoError(t, ratio_setting.UpdateModelRatioByJSONString(savedRatios)) + }) + ratios := ratio_setting.GetModelRatioCopy() + ratios["gemini-2.5-flash"] = 0.15 + ratios["gemini-2.5-flash-thinking-*"] = 0.075 + ratioJSON, err := common.Marshal(ratios) + require.NoError(t, err) + require.NoError(t, ratio_setting.UpdateModelRatioByJSONString(string(ratioJSON))) + + oldSelfUse := operation_setting.SelfUseModeEnabled + operation_setting.SelfUseModeEnabled = true + t.Cleanup(func() { operation_setting.SelfUseModeEnabled = oldSelfUse }) + + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Set("group", "default") + + suffixed := &relaycommon.RelayInfo{ + OriginModelName: "gemini-2.5-flash-thinking-8192", + UserGroup: "default", + UsingGroup: "default", + } + suffixedPrice, err := ModelPriceHelper(ctx, suffixed, 1000, &types.TokenCountMeta{}) + require.NoError(t, err) + assert.Empty(t, suffixed.BillingModelName) + assert.Equal(t, "gemini-2.5-flash-thinking-8192", suffixed.GetBillingModelName()) + assert.Equal(t, 0.075, suffixedPrice.ModelRatio) + + base := &relaycommon.RelayInfo{ + OriginModelName: "gemini-2.5-flash", + UserGroup: "default", + UsingGroup: "default", + } + basePrice, err := ModelPriceHelper(ctx, base, 1000, &types.TokenCountMeta{}) + require.NoError(t, err) + assert.Empty(t, base.BillingModelName) + assert.Equal(t, "gemini-2.5-flash", base.GetBillingModelName()) + assert.Equal(t, 0.15, basePrice.ModelRatio) +} + +func TestModelPriceHelperNativeGeminiNoThinkingDoesNotAliasBillingModel(t *testing.T) { + gin.SetMode(gin.TestMode) + + savedRatios := ratio_setting.ModelRatio2JSONString() + t.Cleanup(func() { + require.NoError(t, ratio_setting.UpdateModelRatioByJSONString(savedRatios)) + }) + ratios := ratio_setting.GetModelRatioCopy() + ratios["gemini-3-pro"] = 1.25 + ratioJSON, err := common.Marshal(ratios) + require.NoError(t, err) + require.NoError(t, ratio_setting.UpdateModelRatioByJSONString(string(ratioJSON))) + + oldSelfUse := operation_setting.SelfUseModeEnabled + operation_setting.SelfUseModeEnabled = true + t.Cleanup(func() { operation_setting.SelfUseModeEnabled = oldSelfUse }) + + geminiSettings := model_setting.GetGeminiSettings() + oldThinking := geminiSettings.ThinkingAdapterEnabled + geminiSettings.ThinkingAdapterEnabled = true + t.Cleanup(func() { geminiSettings.ThinkingAdapterEnabled = oldThinking }) + + budget := 0 + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Set("group", "default") + info := &relaycommon.RelayInfo{ + OriginModelName: "gemini-3-pro", + UserGroup: "default", + UsingGroup: "default", + Request: &dto.GeminiChatRequest{ + GenerationConfig: dto.GeminiChatGenerationConfig{ + ThinkingConfig: &dto.GeminiThinkingConfig{ + ThinkingBudget: &budget, + }, + }, + }, + } + + priceData, err := ModelPriceHelper(ctx, info, 1000, &types.TokenCountMeta{}) + require.NoError(t, err) + assert.Empty(t, info.BillingModelName) + assert.Equal(t, "gemini-3-pro", info.GetBillingModelName()) + assert.Equal(t, 1.25, priceData.ModelRatio) + assert.NotEqual(t, 37.5, priceData.ModelRatio) +} diff --git a/relay/helper/reasoning_suffix.go b/relay/helper/reasoning_suffix.go new file mode 100644 index 0000000000..b5c1ca4329 --- /dev/null +++ b/relay/helper/reasoning_suffix.go @@ -0,0 +1,145 @@ +package helper + +import ( + "strings" + + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" + "github.com/QuantumNous/new-api/setting/model_setting" +) + +// ApplyReasoningModelSuffix parses host-private reasoning suffixes from the +// origin and mapped model names, attaches the resulting intent to RelayInfo, +// and normalizes UpstreamModelName to the unsuffixed base. Optional outbound +// requests are the DeepCopy the handler will send upstream; they must be +// synced here because info.Request is the original, not that copy. Conflict +// between an explicit request field and a suffix is a client error. +func ApplyReasoningModelSuffix(info *relaycommon.RelayInfo, outbound ...dto.Request) error { + if info == nil { + return nil + } + passThrough := model_setting.GetGlobalSettings().PassThroughRequestEnabled + if info.ChannelMeta != nil && info.ChannelSetting.PassThroughBodyEnabled { + passThrough = true + } + if passThrough { + return nil + } + + opts := info.ConvOptions() + origin := info.GetOriginModelName() + upstream := "" + if info.ChannelMeta != nil { + upstream = info.UpstreamModelName + } + if opts.ShouldPreserveThinkingSuffix(origin) || opts.ShouldPreserveThinkingSuffix(upstream) { + return nil + } + + originBase, originIntent, originFound, err := parseHostModelSuffix(origin, opts) + if err != nil { + return reasoning.AsClientError(err) + } + upstreamBase, upstreamIntent, upstreamFound, err := parseHostModelSuffix(upstream, opts) + if err != nil { + return reasoning.AsClientError(err) + } + + suffix := originIntent + if originFound && upstreamFound { + suffix, err = reasoning.MergeExplicitAndSuffix(originIntent, upstreamIntent, origin) + if err != nil { + return reasoning.AsClientError(err) + } + } else if upstreamFound { + suffix = upstreamIntent + } + + explicit, err := explicitIntentFromRequest(info.Request) + if err != nil { + return reasoning.AsClientError(err) + } + conflictModel := upstream + if conflictModel == "" { + conflictModel = origin + } + if _, err = reasoning.MergeExplicitAndSuffix(explicit, suffix, conflictModel); err != nil { + return reasoning.AsClientError(err) + } + + if !suffix.IsEmpty() { + info.ReasoningConversion = reasoning.StateFromIntent(suffix) + } + + if upstreamFound && info.ChannelMeta != nil { + info.UpstreamModelName = upstreamBase + } else if !info.IsModelMapped && originFound && info.ChannelMeta != nil { + info.UpstreamModelName = originBase + } + // Handlers DeepCopy before this helper; info.Request is the original. + // Sync every outbound copy the caller is about to send upstream. + for _, outbound := range outbound { + if outbound != nil { + outbound.SetModelName(info.UpstreamModelName) + } + } + if info.Request != nil { + info.Request.SetModelName(info.UpstreamModelName) + } + return nil +} + +func parseHostModelSuffix(name string, opts *convmeta.Options) (string, reasoning.Intent, bool, error) { + if name == "" { + return name, reasoning.Intent{}, false, nil + } + if strings.HasPrefix(name, "claude-") { + return reasoning.ParseClaudeModelSuffix(name, opts.Claude.ThinkingAdapterEnabled) + } + if strings.HasPrefix(name, "gemini-") { + if !opts.Gemini.ThinkingAdapterEnabled { + return name, reasoning.Intent{}, false, nil + } + return reasoning.ParseGeminiModelSuffix(name, true) + } + // deepseek-v4 effort tails are consumed by ParseDeepSeekV4ThinkingSuffix + // in the DeepSeek adaptor; stripping them here drops THINKING+effort. + if strings.HasPrefix(name, "deepseek-v4-") { + return name, reasoning.Intent{}, false, nil + } + effort, base := reasoning.ParseOpenAIReasoningEffortFromModelSuffix(name, opts.PreserveEffortTail) + if effort != "" { + parsed, err := reasoning.ParseEffort(effort) + if err != nil { + return name, reasoning.Intent{}, false, err + } + mode := reasoning.ModeEnabled + if parsed == reasoning.EffortNone { + mode = reasoning.ModeDisabled + } + return base, reasoning.Intent{Mode: mode, Effort: parsed, Source: reasoning.SourceSuffix}, true, nil + } + // Generic -thinking trim is OpenRouter-only. Volcengine/DeepSeek adaptors + // read the suffix off UpstreamModelName themselves. + if opts != nil && opts.OpenRouterDialect && strings.HasSuffix(name, "-thinking") { + return strings.TrimSuffix(name, "-thinking"), reasoning.Intent{Mode: reasoning.ModeEnabled, Source: reasoning.SourceSuffix}, true, nil + } + return name, reasoning.Intent{}, false, nil +} + +func explicitIntentFromRequest(req dto.Request) (reasoning.Intent, error) { + switch r := req.(type) { + case *dto.ClaudeRequest: + return reasoning.FromClaude(r) + case *dto.GeminiChatRequest: + return reasoning.FromGemini(r) + case *dto.GeneralOpenAIRequest: + return reasoning.FromOpenAIChat(r) + case *dto.OpenAIResponsesRequest: + return reasoning.FromOpenAIResponses(r) + default: + return reasoning.Intent{}, nil + } +} diff --git a/relay/helper/reasoning_suffix_test.go b/relay/helper/reasoning_suffix_test.go new file mode 100644 index 0000000000..e07b8873b0 --- /dev/null +++ b/relay/helper/reasoning_suffix_test.go @@ -0,0 +1,218 @@ +package helper + +import ( + "net/http/httptest" + "testing" + + "github.com/QuantumNous/new-api/common" + "github.com/QuantumNous/new-api/constant" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/setting/model_setting" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplyReasoningModelSuffixTrimsUpstreamAndAttachesState(t *testing.T) { + info := &relaycommon.RelayInfo{ + OriginModelName: "claude-3-7-sonnet-thinking", + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "claude-3-7-sonnet-thinking", + }, + } + + require.NoError(t, ApplyReasoningModelSuffix(info)) + assert.Equal(t, "claude-3-7-sonnet", info.UpstreamModelName) + require.NotNil(t, info.ReasoningConversion) + assert.Equal(t, "enabled", info.ReasoningConversion.Mode) +} + +func TestApplyReasoningModelSuffixRetryKeepsEquivalentState(t *testing.T) { + info := &relaycommon.RelayInfo{ + OriginModelName: "claude-opus-4-8-high", + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "claude-opus-4-8-high", + }, + } + + require.NoError(t, ApplyReasoningModelSuffix(info)) + require.NotNil(t, info.ReasoningConversion) + firstMode := info.ReasoningConversion.Mode + firstEffort := info.ReasoningConversion.Effort + + info.UpstreamModelName = info.OriginModelName + require.NoError(t, ApplyReasoningModelSuffix(info)) + require.NotNil(t, info.ReasoningConversion) + assert.Equal(t, firstMode, info.ReasoningConversion.Mode) + assert.Equal(t, firstEffort, info.ReasoningConversion.Effort) + assert.Equal(t, "claude-opus-4-8", info.UpstreamModelName) +} + +func TestApplyReasoningModelSuffixRetryClearsStateWhenNewChannelHasNoSuffix(t *testing.T) { + gin.SetMode(gin.TestMode) + req := &dto.ClaudeRequest{Model: "claude-3-7-sonnet"} + info := &relaycommon.RelayInfo{ + OriginModelName: "claude-3-7-sonnet", + Request: req, + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "claude-3-7-sonnet-thinking", + IsModelMapped: true, + }, + } + require.NoError(t, ApplyReasoningModelSuffix(info)) + require.NotNil(t, info.ReasoningState()) + + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = httptest.NewRequest("POST", "/v1/messages", nil) + common.SetContextKey(ctx, constant.ContextKeyOriginalModel, "claude-3-7-sonnet") + common.SetContextKey(ctx, constant.ContextKeyChannelType, constant.ChannelTypeAnthropic) + info.InitChannelMeta(ctx) + assert.Nil(t, info.ReasoningState()) + + require.NoError(t, ApplyReasoningModelSuffix(info)) + assert.Nil(t, info.ReasoningState()) +} + +func TestApplyReasoningModelSuffixPassThroughDoesNotTrim(t *testing.T) { + settings := model_setting.GetGlobalSettings() + original := settings.PassThroughRequestEnabled + t.Cleanup(func() { settings.PassThroughRequestEnabled = original }) + settings.PassThroughRequestEnabled = true + + info := &relaycommon.RelayInfo{ + OriginModelName: "claude-3-7-sonnet-thinking", + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "claude-3-7-sonnet-thinking", + }, + } + + require.NoError(t, ApplyReasoningModelSuffix(info)) + assert.Equal(t, "claude-3-7-sonnet-thinking", info.UpstreamModelName) + assert.Nil(t, info.ReasoningConversion) +} + +func TestApplyReasoningModelSuffixBlacklistDoesNotTrim(t *testing.T) { + settings := model_setting.GetGlobalSettings() + original := append([]string(nil), settings.ThinkingModelBlacklist...) + t.Cleanup(func() { settings.ThinkingModelBlacklist = original }) + settings.ThinkingModelBlacklist = append(settings.ThinkingModelBlacklist, "claude-3-7-sonnet-thinking") + + info := &relaycommon.RelayInfo{ + OriginModelName: "claude-3-7-sonnet-thinking", + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "claude-3-7-sonnet-thinking", + }, + } + + require.NoError(t, ApplyReasoningModelSuffix(info)) + assert.Equal(t, "claude-3-7-sonnet-thinking", info.UpstreamModelName) + assert.Nil(t, info.ReasoningConversion) +} + +func TestApplyReasoningModelSuffixRejectsExplicitSuffixConflict(t *testing.T) { + info := &relaycommon.RelayInfo{ + OriginModelName: "claude-3-7-sonnet-thinking", + Request: &dto.ClaudeRequest{ + Model: "claude-3-7-sonnet-thinking", + Thinking: &dto.Thinking{Type: "disabled"}, + }, + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "claude-3-7-sonnet-thinking", + }, + } + + err := ApplyReasoningModelSuffix(info) + require.Error(t, err) +} + +func TestApplyReasoningModelSuffixGeminiNoThinkingWhenAdapterEnabled(t *testing.T) { + settings := model_setting.GetGeminiSettings() + original := settings.ThinkingAdapterEnabled + t.Cleanup(func() { settings.ThinkingAdapterEnabled = original }) + settings.ThinkingAdapterEnabled = true + + info := &relaycommon.RelayInfo{ + OriginModelName: "gemini-2.5-flash-nothinking", + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "gemini-2.5-flash-nothinking", + }, + } + + require.NoError(t, ApplyReasoningModelSuffix(info)) + assert.Equal(t, "gemini-2.5-flash", info.UpstreamModelName) + require.NotNil(t, info.ReasoningConversion) + assert.Equal(t, "disabled", info.ReasoningConversion.Mode) + assert.Equal(t, "none", info.ReasoningConversion.Effort) +} + +func TestApplyReasoningModelSuffixPreservesEffortTailModelID(t *testing.T) { + info := &relaycommon.RelayInfo{ + OriginModelName: "qwen-max", + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "qwen-max", + }, + } + + require.NoError(t, ApplyReasoningModelSuffix(info)) + assert.Equal(t, "qwen-max", info.UpstreamModelName) + assert.Nil(t, info.ReasoningConversion) +} + +func TestApplyReasoningModelSuffixLeavesDeepSeekV4SuffixForAdaptor(t *testing.T) { + info := &relaycommon.RelayInfo{ + OriginModelName: "deepseek-v4-chat-max", + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelType: constant.ChannelTypeDeepSeek, + UpstreamModelName: "deepseek-v4-chat-max", + }, + } + + require.NoError(t, ApplyReasoningModelSuffix(info)) + assert.Equal(t, "deepseek-v4-chat-max", info.UpstreamModelName) + assert.Nil(t, info.ReasoningConversion) +} + +func TestApplyReasoningModelSuffixLeavesVolcengineDeepSeekThinkingForAdaptor(t *testing.T) { + info := &relaycommon.RelayInfo{ + OriginModelName: "deepseek-r1-thinking", + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelType: constant.ChannelTypeVolcEngine, + UpstreamModelName: "deepseek-r1-thinking", + }, + } + + require.NoError(t, ApplyReasoningModelSuffix(info)) + assert.Equal(t, "deepseek-r1-thinking", info.UpstreamModelName) + assert.Nil(t, info.ReasoningConversion) +} + +func TestApplyReasoningModelSuffixStillParsesOpenAIEffortTail(t *testing.T) { + info := &relaycommon.RelayInfo{ + OriginModelName: "gpt-5.1-high", + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelType: constant.ChannelTypeOpenAI, + UpstreamModelName: "gpt-5.1-high", + }, + } + + require.NoError(t, ApplyReasoningModelSuffix(info)) + assert.Equal(t, "gpt-5.1", info.UpstreamModelName) + require.NotNil(t, info.ReasoningConversion) + assert.Equal(t, "enabled", info.ReasoningConversion.Mode) + assert.Equal(t, "high", info.ReasoningConversion.Effort) +} + +func TestApplyReasoningModelSuffixTrimsOpenRouterThinkingOnly(t *testing.T) { + openRouter := &relaycommon.RelayInfo{ + OriginModelName: "some-model-thinking", + ChannelMeta: &relaycommon.ChannelMeta{ + ChannelType: constant.ChannelTypeOpenRouter, + UpstreamModelName: "some-model-thinking", + }, + } + require.NoError(t, ApplyReasoningModelSuffix(openRouter)) + assert.Equal(t, "some-model", openRouter.UpstreamModelName) + require.NotNil(t, openRouter.ReasoningConversion) + assert.Equal(t, "enabled", openRouter.ReasoningConversion.Mode) +} diff --git a/relay/responses_handler.go b/relay/responses_handler.go index 4321c7a4b6..13eeeb8f00 100644 --- a/relay/responses_handler.go +++ b/relay/responses_handler.go @@ -70,6 +70,9 @@ func ResponsesHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError * if err != nil { return types.NewError(err, types.ErrorCodeChannelModelMappedError, types.ErrOptionWithSkipRetry()) } + if err = helper.ApplyReasoningModelSuffix(info, request); err != nil { + return newConvertRequestFailedError(c, info, err) + } adaptor := GetAdaptor(info.ApiType) if adaptor == nil { @@ -86,7 +89,7 @@ func ResponsesHelper(c *gin.Context, info *relaycommon.RelayInfo) (newAPIError * } else { convertedRequest, err := adaptor.ConvertOpenAIResponsesRequest(c, info, *request) if err != nil { - return types.NewError(err, types.ErrorCodeConvertRequestFailed, types.ErrOptionWithSkipRetry()) + return newConvertRequestFailedError(c, info, err) } relaycommon.AppendRequestConversionFromRequest(info, convertedRequest) jsonData, err := common.Marshal(convertedRequest) diff --git a/relaykit/README.md b/relaykit/README.md index 4a26376b5b..cd534c8c36 100644 --- a/relaykit/README.md +++ b/relaykit/README.md @@ -199,6 +199,9 @@ meta := &convmeta.Values{ - OpenAI Chat 或 OpenAI Responses 转 Claude 时，Claude 请求必须具有 `max_tokens`。源请求未提供时，需要配置 `Claude.DefaultMaxTokens`，否则转换会返回错误。 - RelayKit 不负责选择渠道或映射模型名。调用转换前，应将请求中的 `Model` 设置为目标上游使用的模型名。 - 自定义 `convmeta.Meta` 的指针实现必须保证所有方法对 nil receiver 安全，完整约束见 `convmeta.Meta` 的接口注释。 +- 工具损耗策略默认是 `allow`：跨协议转换会成功，损耗以诊断形式返回。`safe` / `strict` 只在请求阶段 opt-in 拒绝；响应和流式转换无论策略如何都不会因损耗失败。 +- `ThinkingAdapterEnabled` 只控制是否把已解析的推理意图渲染到 Claude / Gemini 请求上。`-thinking` / `-nothinking` / effort 尾缀等命名约定不再由转换器自动解释。 +- 若你的入口仍使用这些模型名后缀，请在调用转换前自行调用 `relayconvert/reasoning` 的 `Parse*` 帮助函数，把结果写成 `dto.ReasoningConversionState`，并通过 `convmeta.Meta.ReasoningState()`（`convmeta.Values.ReasoningConversion`）传入。同时把发给上游的模型名裁成无后缀基础名。 ## 多模态内容 diff --git a/relaykit/dto/billing_usage.go b/relaykit/dto/billing_usage.go index 075bce41e6..ac3535e8bf 100644 --- a/relaykit/dto/billing_usage.go +++ b/relaykit/dto/billing_usage.go @@ -1,5 +1,7 @@ package dto +import "strings" + const ( BillingUsageSourceClaudeMessages = "claude_messages" BillingUsageSourceGeminiChat = "gemini_chat" @@ -100,7 +102,15 @@ func HasOpenAIUsageTokens(usage *Usage) bool { usage.CompletionTokenDetails.AudioTokens != 0 { return true } - return usage.InputTokensDetails != nil + if usage.InputTokensDetails == nil { + return false + } + return usage.InputTokensDetails.CachedTokens != 0 || + usage.InputTokensDetails.CachedCreationTokens != 0 || + usage.InputTokensDetails.CacheWriteTokens != 0 || + usage.InputTokensDetails.TextTokens != 0 || + usage.InputTokensDetails.ImageTokens != 0 || + usage.InputTokensDetails.AudioTokens != 0 } func NewGeminiChatBillingUsage(metadata *GeminiUsageMetadata) *BillingUsage { @@ -111,15 +121,92 @@ func NewEstimatedGeminiChatBillingUsage(usage *Usage) *BillingUsage { if usage == nil { return nil } + reasoningTokens := usage.CompletionTokenDetails.ReasoningTokens + candidateTokens := usage.CompletionTokens - reasoningTokens + if candidateTokens < 0 { + candidateTokens = 0 + } totalTokens := usage.TotalTokens if totalTokens == 0 { totalTokens = usage.PromptTokens + usage.CompletionTokens } - return newGeminiChatBillingUsage(&GeminiUsageMetadata{ - PromptTokenCount: usage.PromptTokens, - CandidatesTokenCount: usage.CompletionTokens, - TotalTokenCount: totalTokens, - }, true) + metadata := &GeminiUsageMetadata{ + PromptTokenCount: usage.PromptTokens, + CandidatesTokenCount: candidateTokens, + TotalTokenCount: totalTokens, + ThoughtsTokenCount: reasoningTokens, + CachedContentTokenCount: usage.PromptTokensDetails.CachedTokens, + } + for _, detail := range []GeminiPromptTokensDetails{ + {Modality: "TEXT", TokenCount: usage.PromptTokensDetails.TextTokens}, + {Modality: "IMAGE", TokenCount: usage.PromptTokensDetails.ImageTokens}, + {Modality: "AUDIO", TokenCount: usage.PromptTokensDetails.AudioTokens}, + } { + if detail.TokenCount != 0 { + metadata.PromptTokensDetails = append(metadata.PromptTokensDetails, detail) + } + } + for _, detail := range []GeminiPromptTokensDetails{ + {Modality: "TEXT", TokenCount: usage.CompletionTokenDetails.TextTokens}, + {Modality: "IMAGE", TokenCount: usage.CompletionTokenDetails.ImageTokens}, + {Modality: "AUDIO", TokenCount: usage.CompletionTokenDetails.AudioTokens}, + } { + if detail.TokenCount != 0 { + metadata.CandidatesTokensDetails = append(metadata.CandidatesTokensDetails, detail) + } + } + return newGeminiChatBillingUsage(metadata, true) +} + +// CloneBillingUsageWithEstimatedCompletion preserves the original upstream +// billing dialect and fills a missing completion count without rebuilding the +// payload from a converted, potentially lossy Usage value. +func CloneBillingUsageWithEstimatedCompletion(usage *BillingUsage, completionTokens int) *BillingUsage { + clone := CloneBillingUsage(usage) + if clone == nil || completionTokens <= 0 { + return clone + } + + updated := false + switch { + case clone.OpenAIUsage != nil: + openAIUsage := clone.OpenAIUsage + if openAIUsage.CompletionTokens == 0 && openAIUsage.OutputTokens == 0 { + openAIUsage.CompletionTokens = completionTokens + openAIUsage.OutputTokens = completionTokens + inputTokens := openAIUsage.PromptTokens + if inputTokens == 0 { + inputTokens = openAIUsage.InputTokens + } + if totalTokens := inputTokens + completionTokens; openAIUsage.TotalTokens < totalTokens { + openAIUsage.TotalTokens = totalTokens + } + updated = true + } + case clone.ClaudeUsage != nil: + if clone.ClaudeUsage.OutputTokens == 0 { + clone.ClaudeUsage.OutputTokens = completionTokens + updated = true + } + case clone.GeminiUsageMetadata != nil: + metadata := clone.GeminiUsageMetadata + if metadata.CandidatesTokenCount == 0 { + candidateTokens := completionTokens - metadata.ThoughtsTokenCount + if candidateTokens < 0 { + candidateTokens = 0 + } + metadata.CandidatesTokenCount = candidateTokens + totalTokens := metadata.PromptTokenCount + metadata.ToolUsePromptTokenCount + metadata.CandidatesTokenCount + metadata.ThoughtsTokenCount + if metadata.TotalTokenCount < totalTokens { + metadata.TotalTokenCount = totalTokens + } + updated = true + } + } + if updated { + clone.Estimated = true + } + return clone } func newGeminiChatBillingUsage(metadata *GeminiUsageMetadata, estimated bool) *BillingUsage { @@ -149,6 +236,165 @@ func CloneBillingUsage(usage *BillingUsage) *BillingUsage { return &clone } +// CanonicalUsage decodes the original provider usage carried across relay +// hops into the shared accounting shape. The BillingUsage snapshot remains the +// source of truth and is cloned onto the returned value for further relays. +func (usage *BillingUsage) CanonicalUsage() (*Usage, bool) { + if usage == nil { + return nil, false + } + source := strings.TrimSpace(usage.Source) + semantic := strings.TrimSpace(usage.Semantic) + + // A structurally recognized but all-zero payload must not become the + // settlement source of truth; rejecting it lets settlement fall back to a + // non-zero top-level usage. + if HasOpenAIUsageTokens(usage.OpenAIUsage) && + (strings.EqualFold(source, BillingUsageSourceOAIChat) || + strings.EqualFold(source, BillingUsageSourceOAIResponses) || + strings.EqualFold(semantic, BillingUsageSemanticOpenAI)) { + return usage.canonicalOpenAIUsage(), true + } + if HasClaudeUsageTokens(usage.ClaudeUsage) && + (strings.EqualFold(source, BillingUsageSourceClaudeMessages) || + strings.EqualFold(semantic, BillingUsageSemanticAnthropic)) { + return usage.canonicalClaudeUsage(), true + } + if HasGeminiUsageMetadataTokens(usage.GeminiUsageMetadata) && + (strings.EqualFold(source, BillingUsageSourceGeminiChat) || + strings.EqualFold(semantic, BillingUsageSemanticGemini)) { + return usage.canonicalGeminiUsage(), true + } + return nil, false +} + +func (usage *BillingUsage) canonicalOpenAIUsage() *Usage { + canonical := cloneOpenAIUsage(usage.OpenAIUsage) + if inputDetails := canonical.InputTokensDetails; inputDetails != nil { + if canonical.PromptTokensDetails.CachedTokens == 0 && inputDetails.CachedTokens > 0 { + canonical.PromptTokensDetails.CachedTokens = inputDetails.CachedTokens + } + if canonical.PromptTokensDetails.CachedCreationTokens == 0 && inputDetails.CachedCreationTokens > 0 { + canonical.PromptTokensDetails.CachedCreationTokens = inputDetails.CachedCreationTokens + } + if canonical.PromptTokensDetails.CacheWriteTokens == 0 && inputDetails.CacheWriteTokens > 0 { + canonical.PromptTokensDetails.CacheWriteTokens = inputDetails.CacheWriteTokens + } + if canonical.PromptTokensDetails.TextTokens == 0 && inputDetails.TextTokens > 0 { + canonical.PromptTokensDetails.TextTokens = inputDetails.TextTokens + } + if canonical.PromptTokensDetails.ImageTokens == 0 && inputDetails.ImageTokens > 0 { + canonical.PromptTokensDetails.ImageTokens = inputDetails.ImageTokens + } + if canonical.PromptTokensDetails.AudioTokens == 0 && inputDetails.AudioTokens > 0 { + canonical.PromptTokensDetails.AudioTokens = inputDetails.AudioTokens + } + } + if canonical.PromptTokensDetails.CachedTokens == 0 && canonical.PromptCacheHitTokens > 0 { + canonical.PromptTokensDetails.CachedTokens = canonical.PromptCacheHitTokens + } + if canonical.PromptTokens == 0 && canonical.InputTokens > 0 { + canonical.PromptTokens = canonical.InputTokens + } + if canonical.CompletionTokens == 0 && canonical.OutputTokens > 0 { + canonical.CompletionTokens = canonical.OutputTokens + } + if canonical.InputTokens == 0 && canonical.PromptTokens > 0 { + canonical.InputTokens = canonical.PromptTokens + } + if canonical.OutputTokens == 0 && canonical.CompletionTokens > 0 { + canonical.OutputTokens = canonical.CompletionTokens + } + if canonical.TotalTokens == 0 { + canonical.TotalTokens = canonical.PromptTokens + canonical.CompletionTokens + } + canonical.UsageSemantic = BillingUsageSemanticOpenAI + canonical.UsageSource = usage.Source + canonical.BillingUsage = CloneBillingUsage(usage) + return canonical +} + +func (usage *BillingUsage) canonicalClaudeUsage() *Usage { + claudeUsage := usage.ClaudeUsage + cacheCreation5m := claudeUsage.GetCacheCreation5mTokens() + if cacheCreation5m == 0 { + cacheCreation5m = claudeUsage.ClaudeCacheCreation5mTokens + } + cacheCreation1h := claudeUsage.GetCacheCreation1hTokens() + if cacheCreation1h == 0 { + cacheCreation1h = claudeUsage.ClaudeCacheCreation1hTokens + } + + canonical := &Usage{ + PromptTokens: claudeUsage.InputTokens, + CompletionTokens: claudeUsage.OutputTokens, + TotalTokens: claudeUsage.InputTokens + claudeUsage.OutputTokens, + InputTokens: claudeUsage.InputTokens + claudeUsage.CacheReadInputTokens + claudeUsage.CacheCreationInputTokens, + OutputTokens: claudeUsage.OutputTokens, + UsageSemantic: BillingUsageSemanticAnthropic, + UsageSource: BillingUsageSourceClaudeMessages, + BillingUsage: CloneBillingUsage(usage), + ClaudeCacheCreation5mTokens: cacheCreation5m, + ClaudeCacheCreation1hTokens: cacheCreation1h, + } + canonical.PromptTokensDetails.CachedTokens = claudeUsage.CacheReadInputTokens + canonical.PromptTokensDetails.CachedCreationTokens = claudeUsage.CacheCreationInputTokens + return canonical +} + +func (usage *BillingUsage) canonicalGeminiUsage() *Usage { + metadata := usage.GeminiUsageMetadata + promptTokens := metadata.PromptTokenCount + metadata.ToolUsePromptTokenCount + canonical := &Usage{ + PromptTokens: promptTokens, + CompletionTokens: metadata.CandidatesTokenCount + metadata.ThoughtsTokenCount, + TotalTokens: metadata.TotalTokenCount, + UsageSemantic: BillingUsageSemanticGemini, + UsageSource: BillingUsageSourceGeminiChat, + BillingUsage: CloneBillingUsage(usage), + } + canonical.CompletionTokenDetails.ReasoningTokens = metadata.ThoughtsTokenCount + canonical.PromptTokensDetails.CachedTokens = metadata.CachedContentTokenCount + + for _, detail := range metadata.PromptTokensDetails { + addGeminiInputTokenDetail(&canonical.PromptTokensDetails, detail) + } + for _, detail := range metadata.ToolUsePromptTokensDetails { + addGeminiInputTokenDetail(&canonical.PromptTokensDetails, detail) + } + for _, detail := range metadata.CandidatesTokensDetails { + switch detail.Modality { + case "IMAGE": + canonical.CompletionTokenDetails.ImageTokens += detail.TokenCount + case "AUDIO": + canonical.CompletionTokenDetails.AudioTokens += detail.TokenCount + case "TEXT": + canonical.CompletionTokenDetails.TextTokens += detail.TokenCount + } + } + + if canonical.TotalTokens == 0 { + canonical.TotalTokens = canonical.PromptTokens + canonical.CompletionTokens + } else if canonical.CompletionTokens <= 0 { + canonical.CompletionTokens = canonical.TotalTokens - canonical.PromptTokens + } + if canonical.PromptTokens > 0 && canonical.PromptTokensDetails.TextTokens == 0 && canonical.PromptTokensDetails.AudioTokens == 0 { + canonical.PromptTokensDetails.TextTokens = canonical.PromptTokens + } + return canonical +} + +func addGeminiInputTokenDetail(details *InputTokenDetails, detail GeminiPromptTokensDetails) { + switch detail.Modality { + case "AUDIO": + details.AudioTokens += detail.TokenCount + case "IMAGE": + details.ImageTokens += detail.TokenCount + case "TEXT": + details.TextTokens += detail.TokenCount + } +} + func cloneOpenAIUsage(usage *Usage) *Usage { if usage == nil { return nil diff --git a/relaykit/dto/channel_settings.go b/relaykit/dto/channel_settings.go index 51f7062a80..b23d52218e 100644 --- a/relaykit/dto/channel_settings.go +++ b/relaykit/dto/channel_settings.go @@ -86,6 +86,10 @@ type ChannelOtherSettings struct { UpstreamModelUpdateLastRemovedModels []string `json:"upstream_model_update_last_removed_models,omitempty"` // 上次检测到的可删除模型 UpstreamModelUpdateIgnoredModels []string `json:"upstream_model_update_ignored_models,omitempty"` // 手动忽略的模型 AdvancedCustom *AdvancedCustomConfig `json:"advanced_custom,omitempty"` + // ToolLossPolicy is a channel-level opt-in for request-phase conversion + // rejection. Empty follows the default allow policy. Accepted values: + // "", "allow", "safe", "strict". + ToolLossPolicy string `json:"tool_loss_policy,omitempty"` } func (s *ChannelOtherSettings) IsOpenRouterEnterprise() bool { @@ -95,6 +99,20 @@ func (s *ChannelOtherSettings) IsOpenRouterEnterprise() bool { return *s.OpenRouterEnterprise } +// ValidateToolLossPolicy validates the channel-level request-phase tool-loss +// policy. Empty keeps the default allow policy. +func (s *ChannelOtherSettings) ValidateToolLossPolicy() error { + if s == nil { + return nil + } + switch strings.TrimSpace(s.ToolLossPolicy) { + case "", string(types.ConversionLossPolicyAllow), string(types.ConversionLossPolicySafe), string(types.ConversionLossPolicyStrict): + return nil + default: + return fmt.Errorf("invalid tool_loss_policy: %s", s.ToolLossPolicy) + } +} + const ( advancedCustomConverterNone = "none" advancedCustomConverterClaudeMessagesToOpenAIChat = "anthropic_messages_to_openai_chat_completions" diff --git a/relaykit/dto/channel_settings_test.go b/relaykit/dto/channel_settings_test.go index e84988731b..0f970a1108 100644 --- a/relaykit/dto/channel_settings_test.go +++ b/relaykit/dto/channel_settings_test.go @@ -642,3 +642,15 @@ func TestChannelSettingsValidateHTTPTransport(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "http2_connection_shards") } + +func TestChannelOtherSettingsValidateToolLossPolicy(t *testing.T) { + require.NoError(t, (*ChannelOtherSettings)(nil).ValidateToolLossPolicy()) + require.NoError(t, (&ChannelOtherSettings{}).ValidateToolLossPolicy()) + require.NoError(t, (&ChannelOtherSettings{ToolLossPolicy: "allow"}).ValidateToolLossPolicy()) + require.NoError(t, (&ChannelOtherSettings{ToolLossPolicy: "safe"}).ValidateToolLossPolicy()) + require.NoError(t, (&ChannelOtherSettings{ToolLossPolicy: "strict"}).ValidateToolLossPolicy()) + + err := (&ChannelOtherSettings{ToolLossPolicy: "drop"}).ValidateToolLossPolicy() + require.Error(t, err) + assert.Contains(t, err.Error(), "tool_loss_policy") +} diff --git a/relaykit/dto/claude.go b/relaykit/dto/claude.go index a3a41e7501..46a2f87eae 100644 --- a/relaykit/dto/claude.go +++ b/relaykit/dto/claude.go @@ -24,10 +24,24 @@ type ClaudeMediaMessage struct { PartialJson *string `json:"partial_json,omitempty"` Role string `json:"role,omitempty"` Thinking *string `json:"thinking,omitempty"` + Data string `json:"data,omitempty"` Signature string `json:"signature,omitempty"` Delta string `json:"delta,omitempty"` CacheControl json.RawMessage `json:"cache_control,omitempty"` - // tool_calls + + // Text blocks and citations_delta events. + Citations json.RawMessage `json:"citations,omitempty"` + Citation json.RawMessage `json:"citation,omitempty"` + + // Server-tool and tool-result blocks. + Caller json.RawMessage `json:"caller,omitempty"` + ServerName string `json:"server_name,omitempty"` + IsError *bool `json:"is_error,omitempty"` + // ErrorCode is a relaykit compatibility extension. Claude places provider + // error codes inside nested tool-result error content. + ErrorCode string `json:"error_code,omitempty"` + + // Tool-use and tool-result blocks. Id string `json:"id,omitempty"` Name string `json:"name,omitempty"` Input any `json:"input,omitempty"` @@ -173,6 +187,7 @@ type Tool struct { Name string `json:"name"` Description string `json:"description,omitempty"` InputSchema map[string]interface{} `json:"input_schema"` + Strict *bool `json:"strict,omitempty"` } type InputSchema struct { @@ -182,10 +197,14 @@ type InputSchema struct { } type ClaudeWebSearchTool struct { - Type string `json:"type"` - Name string `json:"name"` - MaxUses int `json:"max_uses,omitempty"` - UserLocation *ClaudeWebSearchUserLocation `json:"user_location,omitempty"` + Type string `json:"type"` + Name string `json:"name"` + MaxUses int `json:"max_uses,omitempty"` + AllowedDomains []string `json:"allowed_domains,omitempty"` + BlockedDomains []string `json:"blocked_domains,omitempty"` + AllowedCallers []string `json:"allowed_callers,omitempty"` + ResponseInclusion string `json:"response_inclusion,omitempty"` + UserLocation *ClaudeWebSearchUserLocation `json:"user_location,omitempty"` } type ClaudeWebSearchUserLocation struct { @@ -413,7 +432,7 @@ func (c *ClaudeRequest) GetTools() []any { func (c *ClaudeRequest) GetEfforts() string { var OutputConfig OutputConfigForEffort - if err := json.Unmarshal(c.OutputConfig, &OutputConfig); err == nil { + if err := kitutil.Unmarshal(c.OutputConfig, &OutputConfig); err == nil { effort := OutputConfig.Effort return effort } @@ -596,5 +615,8 @@ func (u *ClaudeUsage) GetCacheCreationTotalTokens() int { } type ClaudeServerToolUse struct { - WebSearchRequests int `json:"web_search_requests"` + WebSearchRequests int `json:"web_search_requests,omitempty"` + WebFetchRequests int `json:"web_fetch_requests,omitempty"` + CodeExecutionRequests int `json:"code_execution_requests,omitempty"` + ToolSearchRequests int `json:"tool_search_requests,omitempty"` } diff --git a/relaykit/dto/gemini.go b/relaykit/dto/gemini.go index 033fcc0f8b..3a68a0203a 100644 --- a/relaykit/dto/gemini.go +++ b/relaykit/dto/gemini.go @@ -48,8 +48,9 @@ type ToolConfig struct { } type FunctionCallingConfig struct { - Mode FunctionCallingConfigMode `json:"mode,omitempty"` - AllowedFunctionNames []string `json:"allowedFunctionNames,omitempty"` + Mode FunctionCallingConfigMode `json:"mode,omitempty"` + AllowedFunctionNames []string `json:"allowedFunctionNames,omitempty"` + StreamFunctionCallArguments *bool `json:"streamFunctionCallArguments,omitempty"` } type FunctionCallingConfigMode string @@ -161,8 +162,8 @@ func (r *GeminiChatRequest) SetTools(tools []GeminiChatTool) { } type GeminiThinkingConfig struct { - IncludeThoughts bool `json:"includeThoughts,omitempty"` - ThinkingBudget *int `json:"thinkingBudget,omitempty"` + IncludeThoughts *bool `json:"includeThoughts,omitempty"` + ThinkingBudget *int `json:"thinkingBudget,omitempty"` // TODO Conflict with thinkingbudget. ThinkingLevel string `json:"thinkingLevel,omitempty"` } @@ -184,7 +185,7 @@ func (c *GeminiThinkingConfig) UnmarshalJSON(data []byte) error { *c = GeminiThinkingConfig(aux.Alias) if aux.IncludeThoughtsSnake != nil { - c.IncludeThoughts = *aux.IncludeThoughtsSnake + c.IncludeThoughts = aux.IncludeThoughtsSnake } if aux.ThinkingBudgetSnake != nil { @@ -239,8 +240,21 @@ func (g *GeminiInlineData) UnmarshalJSON(data []byte) error { } type FunctionCall struct { - FunctionName string `json:"name"` - Arguments any `json:"args"` + // ID is optional in the Gemini protocol and identifies the matching function response. + ID string `json:"id,omitempty"` + FunctionName string `json:"name"` + Arguments any `json:"args"` + PartialArgs []GeminiPartialArg `json:"partialArgs,omitempty"` + WillContinue *bool `json:"willContinue,omitempty"` +} + +type GeminiPartialArg struct { + JSONPath string `json:"jsonPath"` + NumberValue *float64 `json:"numberValue,omitempty"` + StringValue *string `json:"stringValue,omitempty"` + BoolValue *bool `json:"boolValue,omitempty"` + NullValue json.RawMessage `json:"nullValue,omitempty"` + WillContinue *bool `json:"willContinue,omitempty"` } type GeminiFunctionResponse struct { @@ -320,11 +334,16 @@ type GeminiChatSafetySettings struct { } type GeminiChatTool struct { - GoogleSearch any `json:"googleSearch,omitempty"` - GoogleSearchRetrieval any `json:"googleSearchRetrieval,omitempty"` - CodeExecution any `json:"codeExecution,omitempty"` - FunctionDeclarations any `json:"functionDeclarations,omitempty"` - URLContext any `json:"urlContext,omitempty"` + GoogleSearch any `json:"googleSearch,omitempty"` + GoogleSearchRetrieval any `json:"googleSearchRetrieval,omitempty"` + GoogleMaps json.RawMessage `json:"googleMaps,omitempty"` + EnterpriseWebSearch json.RawMessage `json:"enterpriseWebSearch,omitempty"` + CodeExecution any `json:"codeExecution,omitempty"` + FunctionDeclarations any `json:"functionDeclarations,omitempty"` + URLContext any `json:"urlContext,omitempty"` + FileSearch json.RawMessage `json:"fileSearch,omitempty"` + ComputerUse json.RawMessage `json:"computerUse,omitempty"` + Retrieval json.RawMessage `json:"retrieval,omitempty"` } type GeminiChatGenerationConfig struct { @@ -447,7 +466,14 @@ type GeminiChatCandidate struct { } type GeminiGroundingMetadata struct { - WebSearchQueries []string `json:"webSearchQueries,omitempty"` + WebSearchQueries []string `json:"webSearchQueries,omitempty"` + RetrievalQueries []string `json:"retrievalQueries,omitempty"` + GroundingChunks json.RawMessage `json:"groundingChunks,omitempty"` + GroundingSupports json.RawMessage `json:"groundingSupports,omitempty"` + SearchEntryPoint json.RawMessage `json:"searchEntryPoint,omitempty"` + RetrievalMetadata json.RawMessage `json:"retrievalMetadata,omitempty"` + SourceFlaggingUris json.RawMessage `json:"sourceFlaggingUris,omitempty"` + GoogleMapsWidgetContextToken string `json:"googleMapsWidgetContextToken,omitempty"` } type GeminiChatSafetyRating struct { diff --git a/relaykit/dto/openai_request.go b/relaykit/dto/openai_request.go index d54ac0d15b..1d89361e90 100644 --- a/relaykit/dto/openai_request.go +++ b/relaykit/dto/openai_request.go @@ -81,7 +81,7 @@ type GeneralOpenAIRequest struct { ExtraBody json.RawMessage `json:"extra_body,omitempty"` //xai SearchParameters json.RawMessage `json:"search_parameters,omitempty"` - // claude + // OpenAI Chat web search. WebSearchOptions *WebSearchOptions `json:"web_search_options,omitempty"` // OpenRouter Params Usage json.RawMessage `json:"usage,omitempty"` @@ -108,6 +108,9 @@ type GeneralOpenAIRequest struct { ReasoningSplit json.RawMessage `json:"reasoning_split,omitempty"` // vLLM ThinkingTokenBudget json.RawMessage `json:"thinking_token_budget,omitempty"` + + // Internal conversion state; never serialized to an upstream protocol. + ReasoningConversion *ReasoningConversionState `json:"-"` } func (r GeneralOpenAIRequest) MarshalJSON() ([]byte, error) { @@ -266,6 +269,7 @@ type FunctionRequest struct { Name string `json:"name"` Parameters any `json:"parameters,omitempty"` Arguments string `json:"arguments,omitempty"` + Strict *bool `json:"strict,omitempty"` } type StreamOptions struct { @@ -311,7 +315,10 @@ type Message struct { Reasoning *string `json:"reasoning,omitempty"` ToolCalls json.RawMessage `json:"tool_calls,omitempty"` ToolCallId string `json:"tool_call_id,omitempty"` - parsedContent []MediaContent + // Annotations is an official Chat response field. Keeping it on the shared + // message type also preserves annotations when clients replay assistant output. + Annotations json.RawMessage `json:"annotations,omitempty"` + parsedContent []MediaContent //parsedStringContent *string } @@ -485,14 +492,14 @@ func (m *Message) ParseToolCalls() []ToolCallRequest { return nil } var toolCalls []ToolCallRequest - if err := json.Unmarshal(m.ToolCalls, &toolCalls); err == nil { + if err := kitutil.Unmarshal(m.ToolCalls, &toolCalls); err == nil { return toolCalls } return toolCalls } func (m *Message) SetToolCalls(toolCalls any) { - toolCallsJson, _ := json.Marshal(toolCalls) + toolCallsJson, _ := kitutil.Marshal(toolCalls) m.ToolCalls = toolCallsJson } @@ -562,6 +569,11 @@ func (m *Message) ParseContent() []MediaContent { return contentList } + if content, ok := m.Content.([]MediaContent); ok { + m.parsedContent = content + return content + } + // 尝试解析为数组 //var arrayContent []map[string]interface{} @@ -682,7 +694,7 @@ func (m *Message) ParseContent() []MediaContent { } var stringContent string - if err := json.Unmarshal(m.Content, &stringContent); err == nil { + if err := kitutil.Unmarshal(m.Content, &stringContent); err == nil { m.parsedStringContent = &stringContent return stringContent } @@ -707,14 +719,14 @@ func (m *Message) SetNullContent() { } func (m *Message) SetStringContent(content string) { - jsonContent, _ := json.Marshal(content) + jsonContent, _ := kitutil.Marshal(content) m.Content = jsonContent m.parsedStringContent = &content m.parsedContent = nil } func (m *Message) SetMediaContent(content []MediaContent) { - jsonContent, _ := json.Marshal(content) + jsonContent, _ := kitutil.Marshal(content) m.Content = jsonContent m.parsedContent = nil m.parsedStringContent = nil @@ -725,7 +737,7 @@ func (m *Message) IsStringContent() bool { return true } var stringContent string - if err := json.Unmarshal(m.Content, &stringContent); err == nil { + if err := kitutil.Unmarshal(m.Content, &stringContent); err == nil { m.parsedStringContent = &stringContent return true } @@ -741,7 +753,7 @@ func (m *Message) ParseContent() []MediaContent { // 先尝试解析为字符串 var stringContent string - if err := json.Unmarshal(m.Content, &stringContent); err == nil { + if err := kitutil.Unmarshal(m.Content, &stringContent); err == nil { contentList = []MediaContent{{ Type: ContentTypeText, Text: stringContent, @@ -752,7 +764,7 @@ func (m *Message) ParseContent() []MediaContent { // 尝试解析为数组 var arrayContent []map[string]interface{} - if err := json.Unmarshal(m.Content, &arrayContent); err == nil { + if err := kitutil.Unmarshal(m.Content, &arrayContent); err == nil { for _, contentItem := range arrayContent { contentType, ok := contentItem["type"].(string) if !ok { @@ -907,6 +919,9 @@ type OpenAIResponsesRequest struct { ThinkingBudget json.RawMessage `json:"thinking_budget,omitempty"` // perplexity Preset json.RawMessage `json:"preset,omitempty"` + + // Internal conversion state; never serialized to an upstream protocol. + ReasoningConversion *ReasoningConversionState `json:"-"` } func (r OpenAIResponsesRequest) MarshalJSON() ([]byte, error) { diff --git a/relaykit/dto/openai_response.go b/relaykit/dto/openai_response.go index 945d0a8699..6ab5c6fcf0 100644 --- a/relaykit/dto/openai_response.go +++ b/relaykit/dto/openai_response.go @@ -3,6 +3,7 @@ package dto import ( "encoding/json" "fmt" + "strings" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" "github.com/QuantumNous/new-api/relaykit/types" @@ -91,6 +92,10 @@ type ChatCompletionsStreamResponseChoiceDelta struct { Reasoning *string `json:"reasoning,omitempty"` Role string `json:"role,omitempty"` ToolCalls []ToolCallResponse `json:"tool_calls,omitempty"` + // Annotations is an OpenAI-compatible streaming extension supported by + // providers such as OpenRouter. Relaykit uses it to preserve streaming URL + // citations, including Claude round-trip metadata. + Annotations json.RawMessage `json:"annotations,omitempty"` } func (c *ChatCompletionsStreamResponseChoiceDelta) SetContentString(s string) { @@ -325,17 +330,143 @@ type IncompleteDetails struct { } type ResponsesOutput struct { - Type string `json:"type"` - ID string `json:"id"` - Status string `json:"status"` - Role string `json:"role"` - Content []ResponsesOutputContent `json:"content"` - Quality string `json:"quality"` - Size string `json:"size"` - Result string `json:"result,omitempty"` - CallId string `json:"call_id,omitempty"` - Name string `json:"name,omitempty"` - Arguments json.RawMessage `json:"arguments,omitempty"` + Type string `json:"type"` + ID string `json:"id"` + Status string `json:"status"` + Role string `json:"role"` + Content []ResponsesOutputContent `json:"content"` + Summary []ResponsesReasoningSummaryPart `json:"summary,omitempty"` + Quality string `json:"quality"` + Size string `json:"size"` + Result string `json:"result,omitempty"` + CallId string `json:"call_id,omitempty"` + Name string `json:"name,omitempty"` + Arguments json.RawMessage `json:"arguments,omitempty"` + Action json.RawMessage `json:"action,omitempty"` + Queries json.RawMessage `json:"queries,omitempty"` + Results json.RawMessage `json:"results,omitempty"` + Sources json.RawMessage `json:"sources,omitempty"` + Code json.RawMessage `json:"code,omitempty"` + Outputs json.RawMessage `json:"outputs,omitempty"` + ContainerID string `json:"container_id,omitempty"` + PendingSafetyChecks json.RawMessage `json:"pending_safety_checks,omitempty"` + Caller json.RawMessage `json:"caller,omitempty"` + ServerLabel string `json:"server_label,omitempty"` + Output json.RawMessage `json:"output,omitempty"` + ItemError json.RawMessage `json:"error,omitempty"` + ApprovalRequestID string `json:"approval_request_id,omitempty"` + MCPTools json.RawMessage `json:"tools,omitempty"` +} + +// MarshalJSON keeps hosted-tool variants within their protocol-specific +// schemas. ResponsesOutput also represents messages, images, and function +// calls, whose fields must not leak into web_search_call or mcp_call items. +func (r ResponsesOutput) MarshalJSON() ([]byte, error) { + switch r.Type { + case "web_search_call": + return kitutil.Marshal(struct { + Type string `json:"type"` + ID string `json:"id"` + Status string `json:"status,omitempty"` + Action json.RawMessage `json:"action,omitempty"` + }{Type: r.Type, ID: r.ID, Status: r.Status, Action: r.Action}) + case "mcp_call": + return kitutil.Marshal(struct { + Type string `json:"type"` + ID string `json:"id"` + Name string `json:"name"` + ServerLabel string `json:"server_label"` + Arguments json.RawMessage `json:"arguments"` + Status string `json:"status,omitempty"` + Output json.RawMessage `json:"output,omitempty"` + Error json.RawMessage `json:"error,omitempty"` + ApprovalRequestID string `json:"approval_request_id,omitempty"` + }{ + Type: r.Type, + ID: r.ID, + Name: r.Name, + ServerLabel: r.ServerLabel, + Arguments: r.Arguments, + Status: r.Status, + Output: r.Output, + Error: r.ItemError, + ApprovalRequestID: r.ApprovalRequestID, + }) + default: + type responsesOutputAlias ResponsesOutput + return kitutil.Marshal(responsesOutputAlias(r)) + } +} + +// NormalizeResponsesWebSearchAction validates and canonicalizes the current +// Responses web_search_call action union. Claude emits {"query": ...}; the +// Responses representation additionally requires a discriminator. +func NormalizeResponsesWebSearchAction(raw json.RawMessage) (json.RawMessage, error) { + var action struct { + Type string `json:"type"` + Query string `json:"query"` + Queries []string `json:"queries"` + Sources json.RawMessage `json:"sources"` + URL string `json:"url"` + Pattern string `json:"pattern"` + } + if err := kitutil.Unmarshal(raw, &action); err != nil { + return nil, fmt.Errorf("decode Responses web-search action: %w", err) + } + action.Type = strings.TrimSpace(action.Type) + action.Query = strings.TrimSpace(action.Query) + action.URL = strings.TrimSpace(action.URL) + action.Pattern = strings.TrimSpace(action.Pattern) + for index := range action.Queries { + action.Queries[index] = strings.TrimSpace(action.Queries[index]) + if action.Queries[index] == "" { + return nil, fmt.Errorf("Responses web-search action queries[%d] must not be empty", index) + } + } + if action.Type == "" && (action.Query != "" || len(action.Queries) > 0) { + action.Type = "search" + } + + var canonical any + switch action.Type { + case "search": + if action.Query == "" && len(action.Queries) == 0 { + return nil, fmt.Errorf("Responses web-search action %q requires query or queries", action.Type) + } + if len(action.Sources) > 0 && kitutil.GetJsonType(action.Sources) != "array" && kitutil.GetJsonType(action.Sources) != "null" { + return nil, fmt.Errorf("Responses web-search action sources must be an array") + } + canonical = struct { + Type string `json:"type"` + Query string `json:"query,omitempty"` + Queries []string `json:"queries,omitempty"` + Sources json.RawMessage `json:"sources,omitempty"` + }{Type: action.Type, Query: action.Query, Queries: action.Queries, Sources: action.Sources} + case "open_page": + if action.URL == "" { + return nil, fmt.Errorf("Responses web-search action %q requires url", action.Type) + } + canonical = struct { + Type string `json:"type"` + URL string `json:"url"` + }{Type: action.Type, URL: action.URL} + case "find", "find_in_page": + if action.URL == "" || action.Pattern == "" { + return nil, fmt.Errorf("Responses web-search action %q requires url and pattern", action.Type) + } + canonical = struct { + Type string `json:"type"` + URL string `json:"url"` + Pattern string `json:"pattern"` + }{Type: "find_in_page", URL: action.URL, Pattern: action.Pattern} + default: + return nil, fmt.Errorf("unsupported Responses web-search action type %q", action.Type) + } + encoded, err := kitutil.Marshal(canonical) + if err != nil { + return nil, fmt.Errorf("encode Responses web-search action: %w", err) + } + return encoded, nil } // ArgumentsString returns function call arguments in the string form expected by Chat Completions. @@ -384,10 +515,20 @@ const ( // ResponsesStreamResponse 用于处理 /v1/responses 流式响应 type ResponsesStreamResponse struct { - Type string `json:"type"` - Response *OpenAIResponsesResponse `json:"response,omitempty"` - Delta string `json:"delta,omitempty"` - Item *ResponsesOutput `json:"item,omitempty"` + Type string `json:"type"` + Response *OpenAIResponsesResponse `json:"response,omitempty"` + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` + Param string `json:"param,omitempty"` + Delta string `json:"delta,omitempty"` + Arguments *string `json:"arguments,omitempty"` + Name string `json:"name,omitempty"` + Text *string `json:"text,omitempty"` + Item *ResponsesOutput `json:"item,omitempty"` + SequenceNumber *int `json:"sequence_number,omitempty"` + Annotation json.RawMessage `json:"annotation,omitempty"` + AnnotationIndex *int `json:"annotation_index,omitempty"` + Obfuscation string `json:"obfuscation,omitempty"` // - response.function_call_arguments.delta // - response.function_call_arguments.done OutputIndex *int `json:"output_index,omitempty"` diff --git a/relaykit/dto/reasoning_state.go b/relaykit/dto/reasoning_state.go new file mode 100644 index 0000000000..8a588caa77 --- /dev/null +++ b/relaykit/dto/reasoning_state.go @@ -0,0 +1,14 @@ +package dto + +// ReasoningConversionState carries provider-native reasoning controls between +// in-process conversion steps. It is not part of any provider wire protocol; +// request fields that reference it must use json:"-". +// +// Converters that rebuild an OpenAI request must copy this state so exact +// budgets and explicit include-thoughts choices survive multi-step routes. +type ReasoningConversionState struct { + Mode string + Effort string + BudgetTokens *int + IncludeThoughts *bool +} diff --git a/relaykit/dto/usage_merge.go b/relaykit/dto/usage_merge.go new file mode 100644 index 0000000000..39383b1b77 --- /dev/null +++ b/relaykit/dto/usage_merge.go @@ -0,0 +1,281 @@ +package dto + +import ( + "reflect" + "strings" +) + +// MergeUsageNonZero overlays usage snapshots: a later non-zero field +// overwrites the current value, while a later zero value never erases an +// earlier positive count. Compatible BillingUsage snapshots follow the same +// rule within their provider-native payload. +func MergeUsageNonZero(current *Usage, incoming *Usage) *Usage { + if current == nil { + current = &Usage{} + } + if incoming == nil { + return current + } + + if incoming.PromptTokens > 0 { + current.PromptTokens = incoming.PromptTokens + } + if incoming.CompletionTokens > 0 { + current.CompletionTokens = incoming.CompletionTokens + } + if incoming.TotalTokens > 0 { + current.TotalTokens = incoming.TotalTokens + } + if incoming.PromptCacheHitTokens > 0 { + current.PromptCacheHitTokens = incoming.PromptCacheHitTokens + } + if incoming.InputTokens > 0 { + current.InputTokens = incoming.InputTokens + } + if incoming.OutputTokens > 0 { + current.OutputTokens = incoming.OutputTokens + } + if incoming.ClaudeCacheCreation5mTokens > 0 { + current.ClaudeCacheCreation5mTokens = incoming.ClaudeCacheCreation5mTokens + } + if incoming.ClaudeCacheCreation1hTokens > 0 { + current.ClaudeCacheCreation1hTokens = incoming.ClaudeCacheCreation1hTokens + } + + mergeInputTokenDetails(¤t.PromptTokensDetails, incoming.PromptTokensDetails) + if incoming.InputTokensDetails != nil { + details := *incoming.InputTokensDetails + if details.CachedTokens > 0 || + details.CachedCreationTokens > 0 || + details.CacheWriteTokens > 0 || + details.TextTokens > 0 || + details.AudioTokens > 0 || + details.ImageTokens > 0 { + if current.InputTokensDetails == nil { + current.InputTokensDetails = &InputTokenDetails{} + } + mergeInputTokenDetails(current.InputTokensDetails, details) + } + } + + if incoming.CompletionTokenDetails.TextTokens > 0 { + current.CompletionTokenDetails.TextTokens = incoming.CompletionTokenDetails.TextTokens + } + if incoming.CompletionTokenDetails.AudioTokens > 0 { + current.CompletionTokenDetails.AudioTokens = incoming.CompletionTokenDetails.AudioTokens + } + if incoming.CompletionTokenDetails.ImageTokens > 0 { + current.CompletionTokenDetails.ImageTokens = incoming.CompletionTokenDetails.ImageTokens + } + if incoming.CompletionTokenDetails.ReasoningTokens > 0 { + current.CompletionTokenDetails.ReasoningTokens = incoming.CompletionTokenDetails.ReasoningTokens + } + + if incoming.UsageSemantic != "" { + current.UsageSemantic = incoming.UsageSemantic + } + if incoming.UsageSource != "" { + current.UsageSource = incoming.UsageSource + } + if incoming.BillingUsage != nil { + current.BillingUsage = MergeBillingUsageNonZero(current.BillingUsage, incoming.BillingUsage) + } + if incoming.Cost != nil && !reflect.ValueOf(incoming.Cost).IsZero() { + current.Cost = incoming.Cost + } + if total := current.PromptTokens + current.CompletionTokens; total > current.TotalTokens { + current.TotalTokens = total + } + if total := current.InputTokens + current.OutputTokens; total > current.TotalTokens { + current.TotalTokens = total + } + + return current +} + +// MergeBillingUsageNonZero preserves non-zero provider-native fields across +// partial stream snapshots. A snapshot from a different billing dialect +// remains authoritative and replaces the previous payload. +func MergeBillingUsageNonZero(current *BillingUsage, incoming *BillingUsage) *BillingUsage { + if incoming == nil { + return CloneBillingUsage(current) + } + if current == nil || !sameBillingUsageDialect(current, incoming) { + return CloneBillingUsage(incoming) + } + + merged := CloneBillingUsage(current) + if incoming.Source != "" { + merged.Source = incoming.Source + } + if incoming.Semantic != "" { + merged.Semantic = incoming.Semantic + } + merged.Estimated = current.Estimated || incoming.Estimated + + switch { + case current.OpenAIUsage != nil && incoming.OpenAIUsage != nil: + merged.OpenAIUsage = MergeUsageNonZero( + cloneOpenAIUsage(current.OpenAIUsage), + cloneOpenAIUsage(incoming.OpenAIUsage), + ) + case current.ClaudeUsage != nil && incoming.ClaudeUsage != nil: + merged.ClaudeUsage = mergeClaudeUsageNonZero(current.ClaudeUsage, incoming.ClaudeUsage) + case current.GeminiUsageMetadata != nil && incoming.GeminiUsageMetadata != nil: + merged.GeminiUsageMetadata = MergeGeminiUsageMetadataNonZero(current.GeminiUsageMetadata, incoming.GeminiUsageMetadata) + } + + return merged +} + +func sameBillingUsageDialect(current *BillingUsage, incoming *BillingUsage) bool { + if current.Source != "" && incoming.Source != "" && !strings.EqualFold(current.Source, incoming.Source) { + return false + } + if current.Semantic != "" && incoming.Semantic != "" && !strings.EqualFold(current.Semantic, incoming.Semantic) { + return false + } + return current.OpenAIUsage != nil && incoming.OpenAIUsage != nil || + current.ClaudeUsage != nil && incoming.ClaudeUsage != nil || + current.GeminiUsageMetadata != nil && incoming.GeminiUsageMetadata != nil +} + +func mergeClaudeUsageNonZero(current *ClaudeUsage, incoming *ClaudeUsage) *ClaudeUsage { + merged := cloneClaudeUsage(current) + if merged == nil { + merged = &ClaudeUsage{} + } + if incoming == nil { + return merged + } + if incoming.InputTokens > 0 { + merged.InputTokens = incoming.InputTokens + } + if incoming.CacheCreationInputTokens > 0 { + merged.CacheCreationInputTokens = incoming.CacheCreationInputTokens + } + if incoming.CacheReadInputTokens > 0 { + merged.CacheReadInputTokens = incoming.CacheReadInputTokens + } + if incoming.OutputTokens > 0 { + merged.OutputTokens = incoming.OutputTokens + } + if incoming.ClaudeCacheCreation5mTokens > 0 { + merged.ClaudeCacheCreation5mTokens = incoming.ClaudeCacheCreation5mTokens + } + if incoming.ClaudeCacheCreation1hTokens > 0 { + merged.ClaudeCacheCreation1hTokens = incoming.ClaudeCacheCreation1hTokens + } + if incoming.CacheCreation != nil { + cacheCreation := *incoming.CacheCreation + merged.CacheCreation = &cacheCreation + } + if incoming.ServerToolUse != nil { + if merged.ServerToolUse == nil { + merged.ServerToolUse = &ClaudeServerToolUse{} + } + if incoming.ServerToolUse.WebSearchRequests > 0 { + merged.ServerToolUse.WebSearchRequests = incoming.ServerToolUse.WebSearchRequests + } + if incoming.ServerToolUse.WebFetchRequests > 0 { + merged.ServerToolUse.WebFetchRequests = incoming.ServerToolUse.WebFetchRequests + } + if incoming.ServerToolUse.CodeExecutionRequests > 0 { + merged.ServerToolUse.CodeExecutionRequests = incoming.ServerToolUse.CodeExecutionRequests + } + if incoming.ServerToolUse.ToolSearchRequests > 0 { + merged.ServerToolUse.ToolSearchRequests = incoming.ServerToolUse.ToolSearchRequests + } + } + return merged +} + +// MergeGeminiUsageMetadataNonZero overlays Gemini's cumulative usage +// snapshots: a later non-zero field overwrites the current value without +// dropping fields omitted by a later chunk. +func MergeGeminiUsageMetadataNonZero(current *GeminiUsageMetadata, incoming *GeminiUsageMetadata) *GeminiUsageMetadata { + if current == nil && incoming == nil { + return nil + } + if current == nil { + metadata := cloneGeminiUsageMetadata(*incoming) + metadata.BillingUsage = CloneBillingUsage(incoming.BillingUsage) + return &metadata + } + + merged := cloneGeminiUsageMetadata(*current) + merged.BillingUsage = CloneBillingUsage(current.BillingUsage) + if incoming == nil { + return &merged + } + if incoming.PromptTokenCount > 0 { + merged.PromptTokenCount = incoming.PromptTokenCount + } + if incoming.ToolUsePromptTokenCount > 0 { + merged.ToolUsePromptTokenCount = incoming.ToolUsePromptTokenCount + } + if incoming.CandidatesTokenCount > 0 { + merged.CandidatesTokenCount = incoming.CandidatesTokenCount + merged.ThoughtsTokenCount = incoming.ThoughtsTokenCount + } else if incoming.ThoughtsTokenCount > 0 { + merged.ThoughtsTokenCount = incoming.ThoughtsTokenCount + } + if incoming.TotalTokenCount > 0 { + merged.TotalTokenCount = incoming.TotalTokenCount + } + if incoming.CachedContentTokenCount > 0 { + merged.CachedContentTokenCount = incoming.CachedContentTokenCount + } + merged.PromptTokensDetails = mergeGeminiTokenDetails(merged.PromptTokensDetails, incoming.PromptTokensDetails) + merged.ToolUsePromptTokensDetails = mergeGeminiTokenDetails(merged.ToolUsePromptTokensDetails, incoming.ToolUsePromptTokensDetails) + merged.CandidatesTokensDetails = mergeGeminiTokenDetails(merged.CandidatesTokensDetails, incoming.CandidatesTokensDetails) + if incoming.BillingUsage != nil { + merged.BillingUsage = MergeBillingUsageNonZero(merged.BillingUsage, incoming.BillingUsage) + } + if total := merged.PromptTokenCount + merged.ToolUsePromptTokenCount + merged.CandidatesTokenCount + merged.ThoughtsTokenCount; total > merged.TotalTokenCount { + merged.TotalTokenCount = total + } + return &merged +} + +func mergeGeminiTokenDetails(current []GeminiPromptTokensDetails, incoming []GeminiPromptTokensDetails) []GeminiPromptTokensDetails { + merged := append([]GeminiPromptTokensDetails{}, current...) + indexes := make(map[string]int, len(merged)) + for index, detail := range merged { + indexes[strings.ToUpper(strings.TrimSpace(detail.Modality))] = index + } + for _, detail := range incoming { + if detail.TokenCount <= 0 { + continue + } + key := strings.ToUpper(strings.TrimSpace(detail.Modality)) + if index, ok := indexes[key]; ok { + merged[index] = detail + continue + } + indexes[key] = len(merged) + merged = append(merged, detail) + } + return merged +} + +func mergeInputTokenDetails(current *InputTokenDetails, incoming InputTokenDetails) { + if incoming.CachedTokens > 0 { + current.CachedTokens = incoming.CachedTokens + } + if incoming.CachedCreationTokens > 0 { + current.CachedCreationTokens = incoming.CachedCreationTokens + } + if incoming.CacheWriteTokens > 0 { + current.CacheWriteTokens = incoming.CacheWriteTokens + } + if incoming.TextTokens > 0 { + current.TextTokens = incoming.TextTokens + } + if incoming.AudioTokens > 0 { + current.AudioTokens = incoming.AudioTokens + } + if incoming.ImageTokens > 0 { + current.ImageTokens = incoming.ImageTokens + } +} diff --git a/relaykit/dto/usage_merge_test.go b/relaykit/dto/usage_merge_test.go new file mode 100644 index 0000000000..088ff02405 --- /dev/null +++ b/relaykit/dto/usage_merge_test.go @@ -0,0 +1,67 @@ +package dto + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMergeClaudeUsageCacheCreationReplacesWholeObject(t *testing.T) { + t.Parallel() + + merged := mergeClaudeUsageNonZero( + &ClaudeUsage{ + CacheCreation: &ClaudeCacheCreationUsage{Ephemeral1hInputTokens: 1000}, + }, + &ClaudeUsage{ + CacheCreation: &ClaudeCacheCreationUsage{ + Ephemeral5mInputTokens: 1000, + Ephemeral1hInputTokens: 0, + }, + }, + ) + + require.NotNil(t, merged.CacheCreation) + assert.Equal(t, 1000, merged.CacheCreation.Ephemeral5mInputTokens) + assert.Equal(t, 0, merged.CacheCreation.Ephemeral1hInputTokens) +} + +func TestMergeGeminiUsageMetadataCandidatesAndThoughtsReplacedAsPair(t *testing.T) { + t.Parallel() + + merged := MergeGeminiUsageMetadataNonZero( + &GeminiUsageMetadata{ + PromptTokenCount: 10, + ThoughtsTokenCount: 100, + }, + &GeminiUsageMetadata{ + PromptTokenCount: 10, + CandidatesTokenCount: 150, + ThoughtsTokenCount: 0, + TotalTokenCount: 160, + }, + ) + require.NotNil(t, merged) + assert.Equal(t, 150, merged.CandidatesTokenCount) + assert.Equal(t, 0, merged.ThoughtsTokenCount) + + billing := NewGeminiChatBillingUsage(merged) + usage, ok := billing.CanonicalUsage() + require.True(t, ok) + assert.Equal(t, 150, usage.CompletionTokens) +} + +func TestMergeUsageNonZeroKeepsPositiveValuesAndTakesMaxTotal(t *testing.T) { + t.Parallel() + + merged := MergeUsageNonZero( + &Usage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15}, + &Usage{PromptTokens: 0, CompletionTokens: 0, TotalTokens: 20}, + ) + + require.NotNil(t, merged) + assert.Equal(t, 10, merged.PromptTokens) + assert.Equal(t, 5, merged.CompletionTokens) + assert.Equal(t, 20, merged.TotalTokens) +} diff --git a/relaykit/reasonmap/reasonmap.go b/relaykit/reasonmap/reasonmap.go index 8c6f66c618..04d5d2a3e2 100644 --- a/relaykit/reasonmap/reasonmap.go +++ b/relaykit/reasonmap/reasonmap.go @@ -16,6 +16,11 @@ func ClaudeStopReasonToOpenAIFinishReason(stopReason string) string { return "length" case "tool_use": return "tool_calls" + case "pause_turn": + // Responses has no pause_turn finish reason. Treat the provider's + // resumable server-side loop as an incomplete response instead of a + // successful stop; the hosted output items preserve continuation state. + return "length" case "refusal": return types.FinishReasonContentFilter default: diff --git a/relaykit/relayconvert/claude_default_max_tokens_test.go b/relaykit/relayconvert/claude_default_max_tokens_test.go index d97c17b0e8..8cd61c8b96 100644 --- a/relaykit/relayconvert/claude_default_max_tokens_test.go +++ b/relaykit/relayconvert/claude_default_max_tokens_test.go @@ -8,6 +8,7 @@ import ( "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" sharedclaude "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/claude" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -80,6 +81,21 @@ func TestClaudeDefaultMaxTokensPresence(t *testing.T) { require.NotNil(t, got.MaxTokens) assert.Equal(t, clientMaxTokens, *got.MaxTokens) }) + + t.Run("client zero same as absent, hook fills", func(t *testing.T) { + clientMaxTokens := uint(0) + got, err := converter.convert(t, claudeDefaultsMeta(func(string) int { return 512 }), &clientMaxTokens) + require.NoError(t, err) + require.NotNil(t, got.MaxTokens) + assert.Equal(t, uint(512), *got.MaxTokens) + }) + + t.Run("client zero same as absent, no hook fails", func(t *testing.T) { + clientMaxTokens := uint(0) + got, err := converter.convert(t, &convmeta.Values{}, &clientMaxTokens) + require.ErrorIs(t, err, sharedclaude.ErrMissingMaxTokens) + assert.Nil(t, got) + }) }) } } @@ -88,12 +104,18 @@ func TestClaudeDefaultMaxTokensPresence(t *testing.T) { // "-thinking" request without max_tokens must keep converting even when no // DefaultMaxTokens hook is configured. func TestClaudeThinkingAdapterSatisfiesMaxTokensWithoutCallback(t *testing.T) { - meta := &convmeta.Values{Options: &convmeta.Options{ - Claude: convmeta.ClaudeOptions{ - ThinkingAdapterEnabled: true, - ThinkingAdapterBudgetTokensPercentage: 0.8, + _, intent, found, err := reasoning.ParseClaudeModelSuffix("claude-test-thinking", true) + require.NoError(t, err) + require.True(t, found) + meta := &convmeta.Values{ + ReasoningConversion: reasoning.StateFromIntent(intent), + Options: &convmeta.Options{ + Claude: convmeta.ClaudeOptions{ + ThinkingAdapterEnabled: true, + ThinkingAdapterBudgetTokensPercentage: 0.8, + }, }, - }} + } got, err := OpenAIChatRequestToClaudeMessages(context.Background(), meta, dto.GeneralOpenAIRequest{ Model: "claude-test-thinking", Messages: []dto.Message{ diff --git a/relaykit/relayconvert/convmeta/meta.go b/relaykit/relayconvert/convmeta/meta.go index 68848af971..2362e2b71f 100644 --- a/relaykit/relayconvert/convmeta/meta.go +++ b/relaykit/relayconvert/convmeta/meta.go @@ -28,6 +28,10 @@ type Meta interface { // SetReasoningEffort records the effort level a converter derived from a // model-name suffix so downstream billing/logging can see it. SetReasoningEffort(effort string) + // ReasoningState returns the suffix-derived reasoning intent attached at + // the host entry layer. Standalone callers that do not set it receive nil; + // converters then use only explicit request fields. + ReasoningState() *dto.ReasoningConversionState GetEstimatePromptTokens() int // EnsureClaudeConvertInfo lazily creates and returns the mutable @@ -60,6 +64,20 @@ type ClaudeConvertInfo struct { ToolCallBaseIndex int ToolCallMaxIndexOffset int + ToolCalls []*ClaudeStreamToolCall + ToolCallByIndex map[int]*ClaudeStreamToolCall + ToolCallByID map[string]*ClaudeStreamToolCall +} + +// ClaudeStreamToolCall tracks one OpenAI tool_calls entry while it is encoded +// as a Claude tool_use content block. Chat tool indexes and Claude content +// block indexes are separate domains, so the mapping must remain explicit. +type ClaudeStreamToolCall struct { + BlockIndex int + ID string + Name string + PendingArguments string + Started bool } const ( @@ -79,6 +97,7 @@ type Values struct { ChannelType int IsStream bool ReasoningEffort string + ReasoningConversion *dto.ReasoningConversionState EstimatePromptTokens int ClaudeConvertInfo *ClaudeConvertInfo @@ -139,6 +158,13 @@ func (v *Values) SetReasoningEffort(effort string) { } } +func (v *Values) ReasoningState() *dto.ReasoningConversionState { + if v == nil { + return nil + } + return v.ReasoningConversion +} + func (v *Values) GetEstimatePromptTokens() int { if v == nil { return 0 @@ -213,3 +239,11 @@ func OptionsOf(m Meta) *Options { } return m.ConvOptions() } + +// ReasoningStateOf is a nil-safe reader for Meta.ReasoningState. +func ReasoningStateOf(m Meta) *dto.ReasoningConversionState { + if m == nil { + return nil + } + return m.ReasoningState() +} diff --git a/relaykit/relayconvert/convmeta/meta_test.go b/relaykit/relayconvert/convmeta/meta_test.go index 0b055f3fd3..fbd5b654d5 100644 --- a/relaykit/relayconvert/convmeta/meta_test.go +++ b/relaykit/relayconvert/convmeta/meta_test.go @@ -19,6 +19,7 @@ func TestValuesTypedNilMetaIsSafe(t *testing.T) { assert.Zero(t, meta.GetChannelType()) assert.False(t, meta.GetIsStream()) assert.Empty(t, meta.GetReasoningEffort()) + assert.Nil(t, meta.ReasoningState()) assert.Zero(t, meta.GetEstimatePromptTokens()) assert.Zero(t, meta.GetSendResponseCount()) diff --git a/relaykit/relayconvert/convmeta/options.go b/relaykit/relayconvert/convmeta/options.go index af8cfeb664..efafb2ed46 100644 --- a/relaykit/relayconvert/convmeta/options.go +++ b/relaykit/relayconvert/convmeta/options.go @@ -1,5 +1,7 @@ package convmeta +import "github.com/QuantumNous/new-api/relaykit/types" + // Options is the per-request snapshot of host configuration that converters // consult. The host fills it from its settings system when constructing the // Meta (see relaycommon.RelayInfo.ConvOptions); relaykit users fill it @@ -8,6 +10,13 @@ type Options struct { Claude ClaudeOptions Gemini GeminiOptions + // ToolLossPolicy controls whether a cross-protocol conversion may omit or + // approximate built-in-tool semantics. The zero value uses the allow + // policy: conversion succeeds and every loss is returned as a diagnostic. + // safe/strict rejection is request-phase opt-in only; response and stream + // conversion never reject regardless of this field. + ToolLossPolicy types.ConversionLossPolicy + // OpenRouterDialect marks the upstream as OpenRouter's OpenAI-compatible // surface, which accepts extra fields (reasoning config, cache_control on // system parts) that converters emit only for that dialect. The host sets @@ -18,11 +27,16 @@ type Options struct { // suffix must be kept on the outgoing model name (host blacklist lookup). // Nil means "never preserve". PreserveThinkingSuffix func(modelName string) bool + + // PreserveEffortTail reports real model IDs whose names already end in an + // effort-like token (for example qwen-max). Nil means "never preserve". + PreserveEffortTail func(modelName string) bool } type ClaudeOptions struct { - // ThinkingAdapterEnabled turns "-thinking"-suffixed OpenAI model names - // into Claude extended-thinking requests. + // ThinkingAdapterEnabled controls whether suffix-derived reasoning intent + // is rendered onto Claude thinking / output_config. Suffix parsing itself + // is the host entry layer's job (standalone users call Parse* themselves). ThinkingAdapterEnabled bool // ThinkingAdapterBudgetTokensPercentage sizes thinking budget_tokens as a // fraction of max_tokens when the adapter fires. @@ -36,11 +50,16 @@ type ClaudeOptions struct { // standalone relaykit users must supply one or guarantee max_tokens on // every request. DefaultMaxTokens func(modelName string) int + // WebSearchToolVersion selects the Claude hosted web-search tool version + // emitted by cross-protocol conversion. Empty keeps the compatibility + // baseline web_search_20250305. + WebSearchToolVersion string } type GeminiOptions struct { - // ThinkingAdapterEnabled maps -thinking/-nothinking/effort suffixes to - // Gemini thinkingConfig. + // ThinkingAdapterEnabled controls whether suffix-derived reasoning intent + // is rendered onto Gemini thinkingConfig. Suffix parsing itself is the + // host entry layer's job (standalone users call Parse* themselves). ThinkingAdapterEnabled bool // ThinkingAdapterBudgetTokensPercentage sizes thinkingBudget as a fraction // of maxOutputTokens when the adapter fires. @@ -77,3 +96,14 @@ func (o *GeminiOptions) SafetySettingFor(category string) string { func (o *Options) ShouldPreserveThinkingSuffix(modelName string) bool { return o != nil && o.PreserveThinkingSuffix != nil && o.PreserveThinkingSuffix(modelName) } + +func (o *Options) ShouldPreserveEffortTail(modelName string) bool { + return o != nil && o.PreserveEffortTail != nil && o.PreserveEffortTail(modelName) +} + +func (o *Options) EffectiveToolLossPolicy() types.ConversionLossPolicy { + if o == nil || o.ToolLossPolicy == "" { + return types.ConversionLossPolicyAllow + } + return o.ToolLossPolicy +} diff --git a/relaykit/relayconvert/golden_test.go b/relaykit/relayconvert/golden_test.go index 112ba268ff..ef3a0d4224 100644 --- a/relaykit/relayconvert/golden_test.go +++ b/relaykit/relayconvert/golden_test.go @@ -1,8 +1,7 @@ package relayconvert -// golden_test.go pins the byte-level output of every registered (from, to) -// conversion route so the relaykit extraction refactor can prove behavior is -// unchanged at each phase. Run with -update to regenerate testdata/golden. +// golden_test.go pins the byte-level output of selected public conversion +// routes. Run with -update to regenerate testdata/golden. // // Volatile values (generated UUID-based ids, unix timestamps) are normalized // before comparison so the snapshots are deterministic. @@ -69,10 +68,30 @@ func checkGolden(t *testing.T, name string, got []byte) { return } want, err := os.ReadFile(path) - require.NoError(t, err, "golden file missing, run: go test ./service/relayconvert -run TestGolden -update") + require.NoError(t, err, "golden file missing, run: cd relaykit && GOWORK=off go test ./relayconvert -run TestGolden -update") require.Equal(t, string(want), string(got), "conversion output drifted from golden snapshot %s", path) } +func checkStreamEventsGolden(t *testing.T, name string, events []any) { + t.Helper() + got := marshalGolden(t, map[string]any{"events": events}) + path := filepath.Join(goldenDir, name+".golden.json") + if *updateGolden { + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, got, 0o644)) + return + } + + wantData, err := os.ReadFile(path) + require.NoError(t, err, "golden file missing, run: cd relaykit && GOWORK=off go test ./relayconvert -run TestGolden -update") + var wantSnapshot map[string]json.RawMessage + require.NoError(t, json.Unmarshal(wantData, &wantSnapshot)) + wantEvents, ok := wantSnapshot["events"] + require.True(t, ok, "stream golden snapshot %s has no events", path) + want := marshalGolden(t, map[string]json.RawMessage{"events": wantEvents}) + require.Equal(t, string(want), string(got), "conversion events drifted from golden snapshot %s", path) +} + // goldenInfo mirrors the host's default converter options (new-api's // model_setting defaults at the time the snapshots were recorded) so the // golden files stay comparable across the extraction. @@ -120,42 +139,6 @@ func fixtureRequests() map[types.RelayFormat]any { "tool_choice": "auto" }`, openai) - claude := &dto.ClaudeRequest{} - mustUnmarshalFixture(`{ - "model": "claude-test", - "max_tokens": 1024, - "stream": true, - "system": "You are a helpful assistant.", - "messages": [ - {"role": "user", "content": [ - {"type": "text", "text": "What is in this image?"}, - {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGVsbG8="}} - ]}, - {"role": "assistant", "content": [ - {"type": "thinking", "thinking": "Let me look.", "signature": "sig"}, - {"type": "tool_use", "id": "toolu_abc", "name": "get_weather", "input": {"city": "Paris"}} - ]}, - {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_abc", "content": "15 degrees"}]} - ], - "tools": [{"name": "get_weather", "description": "Get weather by city", "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}], - "thinking": {"type": "enabled", "budget_tokens": 512} - }`, claude) - - gemini := &dto.GeminiChatRequest{} - mustUnmarshalFixture(`{ - "contents": [ - {"role": "user", "parts": [ - {"text": "What is in this image?"}, - {"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}} - ]}, - {"role": "model", "parts": [{"functionCall": {"name": "get_weather", "args": {"city": "Paris"}}}]}, - {"role": "user", "parts": [{"functionResponse": {"name": "get_weather", "response": {"result": "15 degrees"}}}]} - ], - "systemInstruction": {"parts": [{"text": "You are a helpful assistant."}]}, - "tools": [{"functionDeclarations": [{"name": "get_weather", "description": "Get weather by city", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}]}], - "generationConfig": {"maxOutputTokens": 1024, "temperature": 0.7} - }`, gemini) - responses := &dto.OpenAIResponsesRequest{} mustUnmarshalFixture(`{ "model": "gpt-test", @@ -175,8 +158,6 @@ func fixtureRequests() map[types.RelayFormat]any { return map[types.RelayFormat]any{ types.RelayFormatOpenAI: openai, - types.RelayFormatClaude: claude, - types.RelayFormatGemini: gemini, types.RelayFormatOpenAIResponses: responses, } } @@ -308,8 +289,17 @@ func allFormats() []types.RelayFormat { func TestGoldenRequestConversionMatrix(t *testing.T) { requests := fixtureRequests() - for _, from := range allFormats() { - for _, to := range allFormats() { + fromFormats := []types.RelayFormat{ + types.RelayFormatOpenAI, + types.RelayFormatOpenAIResponses, + } + toFormats := []types.RelayFormat{ + types.RelayFormatOpenAI, + types.RelayFormatClaude, + types.RelayFormatOpenAIResponses, + } + for _, from := range fromFormats { + for _, to := range toFormats { if from == to { continue } @@ -330,7 +320,8 @@ func TestGoldenResponseConversionMatrix(t *testing.T) { responses := fixtureResponses() for _, from := range allFormats() { for _, to := range allFormats() { - if from == to { + if from == to || to == types.RelayFormatGemini || + (from == types.RelayFormatOpenAI && to == types.RelayFormatClaude) { continue } name := fmt.Sprintf("response/%s_to_%s", from, to) @@ -373,11 +364,9 @@ func TestGoldenStreamConversionMatrix(t *testing.T) { outputs = append(outputs, r.Value) } - snapshot := map[string]any{ - "events": outputs, - "usage": state.Usage(), - } - checkGolden(t, name, marshalGolden(t, snapshot)) + // Billing usage has private, cross-module acceptance coverage. Keep + // the public golden focused on client-visible stream events. + checkStreamEventsGolden(t, name, outputs) }) } } diff --git a/relaykit/relayconvert/internal/claude_messages/citations.go b/relaykit/relayconvert/internal/claude_messages/citations.go new file mode 100644 index 0000000000..cd97eb3029 --- /dev/null +++ b/relaykit/relayconvert/internal/claude_messages/citations.go @@ -0,0 +1,58 @@ +package claudemessages + +import ( + "encoding/json" + "fmt" + "strings" + "unicode/utf8" + + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" +) + +func claudeCitationsToChat(raw json.RawMessage, text string, textOffset int) ([]any, error) { + if len(raw) == 0 { + return nil, nil + } + var citations []map[string]any + if err := kitutil.Unmarshal(raw, &citations); err != nil { + return nil, fmt.Errorf("invalid Claude citations: %w", err) + } + annotations := make([]any, 0, len(citations)) + for _, citation := range citations { + url := strings.TrimSpace(kitutil.Interface2String(citation["url"])) + if url == "" { + continue + } + converted := map[string]any{ + "url": url, + "title": strings.TrimSpace(kitutil.Interface2String(citation["title"])), + } + citedText := kitutil.Interface2String(citation["cited_text"]) + if citedText != "" { + converted["cited_text"] = citedText + if index := strings.Index(text, citedText); index >= 0 { + startIndex := textOffset + utf8.RuneCountInString(text[:index]) + converted["start_index"] = startIndex + converted["end_index"] = startIndex + utf8.RuneCountInString(citedText) + } + } + if encryptedIndex := kitutil.Interface2String(citation["encrypted_index"]); encryptedIndex != "" { + converted["encrypted_index"] = encryptedIndex + } + if converted["title"] == "" { + delete(converted, "title") + } + annotations = append(annotations, map[string]any{ + "type": "url_citation", + "url_citation": converted, + }) + } + return annotations, nil +} + +func marshalChatAnnotations(annotations []any) (json.RawMessage, error) { + if len(annotations) == 0 { + return nil, nil + } + return kitutil.Marshal(annotations) +} diff --git a/relaykit/relayconvert/internal/claude_messages/stream_billing_usage_test.go b/relaykit/relayconvert/internal/claude_messages/stream_billing_usage_test.go new file mode 100644 index 0000000000..971b07ea03 --- /dev/null +++ b/relaykit/relayconvert/internal/claude_messages/stream_billing_usage_test.go @@ -0,0 +1,66 @@ +package claudemessages + +import ( + "testing" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMessageStartZeroOutputSidecarRemainsRefreshable(t *testing.T) { + t.Parallel() + + info := &ClaudeResponseInfo{Usage: &dto.Usage{}} + ok := FormatClaudeResponseInfo(&dto.ClaudeResponse{ + Type: "message_start", + Message: &dto.ClaudeMediaMessage{ + Id: "msg_1", + Model: "claude-test", + Usage: &dto.ClaudeUsage{ + InputTokens: 10, + OutputTokens: 0, + BillingUsage: dto.NewClaudeMessagesBillingUsage(&dto.ClaudeUsage{ + InputTokens: 10, + OutputTokens: 0, + }), + }, + }, + }, nil, info) + require.True(t, ok) + + ok = FormatClaudeResponseInfo(&dto.ClaudeResponse{ + Type: "message_delta", + Usage: &dto.ClaudeUsage{ + OutputTokens: 42, + }, + }, nil, info) + require.True(t, ok) + require.NotNil(t, info.Usage.BillingUsage) + require.NotNil(t, info.Usage.BillingUsage.ClaudeUsage) + assert.Equal(t, 42, info.Usage.BillingUsage.ClaudeUsage.OutputTokens) +} + +func TestTerminalSidecarRemainsAuthoritativeAgainstFinalize(t *testing.T) { + t.Parallel() + + info := &ClaudeResponseInfo{Usage: &dto.Usage{}} + ok := FormatClaudeResponseInfo(&dto.ClaudeResponse{ + Type: "message_delta", + Usage: &dto.ClaudeUsage{ + InputTokens: 10, + OutputTokens: 7, + BillingUsage: dto.NewClaudeMessagesBillingUsage(&dto.ClaudeUsage{ + InputTokens: 10, + OutputTokens: 7, + }), + }, + }, nil, info) + require.True(t, ok) + require.NotNil(t, info.Usage.BillingUsage) + require.NotNil(t, info.Usage.BillingUsage.ClaudeUsage) + + info.Usage.CompletionTokens = 99 + FinalizeClaudeStreamBillingUsage(info) + assert.Equal(t, 7, info.Usage.BillingUsage.ClaudeUsage.OutputTokens) +} diff --git a/relaykit/relayconvert/internal/claude_messages/to_oai_chat_req.go b/relaykit/relayconvert/internal/claude_messages/to_oai_chat_req.go index 56e6663e1a..fa1a3c1d80 100644 --- a/relaykit/relayconvert/internal/claude_messages/to_oai_chat_req.go +++ b/relaykit/relayconvert/internal/claude_messages/to_oai_chat_req.go @@ -7,6 +7,7 @@ import ( "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" ) const ( @@ -16,7 +17,7 @@ const ( ) type openRouterRequestReasoning struct { - Enabled bool `json:"enabled"` + Enabled *bool `json:"enabled,omitempty"` Effort string `json:"effort,omitempty"` MaxTokens int `json:"max_tokens,omitempty"` Exclude bool `json:"exclude,omitempty"` @@ -39,6 +40,10 @@ func ClaudeMessagesRequestToOpenAIChat(claudeRequest dto.ClaudeRequest, info con if claudeRequest.Stream != nil { openAIRequest.Stream = kitutil.GetPointer(*claudeRequest.Stream) } + reasoningIntent, effectiveEffort, err := claudeRequestReasoningIntent(&claudeRequest, info) + if err != nil { + return nil, reasoning.AsClientError(err) + } isOpenRouter := convmeta.OptionsOf(info).OpenRouterDialect if isOpenRouter { @@ -46,17 +51,21 @@ func ClaudeMessagesRequestToOpenAIChat(claudeRequest dto.ClaudeRequest, info con effortBytes, _ := kitutil.Marshal(effort) openAIRequest.Verbosity = effortBytes } - if claudeRequest.Thinking != nil { + if !reasoningIntent.IsEmpty() { var reasoningConfig openRouterRequestReasoning - if claudeRequest.Thinking.Type == "enabled" { + disabled := reasoningIntent.Mode == reasoning.ModeDisabled || reasoningIntent.Effort == reasoning.EffortNone + enabled := !disabled + reasoningConfig.Enabled = &enabled + if enabled && reasoningIntent.BudgetTokens != nil && reasoningIntent.Mode != reasoning.ModeAdaptive { reasoningConfig = openRouterRequestReasoning{ - Enabled: true, - MaxTokens: claudeRequest.Thinking.GetBudgetTokens(), - } - } else if claudeRequest.Thinking.Type == "adaptive" { - reasoningConfig = openRouterRequestReasoning{ - Enabled: true, + Enabled: &enabled, + MaxTokens: *reasoningIntent.BudgetTokens, } + } else if enabled { + reasoningConfig.Effort = string(reasoning.EffectiveEffort(reasoningIntent)) + } + if reasoningIntent.IncludeThoughts != nil { + reasoningConfig.Exclude = !*reasoningIntent.IncludeThoughts } reasoningJSON, err := kitutil.Marshal(reasoningConfig) if err != nil { @@ -64,12 +73,23 @@ func ClaudeMessagesRequestToOpenAIChat(claudeRequest dto.ClaudeRequest, info con } openAIRequest.Reasoning = reasoningJSON } - } else if info != nil { - thinkingSuffix := "-thinking" - if strings.HasSuffix(info.GetOriginModelName(), thinkingSuffix) && - !strings.HasSuffix(openAIRequest.Model, thinkingSuffix) { - openAIRequest.Model = openAIRequest.Model + thinkingSuffix + } else { + if err := reasoning.ApplyToOpenAIChat(&openAIRequest, reasoningIntent); err != nil { + return nil, reasoning.AsClientError(err) } + if info != nil { + // Keep the outgoing -thinking suffix so a cascaded downstream + // new-api can recover reasoning intent from the model name. This + // is an emission-side policy, not converter-side suffix parsing. + thinkingSuffix := "-thinking" + if strings.HasSuffix(info.GetOriginModelName(), thinkingSuffix) && + !strings.HasSuffix(openAIRequest.Model, thinkingSuffix) { + openAIRequest.Model = openAIRequest.Model + thinkingSuffix + } + } + } + if info != nil && effectiveEffort != "" { + info.SetReasoningEffort(string(effectiveEffort)) } if len(claudeRequest.StopSequences) == 1 { diff --git a/relaykit/relayconvert/internal/claude_messages/to_oai_chat_resp.go b/relaykit/relayconvert/internal/claude_messages/to_oai_chat_resp.go index 55762335a3..1e9a5c4a7e 100644 --- a/relaykit/relayconvert/internal/claude_messages/to_oai_chat_resp.go +++ b/relaykit/relayconvert/internal/claude_messages/to_oai_chat_resp.go @@ -1,8 +1,10 @@ package claudemessages import ( + "encoding/json" "fmt" "strings" + "unicode/utf8" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/reasonmap" @@ -19,6 +21,10 @@ type ClaudeResponseInfo struct { ResponseText strings.Builder Usage *dto.Usage Done bool + + // Only snapshots synthesized from partial display usage may be refreshed by + // later display deltas. Serialized BillingUsage always remains authoritative. + billingUsageSynthesized bool } func StopReasonClaudeToOpenAI(reason string) string { @@ -47,6 +53,10 @@ func StreamResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.ChatCo if claudeResponse.ContentBlock != nil { if claudeResponse.ContentBlock.Type == "text" && claudeResponse.ContentBlock.Text != nil { choice.Delta.SetContentString(*claudeResponse.ContentBlock.Text) + annotations, err := claudeCitationsToChat(claudeResponse.ContentBlock.Citations, *claudeResponse.ContentBlock.Text, 0) + if err == nil { + choice.Delta.Annotations, _ = marshalChatAnnotations(annotations) + } } if claudeResponse.ContentBlock.Type == "tool_use" { tools = append(tools, dto.ToolCallResponse{ @@ -79,6 +89,14 @@ func StreamResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.ChatCo choice.Delta.ReasoningContent = &signatureContent case "thinking_delta": choice.Delta.ReasoningContent = claudeResponse.Delta.Thinking + case "citations_delta": + if len(claudeResponse.Delta.Citation) > 0 { + raw, _ := kitutil.Marshal([]json.RawMessage{claudeResponse.Delta.Citation}) + annotations, err := claudeCitationsToChat(raw, "", 0) + if err == nil { + choice.Delta.Annotations, _ = marshalChatAnnotations(annotations) + } + } } } } else if claudeResponse.Type == "message_delta" { @@ -102,6 +120,101 @@ func StreamResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.ChatCo return &response } +// ClaudeToChatStreamState translates Anthropic content block indexes into the +// independent, dense index space used by Chat Completions tool_calls. Text and +// thinking blocks therefore do not create holes in the downstream tool array. +type ClaudeToChatStreamState struct { + toolIndexByContentBlock map[int]int + blockTypeByContentBlock map[int]string + nextToolIndex int +} + +func NewClaudeToChatStreamState() *ClaudeToChatStreamState { + return &ClaudeToChatStreamState{ + toolIndexByContentBlock: make(map[int]int), + blockTypeByContentBlock: make(map[int]string), + } +} + +func (s *ClaudeToChatStreamState) ConvertChunk(claudeResponse *dto.ClaudeResponse) (*dto.ChatCompletionsStreamResponse, error) { + if s == nil { + return nil, fmt.Errorf("Claude-to-Chat stream state is required") + } + if claudeResponse == nil { + return nil, nil + } + if s.toolIndexByContentBlock == nil { + s.toolIndexByContentBlock = make(map[int]int) + } + if s.blockTypeByContentBlock == nil { + s.blockTypeByContentBlock = make(map[int]string) + } + + converted := *claudeResponse + switch claudeResponse.Type { + case "content_block_start": + if claudeResponse.ContentBlock == nil { + break + } + blockType := strings.TrimSpace(claudeResponse.ContentBlock.Type) + if blockType == "" { + break + } + if claudeResponse.Index == nil { + return nil, fmt.Errorf("Claude content block stream start is missing index") + } + contentBlockIndex := *claudeResponse.Index + s.blockTypeByContentBlock[contentBlockIndex] = blockType + if blockType != "tool_use" { + if isClaudeHostedToolStreamBlock(blockType) { + return nil, nil + } + break + } + toolIndex, exists := s.toolIndexByContentBlock[contentBlockIndex] + if !exists { + toolIndex = s.nextToolIndex + s.nextToolIndex++ + s.toolIndexByContentBlock[contentBlockIndex] = toolIndex + } + converted.Index = kitutil.GetPointer(toolIndex) + case "content_block_delta": + if claudeResponse.Delta == nil || claudeResponse.Delta.Type != "input_json_delta" { + break + } + if claudeResponse.Index == nil { + return nil, fmt.Errorf("Claude tool-use stream delta is missing content block index") + } + if claudeResponse.Delta.PartialJson == nil { + return nil, fmt.Errorf("Claude tool-use stream delta is missing partial JSON") + } + contentBlockIndex := *claudeResponse.Index + toolIndex, exists := s.toolIndexByContentBlock[contentBlockIndex] + if !exists { + if isClaudeHostedToolStreamBlock(s.blockTypeByContentBlock[contentBlockIndex]) { + return nil, nil + } + return nil, fmt.Errorf("Claude tool-use stream delta references unknown content block index %d", contentBlockIndex) + } + converted.Index = kitutil.GetPointer(toolIndex) + case "content_block_stop": + if claudeResponse.Index != nil { + delete(s.blockTypeByContentBlock, *claudeResponse.Index) + } + } + + return StreamResponseClaude2OpenAI(&converted), nil +} + +func isClaudeHostedToolStreamBlock(blockType string) bool { + switch blockType { + case "server_tool_use", "mcp_tool_use", "web_search_tool_result", "mcp_tool_result", "code_execution_tool_result", "web_fetch_tool_result": + return true + default: + return false + } +} + func ResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.OpenAITextResponse { choices := make([]dto.OpenAITextResponseChoice, 0) fullTextResponse := dto.OpenAITextResponse{ @@ -109,16 +222,17 @@ func ResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.OpenAITextRe Object: "chat.completion", Created: kitutil.GetTimestamp(), } - var responseText string + var responseText strings.Builder + responseTextOffset := 0 var responseThinking string if len(claudeResponse.Content) > 0 { - responseText = claudeResponse.Content[0].GetText() if claudeResponse.Content[0].Thinking != nil { responseThinking = *claudeResponse.Content[0].Thinking } } tools := make([]dto.ToolCallResponse, 0) thinkingContent := "" + annotations := make([]any, 0) fullTextResponse.Id = claudeResponse.Id for _, message := range claudeResponse.Content { @@ -138,7 +252,14 @@ func ResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.OpenAITextRe thinkingContent = *message.Thinking } case "text": - responseText = message.GetText() + text := message.GetText() + offset := responseTextOffset + responseText.WriteString(text) + responseTextOffset += utf8.RuneCountInString(text) + converted, err := claudeCitationsToChat(message.Citations, text, offset) + if err == nil { + annotations = append(annotations, converted...) + } } } choice := dto.OpenAITextResponseChoice{ @@ -148,7 +269,10 @@ func ResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.OpenAITextRe }, FinishReason: StopReasonClaudeToOpenAI(claudeResponse.StopReason), } - choice.SetStringContent(responseText) + choice.SetStringContent(responseText.String()) + if encodedAnnotations, err := marshalChatAnnotations(annotations); err == nil && len(encodedAnnotations) > 0 { + choice.Message.Annotations = encodedAnnotations + } if len(responseThinking) > 0 { choice.ReasoningContent = &responseThinking } @@ -295,6 +419,45 @@ func claudeBillingUsageFromSemanticUsage(usage *dto.Usage) *dto.BillingUsage { return dto.NewClaudeMessagesBillingUsage(claudeUsage) } +func updateClaudeStreamBillingUsage(claudeUsage *dto.ClaudeUsage, claudeInfo *ClaudeResponseInfo, terminal bool) { + if claudeUsage == nil || claudeInfo == nil || claudeInfo.Usage == nil { + return + } + if billingUsage := dto.CloneBillingUsage(claudeUsage.BillingUsage); billingUsage != nil { + claudeInfo.Usage.BillingUsage = billingUsage + if terminal || claudeUsage.OutputTokens > 0 { + claudeInfo.billingUsageSynthesized = false + return + } + claudeInfo.billingUsageSynthesized = true + return + } + if claudeInfo.Usage.BillingUsage != nil && !claudeInfo.billingUsageSynthesized { + return + } + claudeInfo.Usage.BillingUsage = claudeBillingUsageFromSemanticUsage(claudeInfo.Usage) + claudeInfo.billingUsageSynthesized = claudeInfo.Usage.BillingUsage != nil +} + +// FinalizeClaudeStreamBillingUsage refreshes only a locally synthesized +// snapshot after the host has applied its missing-usage fallback. A snapshot +// received on the wire remains authoritative and is never rewritten. +func FinalizeClaudeStreamBillingUsage(claudeInfo *ClaudeResponseInfo) { + if claudeInfo == nil || claudeInfo.Usage == nil { + return + } + if claudeInfo.Usage.BillingUsage != nil && !claudeInfo.billingUsageSynthesized { + return + } + + billingUsage := claudeBillingUsageFromSemanticUsage(claudeInfo.Usage) + if billingUsage != nil && !claudeInfo.Done { + billingUsage.Estimated = true + } + claudeInfo.Usage.BillingUsage = billingUsage + claudeInfo.billingUsageSynthesized = billingUsage != nil +} + func PatchClaudeMessageDeltaUsageData(data string, usage *dto.ClaudeUsage) string { if data == "" || usage == nil { return data @@ -343,14 +506,15 @@ func FormatClaudeResponseInfo(claudeResponse *dto.ClaudeResponse, oaiResponse *d } if claudeResponse.Message != nil && claudeResponse.Message.Usage != nil { - claudeInfo.Usage.PromptTokens = claudeResponse.Message.Usage.InputTokens + messageUsage := claudeResponse.Message.Usage + claudeInfo.Usage.PromptTokens = messageUsage.InputTokens claudeInfo.Usage.UsageSemantic = "anthropic" - claudeInfo.Usage.PromptTokensDetails.CachedTokens = claudeResponse.Message.Usage.CacheReadInputTokens - claudeInfo.Usage.PromptTokensDetails.CachedCreationTokens = claudeResponse.Message.Usage.CacheCreationInputTokens - claudeInfo.Usage.ClaudeCacheCreation5mTokens = claudeResponse.Message.Usage.GetCacheCreation5mTokens() - claudeInfo.Usage.ClaudeCacheCreation1hTokens = claudeResponse.Message.Usage.GetCacheCreation1hTokens() - claudeInfo.Usage.CompletionTokens = claudeResponse.Message.Usage.OutputTokens - claudeInfo.Usage.BillingUsage = claudeBillingUsageFromSemanticUsage(claudeInfo.Usage) + claudeInfo.Usage.PromptTokensDetails.CachedTokens = messageUsage.CacheReadInputTokens + claudeInfo.Usage.PromptTokensDetails.CachedCreationTokens = messageUsage.CacheCreationInputTokens + claudeInfo.Usage.ClaudeCacheCreation5mTokens = messageUsage.GetCacheCreation5mTokens() + claudeInfo.Usage.ClaudeCacheCreation1hTokens = messageUsage.GetCacheCreation1hTokens() + claudeInfo.Usage.CompletionTokens = messageUsage.OutputTokens + updateClaudeStreamBillingUsage(messageUsage, claudeInfo, false) } } else if claudeResponse.Type == "content_block_delta" { if claudeResponse.Delta != nil { @@ -383,7 +547,7 @@ func FormatClaudeResponseInfo(claudeResponse *dto.ClaudeResponse, oaiResponse *d claudeInfo.Usage.CompletionTokens = claudeResponse.Usage.OutputTokens } claudeInfo.Usage.TotalTokens = claudeInfo.Usage.PromptTokens + claudeInfo.Usage.CompletionTokens - claudeInfo.Usage.BillingUsage = claudeBillingUsageFromSemanticUsage(claudeInfo.Usage) + updateClaudeStreamBillingUsage(claudeResponse.Usage, claudeInfo, true) } claudeInfo.Done = true diff --git a/relaykit/relayconvert/internal/claude_messages/to_oai_responses_hosted_stream.go b/relaykit/relayconvert/internal/claude_messages/to_oai_responses_hosted_stream.go new file mode 100644 index 0000000000..e074e8b61a --- /dev/null +++ b/relaykit/relayconvert/internal/claude_messages/to_oai_responses_hosted_stream.go @@ -0,0 +1,194 @@ +package claudemessages + +import ( + "fmt" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" + oaichat "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/oai_chat" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" +) + +// ClaudeHostedStreamBridge keeps Anthropic server-executed tool blocks out of +// the Chat Completions pivot. Anthropic streams a hosted call's input after the +// content_block_start event, so the bridge owns those input_json_delta events +// until content_block_stop and only then starts the Responses output item. +type ClaudeHostedStreamBridge struct { + pending map[int]*claudeHostedStreamCall +} + +type claudeHostedStreamCall struct { + blockType string + id string + name string + serverName string + caller []byte + startInput []byte + input strings.Builder +} + +func NewClaudeHostedStreamBridge() *ClaudeHostedStreamBridge { + return &ClaudeHostedStreamBridge{pending: make(map[int]*claudeHostedStreamCall)} +} + +// Convert consumes provider-hosted stream frames and reports whether the frame +// must be skipped by the ordinary Claude-to-Chat converter. +func (b *ClaudeHostedStreamBridge) Convert(response *dto.ClaudeResponse, state *oaichat.ChatToResponsesStreamState) ([]oaichat.ChatToResponsesStreamEvent, bool, error) { + if response == nil || state == nil { + return nil, false, nil + } + if b == nil { + return nil, false, fmt.Errorf("Claude hosted stream bridge is required") + } + if b.pending == nil { + b.pending = make(map[int]*claudeHostedStreamCall) + } + index := response.GetIndex() + + switch response.Type { + case "content_block_start": + if response.ContentBlock == nil { + return nil, false, nil + } + block := response.ContentBlock + blockType := strings.TrimSpace(block.Type) + switch blockType { + case "server_tool_use", "mcp_tool_use": + if _, exists := b.pending[index]; exists { + return nil, true, fmt.Errorf("duplicate Claude hosted-tool content block index %d", index) + } + if blockType == "mcp_tool_use" && (strings.TrimSpace(block.Name) == "" || strings.TrimSpace(block.ServerName) == "") { + return nil, true, fmt.Errorf("Claude MCP tool use must include name and server_name") + } + if _, err := claudeHostedCallOutputType(blockType, block.Name); err != nil { + return nil, true, err + } + pending := &claudeHostedStreamCall{ + blockType: blockType, + id: block.Id, + name: block.Name, + serverName: block.ServerName, + caller: append([]byte(nil), block.Caller...), + } + // Non-stream-shaped gateways occasionally include the complete input + // on the start frame. Preserve it as a fallback, while streamed deltas + // replace the placeholder at block completion. + if block.Input != nil { + input, err := kitutil.Marshal(block.Input) + if err != nil { + return nil, true, fmt.Errorf("marshal Claude hosted-tool input: %w", err) + } + if string(input) != "{}" && string(input) != "null" { + pending.startInput = input + } + } + b.pending[index] = pending + return nil, true, nil + case "web_search_tool_result", "mcp_tool_result": + outputType, err := claudeHostedResultOutputType(blockType) + if err != nil { + return nil, true, err + } + var result []byte + if outputType != "web_search_call" { + result, err = kitutil.Marshal(block.Content) + if err != nil { + return nil, true, fmt.Errorf("marshal Claude hosted-tool result: %w", err) + } + } + events, err := state.CompleteHostedTool(oaichat.HostedToolStreamResult{ + Type: outputType, + ID: block.ToolUseId, + Result: result, + ErrorCode: claudeHostedResultErrorCode(block.Content, block.ErrorCode), + IsError: block.IsError != nil && *block.IsError, + }) + return events, true, err + default: + return nil, false, nil + } + case "content_block_delta": + pending := b.pending[index] + if pending == nil { + return nil, false, nil + } + if response.Delta != nil && response.Delta.Type == "input_json_delta" && response.Delta.PartialJson != nil { + pending.input.WriteString(*response.Delta.PartialJson) + } + return nil, true, nil + case "content_block_stop": + pending := b.pending[index] + if pending == nil { + return nil, false, nil + } + delete(b.pending, index) + action := []byte(pending.input.String()) + if len(action) == 0 { + action = pending.startInput + } + if len(action) == 0 { + action = []byte("{}") + } + outputType, err := claudeHostedCallOutputType(pending.blockType, pending.name) + if err != nil { + return nil, true, err + } + events, err := state.StartHostedTool(oaichat.HostedToolStreamStart{ + Type: outputType, + ID: pending.id, + Name: pending.name, + Action: action, + Caller: pending.caller, + ServerLabel: pending.serverName, + }) + return events, true, err + default: + return nil, false, nil + } +} + +func claudeHostedCallOutputType(blockType string, name string) (string, error) { + if blockType == "mcp_tool_use" { + return "mcp_call", nil + } + switch strings.TrimSpace(name) { + case "web_search": + return "web_search_call", nil + case "code_execution": + return "", fmt.Errorf("Claude code_execution has no valid OpenAI Responses mapping without a container_id") + case "web_fetch": + return "", fmt.Errorf("Claude web_fetch has no valid OpenAI Responses hosted-tool mapping") + default: + return "", fmt.Errorf("unknown Claude server tool %q cannot be represented as an OpenAI Responses hosted tool", name) + } +} + +func claudeHostedResultOutputType(blockType string) (string, error) { + switch blockType { + case "web_search_tool_result": + return "web_search_call", nil + case "mcp_tool_result": + return "mcp_call", nil + case "code_execution_tool_result": + return "", fmt.Errorf("Claude code_execution result has no valid OpenAI Responses mapping without a container_id") + case "web_fetch_tool_result": + return "", fmt.Errorf("Claude web_fetch result has no valid OpenAI Responses hosted-tool mapping") + default: + return "", fmt.Errorf("unknown Claude hosted-tool result %q", blockType) + } +} + +func claudeHostedResultErrorCode(content any, fallback string) string { + if strings.TrimSpace(fallback) != "" { + return strings.TrimSpace(fallback) + } + value, ok := content.(map[string]any) + if !ok { + return "" + } + contentType := strings.TrimSpace(kitutil.Interface2String(value["type"])) + if !strings.HasSuffix(contentType, "_error") { + return "" + } + return strings.TrimSpace(kitutil.Interface2String(value["error_code"])) +} diff --git a/relaykit/relayconvert/internal/claude_messages/to_oai_responses_req.go b/relaykit/relayconvert/internal/claude_messages/to_oai_responses_req.go new file mode 100644 index 0000000000..75ae96f36f --- /dev/null +++ b/relaykit/relayconvert/internal/claude_messages/to_oai_responses_req.go @@ -0,0 +1,326 @@ +package claudemessages + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" +) + +func ClaudeMessagesRequestToOpenAIResponses(claudeRequest dto.ClaudeRequest, info convmeta.Meta) (*dto.OpenAIResponsesRequest, error) { + if strings.TrimSpace(claudeRequest.Model) == "" { + return nil, errors.New("model is required") + } + + input, err := claudeMessagesToResponsesInput(claudeRequest.Messages) + if err != nil { + return nil, err + } + instructions, err := claudeSystemToResponsesInstructions(&claudeRequest) + if err != nil { + return nil, err + } + tools, err := claudeToolsToResponsesTools(claudeRequest.Tools) + if err != nil { + return nil, err + } + toolChoice, parallelToolCalls, err := claudeToolChoiceToResponses(claudeRequest.ToolChoice) + if err != nil { + return nil, err + } + + // Claude context_management is an object containing protocol-specific edit + // strategies. Responses expects an array of compaction entries, so copying + // the raw Claude value would produce an invalid upstream request. + responsesRequest := &dto.OpenAIResponsesRequest{ + Model: claudeRequest.Model, + Input: input, + Instructions: instructions, + Metadata: append(json.RawMessage(nil), claudeRequest.Metadata...), + ServiceTier: claudeRequest.ServiceTier, + Stream: claudeRequest.Stream, + Temperature: claudeRequest.Temperature, + Tools: tools, + ToolChoice: toolChoice, + ParallelToolCalls: parallelToolCalls, + TopP: claudeRequest.TopP, + } + if info != nil && !convmeta.OptionsOf(info).OpenRouterDialect { + // Keep the outgoing -thinking suffix so a cascaded downstream new-api + // can recover reasoning intent from the model name. This is an + // emission-side policy, not converter-side suffix parsing. + thinkingSuffix := "-thinking" + if strings.HasSuffix(info.GetOriginModelName(), thinkingSuffix) && !strings.HasSuffix(responsesRequest.Model, thinkingSuffix) { + responsesRequest.Model += thinkingSuffix + } + } + if claudeRequest.MaxTokens != nil { + maxOutputTokens := *claudeRequest.MaxTokens + responsesRequest.MaxOutputTokens = &maxOutputTokens + } else if claudeRequest.MaxTokensToSample != nil { + maxOutputTokens := *claudeRequest.MaxTokensToSample + responsesRequest.MaxOutputTokens = &maxOutputTokens + } + + reasoningIntent, effectiveEffort, err := claudeRequestReasoningIntent(&claudeRequest, info) + if err != nil { + return nil, reasoning.AsClientError(err) + } + if err := reasoning.ApplyToOpenAIResponses(responsesRequest, reasoningIntent); err != nil { + return nil, reasoning.AsClientError(err) + } + if info != nil && effectiveEffort != "" { + info.SetReasoningEffort(string(effectiveEffort)) + } + + return responsesRequest, nil +} + +func claudeRequestReasoningIntent(claudeRequest *dto.ClaudeRequest, info convmeta.Meta) (reasoning.Intent, reasoning.Effort, error) { + reasoningIntent, err := reasoning.FromClaude(claudeRequest) + if err != nil { + return reasoning.Intent{}, "", err + } + sourceModel := claudeRequest.Model + if info != nil && info.GetOriginModelName() != "" { + sourceModel = info.GetOriginModelName() + } + if suffix := reasoning.IntentFromState(convmeta.ReasoningStateOf(info)); !suffix.IsEmpty() { + reasoningIntent, err = reasoning.MergeExplicitAndSuffix(reasoningIntent, suffix, sourceModel) + if err != nil { + return reasoning.Intent{}, "", err + } + } + reasoningIntent = reasoning.ResolveClaudeDefault(sourceModel, reasoningIntent) + return reasoningIntent, reasoning.EffectiveEffort(reasoningIntent), nil +} + +func claudeSystemToResponsesInstructions(request *dto.ClaudeRequest) (json.RawMessage, error) { + if request == nil || request.System == nil { + return nil, nil + } + if request.IsStringSystem() { + return kitutil.Marshal(request.GetStringSystem()) + } + + var instructions strings.Builder + systemBlocks, err := kitutil.Any2Type[[]dto.ClaudeMediaMessage](request.System) + if err != nil { + return nil, fmt.Errorf("invalid Claude system content: %w", err) + } + for _, block := range systemBlocks { + if block.Type == "text" || block.Type == "input_text" || block.Type == "" { + instructions.WriteString(block.GetText()) + } + } + if instructions.Len() == 0 { + return nil, nil + } + return kitutil.Marshal(instructions.String()) +} + +func claudeMessagesToResponsesInput(messages []dto.ClaudeMessage) (json.RawMessage, error) { + input := make([]map[string]any, 0, len(messages)) + for messageIndex := range messages { + message := messages[messageIndex] + role := strings.TrimSpace(message.Role) + if role == "" { + continue + } + if message.IsStringContent() { + input = append(input, map[string]any{ + "role": role, + "content": message.GetStringContent(), + }) + continue + } + + blocks, err := message.ParseContent() + if err != nil { + return nil, fmt.Errorf("messages[%d].content: %w", messageIndex, err) + } + contentParts := make([]map[string]any, 0, len(blocks)) + flushContent := func() { + if len(contentParts) == 0 { + return + } + input = append(input, map[string]any{ + "role": role, + "content": contentParts, + }) + contentParts = nil + } + + for blockIndex := range blocks { + block := blocks[blockIndex] + switch block.Type { + case "text", "input_text": + partType := "input_text" + if role == "assistant" { + partType = "output_text" + } + contentParts = append(contentParts, map[string]any{ + "type": partType, + "text": block.GetText(), + }) + case "image": + if source := claudeSourceURL(block.Source); source != "" { + contentParts = append(contentParts, map[string]any{ + "type": "input_image", + "image_url": source, + }) + } + case "document": + if source := claudeSourceURL(block.Source); source != "" { + contentParts = append(contentParts, map[string]any{ + "type": "input_file", + "file_data": source, + }) + } + case "tool_use": + flushContent() + arguments, err := kitutil.Marshal(block.Input) + if err != nil { + return nil, fmt.Errorf("messages[%d].content[%d].input: %w", messageIndex, blockIndex, err) + } + if block.Input == nil { + arguments = []byte("{}") + } + input = append(input, map[string]any{ + "type": "function_call", + "call_id": block.Id, + "name": block.Name, + "arguments": string(arguments), + }) + case "tool_result": + flushContent() + output, err := claudeToolResultToResponsesOutput(block.Content) + if err != nil { + return nil, fmt.Errorf("messages[%d].content[%d].content: %w", messageIndex, blockIndex, err) + } + input = append(input, map[string]any{ + "type": "function_call_output", + "call_id": block.ToolUseId, + "output": output, + }) + } + } + flushContent() + } + return kitutil.Marshal(input) +} + +func claudeToolsToResponsesTools(value any) (json.RawMessage, error) { + if value == nil { + return nil, nil + } + tools, err := kitutil.Any2Type[[]dto.Tool](value) + if err != nil { + return nil, fmt.Errorf("invalid Claude tools: %w", err) + } + converted := make([]map[string]any, 0, len(tools)) + for _, tool := range tools { + function := map[string]any{ + "type": "function", + "name": tool.Name, + "description": tool.Description, + "parameters": tool.InputSchema, + } + if tool.Strict != nil { + function["strict"] = *tool.Strict + } + converted = append(converted, function) + } + return kitutil.Marshal(converted) +} + +func claudeToolChoiceToResponses(value any) (json.RawMessage, json.RawMessage, error) { + if value == nil { + return nil, nil, nil + } + choice, err := kitutil.Any2Type[dto.ClaudeToolChoice](value) + if err != nil { + return nil, nil, fmt.Errorf("invalid Claude tool_choice: %w", err) + } + + var converted any + switch choice.Type { + case "", "auto": + converted = "auto" + case "any": + converted = "required" + case "none": + converted = "none" + case "tool": + converted = map[string]any{"type": "function", "name": choice.Name} + default: + return nil, nil, fmt.Errorf("unsupported Claude tool_choice type %q", choice.Type) + } + toolChoice, err := kitutil.Marshal(converted) + if err != nil { + return nil, nil, err + } + + var parallelToolCalls json.RawMessage + if choice.DisableParallelToolUse && choice.Type != "none" { + parallelToolCalls, err = kitutil.Marshal(false) + if err != nil { + return nil, nil, err + } + } + return toolChoice, parallelToolCalls, nil +} + +func claudeToolResultToResponsesOutput(content any) (any, error) { + if content == nil { + return "", nil + } + if text, ok := content.(string); ok { + return text, nil + } + blocks, err := kitutil.Any2Type[[]dto.ClaudeMediaMessage](content) + if err != nil { + return content, nil + } + parts := make([]map[string]any, 0, len(blocks)) + for _, block := range blocks { + switch block.Type { + case "text", "input_text": + parts = append(parts, map[string]any{"type": "input_text", "text": block.GetText()}) + case "image": + if source := claudeSourceURL(block.Source); source != "" { + parts = append(parts, map[string]any{"type": "input_image", "image_url": source}) + } + case "document": + if source := claudeSourceURL(block.Source); source != "" { + parts = append(parts, map[string]any{"type": "input_file", "file_data": source}) + } + } + } + if len(parts) == 0 { + return content, nil + } + return parts, nil +} + +func claudeSourceURL(source *dto.ClaudeMessageSource) string { + if source == nil { + return "" + } + if strings.TrimSpace(source.Url) != "" { + return source.Url + } + data := kitutil.Interface2String(source.Data) + if data == "" { + return "" + } + if strings.HasPrefix(data, "data:") { + return data + } + return fmt.Sprintf("data:%s;base64,%s", source.MediaType, data) +} diff --git a/relaykit/relayconvert/internal/gemini_chat/grounding.go b/relaykit/relayconvert/internal/gemini_chat/grounding.go new file mode 100644 index 0000000000..10658d773b --- /dev/null +++ b/relaykit/relayconvert/internal/gemini_chat/grounding.go @@ -0,0 +1,367 @@ +package geminichat + +import ( + "fmt" + "strings" + "unicode/utf8" + + "github.com/QuantumNous/new-api/relaykit/dto" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" +) + +type geminiGroundingChunk struct { + Web *geminiGroundingSource `json:"web,omitempty"` + RetrievedContext *geminiGroundingSource `json:"retrievedContext,omitempty"` +} + +type geminiGroundingSource struct { + URI string `json:"uri,omitempty"` + Title string `json:"title,omitempty"` +} + +type geminiGroundingSupport struct { + Segment struct { + PartIndex *int `json:"partIndex,omitempty"` + StartIndex int `json:"startIndex,omitempty"` + EndIndex int `json:"endIndex,omitempty"` + Text string `json:"text,omitempty"` + } `json:"segment"` + GroundingChunkIndices []int `json:"groundingChunkIndices"` +} + +type renderedGeminiPart struct { + text string + startByte int +} + +type streamedGeminiPartSpan struct { + partStartByte int + partEndByte int + renderedStartByte int +} + +type streamedGeminiPart struct { + text strings.Builder + spans []streamedGeminiPartSpan +} + +// geminiGroundingStreamCandidate retains the protocol state needed to resolve +// grounding metadata emitted after the text it describes. Gemini's streaming +// contract makes grounding chunk indexes cumulative across response chunks and +// keeps segment offsets relative to the accumulated candidate part. +type geminiGroundingStreamCandidate struct { + rendered strings.Builder + parts map[int]*streamedGeminiPart + chunks []geminiGroundingChunk +} + +// GroundingWebSearchQueries returns the distinct hosted-search queries that +// Gemini reports for a response. The provider may repeat metadata across +// candidates or stream chunks, so callers can safely accumulate this result +// without manufacturing duplicate Responses tool calls. +func GroundingWebSearchQueries(response *dto.GeminiChatResponse) []string { + if response == nil { + return nil + } + queries := make([]string, 0) + seen := make(map[string]struct{}) + for candidateIndex := range response.Candidates { + metadata := response.Candidates[candidateIndex].GroundingMetadata + if metadata == nil { + continue + } + for _, query := range metadata.WebSearchQueries { + query = strings.TrimSpace(query) + if query == "" { + continue + } + if _, exists := seen[query]; exists { + continue + } + seen[query] = struct{}{} + queries = append(queries, query) + } + } + return queries +} + +func groundingAnnotationsToChat(metadata *dto.GeminiGroundingMetadata, content dto.GeminiChatContent, rendered string) []byte { + if metadata == nil || len(metadata.GroundingChunks) == 0 || len(metadata.GroundingSupports) == 0 { + return nil + } + var chunks []geminiGroundingChunk + if err := kitutil.Unmarshal(metadata.GroundingChunks, &chunks); err != nil { + return nil + } + var supports []geminiGroundingSupport + if err := kitutil.Unmarshal(metadata.GroundingSupports, &supports); err != nil { + return nil + } + + parts := locateRenderedGeminiParts(content, rendered) + textPartCount := 0 + soleTextPart := -1 + for index := range parts { + if parts[index].startByte < 0 { + continue + } + textPartCount++ + soleTextPart = index + } + + annotations := make([]any, 0) + seen := make(map[string]struct{}) + for _, support := range supports { + partIndex := soleTextPart + if support.Segment.PartIndex != nil { + partIndex = *support.Segment.PartIndex + } else if textPartCount != 1 { + continue + } + if partIndex < 0 || partIndex >= len(parts) || parts[partIndex].startByte < 0 { + continue + } + part := parts[partIndex] + start, end, ok := groundingRuneRange(rendered, part, support.Segment.StartIndex, support.Segment.EndIndex) + if !ok { + continue + } + if support.Segment.Text != "" && part.text[support.Segment.StartIndex:support.Segment.EndIndex] != support.Segment.Text { + continue + } + annotations = appendGroundingAnnotations(annotations, chunks, support, start, end, "", seen) + } + return marshalGroundingAnnotations(annotations) +} + +func newGeminiGroundingStreamCandidate() *geminiGroundingStreamCandidate { + return &geminiGroundingStreamCandidate{parts: make(map[int]*streamedGeminiPart)} +} + +func (s *geminiGroundingStreamCandidate) appendContent(content dto.GeminiChatContent, rendered string) { + if s == nil { + return + } + if s.parts == nil { + s.parts = make(map[int]*streamedGeminiPart) + } + + renderedParts := locateRenderedGeminiParts(content, rendered) + renderedBase := s.rendered.Len() + for index := range content.Parts { + partContent := content.Parts[index] + text := partContent.Text + if text == "" || partContent.Thought { + continue + } + part := s.parts[index] + if part == nil { + part = &streamedGeminiPart{} + s.parts[index] = part + } + partStart := part.text.Len() + part.text.WriteString(text) + + // A standalone newline is intentionally omitted by the existing Gemini + // renderer. Keep it in the source part so later byte offsets stay correct, + // but do not claim that it has a corresponding rendered span. + if text == "\n" || index >= len(renderedParts) || renderedParts[index].startByte < 0 { + continue + } + renderedStart := renderedBase + renderedParts[index].startByte + part.spans = append(part.spans, streamedGeminiPartSpan{ + partStartByte: partStart, + partEndByte: partStart + len(text), + renderedStartByte: renderedStart, + }) + } + s.rendered.WriteString(rendered) +} + +func (s *geminiGroundingStreamCandidate) appendGroundingChunks(metadata *dto.GeminiGroundingMetadata) { + if s == nil || metadata == nil || len(metadata.GroundingChunks) == 0 { + return + } + var chunks []geminiGroundingChunk + if err := kitutil.Unmarshal(metadata.GroundingChunks, &chunks); err != nil { + return + } + s.chunks = append(s.chunks, chunks...) +} + +func (s *geminiGroundingStreamCandidate) groundingAnnotations( + metadata *dto.GeminiGroundingMetadata, + candidateIndex int64, + seen map[string]struct{}, +) []byte { + if s == nil || metadata == nil { + return nil + } + s.appendGroundingChunks(metadata) + if len(s.chunks) == 0 || len(metadata.GroundingSupports) == 0 { + return nil + } + var supports []geminiGroundingSupport + if err := kitutil.Unmarshal(metadata.GroundingSupports, &supports); err != nil { + return nil + } + + annotations := make([]any, 0) + keyPrefix := fmt.Sprintf("%d:", candidateIndex) + for _, support := range supports { + partIndex, ok := s.groundingPartIndex(support) + if !ok { + continue + } + start, end, ok := s.groundingRuneRange(partIndex, support.Segment.StartIndex, support.Segment.EndIndex) + if !ok { + continue + } + part := s.parts[partIndex] + if support.Segment.Text != "" && part.text.String()[support.Segment.StartIndex:support.Segment.EndIndex] != support.Segment.Text { + continue + } + annotations = appendGroundingAnnotations(annotations, s.chunks, support, start, end, keyPrefix, seen) + } + return marshalGroundingAnnotations(annotations) +} + +func (s *geminiGroundingStreamCandidate) groundingPartIndex(support geminiGroundingSupport) (int, bool) { + if support.Segment.PartIndex != nil { + partIndex := *support.Segment.PartIndex + part := s.parts[partIndex] + return partIndex, part != nil && len(part.spans) > 0 + } + solePartIndex := -1 + for partIndex, part := range s.parts { + if part == nil || len(part.spans) == 0 { + continue + } + if solePartIndex >= 0 { + return 0, false + } + solePartIndex = partIndex + } + return solePartIndex, solePartIndex >= 0 +} + +func (s *geminiGroundingStreamCandidate) groundingRuneRange(partIndex int, startByte int, endByte int) (int, int, bool) { + if s == nil { + return 0, 0, false + } + part := s.parts[partIndex] + if part == nil { + return 0, 0, false + } + partText := part.text.String() + if startByte < 0 || endByte <= startByte || endByte > len(partText) { + return 0, 0, false + } + if !utf8.ValidString(partText[:startByte]) || !utf8.ValidString(partText[:endByte]) { + return 0, 0, false + } + + renderedStart, renderedEnd := -1, -1 + for _, span := range part.spans { + if renderedStart < 0 && startByte >= span.partStartByte && startByte < span.partEndByte { + renderedStart = span.renderedStartByte + startByte - span.partStartByte + } + if endByte > span.partStartByte && endByte <= span.partEndByte { + renderedEnd = span.renderedStartByte + endByte - span.partStartByte + } + } + if renderedStart < 0 || renderedEnd <= renderedStart { + return 0, 0, false + } + rendered := s.rendered.String() + if renderedEnd > len(rendered) || rendered[renderedStart:renderedEnd] != partText[startByte:endByte] { + return 0, 0, false + } + if !utf8.ValidString(rendered[:renderedStart]) || !utf8.ValidString(rendered[:renderedEnd]) { + return 0, 0, false + } + return utf8.RuneCountInString(rendered[:renderedStart]), utf8.RuneCountInString(rendered[:renderedEnd]), true +} + +func appendGroundingAnnotations( + annotations []any, + chunks []geminiGroundingChunk, + support geminiGroundingSupport, + start int, + end int, + keyPrefix string, + seen map[string]struct{}, +) []any { + for _, chunkIndex := range support.GroundingChunkIndices { + if chunkIndex < 0 || chunkIndex >= len(chunks) { + continue + } + source := chunks[chunkIndex].Web + if source == nil { + source = chunks[chunkIndex].RetrievedContext + } + if source == nil || source.URI == "" { + continue + } + key := fmt.Sprintf("%s%d:%d:%s", keyPrefix, start, end, source.URI) + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + annotations = append(annotations, map[string]any{ + "type": "url_citation", + "url_citation": map[string]any{ + "start_index": start, + "end_index": end, + "url": source.URI, + "title": source.Title, + }, + }) + } + return annotations +} + +func marshalGroundingAnnotations(annotations []any) []byte { + if len(annotations) == 0 { + return nil + } + encoded, err := kitutil.Marshal(annotations) + if err != nil { + return nil + } + return encoded +} + +func locateRenderedGeminiParts(content dto.GeminiChatContent, rendered string) []renderedGeminiPart { + parts := make([]renderedGeminiPart, len(content.Parts)) + cursor := 0 + for index := range content.Parts { + part := content.Parts[index] + text := part.Text + parts[index] = renderedGeminiPart{text: text, startByte: -1} + if text == "" || part.Thought || cursor > len(rendered) { + continue + } + relative := strings.Index(rendered[cursor:], text) + if relative < 0 { + continue + } + start := cursor + relative + parts[index].startByte = start + cursor = start + len(text) + } + return parts +} + +func groundingRuneRange(rendered string, part renderedGeminiPart, startByte int, endByte int) (int, int, bool) { + if startByte < 0 || endByte <= startByte || endByte > len(part.text) { + return 0, 0, false + } + if !utf8.ValidString(part.text[:startByte]) || !utf8.ValidString(part.text[:endByte]) { + return 0, 0, false + } + partStartRunes := utf8.RuneCountInString(rendered[:part.startByte]) + start := partStartRunes + utf8.RuneCountInString(part.text[:startByte]) + end := partStartRunes + utf8.RuneCountInString(part.text[:endByte]) + return start, end, true +} diff --git a/relaykit/relayconvert/internal/gemini_chat/to_oai_chat_req.go b/relaykit/relayconvert/internal/gemini_chat/to_oai_chat_req.go index b614bb8d4b..04b6b74e33 100644 --- a/relaykit/relayconvert/internal/gemini_chat/to_oai_chat_req.go +++ b/relaykit/relayconvert/internal/gemini_chat/to_oai_chat_req.go @@ -8,6 +8,7 @@ import ( "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/jsonutil" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" ) func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatRequest, info convmeta.Meta) (*dto.GeneralOpenAIRequest, error) { @@ -21,7 +22,41 @@ func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatReque Model: modelName, Stream: kitutil.GetPointer(isStream), } + reasoningIntent, err := reasoning.FromGemini(geminiRequest) + if err != nil { + return nil, reasoning.AsClientError(err) + } + sourceModelName := modelName + if info != nil && info.GetOriginModelName() != "" { + sourceModelName = info.GetOriginModelName() + } + baseSourceModel := sourceModelName + opts := convmeta.OptionsOf(info) + preserveSuffix := opts.ShouldPreserveThinkingSuffix(sourceModelName) + if !preserveSuffix { + if suffix := reasoning.IntentFromState(convmeta.ReasoningStateOf(info)); !suffix.IsEmpty() { + reasoningIntent, err = reasoning.MergeExplicitAndSuffix(reasoningIntent, suffix, sourceModelName) + if err != nil { + return nil, reasoning.AsClientError(err) + } + } + } + if baseSourceModel != "" && geminiRequest.GenerationConfig.ThinkingConfig != nil { + _, err = reasoning.ValidateGeminiThinkingConfig(baseSourceModel, geminiRequest.GenerationConfig.ThinkingConfig) + if err != nil { + return nil, reasoning.AsClientError(err) + } + } + reasoningIntent = reasoning.ResolveGeminiDefault(baseSourceModel, reasoningIntent) + effectiveEffort := reasoning.EffectiveEffort(reasoningIntent) + if err := reasoning.ApplyToOpenAIChat(openaiRequest, reasoningIntent); err != nil { + return nil, reasoning.AsClientError(err) + } + if effectiveEffort != "" && info != nil { + info.SetReasoningEffort(string(effectiveEffort)) + } + callHistory := newGeminiFunctionCallHistory(geminiRequest.Contents) var messages []dto.Message for _, content := range geminiRequest.Contents { message := dto.Message{ @@ -30,8 +65,13 @@ func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatReque var mediaContents []dto.MediaContent var toolCalls []dto.ToolCallRequest + var reasoningTexts []string for _, part := range content.Parts { if part.Text != "" { + if part.Thought { + reasoningTexts = append(reasoningTexts, part.Text) + continue + } mediaContent := dto.MediaContent{ Type: "text", Text: part.Text, @@ -59,7 +99,7 @@ func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatReque mediaContents = append(mediaContents, mediaContent) } else if part.FunctionCall != nil { toolCall := dto.ToolCallRequest{ - ID: fmt.Sprintf("call_%d", len(toolCalls)+1), + ID: callHistory.add(part.FunctionCall), Type: "function", Function: dto.FunctionRequest{ Name: part.FunctionCall.FunctionName, @@ -70,7 +110,7 @@ func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatReque } else if part.FunctionResponse != nil { toolMessage := dto.Message{ Role: "tool", - ToolCallId: fmt.Sprintf("call_%d", len(toolCalls)), + ToolCallId: callHistory.match(part.FunctionResponse), } toolMessage.SetStringContent(jsonutil.ToJSONString(part.FunctionResponse.Response)) messages = append(messages, toolMessage) @@ -84,8 +124,12 @@ func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatReque } else if len(mediaContents) > 0 { message.SetMediaContent(mediaContents) } + if len(reasoningTexts) > 0 { + reasoningContent := strings.Join(reasoningTexts, "\n") + message.ReasoningContent = &reasoningContent + } - if len(message.ParseContent()) > 0 || len(message.ToolCalls) > 0 { + if len(message.ParseContent()) > 0 || len(message.ToolCalls) > 0 || message.ReasoningContent != nil { messages = append(messages, message) } } @@ -95,19 +139,19 @@ func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatReque if geminiRequest.GenerationConfig.Temperature != nil { openaiRequest.Temperature = geminiRequest.GenerationConfig.Temperature } - if geminiRequest.GenerationConfig.TopP != nil && *geminiRequest.GenerationConfig.TopP > 0 { + if geminiRequest.GenerationConfig.TopP != nil { openaiRequest.TopP = kitutil.GetPointer(*geminiRequest.GenerationConfig.TopP) } - if geminiRequest.GenerationConfig.TopK != nil && *geminiRequest.GenerationConfig.TopK > 0 { + if geminiRequest.GenerationConfig.TopK != nil { openaiRequest.TopK = kitutil.GetPointer(int(*geminiRequest.GenerationConfig.TopK)) } - if geminiRequest.GenerationConfig.MaxOutputTokens != nil && *geminiRequest.GenerationConfig.MaxOutputTokens > 0 { + if geminiRequest.GenerationConfig.MaxOutputTokens != nil { openaiRequest.MaxTokens = kitutil.GetPointer(*geminiRequest.GenerationConfig.MaxOutputTokens) } if len(geminiRequest.GenerationConfig.StopSequences) > 0 { openaiRequest.Stop = geminiRequest.GenerationConfig.StopSequences[:min(len(geminiRequest.GenerationConfig.StopSequences), 4)] } - if geminiRequest.GenerationConfig.CandidateCount != nil && *geminiRequest.GenerationConfig.CandidateCount > 0 { + if geminiRequest.GenerationConfig.CandidateCount != nil { openaiRequest.N = kitutil.GetPointer(*geminiRequest.GenerationConfig.CandidateCount) } @@ -150,6 +194,88 @@ func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatReque return openaiRequest, nil } +type geminiPendingFunctionCall struct { + id string + name string +} + +// geminiFunctionCallHistory keeps legacy Gemini histories without call IDs +// correlated across content boundaries. Named matching permits results for +// different parallel functions to arrive out of order; same-name calls use +// their original call order because old payloads contain no stronger identity. +type geminiFunctionCallHistory struct { + reservedIDs map[string]struct{} + pending []geminiPendingFunctionCall + nextID int +} + +func newGeminiFunctionCallHistory(contents []dto.GeminiChatContent) *geminiFunctionCallHistory { + history := &geminiFunctionCallHistory{ + reservedIDs: make(map[string]struct{}), + nextID: 1, + } + for _, content := range contents { + for _, part := range content.Parts { + if part.FunctionCall != nil && part.FunctionCall.ID != "" { + history.reservedIDs[part.FunctionCall.ID] = struct{}{} + } + if part.FunctionResponse != nil { + if id := kitutil.JsonRawMessageToString(part.FunctionResponse.ID); id != "" { + history.reservedIDs[id] = struct{}{} + } + } + } + } + return history +} + +func (h *geminiFunctionCallHistory) add(call *dto.FunctionCall) string { + id := call.ID + if id == "" { + id = h.newFallbackID() + } + h.pending = append(h.pending, geminiPendingFunctionCall{id: id, name: call.FunctionName}) + return id +} + +func (h *geminiFunctionCallHistory) match(response *dto.GeminiFunctionResponse) string { + if id := kitutil.JsonRawMessageToString(response.ID); id != "" { + h.removePendingByID(id) + return id + } + + for i, call := range h.pending { + if response.Name != "" && call.name != response.Name { + continue + } + h.pending = append(h.pending[:i], h.pending[i+1:]...) + return call.id + } + return h.newFallbackID() +} + +func (h *geminiFunctionCallHistory) removePendingByID(id string) { + for i, call := range h.pending { + if call.id != id { + continue + } + h.pending = append(h.pending[:i], h.pending[i+1:]...) + return + } +} + +func (h *geminiFunctionCallHistory) newFallbackID() string { + for { + id := fmt.Sprintf("call_%d", h.nextID) + h.nextID++ + if _, exists := h.reservedIDs[id]; exists { + continue + } + h.reservedIDs[id] = struct{}{} + return id + } +} + func convertGeminiRoleToOpenAI(geminiRole string) string { switch geminiRole { case "user": diff --git a/relaykit/relayconvert/internal/gemini_chat/to_oai_chat_resp.go b/relaykit/relayconvert/internal/gemini_chat/to_oai_chat_resp.go index 81f8078d35..c74af6d577 100644 --- a/relaykit/relayconvert/internal/gemini_chat/to_oai_chat_resp.go +++ b/relaykit/relayconvert/internal/gemini_chat/to_oai_chat_resp.go @@ -2,6 +2,8 @@ package geminichat import ( "fmt" + "sort" + "strconv" "strings" "github.com/QuantumNous/new-api/relaykit/dto" @@ -176,6 +178,7 @@ func ResponseGeminiChat2OpenAI(id string, created int64, response *dto.GeminiCha if isToolCall { choice.FinishReason = types.FinishReasonToolCalls } + choice.Message.Annotations = groundingAnnotationsToChat(candidate.GroundingMetadata, candidate.Content, choice.Message.StringContent()) fullTextResponse.Choices = append(fullTextResponse.Choices, choice) } @@ -272,6 +275,7 @@ func StreamResponseGeminiChat2OpenAI(geminiResponse *dto.GeminiChatResponse) (*d if isTools { choice.FinishReason = &types.FinishReasonToolCalls } + choice.Delta.Annotations = groundingAnnotationsToChat(candidate.GroundingMetadata, candidate.Content, content.String()) choices = append(choices, choice) } @@ -288,8 +292,29 @@ type GeminiToChatStreamState struct { sawToolCall bool finishEmitted bool latestUsage *dto.Usage + // Gemini generateContent streams complete function calls. Keep their + // occurrence indexes monotonic because chunk-local indexes restart at zero. + nextToolIndexByCandidate map[int64]int + toolIndexByCandidateID map[int64]map[string]int + partialToolByCandidate map[int64]*geminiPartialToolCall + groundingByCandidate map[int64]*geminiGroundingStreamCandidate + sentGroundingAnnotations map[string]struct{} } +type geminiPartialToolCall struct { + id string + name string + arguments map[string]interface{} +} + +type geminiPartialArgPathSegment struct { + member string + index int + isIndex bool +} + +const maxGeminiPartialArgArrayIndex = 4095 + func NewGeminiToChatStreamState(id string, created int64) *GeminiToChatStreamState { id = strings.TrimSpace(id) if id == "" { @@ -298,12 +323,40 @@ func NewGeminiToChatStreamState(id string, created int64) *GeminiToChatStreamSta if created == 0 { created = kitutil.GetTimestamp() } - return &GeminiToChatStreamState{id: id, created: created} + return &GeminiToChatStreamState{ + id: id, + created: created, + nextToolIndexByCandidate: make(map[int64]int), + toolIndexByCandidateID: make(map[int64]map[string]int), + partialToolByCandidate: make(map[int64]*geminiPartialToolCall), + groundingByCandidate: make(map[int64]*geminiGroundingStreamCandidate), + sentGroundingAnnotations: make(map[string]struct{}), + } } -func (s *GeminiToChatStreamState) ConvertChunk(geminiResponse *dto.GeminiChatResponse, model string, usage *dto.Usage) []*dto.ChatCompletionsStreamResponse { +func (s *GeminiToChatStreamState) ConvertChunk(geminiResponse *dto.GeminiChatResponse, model string, usage *dto.Usage) ([]*dto.ChatCompletionsStreamResponse, error) { if s == nil || geminiResponse == nil { - return nil + return nil, nil + } + if s.groundingByCandidate == nil { + s.groundingByCandidate = make(map[int64]*geminiGroundingStreamCandidate) + } + if s.sentGroundingAnnotations == nil { + s.sentGroundingAnnotations = make(map[string]struct{}) + } + if s.nextToolIndexByCandidate == nil { + s.nextToolIndexByCandidate = make(map[int64]int) + } + if s.toolIndexByCandidateID == nil { + s.toolIndexByCandidateID = make(map[int64]map[string]int) + } + if s.partialToolByCandidate == nil { + s.partialToolByCandidate = make(map[int64]*geminiPartialToolCall) + } + var err error + geminiResponse, err = s.preparePartialFunctionCalls(geminiResponse) + if err != nil { + return nil, err } hasNonStopFinish := false for _, candidate := range geminiResponse.Candidates { @@ -314,12 +367,47 @@ func (s *GeminiToChatStreamState) ConvertChunk(geminiResponse *dto.GeminiChatRes } response, isStop := StreamResponseGeminiChat2OpenAI(geminiResponse) if response == nil { - return nil + return nil, nil } response.Id = s.id response.Created = s.created response.Model = model response.Usage = usage + for index := range geminiResponse.Candidates { + if index >= len(response.Choices) { + break + } + candidate := &geminiResponse.Candidates[index] + choice := &response.Choices[index] + for toolIndex := range choice.Delta.ToolCalls { + callID := strings.TrimSpace(choice.Delta.ToolCalls[toolIndex].ID) + indexesByID := s.toolIndexByCandidateID[candidate.Index] + if indexesByID == nil { + indexesByID = make(map[string]int) + s.toolIndexByCandidateID[candidate.Index] = indexesByID + } + stableIndex, exists := indexesByID[callID] + if callID == "" || !exists { + stableIndex = s.nextToolIndexByCandidate[candidate.Index] + s.nextToolIndexByCandidate[candidate.Index] = stableIndex + 1 + if callID != "" { + indexesByID[callID] = stableIndex + } + } + choice.Delta.ToolCalls[toolIndex].SetIndex(stableIndex) + } + grounding := s.groundingByCandidate[candidate.Index] + if grounding == nil { + grounding = newGeminiGroundingStreamCandidate() + s.groundingByCandidate[candidate.Index] = grounding + } + grounding.appendContent(candidate.Content, response.Choices[index].Delta.GetContentString()) + response.Choices[index].Delta.Annotations = grounding.groundingAnnotations( + candidate.GroundingMetadata, + candidate.Index, + s.sentGroundingAnnotations, + ) + } if response.IsToolCall() { s.sawToolCall = true @@ -345,14 +433,29 @@ func (s *GeminiToChatStreamState) ConvertChunk(geminiResponse *dto.GeminiChatRes if isStop && !s.finishEmitted { responses = append(responses, s.terminalChunk(model)) } - return responses + return responses, nil } -func (s *GeminiToChatStreamState) Finalize(model string) []*dto.ChatCompletionsStreamResponse { - if s == nil || s.finishEmitted { - return nil +func (s *GeminiToChatStreamState) Finalize(model string) ([]*dto.ChatCompletionsStreamResponse, error) { + if s == nil { + return nil, nil } - return []*dto.ChatCompletionsStreamResponse{s.terminalChunk(model)} + if len(s.partialToolByCandidate) > 0 { + candidateIndexes := make([]int64, 0, len(s.partialToolByCandidate)) + for candidateIndex := range s.partialToolByCandidate { + candidateIndexes = append(candidateIndexes, candidateIndex) + } + sort.Slice(candidateIndexes, func(i, j int) bool { + return candidateIndexes[i] < candidateIndexes[j] + }) + candidateIndex := candidateIndexes[0] + partial := s.partialToolByCandidate[candidateIndex] + return nil, fmt.Errorf("Gemini stream ended with an incomplete function call for candidate %d (id %q, name %q)", candidateIndex, partial.id, partial.name) + } + if s.finishEmitted { + return nil, nil + } + return []*dto.ChatCompletionsStreamResponse{s.terminalChunk(model)}, nil } func (s *GeminiToChatStreamState) Usage() *dto.Usage { @@ -362,6 +465,234 @@ func (s *GeminiToChatStreamState) Usage() *dto.Usage { return s.latestUsage } +func (s *GeminiToChatStreamState) preparePartialFunctionCalls(response *dto.GeminiChatResponse) (*dto.GeminiChatResponse, error) { + prepared := *response + prepared.Candidates = append([]dto.GeminiChatCandidate(nil), response.Candidates...) + for candidateIndex := range prepared.Candidates { + candidate := &prepared.Candidates[candidateIndex] + parts := make([]dto.GeminiPart, 0, len(candidate.Content.Parts)) + for _, part := range candidate.Content.Parts { + call := part.FunctionCall + if call == nil || (s.partialToolByCandidate[candidate.Index] == nil && call.WillContinue == nil && len(call.PartialArgs) == 0) { + parts = append(parts, part) + continue + } + completed, ready, err := s.appendPartialFunctionCall(candidate.Index, call) + if err != nil { + return nil, fmt.Errorf("reconstruct Gemini streamed function arguments: %w", err) + } + if ready { + part.FunctionCall = completed + parts = append(parts, part) + } + } + candidate.Content.Parts = parts + } + return &prepared, nil +} + +func (s *GeminiToChatStreamState) appendPartialFunctionCall(candidateIndex int64, call *dto.FunctionCall) (*dto.FunctionCall, bool, error) { + current := s.partialToolByCandidate[candidateIndex] + if current == nil { + current = &geminiPartialToolCall{arguments: make(map[string]interface{})} + s.partialToolByCandidate[candidateIndex] = current + } + if id := strings.TrimSpace(call.ID); id != "" { + if current.id != "" && current.id != id { + return nil, false, fmt.Errorf("candidate %d function call changed id from %q to %q", candidateIndex, current.id, id) + } + current.id = id + } + if name := strings.TrimSpace(call.FunctionName); name != "" { + if current.name != "" && current.name != name { + return nil, false, fmt.Errorf("candidate %d function call changed name from %q to %q", candidateIndex, current.name, name) + } + current.name = name + } + for _, partial := range call.PartialArgs { + path, err := parseGeminiPartialArgPath(partial.JSONPath) + if err != nil { + return nil, false, err + } + value, present := geminiPartialArgValue(partial) + if !present { + continue + } + updated, err := setGeminiPartialArgValue(current.arguments, path, value, partial.StringValue != nil) + if err != nil { + return nil, false, fmt.Errorf("set partial argument %q: %w", partial.JSONPath, err) + } + arguments, ok := updated.(map[string]interface{}) + if !ok { + return nil, false, fmt.Errorf("partial argument path %q replaced the arguments object", partial.JSONPath) + } + current.arguments = arguments + } + if call.WillContinue != nil && *call.WillContinue { + return nil, false, nil + } + if current.name == "" { + return nil, false, fmt.Errorf("candidate %d completed a partial function call without a name", candidateIndex) + } + completed := &dto.FunctionCall{ID: current.id, FunctionName: current.name, Arguments: current.arguments} + delete(s.partialToolByCandidate, candidateIndex) + return completed, true, nil +} + +func parseGeminiPartialArgPath(jsonPath string) ([]geminiPartialArgPathSegment, error) { + path := strings.TrimSpace(jsonPath) + if path == "" || path[0] != '$' { + return nil, fmt.Errorf("unsupported Gemini partial argument path %q", jsonPath) + } + segments := make([]geminiPartialArgPathSegment, 0) + for offset := 1; offset < len(path); { + switch path[offset] { + case '.': + offset++ + start := offset + for offset < len(path) && path[offset] != '.' && path[offset] != '[' { + offset++ + } + if start == offset { + return nil, fmt.Errorf("empty member in Gemini partial argument path %q", jsonPath) + } + member := path[start:offset] + if strings.ContainsAny(member, "]*?") { + return nil, fmt.Errorf("unsupported member %q in Gemini partial argument path", member) + } + segments = append(segments, geminiPartialArgPathSegment{member: member}) + case '[': + offset++ + if offset >= len(path) { + return nil, fmt.Errorf("unterminated selector in Gemini partial argument path %q", jsonPath) + } + if path[offset] == '\'' || path[offset] == '"' { + member, next, err := parseGeminiPartialArgMember(path, offset) + if err != nil { + return nil, fmt.Errorf("invalid Gemini partial argument path %q: %w", jsonPath, err) + } + offset = next + if offset >= len(path) || path[offset] != ']' { + return nil, fmt.Errorf("unterminated member selector in Gemini partial argument path %q", jsonPath) + } + offset++ + segments = append(segments, geminiPartialArgPathSegment{member: member}) + continue + } + start := offset + for offset < len(path) && path[offset] >= '0' && path[offset] <= '9' { + offset++ + } + if start == offset || offset >= len(path) || path[offset] != ']' { + return nil, fmt.Errorf("unsupported selector in Gemini partial argument path %q", jsonPath) + } + index, err := strconv.Atoi(path[start:offset]) + if err != nil { + return nil, fmt.Errorf("invalid array index in Gemini partial argument path %q: %w", jsonPath, err) + } + if index > maxGeminiPartialArgArrayIndex { + return nil, fmt.Errorf("array index %d exceeds Gemini partial argument materialization limit %d", index, maxGeminiPartialArgArrayIndex) + } + offset++ + segments = append(segments, geminiPartialArgPathSegment{index: index, isIndex: true}) + default: + return nil, fmt.Errorf("unsupported selector at offset %d in Gemini partial argument path %q", offset, jsonPath) + } + } + if len(segments) == 0 { + return nil, fmt.Errorf("Gemini partial argument path %q targets the arguments root", jsonPath) + } + return segments, nil +} + +func geminiPartialArgValue(partial dto.GeminiPartialArg) (any, bool) { + switch { + case partial.StringValue != nil: + return *partial.StringValue, true + case partial.NumberValue != nil: + return *partial.NumberValue, true + case partial.BoolValue != nil: + return *partial.BoolValue, true + case partial.NullValue != nil: + return nil, true + default: + return nil, false + } +} + +func parseGeminiPartialArgMember(path string, offset int) (string, int, error) { + quote := path[offset] + start := offset + offset++ + for offset < len(path) { + if path[offset] == '\\' { + offset += 2 + continue + } + if path[offset] == quote { + raw := path[start : offset+1] + if quote == '\'' { + raw = `"` + strings.ReplaceAll(strings.ReplaceAll(raw[1:len(raw)-1], `"`, `\"`), `\'`, `'`) + `"` + } + var member string + if err := kitutil.Unmarshal([]byte(raw), &member); err != nil { + return "", 0, err + } + return member, offset + 1, nil + } + offset++ + } + return "", 0, fmt.Errorf("unterminated quoted member") +} + +func setGeminiPartialArgValue(current any, path []geminiPartialArgPathSegment, value any, appendString bool) (any, error) { + if len(path) == 0 { + if appendString { + if existing, ok := current.(string); ok { + return existing + value.(string), nil + } + } + return value, nil + } + segment := path[0] + if segment.isIndex { + var array []interface{} + switch typed := current.(type) { + case nil: + array = make([]interface{}, segment.index+1) + case []interface{}: + array = typed + if len(array) <= segment.index { + array = append(array, make([]interface{}, segment.index-len(array)+1)...) + } + default: + return nil, fmt.Errorf("array index %d traverses %T", segment.index, current) + } + updated, err := setGeminiPartialArgValue(array[segment.index], path[1:], value, appendString) + if err != nil { + return nil, err + } + array[segment.index] = updated + return array, nil + } + + var object map[string]interface{} + switch typed := current.(type) { + case nil: + object = make(map[string]interface{}) + case map[string]interface{}: + object = typed + default: + return nil, fmt.Errorf("member %q traverses %T", segment.member, current) + } + updated, err := setGeminiPartialArgValue(object[segment.member], path[1:], value, appendString) + if err != nil { + return nil, err + } + object[segment.member] = updated + return object, nil +} + func (s *GeminiToChatStreamState) terminalChunk(model string) *dto.ChatCompletionsStreamResponse { finishReason := types.FinishReasonStop if s.sawToolCall { @@ -388,8 +719,12 @@ func geminiResponseToolCall(item *dto.GeminiPart) *dto.ToolCallResponse { if err != nil { return nil } + callID := strings.TrimSpace(item.FunctionCall.ID) + if callID == "" { + callID = fmt.Sprintf("call_%s", kitutil.GetUUID()) + } return &dto.ToolCallResponse{ - ID: fmt.Sprintf("call_%s", kitutil.GetUUID()), + ID: callID, Type: "function", Function: dto.FunctionResponse{ Arguments: string(argsBytes), diff --git a/relaykit/relayconvert/internal/gemini_chat/to_oai_responses_hosted_stream.go b/relaykit/relayconvert/internal/gemini_chat/to_oai_responses_hosted_stream.go new file mode 100644 index 0000000000..473b9766c2 --- /dev/null +++ b/relaykit/relayconvert/internal/gemini_chat/to_oai_responses_hosted_stream.go @@ -0,0 +1,71 @@ +package geminichat + +import ( + "fmt" + + "github.com/QuantumNous/new-api/relaykit/dto" + oaichat "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/oai_chat" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" +) + +// GeminiHostedStreamBridge accumulates grounding metadata until the provider +// stream ends. Gemini commonly reports the search queries after emitting the +// answer text; delaying the synthetic Responses item keeps one complete, +// canonical action instead of emitting partial or duplicate tool calls. +type GeminiHostedStreamBridge struct { + queries []string + seen map[string]struct{} +} + +func NewGeminiHostedStreamBridge() *GeminiHostedStreamBridge { + return &GeminiHostedStreamBridge{seen: make(map[string]struct{})} +} + +func (b *GeminiHostedStreamBridge) Observe(response *dto.GeminiChatResponse) { + if b == nil { + return + } + if b.seen == nil { + b.seen = make(map[string]struct{}) + } + for _, query := range GroundingWebSearchQueries(response) { + if _, exists := b.seen[query]; exists { + continue + } + b.seen[query] = struct{}{} + b.queries = append(b.queries, query) + } +} + +func (b *GeminiHostedStreamBridge) Finalize(state *oaichat.ChatToResponsesStreamState) ([]oaichat.ChatToResponsesStreamEvent, error) { + if b == nil || len(b.queries) == 0 { + return nil, nil + } + if state == nil { + return nil, fmt.Errorf("Chat-to-Responses stream state is required") + } + action, err := kitutil.Marshal(map[string]any{ + "type": "search", + "queries": append([]string(nil), b.queries...), + }) + if err != nil { + return nil, fmt.Errorf("marshal Gemini web-search action: %w", err) + } + callID := fmt.Sprintf("ws_%s", kitutil.GetUUID()) + events, err := state.StartHostedTool(oaichat.HostedToolStreamStart{ + Type: "web_search_call", + ID: callID, + Action: action, + }) + if err != nil { + return nil, err + } + completed, err := state.CompleteHostedTool(oaichat.HostedToolStreamResult{ + Type: "web_search_call", + ID: callID, + }) + if err != nil { + return nil, err + } + return append(events, completed...), nil +} diff --git a/relaykit/relayconvert/internal/oai_chat/citations.go b/relaykit/relayconvert/internal/oai_chat/citations.go new file mode 100644 index 0000000000..a2e056bbbd --- /dev/null +++ b/relaykit/relayconvert/internal/oai_chat/citations.go @@ -0,0 +1,81 @@ +package oaichat + +import ( + "encoding/json" + "strings" + "unicode/utf8" + + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" +) + +func chatAnnotationsToClaude(raw json.RawMessage, text string) []json.RawMessage { + if len(raw) == 0 { + return nil + } + var annotations []map[string]any + if err := kitutil.Unmarshal(raw, &annotations); err != nil { + return nil + } + citations := make([]json.RawMessage, 0, len(annotations)) + for _, annotation := range annotations { + if strings.TrimSpace(kitutil.Interface2String(annotation["type"])) != "url_citation" { + continue + } + citation, ok := annotation["url_citation"].(map[string]any) + if !ok { + citation = annotation + } + url := strings.TrimSpace(kitutil.Interface2String(citation["url"])) + if url == "" { + continue + } + converted := map[string]any{ + "type": "web_search_result_location", + "url": url, + "title": strings.TrimSpace(kitutil.Interface2String(citation["title"])), + } + if citedText := kitutil.Interface2String(citation["cited_text"]); citedText != "" { + converted["cited_text"] = citedText + } else if citedText := citedTextFromAnnotation(text, citation); citedText != "" { + converted["cited_text"] = citedText + } + if encryptedIndex := kitutil.Interface2String(citation["encrypted_index"]); encryptedIndex != "" { + converted["encrypted_index"] = encryptedIndex + } + if converted["title"] == "" { + delete(converted, "title") + } + encoded, err := kitutil.Marshal(converted) + if err == nil { + citations = append(citations, encoded) + } + } + return citations +} + +func citedTextFromAnnotation(text string, citation map[string]any) string { + start, startOK := annotationIndex(citation["start_index"]) + end, endOK := annotationIndex(citation["end_index"]) + if !startOK || !endOK || start < 0 || end <= start { + return "" + } + if end > utf8.RuneCountInString(text) { + return "" + } + runes := []rune(text) + return string(runes[start:end]) +} + +func annotationIndex(value any) (int, bool) { + switch number := value.(type) { + case float64: + return int(number), number >= 0 && number == float64(int(number)) + case int: + return number, number >= 0 + case json.Number: + parsed, err := number.Int64() + return int(parsed), err == nil && parsed >= 0 + default: + return 0, false + } +} diff --git a/relaykit/relayconvert/internal/oai_chat/to_claude_messages_req.go b/relaykit/relayconvert/internal/oai_chat/to_claude_messages_req.go index ee4722f571..f071bc0eba 100644 --- a/relaykit/relayconvert/internal/oai_chat/to_claude_messages_req.go +++ b/relaykit/relayconvert/internal/oai_chat/to_claude_messages_req.go @@ -1,7 +1,6 @@ package oaichat import ( - "encoding/json" "fmt" "strings" @@ -14,19 +13,6 @@ import ( "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" ) -const ( - webSearchMaxUsesLow = 1 - webSearchMaxUsesMedium = 5 - webSearchMaxUsesHigh = 10 -) - -type openRouterRequestReasoning struct { - Enabled bool `json:"enabled"` - Effort string `json:"effort,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - Exclude bool `json:"exclude,omitempty"` -} - func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, textRequest dto.GeneralOpenAIRequest) (*dto.ClaudeRequest, error) { opts := convmeta.OptionsOf(info) claudeTools := make([]any, 0, len(textRequest.Tools)) @@ -74,15 +60,6 @@ func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, te webSearchTool.UserLocation = anthropicUserLocation } - switch textRequest.WebSearchOptions.SearchContextSize { - case "low": - webSearchTool.MaxUses = webSearchMaxUsesLow - case "medium": - webSearchTool.MaxUses = webSearchMaxUsesMedium - case "high": - webSearchTool.MaxUses = webSearchMaxUsesHigh - } - claudeTools = append(claudeTools, &webSearchTool) } @@ -94,8 +71,10 @@ func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, te if len(claudeTools) > 0 { claudeRequest.Tools = claudeTools } - if maxTokens := textRequest.GetMaxTokens(); maxTokens > 0 { - claudeRequest.MaxTokens = kitutil.GetPointer(maxTokens) + if textRequest.MaxCompletionTokens != nil && *textRequest.MaxCompletionTokens > 0 { + claudeRequest.MaxTokens = kitutil.GetPointer(*textRequest.MaxCompletionTokens) + } else if textRequest.MaxTokens != nil && *textRequest.MaxTokens > 0 { + claudeRequest.MaxTokens = kitutil.GetPointer(*textRequest.MaxTokens) } if textRequest.TopP != nil { claudeRequest.TopP = kitutil.GetPointer(*textRequest.TopP) @@ -114,95 +93,20 @@ func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, te } } - if claudeRequest.MaxTokens == nil || *claudeRequest.MaxTokens == 0 { - if defaultMaxTokens, configured := opts.Claude.DefaultMaxTokensFor(textRequest.Model); configured { + sourceReasoning, err := reasoning.FromOpenAIChat(&textRequest) + if err != nil { + return nil, reasoning.AsClientError(err) + } + if err := sharedclaude.ApplyReasoning(&claudeRequest, info, sourceReasoning); err != nil { + return nil, reasoning.AsClientError(err) + } + if claudeRequest.MaxTokens == nil { + if defaultMaxTokens, configured := opts.Claude.DefaultMaxTokensFor(claudeRequest.Model); configured { value := uint(defaultMaxTokens) claudeRequest.MaxTokens = &value } } - if baseModel, effortLevel, ok := reasoning.TrimEffortSuffix(textRequest.Model); ok && effortLevel != "" && - (strings.HasPrefix(textRequest.Model, "claude-opus-4-6") || - strings.HasPrefix(textRequest.Model, "claude-opus-4-7") || - strings.HasPrefix(textRequest.Model, "claude-opus-4-8")) { - claudeRequest.Model = baseModel - claudeRequest.Thinking = &dto.Thinking{ - Type: "adaptive", - } - claudeRequest.OutputConfig = json.RawMessage(fmt.Sprintf(`{"effort":"%s"}`, effortLevel)) - if strings.HasPrefix(baseModel, "claude-opus-4-7") || - strings.HasPrefix(baseModel, "claude-opus-4-8") { - claudeRequest.Thinking.Display = "summarized" - claudeRequest.Temperature = nil - claudeRequest.TopP = nil - claudeRequest.TopK = nil - } else { - claudeRequest.TopP = nil - claudeRequest.Temperature = kitutil.GetPointer[float64](1.0) - } - } else if opts.Claude.ThinkingAdapterEnabled && - strings.HasSuffix(textRequest.Model, "-thinking") { - - trimmedModel := strings.TrimSuffix(textRequest.Model, "-thinking") - if strings.HasPrefix(trimmedModel, "claude-opus-4-7") || - strings.HasPrefix(trimmedModel, "claude-opus-4-8") { - claudeRequest.Thinking = &dto.Thinking{Type: "adaptive", Display: "summarized"} - claudeRequest.OutputConfig = json.RawMessage(`{"effort":"high"}`) - claudeRequest.Temperature = nil - claudeRequest.TopP = nil - claudeRequest.TopK = nil - } else { - if claudeRequest.MaxTokens == nil || *claudeRequest.MaxTokens < 1280 { - claudeRequest.MaxTokens = kitutil.GetPointer[uint](1280) - } - - claudeRequest.Thinking = &dto.Thinking{ - Type: "enabled", - BudgetTokens: kitutil.GetPointer[int](int(float64(*claudeRequest.MaxTokens) * opts.Claude.ThinkingAdapterBudgetTokensPercentage)), - } - claudeRequest.TopP = nil - claudeRequest.Temperature = kitutil.GetPointer[float64](1.0) - } - if !opts.ShouldPreserveThinkingSuffix(textRequest.Model) { - claudeRequest.Model = trimmedModel - } - } - - if textRequest.ReasoningEffort != "" { - switch textRequest.ReasoningEffort { - case "low": - claudeRequest.Thinking = &dto.Thinking{ - Type: "enabled", - BudgetTokens: kitutil.GetPointer[int](1280), - } - case "medium": - claudeRequest.Thinking = &dto.Thinking{ - Type: "enabled", - BudgetTokens: kitutil.GetPointer[int](2048), - } - case "high": - claudeRequest.Thinking = &dto.Thinking{ - Type: "enabled", - BudgetTokens: kitutil.GetPointer[int](4096), - } - } - } - - if textRequest.Reasoning != nil { - var reasoningConfig openRouterRequestReasoning - if err := kitutil.Unmarshal(textRequest.Reasoning, &reasoningConfig); err != nil { - return nil, err - } - - budgetTokens := reasoningConfig.MaxTokens - if budgetTokens > 0 { - claudeRequest.Thinking = &dto.Thinking{ - Type: "enabled", - BudgetTokens: &budgetTokens, - } - } - } - if textRequest.Stop != nil { switch stop := textRequest.Stop.(type) { case string: @@ -220,9 +124,25 @@ func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, te lastMessage := dto.Message{ Role: "tool", } - for i, message := range textRequest.Messages { - if message.Role == "" { - textRequest.Messages[i].Role = "user" + for _, message := range textRequest.Messages { + switch message.Role { + case "": + message.Role = "user" + case "developer": + message.Role = "system" + case "function": + if message.ToolCallId != "" { + message.Role = "tool" + } else { + message.Role = "user" + } + case "tool": + if message.ToolCallId == "" { + message.Role = "user" + } + case "system", "user", "assistant": + default: + message.Role = "user" } fmtMessage := dto.Message{ Role: message.Role, @@ -236,7 +156,7 @@ func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, te } if lastMessage.Role == message.Role && lastMessage.Role != "tool" { if lastMessage.IsStringContent() && message.IsStringContent() { - fmtMessage.SetStringContent(strings.Trim(fmt.Sprintf("%s %s", lastMessage.StringContent(), message.StringContent()), "\"")) + fmtMessage.SetStringContent(fmt.Sprintf("%s %s", lastMessage.StringContent(), message.StringContent())) formatMessages = formatMessages[:len(formatMessages)-1] } } @@ -250,6 +170,15 @@ func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, te claudeMessages := make([]dto.ClaudeMessage, 0) isFirstMessage := true var systemMessages []dto.ClaudeMediaMessage + placeholderUserMessage := dto.ClaudeMessage{ + Role: "user", + Content: []dto.ClaudeMediaMessage{ + { + Type: "text", + Text: kitutil.GetPointer[string]("..."), + }, + }, + } for _, message := range formatMessages { if message.Role == "system" { @@ -276,16 +205,7 @@ func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, te if isFirstMessage { isFirstMessage = false if message.Role != "user" { - claudeMessage := dto.ClaudeMessage{ - Role: "user", - Content: []dto.ClaudeMediaMessage{ - { - Type: "text", - Text: kitutil.GetPointer[string]("..."), - }, - }, - } - claudeMessages = append(claudeMessages, claudeMessage) + claudeMessages = append(claudeMessages, placeholderUserMessage) } } @@ -384,6 +304,9 @@ func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, te } claudeMessages = append(claudeMessages, claudeMessage) } + if len(claudeMessages) == 0 && len(systemMessages) > 0 { + claudeMessages = append(claudeMessages, placeholderUserMessage) + } if len(systemMessages) > 0 { claudeRequest.System = systemMessages diff --git a/relaykit/relayconvert/internal/oai_chat/to_claude_messages_resp.go b/relaykit/relayconvert/internal/oai_chat/to_claude_messages_resp.go index 554e70a9d5..78fb875bc3 100644 --- a/relaykit/relayconvert/internal/oai_chat/to_claude_messages_resp.go +++ b/relaykit/relayconvert/internal/oai_chat/to_claude_messages_resp.go @@ -1,13 +1,14 @@ package oaichat import ( + "fmt" "strings" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/reasonmap" "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + sharedclaude "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/claude" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" - "github.com/samber/lo" ) func generateStopBlock(index int) *dto.ClaudeResponse { @@ -25,9 +26,18 @@ func stopOpenBlocks(state *convmeta.ClaudeConvertInfo) []*dto.ClaudeResponse { case convmeta.LastMessageTypeText, convmeta.LastMessageTypeThinking: return []*dto.ClaudeResponse{generateStopBlock(state.Index)} case convmeta.LastMessageTypeTools: - responses := make([]*dto.ClaudeResponse, 0, state.ToolCallMaxIndexOffset+1) - for offset := 0; offset <= state.ToolCallMaxIndexOffset; offset++ { - responses = append(responses, generateStopBlock(state.ToolCallBaseIndex+offset)) + if len(state.ToolCalls) == 0 { + responses := make([]*dto.ClaudeResponse, 0, state.ToolCallMaxIndexOffset+1) + for offset := 0; offset <= state.ToolCallMaxIndexOffset; offset++ { + responses = append(responses, generateStopBlock(state.ToolCallBaseIndex+offset)) + } + return responses + } + responses := make([]*dto.ClaudeResponse, 0, len(state.ToolCalls)) + for _, tool := range state.ToolCalls { + if tool != nil && tool.Started { + responses = append(responses, generateStopBlock(tool.BlockIndex)) + } } return responses default: @@ -35,59 +45,52 @@ func stopOpenBlocks(state *convmeta.ClaudeConvertInfo) []*dto.ClaudeResponse { } } -func buildClaudeUsageFromOpenAIUsage(oaiUsage *dto.Usage) *dto.ClaudeUsage { - if oaiUsage == nil { +func startPendingToolBlocks(state *convmeta.ClaudeConvertInfo) []*dto.ClaudeResponse { + if state == nil || state.LastMessagesType != convmeta.LastMessageTypeTools { return nil } - if billingUsage := dto.CloneBillingUsage(oaiUsage.BillingUsage); billingUsage != nil && billingUsage.ClaudeUsage != nil { - if billingUsage.Source == dto.BillingUsageSourceClaudeMessages || billingUsage.Semantic == dto.BillingUsageSemanticAnthropic { - return billingUsage.ClaudeUsage + responses := make([]*dto.ClaudeResponse, 0) + for _, tool := range state.ToolCalls { + if tool == nil || tool.Started || tool.Name == "" { + continue + } + if tool.ID == "" { + tool.ID = fmt.Sprintf("toolu_%s", kitutil.GetUUID()) + } + idx := tool.BlockIndex + responses = append(responses, &dto.ClaudeResponse{ + Index: &idx, + Type: "content_block_start", + ContentBlock: &dto.ClaudeMediaMessage{ + Id: tool.ID, + Type: "tool_use", + Name: tool.Name, + Input: map[string]interface{}{}, + }, + }) + tool.Started = true + if tool.PendingArguments != "" { + arguments := tool.PendingArguments + responses = append(responses, &dto.ClaudeResponse{ + Index: &idx, + Type: "content_block_delta", + Delta: &dto.ClaudeMediaMessage{ + Type: "input_json_delta", + PartialJson: &arguments, + }, + }) + tool.PendingArguments = "" } } - billingUsage := dto.NewOpenAIChatBillingUsage(oaiUsage) - if existingBillingUsage := dto.CloneBillingUsage(oaiUsage.BillingUsage); existingBillingUsage != nil && existingBillingUsage.OpenAIUsage != nil { - if existingBillingUsage.Source == dto.BillingUsageSourceOAIChat || - existingBillingUsage.Source == dto.BillingUsageSourceOAIResponses || - existingBillingUsage.Semantic == dto.BillingUsageSemanticOpenAI { - billingUsage = existingBillingUsage - } - } - cacheCreation5m, cacheCreation1h := NormalizeCacheCreationSplit( - oaiUsage.PromptTokensDetails.CachedCreationTokens, - oaiUsage.ClaudeCacheCreation5mTokens, - oaiUsage.ClaudeCacheCreation1hTokens, - ) - cacheCreationTokens := oaiUsage.PromptTokensDetails.CacheCreationTokensTotal() - inputTokens := oaiUsage.PromptTokens - if oaiUsage.PromptTokensDetails.CacheWriteTokens > 0 { - // OpenAI native cache-write usage counts cached and cache-write tokens - // inside prompt_tokens, while Claude semantics reports input_tokens - // excluding both. Both counts are unadjusted prefixes and may overlap, - // so clamp a negative remainder at zero. - inputTokens = oaiUsage.PromptTokens - oaiUsage.PromptTokensDetails.CachedTokens - cacheCreationTokens - if inputTokens < 0 { - inputTokens = 0 - } - } - usage := &dto.ClaudeUsage{ - InputTokens: inputTokens, - OutputTokens: oaiUsage.CompletionTokens, - CacheCreationInputTokens: cacheCreationTokens, - CacheReadInputTokens: oaiUsage.PromptTokensDetails.CachedTokens, - BillingUsage: billingUsage, - } - if cacheCreation5m > 0 || cacheCreation1h > 0 { - usage.CacheCreation = &dto.ClaudeCacheCreationUsage{ - Ephemeral5mInputTokens: cacheCreation5m, - Ephemeral1hInputTokens: cacheCreation1h, - } - } - return usage + return responses +} + +func buildClaudeUsageFromOpenAIUsage(oaiUsage *dto.Usage) *dto.ClaudeUsage { + return sharedclaude.UsageFromOpenAI(oaiUsage) } func NormalizeCacheCreationSplit(totalTokens int, tokens5m int, tokens1h int) (int, int) { - remainder := lo.Max([]int{totalTokens - tokens5m - tokens1h, 0}) - return tokens5m + remainder, tokens1h + return sharedclaude.NormalizeCacheCreationSplit(totalTokens, tokens5m, tokens1h) } func StreamResponseOpenAI2Claude(openAIResponse *dto.ChatCompletionsStreamResponse, info convmeta.Meta) []*dto.ClaudeResponse { @@ -108,6 +111,7 @@ func StreamResponseOpenAI2Claude(openAIResponse *dto.ChatCompletionsStreamRespon // For tools, OpenAI tool_calls can stream multiple parallel tool_use blocks (indexed from 0), // so we may have multiple open blocks and must stop each one explicitly. appendStopOpenBlocks := func() { + claudeResponses = append(claudeResponses, startPendingToolBlocks(state)...) claudeResponses = append(claudeResponses, stopOpenBlocks(state)...) } // stopOpenBlocksAndAdvance closes the currently open block(s) and advances the content block index @@ -122,14 +126,47 @@ func StreamResponseOpenAI2Claude(openAIResponse *dto.ChatCompletionsStreamRespon appendStopOpenBlocks() switch state.LastMessagesType { case convmeta.LastMessageTypeTools: - state.Index = state.ToolCallBaseIndex + state.ToolCallMaxIndexOffset + 1 + state.Index = state.ToolCallBaseIndex + len(state.ToolCalls) state.ToolCallBaseIndex = 0 state.ToolCallMaxIndexOffset = 0 + state.ToolCalls = nil + state.ToolCallByIndex = nil + state.ToolCallByID = nil default: state.Index++ } state.LastMessagesType = convmeta.LastMessageTypeNone } + appendCitationDeltas := func(raw []byte) { + citations := chatAnnotationsToClaude(raw, "") + if len(citations) == 0 { + return + } + if state.LastMessagesType != convmeta.LastMessageTypeText { + stopOpenBlocksAndAdvance() + idx := state.Index + claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ + Index: &idx, + Type: "content_block_start", + ContentBlock: &dto.ClaudeMediaMessage{ + Type: "text", + Text: kitutil.GetPointer[string](""), + }, + }) + state.LastMessagesType = convmeta.LastMessageTypeText + } + for _, citation := range citations { + idx := state.Index + claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ + Index: &idx, + Type: "content_block_delta", + Delta: &dto.ClaudeMediaMessage{ + Type: "citations_delta", + Citation: citation, + }, + }) + } + } if info.GetSendResponseCount() == 1 { msg := &dto.ClaudeMediaMessage{ Id: openAIResponse.Id, @@ -146,128 +183,6 @@ func StreamResponseOpenAI2Claude(openAIResponse *dto.ChatCompletionsStreamRespon Type: "message_start", Message: msg, }) - //claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ - // Type: "ping", - //}) - if openAIResponse.IsToolCall() { - state.LastMessagesType = convmeta.LastMessageTypeTools - state.ToolCallBaseIndex = 0 - state.ToolCallMaxIndexOffset = 0 - var toolCall dto.ToolCallResponse - if len(openAIResponse.Choices) > 0 && len(openAIResponse.Choices[0].Delta.ToolCalls) > 0 { - toolCall = openAIResponse.Choices[0].Delta.ToolCalls[0] - } else { - first := openAIResponse.GetFirstToolCall() - if first != nil { - toolCall = *first - } else { - toolCall = dto.ToolCallResponse{} - } - } - resp := &dto.ClaudeResponse{ - Type: "content_block_start", - ContentBlock: &dto.ClaudeMediaMessage{ - Id: toolCall.ID, - Type: "tool_use", - Name: toolCall.Function.Name, - Input: map[string]interface{}{}, - }, - } - resp.SetIndex(0) - claudeResponses = append(claudeResponses, resp) - // 首块包含工具 delta，则追加 input_json_delta - if toolCall.Function.Arguments != "" { - idx := 0 - claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ - Index: &idx, - Type: "content_block_delta", - Delta: &dto.ClaudeMediaMessage{ - Type: "input_json_delta", - PartialJson: &toolCall.Function.Arguments, - }, - }) - } - } else { - - } - // 判断首个响应是否存在内容（非标准的 OpenAI 响应） - if len(openAIResponse.Choices) > 0 { - reasoning := openAIResponse.Choices[0].Delta.GetReasoningContent() - content := openAIResponse.Choices[0].Delta.GetContentString() - - if reasoning != "" { - if state.LastMessagesType != convmeta.LastMessageTypeThinking { - stopOpenBlocksAndAdvance() - } - idx := state.Index - claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ - Index: &idx, - Type: "content_block_start", - ContentBlock: &dto.ClaudeMediaMessage{ - Type: "thinking", - Thinking: kitutil.GetPointer[string](""), - }, - }) - idx2 := idx - claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ - Index: &idx2, - Type: "content_block_delta", - Delta: &dto.ClaudeMediaMessage{ - Type: "thinking_delta", - Thinking: &reasoning, - }, - }) - state.LastMessagesType = convmeta.LastMessageTypeThinking - } else if content != "" { - if state.LastMessagesType != convmeta.LastMessageTypeText { - stopOpenBlocksAndAdvance() - } - idx := state.Index - claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ - Index: &idx, - Type: "content_block_start", - ContentBlock: &dto.ClaudeMediaMessage{ - Type: "text", - Text: kitutil.GetPointer[string](""), - }, - }) - idx2 := idx - claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ - Index: &idx2, - Type: "content_block_delta", - Delta: &dto.ClaudeMediaMessage{ - Type: "text_delta", - Text: kitutil.GetPointer[string](content), - }, - }) - state.LastMessagesType = convmeta.LastMessageTypeText - } - } - - // A first chunk can carry finish_reason before usage; defer terminal events until usage arrives. - if len(openAIResponse.Choices) > 0 && openAIResponse.Choices[0].FinishReason != nil && *openAIResponse.Choices[0].FinishReason != "" { - state.FinishReason = *openAIResponse.Choices[0].FinishReason - oaiUsage := openAIResponse.Usage - if oaiUsage == nil { - oaiUsage = state.Usage - } - if oaiUsage == nil { - return claudeResponses - } - appendStopOpenBlocks() - claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ - Type: "message_delta", - Usage: buildClaudeUsageFromOpenAIUsage(oaiUsage), - Delta: &dto.ClaudeMediaMessage{ - StopReason: kitutil.GetPointer[string](stopReasonOpenAI2Claude(state.FinishReason)), - }, - }) - claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ - Type: "message_stop", - }) - state.Done = true - } - return claudeResponses } if len(openAIResponse.Choices) == 0 { @@ -300,13 +215,6 @@ func StreamResponseOpenAI2Claude(openAIResponse *dto.ChatCompletionsStreamRespon doneChunk := chosenChoice.FinishReason != nil && *chosenChoice.FinishReason != "" if doneChunk { state.FinishReason = *chosenChoice.FinishReason - oaiUsage := openAIResponse.Usage - if oaiUsage == nil { - oaiUsage = state.Usage - // Some upstreams emit finish_reason first, then send a final usage-only chunk. - // Defer closing until usage is available so the final message_delta carries it. - return claudeResponses - } } var claudeResponse dto.ClaudeResponse @@ -318,38 +226,80 @@ func StreamResponseOpenAI2Claude(openAIResponse *dto.ChatCompletionsStreamRespon stopOpenBlocksAndAdvance() state.ToolCallBaseIndex = state.Index state.ToolCallMaxIndexOffset = 0 + state.ToolCalls = nil + state.ToolCallByIndex = make(map[int]*convmeta.ClaudeStreamToolCall) + state.ToolCallByID = make(map[string]*convmeta.ClaudeStreamToolCall) } state.LastMessagesType = convmeta.LastMessageTypeTools - base := state.ToolCallBaseIndex - maxOffset := state.ToolCallMaxIndexOffset - + if state.ToolCallByIndex == nil { + state.ToolCallByIndex = make(map[int]*convmeta.ClaudeStreamToolCall) + } + if state.ToolCallByID == nil { + state.ToolCallByID = make(map[string]*convmeta.ClaudeStreamToolCall) + } for i, toolCall := range toolCalls { - offset := 0 + toolIndex := i if toolCall.Index != nil { - offset = *toolCall.Index - } else { - offset = i + toolIndex = *toolCall.Index } - if offset > maxOffset { - maxOffset = offset + incomingID := strings.TrimSpace(toolCall.ID) + var tool *convmeta.ClaudeStreamToolCall + if incomingID != "" { + tool = state.ToolCallByID[incomingID] + } + if tool == nil { + tool = state.ToolCallByIndex[toolIndex] + } + if tool != nil && incomingID != "" && tool.ID != "" && tool.ID != incomingID { + tool = nil + } + if tool == nil { + tool = &convmeta.ClaudeStreamToolCall{ + BlockIndex: state.ToolCallBaseIndex + len(state.ToolCalls), + } + state.ToolCalls = append(state.ToolCalls, tool) + } + state.ToolCallByIndex[toolIndex] = tool + if tool.ID == "" && incomingID != "" { + tool.ID = incomingID + state.ToolCallByID[incomingID] = tool + } + if tool.Name == "" && strings.TrimSpace(toolCall.Function.Name) != "" { + tool.Name = strings.TrimSpace(toolCall.Function.Name) + } + if !tool.Started { + tool.PendingArguments += toolCall.Function.Arguments } - blockIndex := base + offset - idx := blockIndex - if toolCall.Function.Name != "" { + idx := tool.BlockIndex + if !tool.Started && tool.ID != "" && tool.Name != "" { claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ Index: &idx, Type: "content_block_start", ContentBlock: &dto.ClaudeMediaMessage{ - Id: toolCall.ID, + Id: tool.ID, Type: "tool_use", - Name: toolCall.Function.Name, + Name: tool.Name, Input: map[string]interface{}{}, }, }) + tool.Started = true + if tool.PendingArguments != "" { + arguments := tool.PendingArguments + claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ + Index: &idx, + Type: "content_block_delta", + Delta: &dto.ClaudeMediaMessage{ + Type: "input_json_delta", + PartialJson: &arguments, + }, + }) + tool.PendingArguments = "" + } + continue } - if len(toolCall.Function.Arguments) > 0 { + if tool.Started && toolCall.Function.Arguments != "" { claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ Index: &idx, Type: "content_block_delta", @@ -360,8 +310,10 @@ func StreamResponseOpenAI2Claude(openAIResponse *dto.ChatCompletionsStreamRespon }) } } - state.ToolCallMaxIndexOffset = maxOffset - state.Index = base + maxOffset + state.ToolCallMaxIndexOffset = len(state.ToolCalls) - 1 + if len(state.ToolCalls) > 0 { + state.Index = state.ToolCallBaseIndex + len(state.ToolCalls) - 1 + } } else { reasoning := chosenChoice.Delta.GetReasoningContent() textContent := chosenChoice.Delta.GetContentString() @@ -412,22 +364,27 @@ func StreamResponseOpenAI2Claude(openAIResponse *dto.ChatCompletionsStreamRespon if !isEmpty && claudeResponse.Delta != nil { claudeResponses = append(claudeResponses, &claudeResponse) } + appendCitationDeltas(chosenChoice.Delta.Annotations) if doneChunk || state.Done { - appendStopOpenBlocks() oaiUsage := openAIResponse.Usage if oaiUsage == nil { oaiUsage = state.Usage } - if oaiUsage != nil { - claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ - Type: "message_delta", - Usage: buildClaudeUsageFromOpenAIUsage(oaiUsage), - Delta: &dto.ClaudeMediaMessage{ - StopReason: kitutil.GetPointer[string](stopReasonOpenAI2Claude(state.FinishReason)), - }, - }) + if oaiUsage == nil { + // Some upstreams emit finish_reason first, then send a final usage-only chunk. + // Keep content blocks open until usage is available so the terminal message_delta + // can carry both usage and the final stop reason. + return claudeResponses } + appendStopOpenBlocks() + claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ + Type: "message_delta", + Usage: buildClaudeUsageFromOpenAIUsage(oaiUsage), + Delta: &dto.ClaudeMediaMessage{ + StopReason: kitutil.GetPointer[string](stopReasonOpenAI2Claude(state.FinishReason)), + }, + }) claudeResponses = append(claudeResponses, &dto.ClaudeResponse{ Type: "message_stop", }) @@ -452,7 +409,8 @@ func FinalizeStreamResponseOpenAI2Claude(info convmeta.Meta) []*dto.ClaudeRespon if stopReason == "" { stopReason = "end_turn" } - responses := stopOpenBlocks(state) + responses := startPendingToolBlocks(state) + responses = append(responses, stopOpenBlocks(state)...) responses = append(responses, &dto.ClaudeResponse{ Type: "message_delta", @@ -478,12 +436,22 @@ func ResponseOpenAI2Claude(openAIResponse *dto.OpenAITextResponse, info convmeta } for _, choice := range openAIResponse.Choices { stopReason = stopReasonOpenAI2Claude(choice.FinishReason) + reasoningContent := choice.Message.GetReasoningContent() textContent := choice.Message.StringContent() toolCalls := choice.Message.ParseToolCalls() - if textContent != "" || len(toolCalls) == 0 { + if reasoningContent != "" { + claudeContent := dto.ClaudeMediaMessage{Type: "thinking"} + claudeContent.Thinking = kitutil.GetPointer(reasoningContent) + contents = append(contents, claudeContent) + } + if textContent != "" || (reasoningContent == "" && len(toolCalls) == 0) { claudeContent := dto.ClaudeMediaMessage{} claudeContent.Type = "text" claudeContent.SetText(textContent) + citations := chatAnnotationsToClaude(choice.Message.Annotations, textContent) + if len(citations) > 0 { + claudeContent.Citations, _ = kitutil.Marshal(citations) + } contents = append(contents, claudeContent) } for _, toolUse := range toolCalls { diff --git a/relaykit/relayconvert/internal/oai_chat/to_claude_messages_resp_test.go b/relaykit/relayconvert/internal/oai_chat/to_claude_messages_resp_test.go index 1ccdd00b25..976e725919 100644 --- a/relaykit/relayconvert/internal/oai_chat/to_claude_messages_resp_test.go +++ b/relaykit/relayconvert/internal/oai_chat/to_claude_messages_resp_test.go @@ -79,6 +79,25 @@ func TestResponseOpenAI2ClaudeUsageCarriesOpenAIBillingUsage(t *testing.T) { assert.Nil(t, resp.Usage.BillingUsage.OpenAIUsage.BillingUsage) } +func TestResponseOpenAI2ClaudePreservesReasoningBeforeText(t *testing.T) { + message := dto.Message{Role: "assistant", Content: "final answer"} + message.ReasoningContent = ptr("considering the request") + resp := ResponseOpenAI2Claude(&dto.OpenAITextResponse{ + Id: "chatcmpl_1", + Model: "gpt-test", + Choices: []dto.OpenAITextResponseChoice{ + {Message: message, FinishReason: "stop"}, + }, + }, nil) + + require.Len(t, resp.Content, 2) + assert.Equal(t, "thinking", resp.Content[0].Type) + require.NotNil(t, resp.Content[0].Thinking) + assert.Equal(t, "considering the request", *resp.Content[0].Thinking) + assert.Equal(t, "text", resp.Content[1].Type) + assert.Equal(t, "final answer", resp.Content[1].GetText()) +} + func TestBuildClaudeUsageFromOpenAICacheWriteUsage(t *testing.T) { usage := buildClaudeUsageFromOpenAIUsage(&dto.Usage{ PromptTokens: 3619, diff --git a/relaykit/relayconvert/internal/oai_chat/to_gemini_chat_req.go b/relaykit/relayconvert/internal/oai_chat/to_gemini_chat_req.go index 2ebb9348d4..3828b5fa03 100644 --- a/relaykit/relayconvert/internal/oai_chat/to_gemini_chat_req.go +++ b/relaykit/relayconvert/internal/oai_chat/to_gemini_chat_req.go @@ -3,6 +3,7 @@ package oaichat import ( "errors" "fmt" + "math" "strings" "context" @@ -11,6 +12,7 @@ import ( relaymedia "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/media" sharedgemini "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/gemini" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" ) func OpenAIChatRequestToGeminiGenerateContent(c context.Context, textRequest dto.GeneralOpenAIRequest, info convmeta.Meta) (*dto.GeminiChatRequest, error) { @@ -22,13 +24,15 @@ func OpenAIChatRequestToGeminiGenerateContent(c context.Context, textRequest dto }, } - if textRequest.TopP != nil && *textRequest.TopP > 0 { + if textRequest.TopP != nil { geminiRequest.GenerationConfig.TopP = kitutil.GetPointer(*textRequest.TopP) } - if maxTokens := textRequest.GetMaxTokens(); maxTokens > 0 { - geminiRequest.GenerationConfig.MaxOutputTokens = kitutil.GetPointer(maxTokens) + if textRequest.MaxCompletionTokens != nil { + geminiRequest.GenerationConfig.MaxOutputTokens = kitutil.GetPointer(*textRequest.MaxCompletionTokens) + } else if textRequest.MaxTokens != nil { + geminiRequest.GenerationConfig.MaxOutputTokens = kitutil.GetPointer(*textRequest.MaxTokens) } - if textRequest.Seed != nil && *textRequest.Seed != 0 { + if textRequest.Seed != nil { geminiRequest.GenerationConfig.Seed = kitutil.GetPointer(int64(*textRequest.Seed)) } @@ -50,7 +54,6 @@ func OpenAIChatRequestToGeminiGenerateContent(c context.Context, textRequest dto geminiRequest.GenerationConfig.StopSequences = stopSequences } - adaptorWithExtraBody := false if len(textRequest.ExtraBody) > 0 { var extraBody map[string]interface{} if err := kitutil.Unmarshal(textRequest.ExtraBody, &extraBody); err != nil { @@ -58,61 +61,47 @@ func OpenAIChatRequestToGeminiGenerateContent(c context.Context, textRequest dto } if googleBody, ok := extraBody["google"].(map[string]interface{}); ok { - if !strings.HasSuffix(upstreamModelName, "-nothinking") { - adaptorWithExtraBody = true - if _, hasErrorParam := googleBody["thinkingConfig"]; hasErrorParam { - return nil, errors.New("extra_body.google.thinkingConfig is not supported, use extra_body.google.thinking_config instead") + if _, hasErrorParam := googleBody["thinkingConfig"]; hasErrorParam { + return nil, errors.New("extra_body.google.thinkingConfig is not supported, use extra_body.google.thinking_config instead") + } + + if thinkingConfig, ok := googleBody["thinking_config"].(map[string]interface{}); ok { + if _, hasErrorParam := thinkingConfig["thinkingBudget"]; hasErrorParam { + return nil, errors.New("extra_body.google.thinking_config.thinkingBudget is not supported, use extra_body.google.thinking_config.thinking_budget instead") + } + var hasThinkingConfig bool + var tempThinkingConfig dto.GeminiThinkingConfig + + if thinkingBudget, exists := thinkingConfig["thinking_budget"]; exists { + v, ok := thinkingBudget.(float64) + maxInt := int(^uint(0) >> 1) + if !ok || math.IsNaN(v) || math.IsInf(v, 0) || math.Trunc(v) != v || v > float64(maxInt) || v < float64(-maxInt-1) { + return nil, errors.New("extra_body.google.thinking_config.thinking_budget must be an integer") + } + budgetInt := int(v) + tempThinkingConfig.ThinkingBudget = kitutil.GetPointer(budgetInt) + hasThinkingConfig = true } - if thinkingConfig, ok := googleBody["thinking_config"].(map[string]interface{}); ok { - if _, hasErrorParam := thinkingConfig["thinkingBudget"]; hasErrorParam { - return nil, errors.New("extra_body.google.thinking_config.thinkingBudget is not supported, use extra_body.google.thinking_config.thinking_budget instead") + if includeThoughts, exists := thinkingConfig["include_thoughts"]; exists { + if v, ok := includeThoughts.(bool); ok { + tempThinkingConfig.IncludeThoughts = kitutil.GetPointer(v) + hasThinkingConfig = true + } else { + return nil, errors.New("extra_body.google.thinking_config.include_thoughts must be a boolean") } - var hasThinkingConfig bool - var tempThinkingConfig dto.GeminiThinkingConfig + } + if thinkingLevel, exists := thinkingConfig["thinking_level"]; exists { + if v, ok := thinkingLevel.(string); ok { + tempThinkingConfig.ThinkingLevel = v + hasThinkingConfig = true + } else { + return nil, errors.New("extra_body.google.thinking_config.thinking_level must be a string") + } + } - if thinkingBudget, exists := thinkingConfig["thinking_budget"]; exists { - switch v := thinkingBudget.(type) { - case float64: - budgetInt := int(v) - tempThinkingConfig.ThinkingBudget = kitutil.GetPointer(budgetInt) - tempThinkingConfig.IncludeThoughts = budgetInt > 0 - hasThinkingConfig = true - default: - return nil, errors.New("extra_body.google.thinking_config.thinking_budget must be an integer") - } - } - - if includeThoughts, exists := thinkingConfig["include_thoughts"]; exists { - if v, ok := includeThoughts.(bool); ok { - tempThinkingConfig.IncludeThoughts = v - hasThinkingConfig = true - } else { - return nil, errors.New("extra_body.google.thinking_config.include_thoughts must be a boolean") - } - } - if thinkingLevel, exists := thinkingConfig["thinking_level"]; exists { - if v, ok := thinkingLevel.(string); ok { - tempThinkingConfig.ThinkingLevel = v - hasThinkingConfig = true - } else { - return nil, errors.New("extra_body.google.thinking_config.thinking_level must be a string") - } - } - - if hasThinkingConfig { - if geminiRequest.GenerationConfig.ThinkingConfig == nil { - geminiRequest.GenerationConfig.ThinkingConfig = &tempThinkingConfig - } else { - if tempThinkingConfig.ThinkingBudget != nil { - geminiRequest.GenerationConfig.ThinkingConfig.ThinkingBudget = tempThinkingConfig.ThinkingBudget - } - geminiRequest.GenerationConfig.ThinkingConfig.IncludeThoughts = tempThinkingConfig.IncludeThoughts - if tempThinkingConfig.ThinkingLevel != "" { - geminiRequest.GenerationConfig.ThinkingConfig.ThinkingLevel = tempThinkingConfig.ThinkingLevel - } - } - } + if hasThinkingConfig { + geminiRequest.GenerationConfig.ThinkingConfig = &tempThinkingConfig } } @@ -147,8 +136,8 @@ func OpenAIChatRequestToGeminiGenerateContent(c context.Context, textRequest dto } } - if !adaptorWithExtraBody { - sharedgemini.ApplyThinkingConfig(&geminiRequest, info, textRequest) + if err := sharedgemini.ApplyThinkingConfig(&geminiRequest, info, textRequest); err != nil { + return nil, reasoning.AsClientError(err) } var safetySettings []dto.GeminiChatSafetySettings @@ -270,6 +259,13 @@ func OpenAIChatRequestToGeminiGenerateContent(c context.Context, textRequest dto Name: name, Response: contentMap, } + if message.ToolCallId != "" { + id, err := kitutil.Marshal(message.ToolCallId) + if err != nil { + return nil, fmt.Errorf("failed to marshal function response ID: %w", err) + } + functionResp.ID = id + } *parts = append(*parts, dto.GeminiPart{ FunctionResponse: functionResp, @@ -293,6 +289,7 @@ func OpenAIChatRequestToGeminiGenerateContent(c context.Context, textRequest dto } toolCall := dto.GeminiPart{ FunctionCall: &dto.FunctionCall{ + ID: call.ID, FunctionName: call.Function.Name, Arguments: args, }, diff --git a/relaykit/relayconvert/internal/oai_chat/to_gemini_chat_resp.go b/relaykit/relayconvert/internal/oai_chat/to_gemini_chat_resp.go index 9c54d3c0ae..4671b2dc50 100644 --- a/relaykit/relayconvert/internal/oai_chat/to_gemini_chat_resp.go +++ b/relaykit/relayconvert/internal/oai_chat/to_gemini_chat_resp.go @@ -1,11 +1,53 @@ package oaichat import ( + "fmt" + "sort" + "strings" + "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" ) +type ChatToGeminiStreamState struct { + toolsByChoice map[int][]*chatToGeminiStreamTool + toolByIndex map[chatToGeminiStreamToolKey]*chatToGeminiStreamTool + toolByID map[chatToGeminiStreamToolIDKey]*chatToGeminiStreamTool + finishedChoices map[int]bool + seenChoices map[int]bool + usage *dto.Usage + usageEmitted bool + finalized bool +} + +type chatToGeminiStreamToolKey struct { + ChoiceIndex int + ToolIndex int +} + +type chatToGeminiStreamToolIDKey struct { + ChoiceIndex int + ID string +} + +type chatToGeminiStreamTool struct { + ID string + Name string + Arguments strings.Builder + Emitted bool +} + +func NewChatToGeminiStreamState() *ChatToGeminiStreamState { + return &ChatToGeminiStreamState{ + toolsByChoice: make(map[int][]*chatToGeminiStreamTool), + toolByIndex: make(map[chatToGeminiStreamToolKey]*chatToGeminiStreamTool), + toolByID: make(map[chatToGeminiStreamToolIDKey]*chatToGeminiStreamTool), + finishedChoices: make(map[int]bool), + seenChoices: make(map[int]bool), + } +} + // ResponseOpenAI2Gemini 将 OpenAI 响应转换为 Gemini 格式 func ResponseOpenAI2Gemini(openAIResponse *dto.OpenAITextResponse, info convmeta.Meta) *dto.GeminiChatResponse { totalTokens := openAIResponse.TotalTokens @@ -64,19 +106,11 @@ func ResponseOpenAI2Gemini(openAIResponse *dto.OpenAITextResponse, info convmeta toolCalls := choice.Message.ParseToolCalls() for _, toolCall := range toolCalls { - var args map[string]interface{} - if toolCall.Function.Arguments != "" { - if err := kitutil.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil { - args = map[string]interface{}{"arguments": toolCall.Function.Arguments} - } - } else { - args = make(map[string]interface{}) - } - part := dto.GeminiPart{ FunctionCall: &dto.FunctionCall{ + ID: toolCall.ID, FunctionName: toolCall.Function.Name, - Arguments: args, + Arguments: geminiFunctionArguments(toolCall.Function.Arguments), }, } content.Parts = append(content.Parts, part) @@ -165,20 +199,11 @@ func StreamResponseOpenAI2Gemini(openAIResponse *dto.ChatCompletionsStreamRespon // 处理工具调用 if choice.Delta.ToolCalls != nil { for _, toolCall := range choice.Delta.ToolCalls { - // 解析参数 - var args map[string]interface{} - if toolCall.Function.Arguments != "" { - if err := kitutil.Unmarshal([]byte(toolCall.Function.Arguments), &args); err != nil { - args = map[string]interface{}{"arguments": toolCall.Function.Arguments} - } - } else { - args = make(map[string]interface{}) - } - part := dto.GeminiPart{ FunctionCall: &dto.FunctionCall{ + ID: toolCall.ID, FunctionName: toolCall.Function.Name, - Arguments: args, + Arguments: geminiFunctionArguments(toolCall.Function.Arguments), }, } content.Parts = append(content.Parts, part) @@ -201,6 +226,306 @@ func StreamResponseOpenAI2Gemini(openAIResponse *dto.ChatCompletionsStreamRespon return geminiResponse } +// ConvertChunk accumulates OpenAI tool-call deltas until their choice ends. +// Gemini functionCall parts are atomic, so emitting each OpenAI arguments +// fragment as a separate part would create duplicate calls with invalid input. +func (s *ChatToGeminiStreamState) ConvertChunk(openAIResponse *dto.ChatCompletionsStreamResponse, info convmeta.Meta) ([]*dto.GeminiChatResponse, error) { + if openAIResponse == nil { + return nil, nil + } + if s == nil { + return nil, fmt.Errorf("OpenAI chat to Gemini stream state is required") + } + if s.finalized { + return nil, fmt.Errorf("OpenAI chat to Gemini stream received data after finalization") + } + if s.toolsByChoice == nil { + s.toolsByChoice = make(map[int][]*chatToGeminiStreamTool) + } + if s.toolByIndex == nil { + s.toolByIndex = make(map[chatToGeminiStreamToolKey]*chatToGeminiStreamTool) + } + if s.toolByID == nil { + s.toolByID = make(map[chatToGeminiStreamToolIDKey]*chatToGeminiStreamTool) + } + if s.finishedChoices == nil { + s.finishedChoices = make(map[int]bool) + } + if s.seenChoices == nil { + s.seenChoices = make(map[int]bool) + } + if openAIResponse.Usage != nil { + s.usage = UsageFromChatUsage(openAIResponse.Usage) + } + + candidates := make([]dto.GeminiChatCandidate, 0, len(openAIResponse.Choices)) + for _, choice := range openAIResponse.Choices { + s.seenChoices[choice.Index] = true + hasText := choice.Delta.GetContentString() != "" + hasToolDelta := len(choice.Delta.ToolCalls) > 0 + hasFinish := choice.FinishReason != nil && strings.TrimSpace(*choice.FinishReason) != "" + if s.finishedChoices[choice.Index] { + if hasText || hasToolDelta { + return nil, fmt.Errorf("OpenAI chat choice %d received data after completion", choice.Index) + } + continue + } + + for position, toolCall := range choice.Delta.ToolCalls { + if toolCall.Index == nil { + toolCall.SetIndex(position) + } + if err := s.appendToolCallDelta(choice.Index, toolCall); err != nil { + return nil, err + } + } + + candidate := dto.GeminiChatCandidate{ + Index: int64(choice.Index), + SafetyRatings: []dto.GeminiChatSafetyRating{}, + Content: dto.GeminiChatContent{ + Role: "model", + Parts: make([]dto.GeminiPart, 0), + }, + } + if hasText { + candidate.Content.Parts = append(candidate.Content.Parts, dto.GeminiPart{Text: choice.Delta.GetContentString()}) + } + if hasFinish { + parts, err := s.finishChoice(choice.Index) + if err != nil { + return nil, err + } + candidate.Content.Parts = append(candidate.Content.Parts, parts...) + finishReason := geminiFinishReason(*choice.FinishReason) + candidate.FinishReason = &finishReason + s.finishedChoices[choice.Index] = true + } + if len(candidate.Content.Parts) > 0 || candidate.FinishReason != nil { + candidates = append(candidates, candidate) + } + } + + if len(candidates) == 0 { + if openAIResponse.Usage != nil && len(s.finishedChoices) > 0 { + s.usageEmitted = true + return []*dto.GeminiChatResponse{newGeminiStreamResponse(nil, s.usage, info)}, nil + } + return nil, nil + } + if openAIResponse.Usage != nil { + s.usageEmitted = true + } + return []*dto.GeminiChatResponse{newGeminiStreamResponse(candidates, openAIResponse.Usage, info)}, nil +} + +// Finalize emits any calls left pending when an upstream closes without a +// finish-reason chunk. Calling Finalize more than once is safe. +func (s *ChatToGeminiStreamState) Finalize(info convmeta.Meta) ([]*dto.GeminiChatResponse, error) { + if s == nil || s.finalized { + return nil, nil + } + + choiceIndexes := make(map[int]struct{}) + for choiceIndex, tools := range s.toolsByChoice { + for _, tool := range tools { + if !tool.Emitted { + choiceIndexes[choiceIndex] = struct{}{} + break + } + } + } + for choiceIndex := range s.seenChoices { + if !s.finishedChoices[choiceIndex] { + choiceIndexes[choiceIndex] = struct{}{} + } + } + orderedChoices := make([]int, 0, len(choiceIndexes)) + for choiceIndex := range choiceIndexes { + orderedChoices = append(orderedChoices, choiceIndex) + } + sort.Ints(orderedChoices) + + candidates := make([]dto.GeminiChatCandidate, 0, len(orderedChoices)) + for _, choiceIndex := range orderedChoices { + parts, err := s.finishChoice(choiceIndex) + if err != nil { + return nil, err + } + finishReason := "STOP" + candidates = append(candidates, dto.GeminiChatCandidate{ + Index: int64(choiceIndex), + FinishReason: &finishReason, + SafetyRatings: []dto.GeminiChatSafetyRating{}, + Content: dto.GeminiChatContent{ + Role: "model", + Parts: parts, + }, + }) + } + if len(candidates) == 0 { + s.finalized = true + if s.usage == nil || s.usageEmitted { + return nil, nil + } + s.usageEmitted = true + return []*dto.GeminiChatResponse{newGeminiStreamResponse(nil, s.usage, info)}, nil + } + s.finalized = true + s.usageEmitted = s.usage != nil + return []*dto.GeminiChatResponse{newGeminiStreamResponse(candidates, s.usage, info)}, nil +} + +func (s *ChatToGeminiStreamState) Usage() *dto.Usage { + if s == nil || s.usage == nil { + return nil + } + return UsageFromChatUsage(s.usage) +} + +func (s *ChatToGeminiStreamState) SetUsage(usage *dto.Usage) { + if s == nil || usage == nil { + return + } + s.usage = UsageFromChatUsage(usage) +} + +func (s *ChatToGeminiStreamState) StreamUsage() *dto.Usage { + return s.Usage() +} + +func (s *ChatToGeminiStreamState) SetStreamUsage(usage *dto.Usage) { + s.SetUsage(usage) +} + +func (s *ChatToGeminiStreamState) appendToolCallDelta(choiceIndex int, toolCall dto.ToolCallResponse) error { + toolIndex := 0 + if toolCall.Index != nil { + toolIndex = *toolCall.Index + } + if toolIndex < 0 { + return fmt.Errorf("OpenAI chat choice %d has negative tool-call index %d", choiceIndex, toolIndex) + } + key := chatToGeminiStreamToolKey{ChoiceIndex: choiceIndex, ToolIndex: toolIndex} + incomingID := strings.TrimSpace(toolCall.ID) + var tool *chatToGeminiStreamTool + if incomingID != "" { + tool = s.toolByID[chatToGeminiStreamToolIDKey{ChoiceIndex: choiceIndex, ID: incomingID}] + } + if tool == nil { + tool = s.toolByIndex[key] + } + if tool != nil && incomingID != "" && tool.ID != "" && tool.ID != incomingID { + tool = nil + } + if tool == nil { + tool = &chatToGeminiStreamTool{} + s.toolsByChoice[choiceIndex] = append(s.toolsByChoice[choiceIndex], tool) + } + s.toolByIndex[key] = tool + // Compatibility gateways may reset a source index for the next occurrence. + // Once identity changes, keep the new occurrence active for later metadata-free deltas. + if tool.Emitted { + return fmt.Errorf("OpenAI chat choice %d tool-call index %d received data after completion", choiceIndex, toolIndex) + } + + if incomingID != "" { + if tool.ID != "" && tool.ID != incomingID { + return fmt.Errorf("OpenAI chat choice %d tool-call index %d changed id from %q to %q", choiceIndex, toolIndex, tool.ID, incomingID) + } + tool.ID = incomingID + s.toolByID[chatToGeminiStreamToolIDKey{ChoiceIndex: choiceIndex, ID: incomingID}] = tool + } + incomingName := strings.TrimSpace(toolCall.Function.Name) + if incomingName != "" { + if tool.Name != "" && tool.Name != incomingName { + return fmt.Errorf("OpenAI chat choice %d tool-call index %d changed name from %q to %q", choiceIndex, toolIndex, tool.Name, incomingName) + } + tool.Name = incomingName + } + tool.Arguments.WriteString(toolCall.Function.Arguments) + return nil +} + +func (s *ChatToGeminiStreamState) finishChoice(choiceIndex int) ([]dto.GeminiPart, error) { + tools := s.toolsByChoice[choiceIndex] + pending := make([]*chatToGeminiStreamTool, 0, len(tools)) + for _, tool := range tools { + if !tool.Emitted { + pending = append(pending, tool) + } + } + + parts := make([]dto.GeminiPart, 0, len(pending)) + for _, tool := range pending { + if tool.Name == "" { + return nil, fmt.Errorf("OpenAI chat choice %d has a tool call without a function name", choiceIndex) + } + parts = append(parts, dto.GeminiPart{FunctionCall: &dto.FunctionCall{ + ID: tool.ID, + FunctionName: tool.Name, + Arguments: geminiFunctionArguments(tool.Arguments.String()), + }}) + } + for _, tool := range pending { + tool.Emitted = true + } + return parts, nil +} + +func newGeminiStreamResponse(candidates []dto.GeminiChatCandidate, usage *dto.Usage, info convmeta.Meta) *dto.GeminiChatResponse { + if candidates == nil { + candidates = make([]dto.GeminiChatCandidate, 0) + } + estimatePromptTokens := 0 + if info != nil { + estimatePromptTokens = info.GetEstimatePromptTokens() + } + response := &dto.GeminiChatResponse{ + Candidates: candidates, + HasUsageMetadata: true, + UsageMetadata: dto.GeminiUsageMetadata{ + PromptTokenCount: estimatePromptTokens, + TotalTokenCount: estimatePromptTokens, + }, + } + if usage == nil { + return response + } + response.UsageMetadata.PromptTokenCount = usage.PromptTokens + response.UsageMetadata.CandidatesTokenCount = usage.CompletionTokens + response.UsageMetadata.TotalTokenCount = usage.TotalTokens + response.UsageMetadata.BillingUsage = openAIBillingUsageFromUsage(usage) + if metadata, ok := geminiBillingMetadataFromOpenAIUsage(usage); ok { + response.UsageMetadata = metadata + } + return response +} + +func geminiFunctionArguments(raw string) map[string]interface{} { + if strings.TrimSpace(raw) == "" || strings.TrimSpace(raw) == "null" { + return map[string]interface{}{} + } + var args map[string]interface{} + if err := kitutil.Unmarshal([]byte(raw), &args); err == nil && args != nil { + return args + } + // Preserve historically accepted malformed/non-object input without + // emitting a non-object Gemini args value. + return map[string]interface{}{"arguments": raw} +} + +func geminiFinishReason(finishReason string) string { + switch strings.TrimSpace(finishReason) { + case "length": + return "MAX_TOKENS" + case "content_filter": + return "SAFETY" + default: + return "STOP" + } +} + func geminiBillingMetadataFromOpenAIUsage(usage *dto.Usage) (dto.GeminiUsageMetadata, bool) { if usage == nil || usage.BillingUsage == nil || usage.BillingUsage.GeminiUsageMetadata == nil { return dto.GeminiUsageMetadata{}, false @@ -212,19 +537,22 @@ func geminiBillingMetadataFromOpenAIUsage(usage *dto.Usage) (dto.GeminiUsageMeta if billingUsage == nil || billingUsage.GeminiUsageMetadata == nil { return dto.GeminiUsageMetadata{}, false } - return *billingUsage.GeminiUsageMetadata, true + metadata := *billingUsage.GeminiUsageMetadata + // Restore the sidecar marker on the restored native payload so the next + // hop keeps settling on the original dialect (including Estimated). + metadata.BillingUsage = dto.CloneBillingUsage(usage.BillingUsage) + return metadata, true } func openAIBillingUsageFromUsage(usage *dto.Usage) *dto.BillingUsage { if usage == nil { return nil } - if existingBillingUsage := dto.CloneBillingUsage(usage.BillingUsage); existingBillingUsage != nil && existingBillingUsage.OpenAIUsage != nil { - if existingBillingUsage.Source == dto.BillingUsageSourceOAIChat || - existingBillingUsage.Source == dto.BillingUsageSourceOAIResponses || - existingBillingUsage.Semantic == dto.BillingUsageSemanticOpenAI { - return existingBillingUsage - } + // An existing sidecar snapshots the original provider usage; carry it + // across this bridge unchanged regardless of its dialect. Only synthesize + // an OpenAI snapshot when no sidecar exists yet. + if existingBillingUsage := dto.CloneBillingUsage(usage.BillingUsage); existingBillingUsage != nil { + return existingBillingUsage } return dto.NewOpenAIChatBillingUsage(usage) } diff --git a/relaykit/relayconvert/internal/oai_chat/to_oai_responses_req.go b/relaykit/relayconvert/internal/oai_chat/to_oai_responses_req.go index 104dfe8baa..6bb03a1d5e 100644 --- a/relaykit/relayconvert/internal/oai_chat/to_oai_responses_req.go +++ b/relaykit/relayconvert/internal/oai_chat/to_oai_responses_req.go @@ -8,6 +8,7 @@ import ( "github.com/QuantumNous/new-api/relaykit/dto" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" "github.com/samber/lo" ) @@ -358,9 +359,8 @@ func ChatCompletionsRequestToResponsesRequest(req *dto.GeneralOpenAIRequest) (*d textRaw := convertChatResponseFormatToResponsesText(req.ResponseFormat) maxOutputTokens := lo.FromPtrOr(req.MaxTokens, uint(0)) - maxCompletionTokens := lo.FromPtrOr(req.MaxCompletionTokens, uint(0)) - if maxCompletionTokens > maxOutputTokens { - maxOutputTokens = maxCompletionTokens + if req.MaxCompletionTokens != nil { + maxOutputTokens = *req.MaxCompletionTokens } // OpenAI Responses API rejects max_output_tokens < 16 when explicitly provided. //if maxOutputTokens > 0 && maxOutputTokens < 16 { @@ -412,11 +412,12 @@ func ChatCompletionsRequestToResponsesRequest(req *dto.GeneralOpenAIRequest) (*d out.MaxOutputTokens = lo.ToPtr(maxOutputTokens) } - if req.ReasoningEffort != "" { - out.Reasoning = &dto.Reasoning{ - Effort: req.ReasoningEffort, - Summary: "detailed", - } + reasoningIntent, err := reasoning.FromOpenAIChat(req) + if err != nil { + return nil, reasoning.AsClientError(err) + } + if err := reasoning.ApplyToOpenAIResponses(out, reasoningIntent); err != nil { + return nil, reasoning.AsClientError(err) } return out, nil diff --git a/relaykit/relayconvert/internal/oai_chat/to_oai_responses_resp.go b/relaykit/relayconvert/internal/oai_chat/to_oai_responses_resp.go index e732819d45..5fc35dd3ae 100644 --- a/relaykit/relayconvert/internal/oai_chat/to_oai_responses_resp.go +++ b/relaykit/relayconvert/internal/oai_chat/to_oai_responses_resp.go @@ -14,21 +14,22 @@ const ( chatFinishReasonLength = "length" chatFinishReasonContentFilter = "content_filter" - responsesEventCreated = "response.created" - responsesEventCompleted = "response.completed" - responsesEventIncomplete = "response.incomplete" - responsesEventOutputTextDelta = "response.output_text.delta" - responsesEventOutputItemAdded = "response.output_item.added" - responsesEventOutputItemDone = "response.output_item.done" - responsesEventFunctionArgsDelta = "response.function_call_arguments.delta" - responsesEventFunctionArgsDone = "response.function_call_arguments.done" - responsesEventReasoningSummaryDelta = "response.reasoning_summary_text.delta" - responsesEventReasoningSummaryDone = "response.reasoning_summary_text.done" - responsesOutputTypeFunctionCall = "function_call" - responsesOutputTypeMessage = "message" - responsesOutputTypeReasoning = "reasoning" - responsesIncompleteReasonContentFilter = "content_filter" - responsesIncompleteReasonMaxTokens = "max_output_tokens" + responsesEventCreated = "response.created" + responsesEventCompleted = "response.completed" + responsesEventIncomplete = "response.incomplete" + responsesEventOutputTextDelta = "response.output_text.delta" + responsesEventOutputTextAnnotationAdded = "response.output_text.annotation.added" + responsesEventOutputItemAdded = "response.output_item.added" + responsesEventOutputItemDone = "response.output_item.done" + responsesEventFunctionArgsDelta = "response.function_call_arguments.delta" + responsesEventFunctionArgsDone = "response.function_call_arguments.done" + responsesEventReasoningSummaryDelta = "response.reasoning_summary_text.delta" + responsesEventReasoningSummaryDone = "response.reasoning_summary_text.done" + responsesOutputTypeFunctionCall = "function_call" + responsesOutputTypeMessage = "message" + responsesOutputTypeReasoning = "reasoning" + responsesIncompleteReasonContentFilter = "content_filter" + responsesIncompleteReasonMaxTokens = "max_output_tokens" ) func ChatCompletionsResponseToResponsesResponse(resp *dto.OpenAITextResponse, id string) (*dto.OpenAIResponsesResponse, *dto.Usage, error) { @@ -57,7 +58,24 @@ func ChatCompletionsResponseToResponsesResponse(resp *dto.OpenAITextResponse, id out.IncompleteDetails = details } + if reasoning := choice.Message.GetReasoningContent(); reasoning != "" { + out.Output = append(out.Output, dto.ResponsesOutput{ + Type: responsesOutputTypeReasoning, + ID: fmt.Sprintf("%s_reasoning_0", id), + Status: responseOutputStatus(out), + Summary: []dto.ResponsesReasoningSummaryPart{ + { + Type: "summary_text", + Text: reasoning, + }, + }, + }) + } if text := choice.Message.StringContent(); text != "" { + annotations, err := chatAnnotationsToResponses(choice.Message.Annotations) + if err != nil { + return nil, nil, err + } out.Output = append(out.Output, dto.ResponsesOutput{ Type: responsesOutputTypeMessage, ID: fmt.Sprintf("%s_msg_0", id), @@ -67,20 +85,7 @@ func ChatCompletionsResponseToResponsesResponse(resp *dto.OpenAITextResponse, id { Type: "output_text", Text: text, - Annotations: []interface{}{}, - }, - }, - }) - } - if reasoning := choice.Message.GetReasoningContent(); reasoning != "" { - out.Output = append(out.Output, dto.ResponsesOutput{ - Type: responsesOutputTypeReasoning, - ID: fmt.Sprintf("%s_reasoning_0", id), - Status: responseOutputStatus(out), - Content: []dto.ResponsesOutputContent{ - { - Type: "summary_text", - Text: reasoning, + Annotations: annotations, }, }, }) @@ -97,6 +102,35 @@ func ChatCompletionsResponseToResponsesResponse(resp *dto.OpenAITextResponse, id return out, usage, nil } +func chatAnnotationsToResponses(raw []byte) ([]interface{}, error) { + if len(raw) == 0 { + return []interface{}{}, nil + } + var annotations []map[string]any + if err := kitutil.Unmarshal(raw, &annotations); err != nil { + return nil, fmt.Errorf("invalid Chat annotations: %w", err) + } + converted := make([]interface{}, 0, len(annotations)) + for _, annotation := range annotations { + if strings.TrimSpace(kitutil.Interface2String(annotation["type"])) != "url_citation" { + converted = append(converted, annotation) + continue + } + citation, ok := annotation["url_citation"].(map[string]any) + if !ok { + converted = append(converted, annotation) + continue + } + flattened := make(map[string]any, len(citation)+1) + flattened["type"] = "url_citation" + for key, value := range citation { + flattened[key] = value + } + converted = append(converted, flattened) + } + return converted, nil +} + func ResponsesStatusFromChatFinishReason(finishReason string) (string, *dto.IncompleteDetails) { switch strings.TrimSpace(finishReason) { case chatFinishReasonLength: @@ -230,3 +264,7 @@ func responsesStreamEvent(eventType string, payload dto.ResponsesStreamResponse) func intPtr(v int) *int { return &v } + +func stringPtr(v string) *string { + return &v +} diff --git a/relaykit/relayconvert/internal/oai_chat/to_oai_responses_resp_test.go b/relaykit/relayconvert/internal/oai_chat/to_oai_responses_resp_test.go index a6d752c981..34b6a5d9d9 100644 --- a/relaykit/relayconvert/internal/oai_chat/to_oai_responses_resp_test.go +++ b/relaykit/relayconvert/internal/oai_chat/to_oai_responses_resp_test.go @@ -41,6 +41,27 @@ func TestChatCompletionsResponseToResponsesPreservesTextToolCallsAndUsage(t *tes assert.Equal(t, `"{\"q\":\"x\"}"`, string(resp.Output[1].Arguments)) } +func TestChatCompletionsResponseToResponsesEmitsReasoningSummaryBeforeText(t *testing.T) { + message := dto.Message{Role: "assistant", Content: "final answer"} + message.ReasoningContent = lo.ToPtr("thinking summary") + resp, _, err := ChatCompletionsResponseToResponsesResponse(&dto.OpenAITextResponse{ + Id: "chatcmpl_1", + Model: "gpt-test", + Choices: []dto.OpenAITextResponseChoice{ + {Message: message, FinishReason: "stop"}, + }, + }, "resp_1") + require.NoError(t, err) + + require.Len(t, resp.Output, 2) + assert.Equal(t, responsesOutputTypeReasoning, resp.Output[0].Type) + require.Len(t, resp.Output[0].Summary, 1) + assert.Equal(t, "thinking summary", resp.Output[0].Summary[0].Text) + assert.Empty(t, resp.Output[0].Content) + assert.Equal(t, responsesOutputTypeMessage, resp.Output[1].Type) + assert.Equal(t, "final answer", resp.Output[1].Content[0].Text) +} + func TestChatCompletionsResponseToResponsesMapsIncompleteFinishReasons(t *testing.T) { tests := []struct { name string diff --git a/relaykit/relayconvert/internal/oai_chat/to_oai_responses_stream_resp.go b/relaykit/relayconvert/internal/oai_chat/to_oai_responses_stream_resp.go index 301afc76d9..da97773594 100644 --- a/relaykit/relayconvert/internal/oai_chat/to_oai_responses_stream_resp.go +++ b/relaykit/relayconvert/internal/oai_chat/to_oai_responses_stream_resp.go @@ -1,12 +1,14 @@ package oaichat import ( + "encoding/json" "fmt" "sort" "strings" "time" "github.com/QuantumNous/new-api/relaykit/dto" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" ) type ChatToResponsesStreamEvent struct { @@ -20,27 +22,35 @@ type ChatToResponsesStreamState struct { Created int64 Usage *dto.Usage - status string - incompleteDetails *dto.IncompleteDetails - sentCreated bool - textOutputIndex int - textStarted bool - textDone bool - reasoningIndex int - reasoningStarted bool - reasoningDone bool - finalized bool - nextOutputIndex int - toolsByIndex map[int]*chatToResponsesStreamTool - outputOrder []chatToResponsesOutputRef - text strings.Builder - reasoning strings.Builder + // EmitSequenceNumber enables the required sequence_number field for current + // Responses API SSE consumers while preserving the legacy relaykit default. + EmitSequenceNumber bool + + status string + incompleteDetails *dto.IncompleteDetails + sentCreated bool + textOutputIndex int + textStarted bool + textDone bool + reasoningIndex int + reasoningStarted bool + reasoningDone bool + finalized bool + nextSequenceNumber int + nextOutputIndex int + toolsByIndex map[int]*chatToResponsesStreamTool + hostedByID map[string]*chatToResponsesHostedTool + outputOrder []chatToResponsesOutputRef + text strings.Builder + annotations []interface{} + reasoning strings.Builder } type chatToResponsesStreamTool struct { ChatIndex int OutputIndex int - ID string + ItemID string + CallID string Name string Arguments strings.Builder Done bool @@ -49,6 +59,34 @@ type chatToResponsesStreamTool struct { type chatToResponsesOutputRef struct { Kind string ToolIndex int + HostedID string +} + +// HostedToolStreamStart describes a provider-hosted tool call that is already +// being executed upstream. It is intentionally separate from function calls: +// hosted calls have their own Responses lifecycle and result fields. +type HostedToolStreamStart struct { + Type string + ID string + Name string + Action []byte + Caller []byte + ServerLabel string +} + +// HostedToolStreamResult completes a previously started hosted tool call. +type HostedToolStreamResult struct { + Type string + ID string + Result []byte + ErrorCode string + IsError bool +} + +type chatToResponsesHostedTool struct { + OutputIndex int + Output dto.ResponsesOutput + Done bool } func NewChatToResponsesStreamState(id string, model string) *ChatToResponsesStreamState { @@ -61,9 +99,213 @@ func NewChatToResponsesStreamState(id string, model string) *ChatToResponsesStre textOutputIndex: -1, reasoningIndex: -1, toolsByIndex: make(map[int]*chatToResponsesStreamTool), + hostedByID: make(map[string]*chatToResponsesHostedTool), } } +func (s *ChatToResponsesStreamState) StreamUsage() *dto.Usage { + if s == nil { + return nil + } + return s.Usage +} + +func (s *ChatToResponsesStreamState) SetStreamUsage(usage *dto.Usage) { + if s != nil && usage != nil { + s.Usage = UsageFromChatUsage(usage) + } +} + +func (s *ChatToResponsesStreamState) StartHostedTool(start HostedToolStreamStart) ([]ChatToResponsesStreamEvent, error) { + if s == nil { + return nil, fmt.Errorf("Chat-to-Responses stream state is required") + } + start.ID = strings.TrimSpace(start.ID) + if start.ID == "" { + return nil, fmt.Errorf("hosted-tool stream call is missing an id") + } + if _, exists := s.hostedByID[start.ID]; exists { + return nil, fmt.Errorf("duplicate hosted-tool stream call id %q", start.ID) + } + if hostedEventPrefix(start.Type) == "" { + return nil, fmt.Errorf("unsupported Responses hosted-tool output type %q", start.Type) + } + caller := strings.TrimSpace(string(start.Caller)) + if caller != "" && caller != "null" { + return nil, fmt.Errorf("Responses %s cannot preserve Claude hosted-tool caller provenance", start.Type) + } + + tool := &chatToResponsesHostedTool{ + Output: dto.ResponsesOutput{ + Type: start.Type, + ID: start.ID, + Status: "in_progress", + }, + } + switch start.Type { + case "web_search_call": + action, err := dto.NormalizeResponsesWebSearchAction(start.Action) + if err != nil { + return nil, err + } + tool.Output.Action = action + case "code_interpreter_call": + return nil, fmt.Errorf("cannot map provider code execution to Responses code_interpreter_call without a container_id") + case "mcp_call": + if strings.TrimSpace(start.Name) == "" || strings.TrimSpace(start.ServerLabel) == "" { + return nil, fmt.Errorf("Responses MCP call requires name and server_label") + } + arguments, err := hostedJSONString(start.Action) + if err != nil { + return nil, fmt.Errorf("encode Responses MCP arguments: %w", err) + } + tool.Output.Name = start.Name + tool.Output.ServerLabel = start.ServerLabel + tool.Output.Arguments = arguments + } + outputIndex := s.nextHostedIndex(start.ID) + tool.OutputIndex = outputIndex + s.hostedByID[start.ID] = tool + + events := s.ensureCreated() + addedItem := cloneHostedOutput(&tool.Output) + if start.Type == "mcp_call" { + addedItem.Arguments = json.RawMessage(`""`) + } + events = append(events, + s.event(responsesEventOutputItemAdded, dto.ResponsesStreamResponse{ + OutputIndex: intPtr(outputIndex), + ItemID: start.ID, + Item: addedItem, + }), + s.event(hostedEventPrefix(start.Type)+".in_progress", dto.ResponsesStreamResponse{ + OutputIndex: intPtr(outputIndex), + ItemID: start.ID, + }), + ) + if start.Type == "web_search_call" { + events = append(events, s.event(hostedEventPrefix(start.Type)+".searching", dto.ResponsesStreamResponse{ + OutputIndex: intPtr(outputIndex), + ItemID: start.ID, + })) + } + if start.Type == "mcp_call" { + arguments := dto.ResponsesArgumentsString(tool.Output.Arguments) + events = append(events, + s.event("response.mcp_call_arguments.delta", dto.ResponsesStreamResponse{ + OutputIndex: intPtr(outputIndex), + ItemID: start.ID, + Delta: arguments, + }), + s.event("response.mcp_call_arguments.done", dto.ResponsesStreamResponse{ + OutputIndex: intPtr(outputIndex), + ItemID: start.ID, + Arguments: kitutil.GetPointer(arguments), + }), + ) + } + return events, nil +} + +func (s *ChatToResponsesStreamState) CompleteHostedTool(result HostedToolStreamResult) ([]ChatToResponsesStreamEvent, error) { + if s == nil { + return nil, fmt.Errorf("Chat-to-Responses stream state is required") + } + result.ID = strings.TrimSpace(result.ID) + tool := s.hostedByID[result.ID] + if tool == nil { + return nil, fmt.Errorf("hosted-tool result references unknown call %q", result.ID) + } + if tool.Done { + return nil, fmt.Errorf("duplicate hosted-tool result for call %q", result.ID) + } + if result.Type != "" && result.Type != tool.Output.Type { + return nil, fmt.Errorf("hosted-tool result type %q does not match call type %q", result.Type, tool.Output.Type) + } + + failed := result.IsError || strings.TrimSpace(result.ErrorCode) != "" + tool.Output.Status = "completed" + switch tool.Output.Type { + case "web_search_call": + // Responses exposes only the action and lifecycle status on a + // web_search_call. Claude's opaque result payload cannot be emitted + // as a top-level `results` field. + case "code_interpreter_call": + return nil, fmt.Errorf("Responses code_interpreter_call is not supported without a container_id") + case "mcp_call": + output, err := hostedResultString(result.Result) + if err != nil { + return nil, fmt.Errorf("encode Responses MCP output: %w", err) + } + tool.Output.Output = output + } + if failed { + tool.Output.Status = "failed" + errorValue := result.ErrorCode + if errorValue == "" { + errorValue = "hosted tool execution failed" + } + if tool.Output.Type == "mcp_call" { + encoded, err := kitutil.Marshal(errorValue) + if err != nil { + return nil, fmt.Errorf("marshal hosted-tool error: %w", err) + } + tool.Output.ItemError = encoded + tool.Output.Output = nil + } + } + tool.Done = true + + events := make([]ChatToResponsesStreamEvent, 0, 2) + if eventType := hostedTerminalEvent(tool.Output.Type, failed); eventType != "" { + events = append(events, s.event(eventType, dto.ResponsesStreamResponse{ + OutputIndex: intPtr(tool.OutputIndex), + ItemID: result.ID, + })) + } + events = append(events, s.event(responsesEventOutputItemDone, dto.ResponsesStreamResponse{ + OutputIndex: intPtr(tool.OutputIndex), + ItemID: result.ID, + Item: cloneHostedOutput(&tool.Output), + })) + return events, nil +} + +// Fail emits a terminal Responses error using the same event allocator as the +// rest of the stream, so callers never have to append a JSON HTTP error to an +// already-started SSE response. +func (s *ChatToResponsesStreamState) Fail(code string, message string, param string) []ChatToResponsesStreamEvent { + if s == nil || s.finalized { + return nil + } + code = strings.TrimSpace(code) + if code == "" { + code = "server_error" + } + message = strings.TrimSpace(message) + if message == "" { + message = "upstream response stream failed" + } + s.status = "failed" + events := s.ensureCreated() + events = append(events, s.doneDeltaEvents()...) + s.finalized = true + events = append(events, s.event("error", dto.ResponsesStreamResponse{ + Code: code, + Message: message, + Param: param, + })) + response := s.finalResponse() + response.Error = map[string]any{ + "code": code, + "message": message, + } + events = append(events, s.event("response.failed", dto.ResponsesStreamResponse{ + Response: response, + })) + return events +} + func ChatCompletionsStreamChunkToResponsesEvents(chunk *dto.ChatCompletionsStreamResponse, state *ChatToResponsesStreamState) ([]ChatToResponsesStreamEvent, error) { if chunk == nil || state == nil { return nil, nil @@ -81,14 +323,7 @@ func ChatCompletionsStreamChunkToResponsesEvents(chunk *dto.ChatCompletionsStrea state.Usage = UsageFromChatUsage(chunk.Usage) } - events := make([]ChatToResponsesStreamEvent, 0) - if !state.sentCreated { - state.sentCreated = true - events = append(events, responsesStreamEvent(responsesEventCreated, dto.ResponsesStreamResponse{ - Type: responsesEventCreated, - Response: state.createdResponse(), - })) - } + events := state.ensureCreated() for _, choice := range chunk.Choices { if choice.Delta.GetReasoningContent() != "" { events = append(events, state.appendReasoningDelta(choice.Delta.GetReasoningContent())...) @@ -96,6 +331,13 @@ func ChatCompletionsStreamChunkToResponsesEvents(chunk *dto.ChatCompletionsStrea if choice.Delta.GetContentString() != "" { events = append(events, state.appendTextDelta(choice.Delta.GetContentString())...) } + if len(choice.Delta.Annotations) > 0 { + annotationEvents, err := state.appendAnnotationDelta(choice.Delta.Annotations) + if err != nil { + return nil, err + } + events = append(events, annotationEvents...) + } for _, toolCall := range choice.Delta.ToolCalls { toolEvents, err := state.appendToolCallDelta(toolCall) if err != nil { @@ -111,6 +353,17 @@ func ChatCompletionsStreamChunkToResponsesEvents(chunk *dto.ChatCompletionsStrea return events, nil } +func (s *ChatToResponsesStreamState) ensureCreated() []ChatToResponsesStreamEvent { + if s.sentCreated { + return nil + } + s.sentCreated = true + return []ChatToResponsesStreamEvent{s.event(responsesEventCreated, dto.ResponsesStreamResponse{ + Type: responsesEventCreated, + Response: s.createdResponse(), + })} +} + func FinalizeChatCompletionsStreamToResponses(state *ChatToResponsesStreamState) []ChatToResponsesStreamEvent { if state == nil || state.finalized { return nil @@ -122,7 +375,7 @@ func FinalizeChatCompletionsStreamToResponses(state *ChatToResponsesStreamState) if state.status == "incomplete" { eventType = responsesEventIncomplete } - events = append(events, responsesStreamEvent(eventType, dto.ResponsesStreamResponse{ + events = append(events, state.event(eventType, dto.ResponsesStreamResponse{ Type: eventType, Response: resp, })) @@ -137,24 +390,9 @@ func (s *ChatToResponsesStreamState) UsageText() string { } func (s *ChatToResponsesStreamState) appendTextDelta(delta string) []ChatToResponsesStreamEvent { - events := make([]ChatToResponsesStreamEvent, 0, 2) - if !s.textStarted { - s.textStarted = true - s.textOutputIndex = s.nextIndex("message", -1) - events = append(events, responsesStreamEvent(responsesEventOutputItemAdded, dto.ResponsesStreamResponse{ - Type: responsesEventOutputItemAdded, - OutputIndex: intPtr(s.textOutputIndex), - Item: &dto.ResponsesOutput{ - Type: responsesOutputTypeMessage, - ID: s.messageID(), - Status: "in_progress", - Role: "assistant", - Content: []dto.ResponsesOutputContent{}, - }, - })) - } + events := s.startText() s.text.WriteString(delta) - events = append(events, responsesStreamEvent(responsesEventOutputTextDelta, dto.ResponsesStreamResponse{ + events = append(events, s.event(responsesEventOutputTextDelta, dto.ResponsesStreamResponse{ Type: responsesEventOutputTextDelta, OutputIndex: intPtr(s.textOutputIndex), ContentIndex: intPtr(0), @@ -164,24 +402,68 @@ func (s *ChatToResponsesStreamState) appendTextDelta(delta string) []ChatToRespo return events } +func (s *ChatToResponsesStreamState) startText() []ChatToResponsesStreamEvent { + if !s.textStarted { + s.textStarted = true + s.textOutputIndex = s.nextIndex("message", -1) + return []ChatToResponsesStreamEvent{s.event(responsesEventOutputItemAdded, dto.ResponsesStreamResponse{ + Type: responsesEventOutputItemAdded, + OutputIndex: intPtr(s.textOutputIndex), + Item: &dto.ResponsesOutput{ + Type: responsesOutputTypeMessage, + ID: s.messageID(), + Status: "in_progress", + Role: "assistant", + Content: []dto.ResponsesOutputContent{}, + }, + })} + } + return nil +} + +func (s *ChatToResponsesStreamState) appendAnnotationDelta(raw []byte) ([]ChatToResponsesStreamEvent, error) { + annotations, err := chatAnnotationsToResponses(raw) + if err != nil { + return nil, err + } + events := s.startText() + for _, annotation := range annotations { + annotationJSON, err := kitutil.Marshal(annotation) + if err != nil { + return nil, fmt.Errorf("marshal Responses annotation: %w", err) + } + annotationIndex := len(s.annotations) + s.annotations = append(s.annotations, annotation) + events = append(events, s.event(responsesEventOutputTextAnnotationAdded, dto.ResponsesStreamResponse{ + Type: responsesEventOutputTextAnnotationAdded, + OutputIndex: intPtr(s.textOutputIndex), + ContentIndex: intPtr(0), + AnnotationIndex: intPtr(annotationIndex), + Annotation: annotationJSON, + ItemID: s.messageID(), + })) + } + return events, nil +} + func (s *ChatToResponsesStreamState) appendReasoningDelta(delta string) []ChatToResponsesStreamEvent { events := make([]ChatToResponsesStreamEvent, 0, 2) if !s.reasoningStarted { s.reasoningStarted = true s.reasoningIndex = s.nextIndex("reasoning", -1) - events = append(events, responsesStreamEvent(responsesEventOutputItemAdded, dto.ResponsesStreamResponse{ + events = append(events, s.event(responsesEventOutputItemAdded, dto.ResponsesStreamResponse{ Type: responsesEventOutputItemAdded, OutputIndex: intPtr(s.reasoningIndex), Item: &dto.ResponsesOutput{ Type: responsesOutputTypeReasoning, ID: s.reasoningID(), Status: "in_progress", - Content: []dto.ResponsesOutputContent{}, + Summary: []dto.ResponsesReasoningSummaryPart{}, }, })) } s.reasoning.WriteString(delta) - events = append(events, responsesStreamEvent(responsesEventReasoningSummaryDelta, dto.ResponsesStreamResponse{ + events = append(events, s.event(responsesEventReasoningSummaryDelta, dto.ResponsesStreamResponse{ Type: responsesEventReasoningSummaryDelta, OutputIndex: intPtr(s.reasoningIndex), SummaryIndex: intPtr(0), @@ -196,45 +478,57 @@ func (s *ChatToResponsesStreamState) appendToolCallDelta(toolCall dto.ToolCallRe if toolCall.Index != nil { chatIndex = *toolCall.Index } + incomingID := strings.TrimSpace(toolCall.ID) tool := s.toolsByIndex[chatIndex] events := make([]ChatToResponsesStreamEvent, 0, 2) if tool == nil { tool = &chatToResponsesStreamTool{ ChatIndex: chatIndex, OutputIndex: s.nextIndex("tool", chatIndex), - ID: strings.TrimSpace(toolCall.ID), + CallID: incomingID, Name: strings.TrimSpace(toolCall.Function.Name), } - if tool.ID == "" { - tool.ID = fmt.Sprintf("%s_call_%d", s.ID, chatIndex) + tool.ItemID = incomingID + if tool.ItemID == "" { + tool.ItemID = fmt.Sprintf("%s_call_%d", s.ID, chatIndex) } s.toolsByIndex[chatIndex] = tool - events = append(events, responsesStreamEvent(responsesEventOutputItemAdded, dto.ResponsesStreamResponse{ + events = append(events, s.event(responsesEventOutputItemAdded, dto.ResponsesStreamResponse{ Type: responsesEventOutputItemAdded, OutputIndex: intPtr(tool.OutputIndex), - ItemID: tool.ID, + ItemID: tool.ItemID, Item: &dto.ResponsesOutput{ Type: responsesOutputTypeFunctionCall, - ID: tool.ID, + ID: tool.ItemID, Status: "in_progress", - CallId: tool.ID, + CallId: tool.callID(), Name: tool.Name, Arguments: []byte(`""`), }, })) } - if strings.TrimSpace(toolCall.ID) != "" { - tool.ID = strings.TrimSpace(toolCall.ID) + if tool.Done { + return nil, fmt.Errorf("tool-call stream index %d received data after completion", chatIndex) } - if strings.TrimSpace(toolCall.Function.Name) != "" { - tool.Name = strings.TrimSpace(toolCall.Function.Name) + if incomingID != "" { + if tool.CallID != "" && tool.CallID != incomingID { + return nil, fmt.Errorf("tool-call stream index %d changed id from %q to %q", chatIndex, tool.CallID, incomingID) + } + tool.CallID = incomingID + } + incomingName := strings.TrimSpace(toolCall.Function.Name) + if incomingName != "" { + if tool.Name != "" && tool.Name != incomingName { + return nil, fmt.Errorf("tool-call stream index %d changed name from %q to %q", chatIndex, tool.Name, incomingName) + } + tool.Name = incomingName } if toolCall.Function.Arguments != "" { tool.Arguments.WriteString(toolCall.Function.Arguments) - events = append(events, responsesStreamEvent(responsesEventFunctionArgsDelta, dto.ResponsesStreamResponse{ + events = append(events, s.event(responsesEventFunctionArgsDelta, dto.ResponsesStreamResponse{ Type: responsesEventFunctionArgsDelta, OutputIndex: intPtr(tool.OutputIndex), - ItemID: tool.ID, + ItemID: tool.ItemID, Delta: toolCall.Function.Arguments, })) } @@ -246,13 +540,17 @@ func (s *ChatToResponsesStreamState) doneDeltaEvents() []ChatToResponsesStreamEv status := s.outputStatus() if s.textStarted && !s.textDone { s.textDone = true - events = append(events, responsesStreamEvent("response.output_text.done", dto.ResponsesStreamResponse{ + textDone := dto.ResponsesStreamResponse{ Type: "response.output_text.done", OutputIndex: intPtr(s.textOutputIndex), ContentIndex: intPtr(0), ItemID: s.messageID(), - })) - events = append(events, responsesStreamEvent(responsesEventOutputItemDone, dto.ResponsesStreamResponse{ + } + if s.EmitSequenceNumber { + textDone.Text = kitutil.GetPointer(s.text.String()) + } + events = append(events, s.event("response.output_text.done", textDone)) + events = append(events, s.event(responsesEventOutputItemDone, dto.ResponsesStreamResponse{ Type: responsesEventOutputItemDone, OutputIndex: intPtr(s.textOutputIndex), Item: s.messageOutput(status), @@ -260,7 +558,7 @@ func (s *ChatToResponsesStreamState) doneDeltaEvents() []ChatToResponsesStreamEv } if s.reasoningStarted && !s.reasoningDone { s.reasoningDone = true - events = append(events, responsesStreamEvent(responsesEventReasoningSummaryDone, dto.ResponsesStreamResponse{ + reasoningDone := dto.ResponsesStreamResponse{ Type: responsesEventReasoningSummaryDone, OutputIndex: intPtr(s.reasoningIndex), SummaryIndex: intPtr(0), @@ -269,8 +567,13 @@ func (s *ChatToResponsesStreamState) doneDeltaEvents() []ChatToResponsesStreamEv Type: "summary_text", Text: s.reasoning.String(), }, - })) - events = append(events, responsesStreamEvent(responsesEventOutputItemDone, dto.ResponsesStreamResponse{ + } + if s.EmitSequenceNumber { + reasoningDone.Text = kitutil.GetPointer(s.reasoning.String()) + reasoningDone.Part = nil + } + events = append(events, s.event(responsesEventReasoningSummaryDone, reasoningDone)) + events = append(events, s.event(responsesEventOutputItemDone, dto.ResponsesStreamResponse{ Type: responsesEventOutputItemDone, OutputIndex: intPtr(s.reasoningIndex), Item: s.reasoningOutput(status), @@ -281,17 +584,55 @@ func (s *ChatToResponsesStreamState) doneDeltaEvents() []ChatToResponsesStreamEv continue } tool.Done = true - events = append(events, responsesStreamEvent(responsesEventFunctionArgsDone, dto.ResponsesStreamResponse{ + argumentsDone := dto.ResponsesStreamResponse{ Type: responsesEventFunctionArgsDone, OutputIndex: intPtr(tool.OutputIndex), - ItemID: tool.ID, - })) - events = append(events, responsesStreamEvent(responsesEventOutputItemDone, dto.ResponsesStreamResponse{ + ItemID: tool.ItemID, + } + if s.EmitSequenceNumber { + argumentsDone.Arguments = kitutil.GetPointer(tool.Arguments.String()) + argumentsDone.Name = tool.Name + } + events = append(events, s.event(responsesEventFunctionArgsDone, argumentsDone)) + events = append(events, s.event(responsesEventOutputItemDone, dto.ResponsesStreamResponse{ Type: responsesEventOutputItemDone, OutputIndex: intPtr(tool.OutputIndex), Item: s.toolOutput(tool, status), })) } + for _, ref := range s.outputOrder { + if ref.Kind != "hosted" { + continue + } + tool := s.hostedByID[ref.HostedID] + if tool == nil || tool.Done { + continue + } + if s.status != "failed" { + s.status = "incomplete" + } + tool.Done = true + tool.Output.Status = "incomplete" + if s.status == "failed" { + tool.Output.Status = "failed" + errorValue, err := kitutil.Marshal("provider stream failed before hosted-tool result") + if err == nil && tool.Output.Type == "mcp_call" { + tool.Output.ItemError = errorValue + tool.Output.Output = nil + } + if eventType := hostedTerminalEvent(tool.Output.Type, true); eventType != "" { + events = append(events, s.event(eventType, dto.ResponsesStreamResponse{ + OutputIndex: intPtr(tool.OutputIndex), + ItemID: tool.Output.ID, + })) + } + } + events = append(events, s.event(responsesEventOutputItemDone, dto.ResponsesStreamResponse{ + OutputIndex: intPtr(tool.OutputIndex), + ItemID: tool.Output.ID, + Item: cloneHostedOutput(&tool.Output), + })) + } return events } @@ -315,6 +656,10 @@ func (s *ChatToResponsesStreamState) finalResponse() *dto.OpenAIResponsesRespons if tool := s.toolsByIndex[ref.ToolIndex]; tool != nil { output = append(output, *s.toolOutput(tool, status)) } + case "hosted": + if tool := s.hostedByID[ref.HostedID]; tool != nil { + output = append(output, *cloneHostedOutput(&tool.Output)) + } } } return &dto.OpenAIResponsesResponse{ @@ -347,6 +692,13 @@ func (s *ChatToResponsesStreamState) nextIndex(kind string, toolIndex int) int { return index } +func (s *ChatToResponsesStreamState) nextHostedIndex(id string) int { + index := s.nextOutputIndex + s.nextOutputIndex++ + s.outputOrder = append(s.outputOrder, chatToResponsesOutputRef{Kind: "hosted", HostedID: id}) + return index +} + func (s *ChatToResponsesStreamState) sortedTools() []*chatToResponsesStreamTool { indexes := make([]int, 0, len(s.toolsByIndex)) for index := range s.toolsByIndex { @@ -361,7 +713,7 @@ func (s *ChatToResponsesStreamState) sortedTools() []*chatToResponsesStreamTool } func (s *ChatToResponsesStreamState) outputStatus() string { - if s.status == "incomplete" { + if s.status == "incomplete" || s.status == "failed" { return "incomplete" } return "completed" @@ -376,6 +728,10 @@ func (s *ChatToResponsesStreamState) reasoningID() string { } func (s *ChatToResponsesStreamState) messageOutput(status string) *dto.ResponsesOutput { + annotations := s.annotations + if annotations == nil { + annotations = []interface{}{} + } return &dto.ResponsesOutput{ Type: responsesOutputTypeMessage, ID: s.messageID(), @@ -385,7 +741,7 @@ func (s *ChatToResponsesStreamState) messageOutput(status string) *dto.Responses { Type: "output_text", Text: s.text.String(), - Annotations: []interface{}{}, + Annotations: annotations, }, }, } @@ -396,7 +752,7 @@ func (s *ChatToResponsesStreamState) reasoningOutput(status string) *dto.Respons Type: responsesOutputTypeReasoning, ID: s.reasoningID(), Status: status, - Content: []dto.ResponsesOutputContent{ + Summary: []dto.ResponsesReasoningSummaryPart{ { Type: "summary_text", Text: s.reasoning.String(), @@ -408,10 +764,99 @@ func (s *ChatToResponsesStreamState) reasoningOutput(status string) *dto.Respons func (s *ChatToResponsesStreamState) toolOutput(tool *chatToResponsesStreamTool, status string) *dto.ResponsesOutput { return &dto.ResponsesOutput{ Type: responsesOutputTypeFunctionCall, - ID: tool.ID, + ID: tool.ItemID, Status: status, - CallId: tool.ID, + CallId: tool.callID(), Name: tool.Name, Arguments: chatArgumentsRawMessage(tool.Arguments.String()), } } + +func (t *chatToResponsesStreamTool) callID() string { + if t == nil { + return "" + } + if t.CallID == "" { + return t.ItemID + } + return t.CallID +} + +func hostedEventPrefix(outputType string) string { + switch outputType { + case "web_search_call": + return "response.web_search_call" + case "mcp_call": + return "response.mcp_call" + default: + return "" + } +} + +func hostedTerminalEvent(outputType string, failed bool) string { + prefix := hostedEventPrefix(outputType) + if prefix == "" { + return "" + } + if !failed { + return prefix + ".completed" + } + // OpenAI currently defines a dedicated failed lifecycle event for MCP. + // Web search and code interpreter surface failure on output_item.done. + if outputType == "mcp_call" { + return prefix + ".failed" + } + return "" +} + +func hostedJSONString(value []byte) (json.RawMessage, error) { + if len(value) == 0 { + return json.RawMessage(`""`), nil + } + if !json.Valid(value) { + return nil, fmt.Errorf("invalid JSON payload") + } + encoded, err := kitutil.Marshal(string(value)) + if err != nil { + return nil, err + } + return encoded, nil +} + +func hostedResultString(value []byte) (json.RawMessage, error) { + if len(value) == 0 { + return json.RawMessage(`""`), nil + } + if !json.Valid(value) { + return nil, fmt.Errorf("invalid JSON payload") + } + if kitutil.GetJsonType(value) == "string" { + return append(json.RawMessage(nil), value...), nil + } + return hostedJSONString(value) +} + +func cloneHostedOutput(output *dto.ResponsesOutput) *dto.ResponsesOutput { + if output == nil { + return nil + } + clone := *output + clone.Action = append([]byte(nil), output.Action...) + clone.Arguments = append([]byte(nil), output.Arguments...) + clone.Code = append([]byte(nil), output.Code...) + clone.Results = append([]byte(nil), output.Results...) + clone.Outputs = append([]byte(nil), output.Outputs...) + clone.Output = append([]byte(nil), output.Output...) + clone.ItemError = append([]byte(nil), output.ItemError...) + clone.Caller = append([]byte(nil), output.Caller...) + return &clone +} + +func (s *ChatToResponsesStreamState) event(eventType string, payload dto.ResponsesStreamResponse) ChatToResponsesStreamEvent { + if s.EmitSequenceNumber { + sequenceNumber := s.nextSequenceNumber + s.nextSequenceNumber++ + payload.SequenceNumber = &sequenceNumber + } + return responsesStreamEvent(eventType, payload) +} diff --git a/relaykit/relayconvert/internal/oai_responses/to_claude_messages_req.go b/relaykit/relayconvert/internal/oai_responses/to_claude_messages_req.go index 3695449f0a..a9116a0f21 100644 --- a/relaykit/relayconvert/internal/oai_responses/to_claude_messages_req.go +++ b/relaykit/relayconvert/internal/oai_responses/to_claude_messages_req.go @@ -1,15 +1,16 @@ package oairesponses import ( + "context" "fmt" "strings" - "context" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" relaymedia "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/media" sharedclaude "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/claude" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" ) func convertOpenAIResponsesRequestToClaudeMessages(c context.Context, info convmeta.Meta, request any) (any, error) { @@ -40,13 +41,6 @@ func OpenAIResponsesRequestToClaudeMessages(c context.Context, info convmeta.Met if req.MaxOutputTokens != nil && *req.MaxOutputTokens > 0 { claudeRequest.MaxTokens = kitutil.GetPointer(*req.MaxOutputTokens) } - if claudeRequest.MaxTokens == nil || *claudeRequest.MaxTokens == 0 { - if defaultMaxTokens, configured := convmeta.OptionsOf(info).Claude.DefaultMaxTokensFor(req.Model); configured { - value := uint(defaultMaxTokens) - claudeRequest.MaxTokens = &value - } - } - functions, err := RequestFunctionDeclarations(req.Tools) if err != nil { return nil, err @@ -62,7 +56,19 @@ func OpenAIResponsesRequestToClaudeMessages(c context.Context, info convmeta.Met if toolChoice != nil || RawJSONPresent(req.ParallelToolCalls) { claudeRequest.ToolChoice = sharedclaude.MapOpenAIToolChoice(toolChoice, ParallelToolCalls(req.ParallelToolCalls)) } - applyResponsesReasoningToClaude(req, claudeRequest) + sourceReasoning, err := reasoning.FromOpenAIResponses(req) + if err != nil { + return nil, reasoning.AsClientError(err) + } + if err := sharedclaude.ApplyReasoning(claudeRequest, info, sourceReasoning); err != nil { + return nil, reasoning.AsClientError(err) + } + if claudeRequest.MaxTokens == nil { + if defaultMaxTokens, configured := convmeta.OptionsOf(info).Claude.DefaultMaxTokensFor(claudeRequest.Model); configured { + value := uint(defaultMaxTokens) + claudeRequest.MaxTokens = &value + } + } systemMessages := make([]dto.ClaudeMediaMessage, 0) if RawJSONPresent(req.Instructions) { @@ -92,13 +98,21 @@ func OpenAIResponsesRequestToClaudeMessages(c context.Context, info convmeta.Met case ResponsesInputTypeFunctionCallOutput, ResponsesInputTypeCustomToolOutput: claudeRequest.Messages = appendClaudeToolResult(claudeRequest.Messages, responsesFunctionOutputItemToClaudeToolResult(item)) default: - role := responsesClaudeRole(item) + sourceRole := strings.TrimSpace(kitutil.Interface2String(item["role"])) + role := responsesClaudeRole(sourceRole) parts, err := responsesInputContentToClaudeMediaMessages(c, item["content"]) if err != nil { return nil, err } + if sourceRole == "" && len(parts) == 0 { + continue + } if role == "system" { - systemMessages = append(systemMessages, parts...) + for _, part := range parts { + if part.Type == "text" { + systemMessages = append(systemMessages, part) + } + } continue } if len(parts) == 0 { @@ -119,7 +133,9 @@ func OpenAIResponsesRequestToClaudeMessages(c context.Context, info convmeta.Met if len(systemMessages) > 0 { claudeRequest.System = systemMessages } - claudeRequest.Messages = ensureClaudeMessagesStartWithUser(claudeRequest.Messages) + if len(claudeRequest.Messages) > 0 || len(systemMessages) > 0 { + claudeRequest.Messages = ensureClaudeMessagesStartWithUser(claudeRequest.Messages) + } // Checked last so every injection path has had its chance to satisfy the // required field. if claudeRequest.MaxTokens == nil { @@ -140,27 +156,6 @@ func responsesFunctionDeclarationsToClaudeTools(functions []dto.FunctionRequest) return tools } -func applyResponsesReasoningToClaude(req *dto.OpenAIResponsesRequest, claudeRequest *dto.ClaudeRequest) { - effort := ReasoningEffort(req) - switch effort { - case "low": - claudeRequest.Thinking = &dto.Thinking{ - Type: "enabled", - BudgetTokens: kitutil.GetPointer(1280), - } - case "medium": - claudeRequest.Thinking = &dto.Thinking{ - Type: "enabled", - BudgetTokens: kitutil.GetPointer(2048), - } - case "high": - claudeRequest.Thinking = &dto.Thinking{ - Type: "enabled", - BudgetTokens: kitutil.GetPointer(4096), - } - } -} - func responsesInputContentToClaudeMediaMessages(c context.Context, content any) ([]dto.ClaudeMediaMessage, error) { contentParts, err := ContentParts(content) if err != nil { @@ -280,8 +275,8 @@ func claudeMessageContentParts(content any) []dto.ClaudeMediaMessage { } } -func responsesClaudeRole(item map[string]any) string { - switch strings.TrimSpace(kitutil.Interface2String(item["role"])) { +func responsesClaudeRole(role string) string { + switch role { case "assistant": return "assistant" case "system", "developer": @@ -292,7 +287,7 @@ func responsesClaudeRole(item map[string]any) string { } func ensureClaudeMessagesStartWithUser(messages []dto.ClaudeMessage) []dto.ClaudeMessage { - if len(messages) == 0 || messages[0].Role == "user" { + if len(messages) > 0 && messages[0].Role == "user" { return messages } return append([]dto.ClaudeMessage{ diff --git a/relaykit/relayconvert/internal/oai_responses/to_claude_messages_resp.go b/relaykit/relayconvert/internal/oai_responses/to_claude_messages_resp.go new file mode 100644 index 0000000000..51482ab188 --- /dev/null +++ b/relaykit/relayconvert/internal/oai_responses/to_claude_messages_resp.go @@ -0,0 +1,156 @@ +package oairesponses + +import ( + "encoding/json" + "errors" + "strings" + "unicode/utf8" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/reasonmap" + sharedclaude "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/claude" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" +) + +func ResponsesResponseToClaudeMessagesResponse(resp *dto.OpenAIResponsesResponse) (*dto.ClaudeResponse, *dto.Usage, error) { + if resp == nil { + return nil, nil, errors.New("response is nil") + } + + usage := UsageFromResponsesUsage(resp.Usage) + claudeResponse := &dto.ClaudeResponse{ + Id: resp.ID, + Type: "message", + Role: "assistant", + Model: resp.Model, + Usage: sharedclaude.UsageFromOpenAI(usage), + } + sawToolCall := false + for index := range resp.Output { + output := resp.Output[index] + if output.Type == responsesOutputTypeMessage && output.Role != "" && output.Role != "assistant" { + continue + } + switch output.Type { + case responsesOutputTypeReasoning: + if thinking := reasoningOutputText(&output); thinking != "" { + claudeResponse.Content = append(claudeResponse.Content, dto.ClaudeMediaMessage{ + Type: "thinking", + Thinking: kitutil.GetPointer(thinking), + }) + } + case responsesOutputTypeMessage: + for _, content := range output.Content { + if content.Type != "output_text" { + continue + } + block := dto.ClaudeMediaMessage{Type: "text", Text: kitutil.GetPointer(content.Text)} + if citations := responsesAnnotationsToClaude(content.Annotations, content.Text); len(citations) > 0 { + block.Citations, _ = kitutil.Marshal(citations) + } + claudeResponse.Content = append(claudeResponse.Content, block) + } + case responsesOutputTypeFunctionCall, responsesOutputTypeCustomToolCall: + sawToolCall = true + callID := strings.TrimSpace(output.CallId) + if callID == "" { + callID = strings.TrimSpace(output.ID) + } + claudeResponse.Content = append(claudeResponse.Content, dto.ClaudeMediaMessage{ + Type: "tool_use", + Id: callID, + Name: output.Name, + Input: responsesArgumentsToClaudeInput(output.ArgumentsString()), + }) + } + } + if len(claudeResponse.Content) == 0 { + claudeResponse.Content = []dto.ClaudeMediaMessage{{Type: "text", Text: kitutil.GetPointer("")}} + } + claudeResponse.StopReason = responsesClaudeStopReason(resp, sawToolCall) + return claudeResponse, usage, nil +} + +func responsesArgumentsToClaudeInput(arguments string) map[string]any { + input := make(map[string]any) + if strings.TrimSpace(arguments) == "" { + return input + } + if err := kitutil.Unmarshal([]byte(arguments), &input); err == nil && input != nil { + return input + } + return map[string]any{"input": arguments} +} + +func responsesClaudeStopReason(resp *dto.OpenAIResponsesResponse, sawToolCall bool) string { + if finishReason, ok := ResponsesFinishReasonFromStatus(resp); ok { + return reasonmap.OpenAIFinishReasonToClaudeStopReason(finishReason) + } + if sawToolCall { + return "tool_use" + } + return "end_turn" +} + +func responsesAnnotationsToClaude(annotations []interface{}, text string) []json.RawMessage { + citations := make([]json.RawMessage, 0, len(annotations)) + for _, rawAnnotation := range annotations { + annotation, err := kitutil.Any2Type[map[string]any](rawAnnotation) + if err != nil || strings.TrimSpace(kitutil.Interface2String(annotation["type"])) != "url_citation" { + continue + } + citation := annotation + if nested, ok := annotation["url_citation"].(map[string]any); ok { + citation = nested + } + url := strings.TrimSpace(kitutil.Interface2String(citation["url"])) + if url == "" { + continue + } + converted := map[string]any{ + "type": "web_search_result_location", + "url": url, + "title": strings.TrimSpace(kitutil.Interface2String(citation["title"])), + } + if citedText := kitutil.Interface2String(citation["cited_text"]); citedText != "" { + converted["cited_text"] = citedText + } else if citedText := responsesCitedText(text, citation); citedText != "" { + converted["cited_text"] = citedText + } + if encryptedIndex := kitutil.Interface2String(citation["encrypted_index"]); encryptedIndex != "" { + converted["encrypted_index"] = encryptedIndex + } + if converted["title"] == "" { + delete(converted, "title") + } + encoded, err := kitutil.Marshal(converted) + if err == nil { + citations = append(citations, encoded) + } + } + return citations +} + +func responsesCitedText(text string, citation map[string]any) string { + start, startOK := responsesAnnotationIndex(citation["start_index"]) + end, endOK := responsesAnnotationIndex(citation["end_index"]) + if !startOK || !endOK || start < 0 || end <= start || end > utf8.RuneCountInString(text) { + return "" + } + runes := []rune(text) + return string(runes[start:end]) +} + +func responsesAnnotationIndex(value any) (int, bool) { + switch number := value.(type) { + case float64: + return int(number), number >= 0 && number == float64(int(number)) + case int: + return number, number >= 0 + case json.Number: + parsed, err := number.Int64() + return int(parsed), err == nil && parsed >= 0 + default: + return 0, false + } +} diff --git a/relaykit/relayconvert/internal/oai_responses/to_claude_messages_stream_resp.go b/relaykit/relayconvert/internal/oai_responses/to_claude_messages_stream_resp.go new file mode 100644 index 0000000000..7721bd80be --- /dev/null +++ b/relaykit/relayconvert/internal/oai_responses/to_claude_messages_stream_resp.go @@ -0,0 +1,500 @@ +package oairesponses + +import ( + "fmt" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" + sharedclaude "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/claude" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" +) + +const responsesEventOutputTextDone = "response.output_text.done" + +type ResponsesToClaudeStreamState struct { + ID string + Model string + Usage *dto.Usage + + sentMessageStart bool + done bool + sawToolCall bool + nextBlockIndex int + blocks []*responsesClaudeStreamBlock + byOutputIndex map[int]*responsesClaudeStreamBlock + byItemID map[string]*responsesClaudeStreamBlock + lastByKind map[string]*responsesClaudeStreamBlock + usageText strings.Builder +} + +type responsesClaudeStreamBlock struct { + Index int + Kind string + ItemID string + CallID string + Name string + Started bool + Stopped bool + Value strings.Builder + SentBytes int + AnnotationCount int + NeedsReasoningBreak bool +} + +func NewResponsesToClaudeStreamState(id string, model string) *ResponsesToClaudeStreamState { + return &ResponsesToClaudeStreamState{ + ID: strings.TrimSpace(id), + Model: strings.TrimSpace(model), + byOutputIndex: make(map[int]*responsesClaudeStreamBlock), + byItemID: make(map[string]*responsesClaudeStreamBlock), + lastByKind: make(map[string]*responsesClaudeStreamBlock), + } +} + +func (s *ResponsesToClaudeStreamState) UsageText() string { + if s == nil { + return "" + } + return s.usageText.String() +} + +func (s *ResponsesToClaudeStreamState) Done() bool { + return s != nil && s.done +} + +func (s *ResponsesToClaudeStreamState) SetUsage(usage *dto.Usage) { + if s != nil && usage != nil { + s.Usage = usage + } +} + +func (s *ResponsesToClaudeStreamState) StreamUsage() *dto.Usage { + if s == nil { + return nil + } + return s.Usage +} + +func (s *ResponsesToClaudeStreamState) SetStreamUsage(usage *dto.Usage) { + s.SetUsage(usage) +} + +func (s *ResponsesToClaudeStreamState) ConvertChunk(event *dto.ResponsesStreamResponse, estimatedInputTokens int) ([]*dto.ClaudeResponse, *dto.Usage, error) { + if s == nil { + return nil, nil, nil + } + if event == nil || s.done { + return nil, s.Usage, nil + } + + s.applyResponseMetadata(event.Response) + switch event.Type { + case responsesEventCreated: + return s.ensureMessageStart(estimatedInputTokens), s.Usage, nil + case responsesEventReasoningSummaryDelta, responsesEventReasoningTextDelta: + block, err := s.ensureBlock(event, "thinking") + if err != nil { + return nil, s.Usage, err + } + delta := event.Delta + if block.NeedsReasoningBreak && delta != "" { + delta = separatedResponsesDelta(delta) + block.NeedsReasoningBreak = false + } + return s.appendDelta(block, delta, estimatedInputTokens), s.Usage, nil + case responsesEventReasoningSummaryDone, responsesEventReasoningTextDone: + block, err := s.ensureBlock(event, "thinking") + if err != nil { + return nil, s.Usage, err + } + var responses []*dto.ClaudeResponse + if event.Text != nil { + responses = append(responses, s.mergeFinalValue(block, *event.Text, estimatedInputTokens)...) + } + if block.Value.Len() > 0 { + block.NeedsReasoningBreak = true + } + return responses, s.Usage, nil + case responsesEventOutputTextDelta: + block, err := s.ensureBlock(event, "text") + if err != nil { + return nil, s.Usage, err + } + return s.appendDelta(block, event.Delta, estimatedInputTokens), s.Usage, nil + case responsesEventOutputTextDone: + block, err := s.ensureBlock(event, "text") + if err != nil { + return nil, s.Usage, err + } + if event.Text == nil { + return nil, s.Usage, nil + } + return s.mergeFinalValue(block, *event.Text, estimatedInputTokens), s.Usage, nil + case responsesEventOutputTextAnnotationAdded: + block, err := s.ensureBlock(event, "text") + if err != nil { + return nil, s.Usage, err + } + var annotation any + if err := kitutil.Unmarshal(event.Annotation, &annotation); err != nil { + return nil, s.Usage, fmt.Errorf("invalid Responses stream annotation: %w", err) + } + return s.appendAnnotations(block, []any{annotation}, estimatedInputTokens, false), s.Usage, nil + case responsesEventOutputItemAdded, responsesEventOutputItemDone: + responses, err := s.applyOutputItem(event, estimatedInputTokens, event.Type == responsesEventOutputItemDone) + return responses, s.Usage, err + case responsesEventFunctionArgsDelta, responsesEventCustomToolInputDelta: + block, err := s.ensureBlock(event, "tool_use") + if err != nil { + return nil, s.Usage, err + } + return s.appendDelta(block, event.Delta, estimatedInputTokens), s.Usage, nil + case responsesEventFunctionArgsDone, responsesEventCustomToolInputDone: + block, err := s.ensureBlock(event, "tool_use") + if err != nil { + return nil, s.Usage, err + } + if event.Arguments == nil { + return nil, s.Usage, nil + } + return s.mergeFinalValue(block, *event.Arguments, estimatedInputTokens), s.Usage, nil + case responsesEventCompleted, responsesEventDone, responsesEventIncomplete: + responses, err := s.finish(event.Response, estimatedInputTokens) + return responses, s.Usage, err + case responsesEventFailed, responsesEventError: + message := strings.TrimSpace(event.Message) + if message == "" { + message = event.Type + } + return nil, s.Usage, fmt.Errorf("responses stream error: %s", message) + default: + return nil, s.Usage, nil + } +} + +func (s *ResponsesToClaudeStreamState) Finalize(estimatedInputTokens int) ([]*dto.ClaudeResponse, error) { + if s == nil || s.done { + return nil, nil + } + return s.finish(nil, estimatedInputTokens) +} + +func (s *ResponsesToClaudeStreamState) applyResponseMetadata(response *dto.OpenAIResponsesResponse) { + if s == nil || response == nil { + return + } + if response.ID != "" { + s.ID = response.ID + } + if response.Model != "" { + s.Model = response.Model + } + if response.Usage != nil { + s.Usage = dto.MergeUsageNonZero(s.Usage, UsageFromResponsesUsage(response.Usage)) + } +} + +func (s *ResponsesToClaudeStreamState) ensureMessageStart(estimatedInputTokens int) []*dto.ClaudeResponse { + if s.sentMessageStart { + return nil + } + s.sentMessageStart = true + inputTokens := estimatedInputTokens + if s.Usage != nil { + if usage := sharedclaude.UsageFromOpenAI(s.Usage); usage != nil { + inputTokens = usage.InputTokens + } + } + message := &dto.ClaudeMediaMessage{ + Id: s.ID, + Type: "message", + Role: "assistant", + Model: s.Model, + Usage: &dto.ClaudeUsage{InputTokens: inputTokens}, + } + message.SetContent(make([]any, 0)) + return []*dto.ClaudeResponse{{Type: "message_start", Message: message}} +} + +func (s *ResponsesToClaudeStreamState) ensureBlock(event *dto.ResponsesStreamResponse, kind string) (*responsesClaudeStreamBlock, error) { + block := s.findBlock(event) + if block == nil { + if last := s.lastByKind[kind]; last != nil && !last.Stopped && event.OutputIndex == nil && responseStreamEventItemID(event) == "" { + block = last + } + } + if block == nil { + block = &responsesClaudeStreamBlock{Index: s.nextBlockIndex, Kind: kind} + s.nextBlockIndex++ + s.blocks = append(s.blocks, block) + } + if block.Kind == "" { + block.Kind = kind + } + if block.Kind != kind { + return nil, fmt.Errorf("Responses output item changed from %s to %s", block.Kind, kind) + } + s.applyBlockMetadata(block, event) + s.lastByKind[kind] = block + return block, nil +} + +func (s *ResponsesToClaudeStreamState) findBlock(event *dto.ResponsesStreamResponse) *responsesClaudeStreamBlock { + if event == nil { + return nil + } + if event.OutputIndex != nil { + if block := s.byOutputIndex[*event.OutputIndex]; block != nil { + return block + } + } + if itemID := responseStreamEventItemID(event); itemID != "" { + return s.byItemID[itemID] + } + return nil +} + +func (s *ResponsesToClaudeStreamState) applyBlockMetadata(block *responsesClaudeStreamBlock, event *dto.ResponsesStreamResponse) { + if block == nil || event == nil { + return + } + if event.OutputIndex != nil { + s.byOutputIndex[*event.OutputIndex] = block + } + if itemID := responseStreamEventItemID(event); itemID != "" { + block.ItemID = itemID + s.byItemID[itemID] = block + } + if event.Item == nil { + return + } + if callID := strings.TrimSpace(event.Item.CallId); callID != "" { + block.CallID = callID + } else if block.CallID == "" { + block.CallID = strings.TrimSpace(event.Item.ID) + } + if name := strings.TrimSpace(event.Item.Name); name != "" { + block.Name = name + } +} + +func (s *ResponsesToClaudeStreamState) startBlock(block *responsesClaudeStreamBlock, estimatedInputTokens int) []*dto.ClaudeResponse { + if block == nil || block.Started || block.Stopped { + return nil + } + var content dto.ClaudeMediaMessage + switch block.Kind { + case "text": + content = dto.ClaudeMediaMessage{Type: "text", Text: kitutil.GetPointer("")} + case "thinking": + content = dto.ClaudeMediaMessage{Type: "thinking", Thinking: kitutil.GetPointer("")} + case "tool_use": + if block.Name == "" { + return nil + } + callID := block.CallID + if callID == "" { + callID = block.ItemID + } + content = dto.ClaudeMediaMessage{Type: "tool_use", Id: callID, Name: block.Name, Input: map[string]any{}} + s.sawToolCall = true + default: + return nil + } + block.Started = true + responses := s.ensureMessageStart(estimatedInputTokens) + index := block.Index + responses = append(responses, &dto.ClaudeResponse{Type: "content_block_start", Index: &index, ContentBlock: &content}) + return responses +} + +func (s *ResponsesToClaudeStreamState) appendDelta(block *responsesClaudeStreamBlock, delta string, estimatedInputTokens int) []*dto.ClaudeResponse { + if block == nil || block.Stopped || delta == "" { + return nil + } + block.Value.WriteString(delta) + return s.flushBlock(block, estimatedInputTokens) +} + +func (s *ResponsesToClaudeStreamState) mergeFinalValue(block *responsesClaudeStreamBlock, finalValue string, estimatedInputTokens int) []*dto.ClaudeResponse { + if block == nil || block.Stopped { + return nil + } + current := block.Value.String() + if current == "" { + block.Value.WriteString(finalValue) + } else if strings.HasPrefix(finalValue, current) { + block.Value.WriteString(finalValue[len(current):]) + } + return s.flushBlock(block, estimatedInputTokens) +} + +func (s *ResponsesToClaudeStreamState) flushBlock(block *responsesClaudeStreamBlock, estimatedInputTokens int) []*dto.ClaudeResponse { + if block == nil || block.Stopped { + return nil + } + responses := s.startBlock(block, estimatedInputTokens) + if !block.Started { + return responses + } + value := block.Value.String() + if block.SentBytes >= len(value) { + return responses + } + delta := value[block.SentBytes:] + block.SentBytes = len(value) + s.usageText.WriteString(delta) + index := block.Index + media := &dto.ClaudeMediaMessage{} + switch block.Kind { + case "text": + media.Type = "text_delta" + media.Text = &delta + case "thinking": + media.Type = "thinking_delta" + media.Thinking = &delta + case "tool_use": + media.Type = "input_json_delta" + media.PartialJson = &delta + } + responses = append(responses, &dto.ClaudeResponse{Type: "content_block_delta", Index: &index, Delta: media}) + return responses +} + +func (s *ResponsesToClaudeStreamState) stopBlock(block *responsesClaudeStreamBlock, estimatedInputTokens int) []*dto.ClaudeResponse { + if block == nil || block.Stopped { + return nil + } + responses := s.flushBlock(block, estimatedInputTokens) + responses = append(responses, s.startBlock(block, estimatedInputTokens)...) + if !block.Started { + return responses + } + block.Stopped = true + index := block.Index + return append(responses, &dto.ClaudeResponse{Type: "content_block_stop", Index: &index}) +} + +func (s *ResponsesToClaudeStreamState) applyOutputItem(event *dto.ResponsesStreamResponse, estimatedInputTokens int, stop bool) ([]*dto.ClaudeResponse, error) { + if event == nil || event.Item == nil { + return nil, nil + } + item := event.Item + var kind string + switch item.Type { + case responsesOutputTypeReasoning: + kind = "thinking" + case responsesOutputTypeMessage: + if item.Role != "" && item.Role != "assistant" { + return nil, nil + } + kind = "text" + case responsesOutputTypeFunctionCall, responsesOutputTypeCustomToolCall: + kind = "tool_use" + default: + return nil, nil + } + block, err := s.ensureBlock(event, kind) + if err != nil { + return nil, err + } + var responses []*dto.ClaudeResponse + switch kind { + case "thinking": + responses = append(responses, s.mergeFinalValue(block, reasoningOutputText(item), estimatedInputTokens)...) + case "text": + var text strings.Builder + var annotations []any + for _, content := range item.Content { + if content.Type != "output_text" { + continue + } + text.WriteString(content.Text) + annotations = append(annotations, content.Annotations...) + } + responses = append(responses, s.mergeFinalValue(block, text.String(), estimatedInputTokens)...) + responses = append(responses, s.appendAnnotations(block, annotations, estimatedInputTokens, true)...) + case "tool_use": + responses = append(responses, s.mergeFinalValue(block, item.ArgumentsString(), estimatedInputTokens)...) + } + if stop { + responses = append(responses, s.stopBlock(block, estimatedInputTokens)...) + } + return responses, nil +} + +func (s *ResponsesToClaudeStreamState) appendAnnotations(block *responsesClaudeStreamBlock, annotations []any, estimatedInputTokens int, snapshot bool) []*dto.ClaudeResponse { + if block == nil || block.Kind != "text" || block.Stopped || len(annotations) == 0 { + return nil + } + remaining := annotations + if snapshot { + if len(annotations) <= block.AnnotationCount { + return nil + } + remaining = annotations[block.AnnotationCount:] + block.AnnotationCount = len(annotations) + } else { + block.AnnotationCount += len(annotations) + } + citations := responsesAnnotationsToClaude(remaining, block.Value.String()) + if len(citations) == 0 { + return nil + } + responses := s.startBlock(block, estimatedInputTokens) + index := block.Index + for _, citation := range citations { + responses = append(responses, &dto.ClaudeResponse{ + Type: "content_block_delta", + Index: &index, + Delta: &dto.ClaudeMediaMessage{Type: "citations_delta", Citation: citation}, + }) + } + return responses +} + +func (s *ResponsesToClaudeStreamState) finish(response *dto.OpenAIResponsesResponse, estimatedInputTokens int) ([]*dto.ClaudeResponse, error) { + if s.done { + return nil, nil + } + s.applyResponseMetadata(response) + responses := make([]*dto.ClaudeResponse, 0) + if response != nil { + for outputIndex := range response.Output { + index := outputIndex + item := response.Output[outputIndex] + event := &dto.ResponsesStreamResponse{OutputIndex: &index, ItemID: item.ID, Item: &item} + itemResponses, err := s.applyOutputItem(event, estimatedInputTokens, true) + if err != nil { + return nil, err + } + responses = append(responses, itemResponses...) + } + } + for _, block := range s.blocks { + responses = append(responses, s.stopBlock(block, estimatedInputTokens)...) + } + responses = append(responses, s.ensureMessageStart(estimatedInputTokens)...) + stopReason := responsesClaudeStopReason(response, s.sawToolCall) + usage := sharedclaude.UsageFromOpenAI(s.Usage) + responses = append(responses, + &dto.ClaudeResponse{ + Type: "message_delta", + Usage: usage, + Delta: &dto.ClaudeMediaMessage{StopReason: &stopReason}, + }, + &dto.ClaudeResponse{Type: "message_stop"}, + ) + s.done = true + return responses, nil +} + +func separatedResponsesDelta(delta string) string { + if strings.HasPrefix(delta, "\n\n") { + return delta + } + if strings.HasPrefix(delta, "\n") { + return "\n" + delta + } + return "\n\n" + delta +} diff --git a/relaykit/relayconvert/internal/oai_responses/to_claude_messages_stream_resp_test.go b/relaykit/relayconvert/internal/oai_responses/to_claude_messages_stream_resp_test.go new file mode 100644 index 0000000000..b20c986069 --- /dev/null +++ b/relaykit/relayconvert/internal/oai_responses/to_claude_messages_stream_resp_test.go @@ -0,0 +1,129 @@ +package oairesponses + +import ( + "testing" + + "github.com/QuantumNous/new-api/relaykit/dto" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResponsesToClaudeStreamDoesNotRepeatBlocksFromDoneAndCompletedEvents(t *testing.T) { + state := NewResponsesToClaudeStreamState("", "") + arguments := `{"q":"x"}` + argumentRaw, err := kitutil.Marshal(arguments) + require.NoError(t, err) + statusRaw, err := kitutil.Marshal("completed") + require.NoError(t, err) + + reasoningItem := dto.ResponsesOutput{ + Type: responsesOutputTypeReasoning, + ID: "rs_1", + Summary: []dto.ResponsesReasoningSummaryPart{{Type: "summary_text", Text: "plan"}}, + } + messageItem := dto.ResponsesOutput{ + Type: responsesOutputTypeMessage, + ID: "msg_1", + Role: "assistant", + Content: []dto.ResponsesOutputContent{{Type: "output_text", Text: "hello"}}, + } + toolItem := dto.ResponsesOutput{ + Type: responsesOutputTypeFunctionCall, + ID: "fc_1", + CallId: "call_1", + Name: "lookup", + Arguments: argumentRaw, + } + + events := []*dto.ResponsesStreamResponse{ + {Type: responsesEventCreated, Response: &dto.OpenAIResponsesResponse{ID: "resp_1", Model: "gpt-test"}}, + {Type: responsesEventOutputItemAdded, OutputIndex: kitutil.GetPointer(0), ItemID: reasoningItem.ID, Item: &dto.ResponsesOutput{Type: reasoningItem.Type, ID: reasoningItem.ID}}, + {Type: responsesEventReasoningSummaryDelta, OutputIndex: kitutil.GetPointer(0), ItemID: reasoningItem.ID, Delta: "plan"}, + {Type: responsesEventReasoningSummaryDone, OutputIndex: kitutil.GetPointer(0), ItemID: reasoningItem.ID, Text: kitutil.GetPointer("plan")}, + {Type: responsesEventOutputItemDone, OutputIndex: kitutil.GetPointer(0), ItemID: reasoningItem.ID, Item: &reasoningItem}, + {Type: responsesEventOutputItemAdded, OutputIndex: kitutil.GetPointer(1), ItemID: messageItem.ID, Item: &dto.ResponsesOutput{Type: messageItem.Type, ID: messageItem.ID, Role: "assistant"}}, + {Type: responsesEventOutputTextDelta, OutputIndex: kitutil.GetPointer(1), ItemID: messageItem.ID, Delta: "hello"}, + {Type: responsesEventOutputTextDone, OutputIndex: kitutil.GetPointer(1), ItemID: messageItem.ID, Text: kitutil.GetPointer("hello")}, + {Type: responsesEventOutputItemDone, OutputIndex: kitutil.GetPointer(1), ItemID: messageItem.ID, Item: &messageItem}, + {Type: responsesEventOutputItemAdded, OutputIndex: kitutil.GetPointer(2), ItemID: toolItem.ID, Item: &dto.ResponsesOutput{Type: toolItem.Type, ID: toolItem.ID, CallId: toolItem.CallId, Name: toolItem.Name}}, + {Type: responsesEventFunctionArgsDelta, OutputIndex: kitutil.GetPointer(2), ItemID: toolItem.ID, Delta: `{"q":`}, + {Type: responsesEventFunctionArgsDelta, OutputIndex: kitutil.GetPointer(2), ItemID: toolItem.ID, Delta: `"x"}`}, + {Type: responsesEventFunctionArgsDone, OutputIndex: kitutil.GetPointer(2), ItemID: toolItem.ID, Arguments: &arguments}, + {Type: responsesEventOutputItemDone, OutputIndex: kitutil.GetPointer(2), ItemID: toolItem.ID, Item: &toolItem}, + { + Type: responsesEventCompleted, + Response: &dto.OpenAIResponsesResponse{ + ID: "resp_1", + Model: "gpt-test", + Status: statusRaw, + Output: []dto.ResponsesOutput{reasoningItem, messageItem, toolItem}, + Usage: &dto.Usage{InputTokens: 11, OutputTokens: 7, TotalTokens: 18}, + }, + }, + } + + var output []*dto.ClaudeResponse + for _, event := range events { + converted, _, err := state.ConvertChunk(event, 9) + require.NoError(t, err) + output = append(output, converted...) + } + + starts := responsesOfType(output, "content_block_start") + stops := responsesOfType(output, "content_block_stop") + require.Len(t, responsesOfType(output, "message_start"), 1) + require.Len(t, starts, 3) + require.Len(t, stops, 3) + require.Len(t, responsesOfType(output, "message_delta"), 1) + require.Len(t, responsesOfType(output, "message_stop"), 1) + assert.Equal(t, []int{0, 1, 2}, []int{starts[0].GetIndex(), starts[1].GetIndex(), starts[2].GetIndex()}) + assert.Equal(t, []string{"thinking", "text", "tool_use"}, []string{starts[0].ContentBlock.Type, starts[1].ContentBlock.Type, starts[2].ContentBlock.Type}) + assert.Equal(t, "plan", joinedClaudeDeltas(output, "thinking_delta")) + assert.Equal(t, "hello", joinedClaudeDeltas(output, "text_delta")) + assert.Equal(t, arguments, joinedClaudeDeltas(output, "input_json_delta")) + messageDelta := responsesOfType(output, "message_delta")[0] + require.NotNil(t, messageDelta.Delta.StopReason) + assert.Equal(t, "tool_use", *messageDelta.Delta.StopReason) + + finalized, err := state.Finalize(9) + require.NoError(t, err) + assert.Empty(t, finalized) + repeated, _, err := state.ConvertChunk(events[len(events)-1], 9) + require.NoError(t, err) + assert.Empty(t, repeated) +} + +func responsesOfType(responses []*dto.ClaudeResponse, responseType string) []*dto.ClaudeResponse { + filtered := make([]*dto.ClaudeResponse, 0) + for _, response := range responses { + if response != nil && response.Type == responseType { + filtered = append(filtered, response) + } + } + return filtered +} + +func joinedClaudeDeltas(responses []*dto.ClaudeResponse, deltaType string) string { + result := "" + for _, response := range responses { + if response == nil || response.Type != "content_block_delta" || response.Delta == nil || response.Delta.Type != deltaType { + continue + } + switch deltaType { + case "thinking_delta": + if response.Delta.Thinking != nil { + result += *response.Delta.Thinking + } + case "text_delta": + if response.Delta.Text != nil { + result += *response.Delta.Text + } + case "input_json_delta": + if response.Delta.PartialJson != nil { + result += *response.Delta.PartialJson + } + } + } + return result +} diff --git a/relaykit/relayconvert/internal/oai_responses/to_gemini_chat_req.go b/relaykit/relayconvert/internal/oai_responses/to_gemini_chat_req.go index 18ddceedba..41fc03fc01 100644 --- a/relaykit/relayconvert/internal/oai_responses/to_gemini_chat_req.go +++ b/relaykit/relayconvert/internal/oai_responses/to_gemini_chat_req.go @@ -10,6 +10,7 @@ import ( relaymedia "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/media" sharedgemini "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/gemini" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" ) func convertOpenAIResponsesRequestToGeminiChat(c context.Context, info convmeta.Meta, request any) (any, error) { @@ -42,10 +43,10 @@ func OpenAIResponsesRequestToGeminiChat(c context.Context, req *dto.OpenAIRespon Temperature: req.Temperature, }, } - if req.TopP != nil && *req.TopP > 0 { + if req.TopP != nil { geminiRequest.GenerationConfig.TopP = kitutil.GetPointer(*req.TopP) } - if req.MaxOutputTokens != nil && *req.MaxOutputTokens > 0 { + if req.MaxOutputTokens != nil { geminiRequest.GenerationConfig.MaxOutputTokens = kitutil.GetPointer(*req.MaxOutputTokens) } @@ -59,11 +60,19 @@ func OpenAIResponsesRequestToGeminiChat(c context.Context, req *dto.OpenAIRespon if err := applyResponsesTextToGemini(req.Text, geminiRequest); err != nil { return nil, err } - sharedgemini.ApplyThinkingConfig(geminiRequest, info, dto.GeneralOpenAIRequest{ - Model: req.Model, - MaxCompletionTokens: req.MaxOutputTokens, - ReasoningEffort: ReasoningEffort(req), - }) + reasoningIntent, err := reasoning.FromOpenAIResponses(req) + if err != nil { + return nil, reasoning.AsClientError(err) + } + var reasoningPivot dto.GeneralOpenAIRequest + if err := reasoning.ApplyToOpenAIChat(&reasoningPivot, reasoningIntent); err != nil { + return nil, reasoning.AsClientError(err) + } + reasoningPivot.Model = req.Model + reasoningPivot.MaxCompletionTokens = req.MaxOutputTokens + if err := sharedgemini.ApplyThinkingConfig(geminiRequest, info, reasoningPivot); err != nil { + return nil, reasoning.AsClientError(err) + } var safetySettings []dto.GeminiChatSafetySettings for _, category := range sharedgemini.SafetySettingCategories { @@ -137,7 +146,10 @@ func OpenAIResponsesRequestToGeminiChat(c context.Context, req *dto.OpenAIRespon } appendGeminiContentPart(geminiRequest, "model", part) case ResponsesInputTypeFunctionCallOutput: - part := responsesFunctionOutputItemToGeminiPart(item, callNames) + part, err := responsesFunctionOutputItemToGeminiPart(item, callNames) + if err != nil { + return nil, err + } appendGeminiContentPart(geminiRequest, "user", part) default: role := responsesGeminiRole(item) @@ -252,24 +264,33 @@ func responsesFunctionCallItemToGeminiPart(item map[string]any) (dto.GeminiPart, callID := CallID(item) return dto.GeminiPart{ FunctionCall: &dto.FunctionCall{ + ID: callID, FunctionName: name, Arguments: ObjectValue(item["arguments"], "arguments"), }, }, callID, nil } -func responsesFunctionOutputItemToGeminiPart(item map[string]any, callNames map[string]string) dto.GeminiPart { +func responsesFunctionOutputItemToGeminiPart(item map[string]any, callNames map[string]string) (dto.GeminiPart, error) { callID := CallID(item) name := strings.TrimSpace(kitutil.Interface2String(item["name"])) if name == "" { name = callNames[callID] } - return dto.GeminiPart{ - FunctionResponse: &dto.GeminiFunctionResponse{ - Name: name, - Response: GeminiResponseMap(item["output"]), - }, + response := &dto.GeminiFunctionResponse{ + Name: name, + Response: GeminiResponseMap(item["output"]), } + if callID != "" { + id, err := kitutil.Marshal(callID) + if err != nil { + return dto.GeminiPart{}, fmt.Errorf("failed to marshal function response ID: %w", err) + } + response.ID = id + } + return dto.GeminiPart{ + FunctionResponse: response, + }, nil } func appendGeminiContentPart(req *dto.GeminiChatRequest, role string, part dto.GeminiPart) { diff --git a/relaykit/relayconvert/internal/oai_responses/to_oai_chat_req.go b/relaykit/relayconvert/internal/oai_responses/to_oai_chat_req.go index 6966095ecc..21a2de7d4f 100644 --- a/relaykit/relayconvert/internal/oai_responses/to_oai_chat_req.go +++ b/relaykit/relayconvert/internal/oai_responses/to_oai_chat_req.go @@ -8,6 +8,7 @@ import ( "github.com/QuantumNous/new-api/relaykit/dto" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" ) const ( @@ -85,8 +86,10 @@ func ResponsesRequestToChatCompletionsRequest(req *dto.OpenAIResponsesRequest) ( return nil, fmt.Errorf("invalid presence_penalty: %w", err) } - if req.Reasoning != nil { - out.ReasoningEffort = req.Reasoning.Effort + if reasoningIntent, err := reasoning.FromOpenAIResponses(req); err != nil { + return nil, reasoning.AsClientError(err) + } else if err := reasoning.ApplyToOpenAIChat(out, reasoningIntent); err != nil { + return nil, reasoning.AsClientError(err) } if req.ServiceTier != "" { out.ServiceTier, _ = kitutil.Marshal(req.ServiceTier) diff --git a/relaykit/relayconvert/internal/oai_responses/to_oai_chat_resp.go b/relaykit/relayconvert/internal/oai_responses/to_oai_chat_resp.go index b97f9116d7..630252487f 100644 --- a/relaykit/relayconvert/internal/oai_responses/to_oai_chat_resp.go +++ b/relaykit/relayconvert/internal/oai_responses/to_oai_chat_resp.go @@ -10,29 +10,30 @@ import ( ) const ( - responsesEventCreated = "response.created" - responsesEventCompleted = "response.completed" - responsesEventDone = "response.done" - responsesEventIncomplete = "response.incomplete" - responsesEventFailed = "response.failed" - responsesEventError = "response.error" - responsesEventOutputTextDelta = "response.output_text.delta" - responsesEventOutputItemAdded = "response.output_item.added" - responsesEventOutputItemDone = "response.output_item.done" - responsesEventFunctionArgsDelta = "response.function_call_arguments.delta" - responsesEventFunctionArgsDone = "response.function_call_arguments.done" - responsesEventCustomToolInputDelta = "response.custom_tool_call_input.delta" - responsesEventCustomToolInputDone = "response.custom_tool_call_input.done" - responsesEventReasoningSummaryDelta = "response.reasoning_summary_text.delta" - responsesEventReasoningSummaryDone = "response.reasoning_summary_text.done" - responsesEventReasoningTextDelta = "response.reasoning_text.delta" - responsesEventReasoningTextDone = "response.reasoning_text.done" - responsesOutputTypeFunctionCall = "function_call" - responsesOutputTypeCustomToolCall = "custom_tool_call" - responsesOutputTypeMessage = "message" - responsesOutputTypeReasoning = "reasoning" - responsesIncompleteReasonContentFilter = "content_filter" - responsesIncompleteReasonMaxTokens = "max_output_tokens" + responsesEventCreated = "response.created" + responsesEventCompleted = "response.completed" + responsesEventDone = "response.done" + responsesEventIncomplete = "response.incomplete" + responsesEventFailed = "response.failed" + responsesEventError = "response.error" + responsesEventOutputTextDelta = "response.output_text.delta" + responsesEventOutputTextAnnotationAdded = "response.output_text.annotation.added" + responsesEventOutputItemAdded = "response.output_item.added" + responsesEventOutputItemDone = "response.output_item.done" + responsesEventFunctionArgsDelta = "response.function_call_arguments.delta" + responsesEventFunctionArgsDone = "response.function_call_arguments.done" + responsesEventCustomToolInputDelta = "response.custom_tool_call_input.delta" + responsesEventCustomToolInputDone = "response.custom_tool_call_input.done" + responsesEventReasoningSummaryDelta = "response.reasoning_summary_text.delta" + responsesEventReasoningSummaryDone = "response.reasoning_summary_text.done" + responsesEventReasoningTextDelta = "response.reasoning_text.delta" + responsesEventReasoningTextDone = "response.reasoning_text.done" + responsesOutputTypeFunctionCall = "function_call" + responsesOutputTypeCustomToolCall = "custom_tool_call" + responsesOutputTypeMessage = "message" + responsesOutputTypeReasoning = "reasoning" + responsesIncompleteReasonContentFilter = "content_filter" + responsesIncompleteReasonMaxTokens = "max_output_tokens" ) func ResponsesFinishReasonFromStatus(resp *dto.OpenAIResponsesResponse) (string, bool) { @@ -103,6 +104,11 @@ func ResponsesResponseToChatCompletionsResponse(resp *dto.OpenAIResponsesRespons Role: "assistant", Content: text, } + if annotations, err := responsesAnnotationsToChat(resp); err != nil { + return nil, nil, err + } else if len(annotations) > 0 { + msg.Annotations = annotations + } if reasoning != "" { msg.ReasoningContent = &reasoning } @@ -128,7 +134,65 @@ func ResponsesResponseToChatCompletionsResponse(resp *dto.OpenAIResponsesRespons return out, usage, nil } +func responsesAnnotationsToChat(resp *dto.OpenAIResponsesResponse) ([]byte, error) { + annotations := make([]any, 0) + for _, output := range resp.Output { + if output.Type != responsesOutputTypeMessage { + continue + } + for _, content := range output.Content { + for _, annotation := range content.Annotations { + converted, err := responseAnnotationToChat(annotation) + if err != nil { + return nil, err + } + annotations = append(annotations, converted) + } + } + } + if len(annotations) == 0 { + return nil, nil + } + return kitutil.Marshal(annotations) +} + +func responseAnnotationToChat(annotation any) (map[string]any, error) { + value, ok := annotation.(map[string]any) + if !ok { + converted, err := kitutil.Any2Type[map[string]any](annotation) + if err != nil { + return nil, fmt.Errorf("invalid Responses annotation: %w", err) + } + value = converted + } + if strings.TrimSpace(kitutil.Interface2String(value["type"])) != "url_citation" { + return value, nil + } + citation := make(map[string]any, len(value)-1) + for key, item := range value { + if key != "type" { + citation[key] = item + } + } + return map[string]any{ + "type": "url_citation", + "url_citation": citation, + }, nil +} + func UsageFromResponsesUsage(src *dto.Usage) *dto.Usage { + return usageFromResponsesUsage(src, true) +} + +// NormalizeResponsesUsage maps Responses usage into the shared accounting +// shape without creating a BillingUsage snapshot. Native Responses handlers +// use it so passthrough traffic preserves an existing snapshot but does not +// introduce a conversion sidecar solely for local settlement. +func NormalizeResponsesUsage(src *dto.Usage) *dto.Usage { + return usageFromResponsesUsage(src, false) +} + +func usageFromResponsesUsage(src *dto.Usage, createBillingSnapshot bool) *dto.Usage { usage := &dto.Usage{} if src == nil { return usage @@ -136,7 +200,7 @@ func UsageFromResponsesUsage(src *dto.Usage) *dto.Usage { usage.UsageSemantic = src.UsageSemantic usage.UsageSource = src.UsageSource usage.BillingUsage = dto.CloneBillingUsage(src.BillingUsage) - if usage.BillingUsage == nil { + if usage.BillingUsage == nil && createBillingSnapshot { usage.BillingUsage = dto.NewOpenAIResponsesBillingUsage(src) } usage.Cost = src.Cost @@ -190,21 +254,25 @@ func ExtractOutputTextFromResponses(resp *dto.OpenAIResponsesResponse) string { if out.Role != "" && out.Role != "assistant" { continue } + var outputText strings.Builder for _, c := range out.Content { if c.Type == "output_text" && c.Text != "" { - sb.WriteString(c.Text) + outputText.WriteString(c.Text) } } + appendSeparatedText(&sb, outputText.String()) } if sb.Len() > 0 { return sb.String() } for _, out := range resp.Output { + var outputText strings.Builder for _, c := range out.Content { if c.Text != "" { - sb.WriteString(c.Text) + outputText.WriteString(c.Text) } } + appendSeparatedText(&sb, outputText.String()) } return sb.String() } @@ -219,15 +287,56 @@ func ExtractReasoningTextFromResponses(resp *dto.OpenAIResponsesResponse) string if out.Type != responsesOutputTypeReasoning { continue } - for _, c := range out.Content { - if c.Text != "" { - sb.WriteString(c.Text) - } - } + appendSeparatedText(&sb, reasoningOutputText(&out)) } return sb.String() } +func reasoningOutputText(output *dto.ResponsesOutput) string { + if output == nil { + return "" + } + var text strings.Builder + hasContentText := false + for _, part := range output.Content { + if part.Text != "" { + hasContentText = true + break + } + } + if hasContentText { + for _, part := range output.Content { + appendSeparatedText(&text, part.Text) + } + return text.String() + } + for _, part := range output.Summary { + appendSeparatedText(&text, part.Text) + } + return text.String() +} + +func appendSeparatedText(builder *strings.Builder, text string) { + if builder == nil || text == "" { + return + } + if builder.Len() > 0 { + current := builder.String() + trailingNewlines := 0 + for index := len(current) - 1; index >= 0 && trailingNewlines < 2 && current[index] == '\n'; index-- { + trailingNewlines++ + } + leadingNewlines := 0 + for leadingNewlines < len(text) && leadingNewlines < 2 && text[leadingNewlines] == '\n' { + leadingNewlines++ + } + for missing := 2 - trailingNewlines - leadingNewlines; missing > 0; missing-- { + builder.WriteByte('\n') + } + } + builder.WriteString(text) +} + func responseStatusString(resp *dto.OpenAIResponsesResponse) string { if resp == nil || len(resp.Status) == 0 { return "" diff --git a/relaykit/relayconvert/internal/oai_responses/to_oai_chat_resp_test.go b/relaykit/relayconvert/internal/oai_responses/to_oai_chat_resp_test.go index 49efa07d16..100289bb64 100644 --- a/relaykit/relayconvert/internal/oai_responses/to_oai_chat_resp_test.go +++ b/relaykit/relayconvert/internal/oai_responses/to_oai_chat_resp_test.go @@ -56,9 +56,9 @@ func TestResponsesResponseToChatCompletionsPreservesReasoningSummary(t *testing. Output: []dto.ResponsesOutput{ { Type: responsesOutputTypeReasoning, - Content: []dto.ResponsesOutputContent{ + Summary: []dto.ResponsesReasoningSummaryPart{ {Type: "summary_text", Text: "first summary"}, - {Type: "summary_text", Text: "\n\nsecond summary"}, + {Type: "summary_text", Text: "second summary"}, }, }, { @@ -77,6 +77,20 @@ func TestResponsesResponseToChatCompletionsPreservesReasoningSummary(t *testing. assert.Equal(t, "final", chat.Choices[0].Message.StringContent()) } +func TestResponsesResponseToChatCompletionsSeparatesInterleavedOutputItems(t *testing.T) { + resp := &dto.OpenAIResponsesResponse{ + ID: "resp_1", + Model: "gpt-test", + Status: []byte(`"completed"`), + Output: interleavedReasoningAndTextOutput(), + } + + chat, _, err := ResponsesResponseToChatCompletionsResponse(resp, "chatcmpl_1") + require.NoError(t, err) + assert.Equal(t, "**Planning file inspection**\n\n**Clarifying environment task requirements**", chat.Choices[0].Message.GetReasoningContent()) + assert.Equal(t, "I’ll inspect the starter repository.\n\nWhat would you like me to build?", chat.Choices[0].Message.StringContent()) +} + func TestResponsesFinishReasonFromIncompleteStatus(t *testing.T) { tests := []struct { name string @@ -433,6 +447,90 @@ func TestResponsesBufferedAccumulatorDoesNotDuplicatePendingArgsWithOutputIndexA assert.Empty(t, acc.pendingByItemID) } +func TestResponsesBufferedAccumulatorPreservesInterleavedReasoningAndTextItems(t *testing.T) { + acc := NewResponsesBufferedAccumulator() + events := []dto.ResponsesStreamResponse{ + bufferedOutputItemAdded(0, "rs_1", responsesOutputTypeReasoning), + {Type: responsesEventReasoningSummaryDelta, OutputIndex: intPointer(0), ItemID: "rs_1", Delta: "**Planning file inspection**"}, + bufferedOutputItemAdded(1, "msg_1", responsesOutputTypeMessage), + {Type: responsesEventOutputTextDelta, OutputIndex: intPointer(1), ItemID: "msg_1", Delta: "I’ll inspect the starter repository."}, + bufferedOutputItemAdded(2, "rs_2", responsesOutputTypeReasoning), + {Type: responsesEventReasoningSummaryDelta, OutputIndex: intPointer(2), ItemID: "rs_2", Delta: "**Clarifying environment task requirements**"}, + bufferedOutputItemAdded(3, "msg_2", responsesOutputTypeMessage), + {Type: responsesEventOutputTextDelta, OutputIndex: intPointer(3), ItemID: "msg_2", Delta: "What would you like me to build?"}, + } + for index := range events { + acc.ProcessEvent(&events[index]) + } + + output := acc.BuildOutput() + require.Len(t, output, 4) + assert.Equal(t, []string{ + responsesOutputTypeReasoning, + responsesOutputTypeMessage, + responsesOutputTypeReasoning, + responsesOutputTypeMessage, + }, []string{output[0].Type, output[1].Type, output[2].Type, output[3].Type}) + assert.Equal(t, "**Planning file inspection**", output[0].Summary[0].Text) + assert.Equal(t, "I’ll inspect the starter repository.", output[1].Content[0].Text) + assert.Equal(t, "**Clarifying environment task requirements**", output[2].Summary[0].Text) + assert.Equal(t, "What would you like me to build?", output[3].Content[0].Text) +} + +func TestResponsesStreamTerminalOutputPreservesInterleavedReasoningAndTextItems(t *testing.T) { + state := newTestResponsesStreamState() + chunks, err := ResponsesStreamEventToChatChunks(&dto.ResponsesStreamResponse{ + Type: responsesEventCompleted, + Response: &dto.OpenAIResponsesResponse{ + Status: []byte(`"completed"`), + Output: interleavedReasoningAndTextOutput(), + }, + }, state) + require.NoError(t, err) + + var deltas []string + for _, chunk := range chunks { + if len(chunk.Choices) == 0 { + continue + } + delta := chunk.Choices[0].Delta + if delta.ReasoningContent != nil { + deltas = append(deltas, "thinking:"+*delta.ReasoningContent) + } + if delta.Content != nil && *delta.Content != "" { + deltas = append(deltas, "text:"+*delta.Content) + } + } + assert.Equal(t, []string{ + "thinking:**Planning file inspection**", + "text:I’ll inspect the starter repository.", + "thinking:**Clarifying environment task requirements**", + "text:What would you like me to build?", + }, deltas) +} + +func bufferedOutputItemAdded(outputIndex int, itemID string, itemType string) dto.ResponsesStreamResponse { + return dto.ResponsesStreamResponse{ + Type: responsesEventOutputItemAdded, + OutputIndex: &outputIndex, + ItemID: itemID, + Item: &dto.ResponsesOutput{ID: itemID, Type: itemType}, + } +} + +func interleavedReasoningAndTextOutput() []dto.ResponsesOutput { + return []dto.ResponsesOutput{ + {Type: responsesOutputTypeReasoning, Summary: []dto.ResponsesReasoningSummaryPart{{Type: "summary_text", Text: "**Planning file inspection**"}}}, + {Type: responsesOutputTypeMessage, Role: "assistant", Content: []dto.ResponsesOutputContent{{Type: "output_text", Text: "I’ll inspect the starter repository."}}}, + {Type: responsesOutputTypeReasoning, Summary: []dto.ResponsesReasoningSummaryPart{{Type: "summary_text", Text: "**Clarifying environment task requirements**"}}}, + {Type: responsesOutputTypeMessage, Role: "assistant", Content: []dto.ResponsesOutputContent{{Type: "output_text", Text: "What would you like me to build?"}}}, + } +} + +func intPointer(value int) *int { + return &value +} + func newTestResponsesStreamState() *ResponsesToChatStreamState { state := NewResponsesToChatStreamState("gpt-test", false) state.ID = "chatcmpl_test" diff --git a/relaykit/relayconvert/internal/oai_responses/to_oai_chat_stream_resp.go b/relaykit/relayconvert/internal/oai_responses/to_oai_chat_stream_resp.go index 6026e3899e..645b169fbb 100644 --- a/relaykit/relayconvert/internal/oai_responses/to_oai_chat_stream_resp.go +++ b/relaykit/relayconvert/internal/oai_responses/to_oai_chat_stream_resp.go @@ -21,6 +21,7 @@ type ResponsesToChatStreamState struct { sentStart bool finalized bool hasSentText bool + sentAnnotationCount int sawToolCall bool hasSentReasoning bool needsReasoningSummaryBreak bool @@ -61,6 +62,19 @@ func NewResponsesToChatStreamState(model string, includeUsage bool) *ResponsesTo } } +func (s *ResponsesToChatStreamState) StreamUsage() *dto.Usage { + if s == nil { + return nil + } + return s.Usage +} + +func (s *ResponsesToChatStreamState) SetStreamUsage(usage *dto.Usage) { + if s != nil && usage != nil { + s.Usage = usage + } +} + func (s *ResponsesToChatStreamState) UsageText() string { if s == nil { return "" @@ -86,6 +100,8 @@ func ResponsesStreamEventToChatChunks(event *dto.ResponsesStreamResponse, state return nil, nil case responsesEventOutputTextDelta: return state.textDelta(event.Delta), nil + case responsesEventOutputTextAnnotationAdded: + return state.annotationRawDelta(event.Annotation) case responsesEventOutputItemAdded, responsesEventOutputItemDone: if event.Item == nil || !isResponsesToolOutputType(event.Item.Type) { return nil, nil @@ -101,7 +117,10 @@ func ResponsesStreamEventToChatChunks(event *dto.ResponsesStreamResponse, state response = ensureIncompleteResponse(response) } state.applyResponseMetadata(response) - chunks := state.terminalOutputChunks(response) + chunks, err := state.terminalOutputChunks(response) + if err != nil { + return nil, err + } chunks = append(chunks, state.finalize(response)...) return chunks, nil case responsesEventFailed, responsesEventError: @@ -132,7 +151,7 @@ func (s *ResponsesToChatStreamState) applyResponseMetadata(response *dto.OpenAIR s.Created = int64(response.CreatedAt) } if response.Usage != nil { - s.Usage = UsageFromResponsesUsage(response.Usage) + s.Usage = dto.MergeUsageNonZero(s.Usage, UsageFromResponsesUsage(response.Usage)) } } @@ -160,16 +179,19 @@ func (s *ResponsesToChatStreamState) textDelta(delta string) []dto.ChatCompletio return chunks } -func (s *ResponsesToChatStreamState) terminalOutputChunks(response *dto.OpenAIResponsesResponse) []dto.ChatCompletionsStreamResponse { +func (s *ResponsesToChatStreamState) terminalOutputChunks(response *dto.OpenAIResponsesResponse) ([]dto.ChatCompletionsStreamResponse, error) { if s == nil || response == nil || len(response.Output) == 0 { - return nil + return nil, nil } var chunks []dto.ChatCompletionsStreamResponse + hadSentText := s.hasSentText + hadSentReasoning := s.hasSentReasoning + annotationOffset := 0 for i := range response.Output { out := &response.Output[i] switch { - case out.Type == responsesOutputTypeMessage && !s.hasSentText: + case out.Type == responsesOutputTypeMessage && !hadSentText: var text strings.Builder for _, c := range out.Content { if c.Type == "output_text" && c.Text != "" { @@ -177,19 +199,88 @@ func (s *ResponsesToChatStreamState) terminalOutputChunks(response *dto.OpenAIRe } } chunks = append(chunks, s.textDelta(text.String())...) - case out.Type == responsesOutputTypeReasoning && !s.hasSentReasoning: - var reasoning strings.Builder - for _, c := range out.Content { - if c.Text != "" { - reasoning.WriteString(c.Text) - } + annotationChunks, err := s.remainingAnnotationChunks(out, annotationOffset) + if err != nil { + return nil, err } - chunks = append(chunks, s.reasoningDelta(reasoning.String())...) + chunks = append(chunks, annotationChunks...) + annotationOffset += responsesOutputAnnotationCount(out) + case out.Type == responsesOutputTypeMessage: + annotationChunks, err := s.remainingAnnotationChunks(out, annotationOffset) + if err != nil { + return nil, err + } + chunks = append(chunks, annotationChunks...) + annotationOffset += responsesOutputAnnotationCount(out) + case out.Type == responsesOutputTypeReasoning && !hadSentReasoning: + chunks = append(chunks, s.reasoningDelta(reasoningOutputText(out))...) case isResponsesToolOutputType(out.Type): chunks = append(chunks, s.toolItem(&dto.ResponsesStreamResponse{Item: out})...) } } - return chunks + return chunks, nil +} + +func (s *ResponsesToChatStreamState) annotationRawDelta(raw []byte) ([]dto.ChatCompletionsStreamResponse, error) { + if len(raw) == 0 { + return nil, nil + } + var annotation map[string]any + if err := kitutil.Unmarshal(raw, &annotation); err != nil { + return nil, fmt.Errorf("invalid Responses stream annotation: %w", err) + } + return s.annotationDelta(annotation) +} + +func (s *ResponsesToChatStreamState) annotationDelta(annotation any) ([]dto.ChatCompletionsStreamResponse, error) { + converted, err := responseAnnotationToChat(annotation) + if err != nil { + return nil, err + } + raw, err := kitutil.Marshal([]any{converted}) + if err != nil { + return nil, fmt.Errorf("marshal Chat annotation: %w", err) + } + s.sentAnnotationCount++ + chunks := s.ensureStart() + chunks = append(chunks, s.makeChunk(dto.ChatCompletionsStreamResponseChoiceDelta{ + Annotations: raw, + }, nil)) + return chunks, nil +} + +func (s *ResponsesToChatStreamState) remainingAnnotationChunks(output *dto.ResponsesOutput, offset int) ([]dto.ChatCompletionsStreamResponse, error) { + if output == nil { + return nil, nil + } + annotations := make([]any, 0) + for _, content := range output.Content { + annotations = append(annotations, content.Annotations...) + } + start := s.sentAnnotationCount - offset + if start < 0 { + start = 0 + } + if start >= len(annotations) { + return nil, nil + } + var chunks []dto.ChatCompletionsStreamResponse + for _, annotation := range annotations[start:] { + converted, err := s.annotationDelta(annotation) + if err != nil { + return nil, err + } + chunks = append(chunks, converted...) + } + return chunks, nil +} + +func responsesOutputAnnotationCount(output *dto.ResponsesOutput) int { + count := 0 + for _, content := range output.Content { + count += len(content.Annotations) + } + return count } func (s *ResponsesToChatStreamState) reasoningDelta(delta string) []dto.ChatCompletionsStreamResponse { @@ -554,8 +645,10 @@ func (s *ResponsesToChatStreamState) keyForEvent(event *dto.ResponsesStreamRespo } type ResponsesBufferedAccumulator struct { - text strings.Builder - reasoning strings.Builder + items []*responsesBufferedItem + outputIndexToItemIdx map[int]int + itemIDToItemIdx map[string]int + lastUnindexedItemIdx int tools []*responsesBufferedTool outputIndexToToolIdx map[int]int itemIDToToolIdx map[string]int @@ -563,6 +656,15 @@ type ResponsesBufferedAccumulator struct { pendingByItemID map[string]string } +type responsesBufferedItem struct { + Type string + ID string + Text strings.Builder + Annotations []interface{} + ToolIndex int + NeedsReasoningBreak bool +} + type responsesBufferedTool struct { CallID string ItemID string @@ -572,6 +674,9 @@ type responsesBufferedTool struct { func NewResponsesBufferedAccumulator() *ResponsesBufferedAccumulator { return &ResponsesBufferedAccumulator{ + outputIndexToItemIdx: make(map[int]int), + itemIDToItemIdx: make(map[string]int), + lastUnindexedItemIdx: -1, outputIndexToToolIdx: make(map[int]int), itemIDToToolIdx: make(map[string]int), pendingByOutputIndex: make(map[int]string), @@ -585,11 +690,55 @@ func (a *ResponsesBufferedAccumulator) ProcessEvent(event *dto.ResponsesStreamRe } switch event.Type { case responsesEventOutputTextDelta: - a.text.WriteString(event.Delta) + item := a.ensureItem(event, responsesOutputTypeMessage) + item.Text.WriteString(event.Delta) + case responsesEventOutputTextAnnotationAdded: + item := a.ensureItem(event, responsesOutputTypeMessage) + var annotation interface{} + if err := kitutil.Unmarshal(event.Annotation, &annotation); err == nil && annotation != nil { + item.Annotations = append(item.Annotations, annotation) + } case responsesEventReasoningSummaryDelta, responsesEventReasoningTextDelta: - a.reasoning.WriteString(event.Delta) + item := a.ensureItem(event, responsesOutputTypeReasoning) + if item.NeedsReasoningBreak { + appendSeparatedText(&item.Text, event.Delta) + item.NeedsReasoningBreak = false + } else { + item.Text.WriteString(event.Delta) + } + case responsesEventReasoningSummaryDone, responsesEventReasoningTextDone: + item := a.ensureItem(event, responsesOutputTypeReasoning) + if item.Text.Len() == 0 && event.Text != nil { + item.Text.WriteString(*event.Text) + } + if item.Text.Len() > 0 { + item.NeedsReasoningBreak = true + } case responsesEventOutputItemAdded, responsesEventOutputItemDone: - if event.Item != nil && isResponsesToolOutputType(event.Item.Type) { + if event.Item == nil { + return + } + switch { + case event.Item.Type == responsesOutputTypeReasoning: + item := a.ensureItem(event, event.Item.Type) + if item.Text.Len() == 0 { + item.Text.WriteString(reasoningOutputText(event.Item)) + } + case event.Item.Type == responsesOutputTypeMessage: + item := a.ensureItem(event, event.Item.Type) + seedText := item.Text.Len() == 0 + seedAnnotations := len(item.Annotations) == 0 + for _, content := range event.Item.Content { + if content.Type == "output_text" { + if seedText { + item.Text.WriteString(content.Text) + } + if seedAnnotations { + item.Annotations = append(item.Annotations, content.Annotations...) + } + } + } + case isResponsesToolOutputType(event.Item.Type): tool := a.ensureTool(event) if args := event.Item.ArgumentsString(); args != "" { tool.Arguments.Reset() @@ -620,50 +769,119 @@ func (a *ResponsesBufferedAccumulator) BuildOutput() []dto.ResponsesOutput { if a == nil { return nil } - out := make([]dto.ResponsesOutput, 0, 2+len(a.tools)) - if a.reasoning.Len() > 0 { - out = append(out, dto.ResponsesOutput{ - Type: responsesOutputTypeReasoning, - Content: []dto.ResponsesOutputContent{ - {Type: "summary_text", Text: a.reasoning.String()}, - }, - }) - } - if a.text.Len() > 0 { - out = append(out, dto.ResponsesOutput{ - Type: responsesOutputTypeMessage, - Role: "assistant", - Content: []dto.ResponsesOutputContent{ - {Type: "output_text", Text: a.text.String()}, - }, - }) - } - for _, tool := range a.tools { - if tool == nil { + out := make([]dto.ResponsesOutput, 0, len(a.items)) + for _, item := range a.items { + if item == nil { continue } - argsRaw, _ := kitutil.Marshal(tool.Arguments.String()) - out = append(out, dto.ResponsesOutput{ - Type: responsesOutputTypeFunctionCall, - ID: tool.ItemID, - CallId: tool.CallID, - Name: tool.Name, - Arguments: argsRaw, - }) + switch item.Type { + case responsesOutputTypeReasoning: + if item.Text.Len() == 0 { + continue + } + out = append(out, dto.ResponsesOutput{ + Type: item.Type, + ID: item.ID, + Summary: []dto.ResponsesReasoningSummaryPart{ + {Type: "summary_text", Text: item.Text.String()}, + }, + }) + case responsesOutputTypeMessage: + if item.Text.Len() == 0 { + continue + } + out = append(out, dto.ResponsesOutput{ + Type: item.Type, + ID: item.ID, + Role: "assistant", + Content: []dto.ResponsesOutputContent{ + {Type: "output_text", Text: item.Text.String(), Annotations: item.Annotations}, + }, + }) + case responsesOutputTypeFunctionCall, responsesOutputTypeCustomToolCall: + if item.ToolIndex < 0 || item.ToolIndex >= len(a.tools) || a.tools[item.ToolIndex] == nil { + continue + } + tool := a.tools[item.ToolIndex] + argsRaw, _ := kitutil.Marshal(tool.Arguments.String()) + out = append(out, dto.ResponsesOutput{ + Type: item.Type, + ID: tool.ItemID, + CallId: tool.CallID, + Name: tool.Name, + Arguments: argsRaw, + }) + } } return out } +func (a *ResponsesBufferedAccumulator) ensureItem(event *dto.ResponsesStreamResponse, itemType string) *responsesBufferedItem { + if idx, ok := a.findItemIndex(event); ok { + item := a.items[idx] + if item.Type == "" { + item.Type = itemType + } + a.applyItemMetadata(idx, item, event) + return item + } + if event != nil && event.OutputIndex == nil && responseStreamEventItemID(event) == "" && a.lastUnindexedItemIdx >= 0 { + item := a.items[a.lastUnindexedItemIdx] + if item != nil && item.Type == itemType { + return item + } + } + item := &responsesBufferedItem{Type: itemType, ToolIndex: -1} + idx := len(a.items) + a.items = append(a.items, item) + a.lastUnindexedItemIdx = idx + a.applyItemMetadata(idx, item, event) + return item +} + +func (a *ResponsesBufferedAccumulator) applyItemMetadata(idx int, item *responsesBufferedItem, event *dto.ResponsesStreamResponse) { + if item == nil || event == nil { + return + } + if event.OutputIndex != nil { + a.outputIndexToItemIdx[*event.OutputIndex] = idx + } + if itemID := responseStreamEventItemID(event); itemID != "" { + item.ID = itemID + a.itemIDToItemIdx[itemID] = idx + } +} + +func (a *ResponsesBufferedAccumulator) findItemIndex(event *dto.ResponsesStreamResponse) (int, bool) { + if event == nil { + return 0, false + } + if event.OutputIndex != nil { + if idx, ok := a.outputIndexToItemIdx[*event.OutputIndex]; ok { + return idx, true + } + } + if itemID := responseStreamEventItemID(event); itemID != "" { + idx, ok := a.itemIDToItemIdx[itemID] + return idx, ok + } + return 0, false +} + func (a *ResponsesBufferedAccumulator) ensureTool(event *dto.ResponsesStreamResponse) *responsesBufferedTool { if idx, ok := a.findToolIndex(event); ok { tool := a.tools[idx] a.applyToolMetadata(tool, event) + item := a.ensureItem(event, event.Item.Type) + item.ToolIndex = idx return tool } tool := &responsesBufferedTool{} a.applyToolMetadata(tool, event) idx := len(a.tools) a.tools = append(a.tools, tool) + item := a.ensureItem(event, event.Item.Type) + item.ToolIndex = idx if event.OutputIndex != nil { a.outputIndexToToolIdx[*event.OutputIndex] = idx if pending := a.pendingByOutputIndex[*event.OutputIndex]; pending != "" { diff --git a/relaykit/relayconvert/internal/shared/claude/reasoning.go b/relaykit/relayconvert/internal/shared/claude/reasoning.go new file mode 100644 index 0000000000..054fda404c --- /dev/null +++ b/relaykit/relayconvert/internal/shared/claude/reasoning.go @@ -0,0 +1,135 @@ +package claude + +import ( + "fmt" + "math" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" +) + +func ApplyReasoning(req *dto.ClaudeRequest, info convmeta.Meta, source reasoning.Intent) error { + if req == nil { + return nil + } + + native, err := reasoning.FromClaude(req) + if err != nil { + return err + } + explicit, err := reasoning.MergeExplicit(native, source, req.Model) + if err != nil { + return err + } + + opts := convmeta.OptionsOf(info) + baseModel := req.Model + capabilityModel := baseModel + suffix := reasoning.IntentFromState(convmeta.ReasoningStateOf(info)) + preserveSuffix := opts.ShouldPreserveThinkingSuffix(req.Model) + if info != nil && opts.ShouldPreserveThinkingSuffix(info.GetOriginModelName()) { + preserveSuffix = true + } + if preserveSuffix { + suffix = reasoning.Intent{} + } + if info != nil && !reasoning.IsKnownClaudeModel(capabilityModel) && reasoning.IsKnownClaudeModel(info.GetOriginModelName()) { + capabilityModel = info.GetOriginModelName() + } + intent, err := reasoning.MergeExplicitAndSuffix(explicit, suffix, req.Model) + if err != nil { + return err + } + knownClaudeModel := reasoning.IsKnownClaudeModel(capabilityModel) + if source.IsEmpty() && suffix.IsEmpty() && !knownClaudeModel { + // A native Messages request can target a non-Anthropic model through a + // Claude-compatible proxy. Its capability vocabulary belongs to that + // upstream, so preserve validated native controls instead of applying + // Anthropic model rules to an unknown model name. + if info != nil { + if effort := reasoning.EffectiveEffort(intent); effort != "" { + info.SetReasoningEffort(string(effort)) + } + } + return nil + } + if !knownClaudeModel && intent.Mode == reasoning.ModeAdaptive { + // Cross-protocol pivots cannot safely assume that an unknown + // Claude-compatible model implements Anthropic's adaptive mode. Render + // the broadly supported manual form while retaining the requested + // strength. Native Claude requests took the passthrough path above. + intent.Mode = reasoning.ModeEnabled + if intent.Effort == "" { + intent.Effort = reasoning.EffortHigh + } + } + if req.MaxTokens == nil && intent.HasStrength() { + // Adapter-provided defaults may be raised to accommodate an exact + // cross-protocol budget. Explicit client max_tokens values are never + // expanded and remain subject to the renderer's strict validation. + minimum := uint(1280) + if configuredDefault, configured := opts.Claude.DefaultMaxTokensFor(capabilityModel); configured && configuredDefault > 0 { + minimum = uint(configuredDefault) + } + if reasoning.ClaudeUsesManualThinking(capabilityModel, intent) && *intent.BudgetTokens >= 0 { + if *intent.BudgetTokens == math.MaxInt { + return fmt.Errorf("thinking budget is too large to derive max_tokens") + } + required := uint(*intent.BudgetTokens) + 1 + const maxDerivedTokens = uint(math.MaxInt32 / 2) + if required > maxDerivedTokens { + return fmt.Errorf("thinking budget %d exceeds the supported conversion limit", *intent.BudgetTokens) + } + if minimum < required { + minimum = required + } + } + req.MaxTokens = &minimum + } + + rendered, err := reasoning.RenderClaude(capabilityModel, intent, req.MaxTokens, opts.Claude.ThinkingAdapterBudgetTokensPercentage) + if err != nil { + return err + } + req.Model = baseModel + if rendered.Thinking != nil { + req.Thinking = rendered.Thinking + } + if rendered.OutputEffort != "" { + outputConfig := make(map[string]any) + if len(req.OutputConfig) > 0 { + if kitutil.GetJsonType(req.OutputConfig) != "object" { + return fmt.Errorf("Claude output_config must be a JSON object") + } + if err := kitutil.Unmarshal(req.OutputConfig, &outputConfig); err != nil { + return fmt.Errorf("invalid Claude output_config: %w", err) + } + if outputConfig == nil { + outputConfig = make(map[string]any) + } + } + outputConfig["effort"] = string(rendered.OutputEffort) + encoded, err := kitutil.Marshal(outputConfig) + if err != nil { + return fmt.Errorf("failed to marshal Claude output_config: %w", err) + } + req.OutputConfig = encoded + } + if rendered.ClearSampling { + req.Temperature = nil + req.TopP = nil + req.TopK = nil + } else if rendered.ConstrainThinkingSampling { + req.Temperature = nil + req.TopK = nil + if req.TopP != nil && (*req.TopP < 0.95 || *req.TopP > 1) { + req.TopP = nil + } + } + if info != nil && rendered.EffectiveEffort != "" { + info.SetReasoningEffort(string(rendered.EffectiveEffort)) + } + return nil +} diff --git a/relaykit/relayconvert/internal/shared/claude/usage.go b/relaykit/relayconvert/internal/shared/claude/usage.go new file mode 100644 index 0000000000..25fe72bb6f --- /dev/null +++ b/relaykit/relayconvert/internal/shared/claude/usage.go @@ -0,0 +1,52 @@ +package claude + +import "github.com/QuantumNous/new-api/relaykit/dto" + +func UsageFromOpenAI(usage *dto.Usage) *dto.ClaudeUsage { + if usage == nil { + return nil + } + // An existing sidecar snapshots the original provider usage; carry it + // across this bridge unchanged regardless of its dialect. Only synthesize + // an OpenAI snapshot when no sidecar exists yet. + existingBillingUsage := dto.CloneBillingUsage(usage.BillingUsage) + if existingBillingUsage != nil && existingBillingUsage.ClaudeUsage != nil && + (existingBillingUsage.Source == dto.BillingUsageSourceClaudeMessages || existingBillingUsage.Semantic == dto.BillingUsageSemanticAnthropic) { + result := existingBillingUsage.ClaudeUsage + result.BillingUsage = dto.CloneBillingUsage(usage.BillingUsage) + return result + } + billingUsage := existingBillingUsage + if billingUsage == nil { + billingUsage = dto.NewOpenAIChatBillingUsage(usage) + } + cacheCreation5m, cacheCreation1h := NormalizeCacheCreationSplit( + usage.PromptTokensDetails.CachedCreationTokens, + usage.ClaudeCacheCreation5mTokens, + usage.ClaudeCacheCreation1hTokens, + ) + cacheCreationTokens := usage.PromptTokensDetails.CacheCreationTokensTotal() + inputTokens := usage.PromptTokens + if usage.UsageSemantic != dto.BillingUsageSemanticAnthropic { + // OpenAI-style prompt/input totals include cache reads and writes, while + // Claude reports both separately from input_tokens. + inputTokens = usage.PromptTokens - usage.PromptTokensDetails.CachedTokens - cacheCreationTokens + if inputTokens < 0 { + inputTokens = 0 + } + } + result := &dto.ClaudeUsage{ + InputTokens: inputTokens, + OutputTokens: usage.CompletionTokens, + CacheCreationInputTokens: cacheCreationTokens, + CacheReadInputTokens: usage.PromptTokensDetails.CachedTokens, + BillingUsage: billingUsage, + } + if cacheCreation5m > 0 || cacheCreation1h > 0 { + result.CacheCreation = &dto.ClaudeCacheCreationUsage{ + Ephemeral5mInputTokens: cacheCreation5m, + Ephemeral1hInputTokens: cacheCreation1h, + } + } + return result +} diff --git a/relaykit/relayconvert/internal/shared/gemini/request.go b/relaykit/relayconvert/internal/shared/gemini/request.go index ac4ce3a694..8845bb242b 100644 --- a/relaykit/relayconvert/internal/shared/gemini/request.go +++ b/relaykit/relayconvert/internal/shared/gemini/request.go @@ -1,12 +1,12 @@ package gemini import ( + "fmt" "strconv" "strings" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" - kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" ) @@ -41,14 +41,6 @@ var SafetySettingCategories = []string{ const ThoughtSignatureBypassValue = "context_engineering_is_the_way_to_go" -const ( - pro25MinBudget = 128 - pro25MaxBudget = 32768 - flash25MaxBudget = 24576 - flash25LiteMinBudget = 512 - flash25LiteMaxBudget = 24576 -) - func ShouldAttachThoughtSignature(opts *convmeta.Options) bool { return opts != nil && opts.Gemini.FunctionCallThoughtSignatureEnabled } @@ -81,70 +73,109 @@ func AttachFirstTextThoughtSignature(opts *convmeta.Options, parts []dto.GeminiP return false } -func ApplyThinkingConfig(geminiRequest *dto.GeminiChatRequest, info convmeta.Meta, oaiRequest ...dto.GeneralOpenAIRequest) { +func ApplyThinkingConfig(geminiRequest *dto.GeminiChatRequest, info convmeta.Meta, oaiRequest ...dto.GeneralOpenAIRequest) error { opts := convmeta.OptionsOf(info) - if geminiRequest == nil || info == nil || !opts.Gemini.ThinkingAdapterEnabled { - return + if geminiRequest == nil { + return nil } modelName := convmeta.UpstreamModelName(info) - isNew25Pro := strings.HasPrefix(modelName, "gemini-2.5-pro") && - !strings.HasPrefix(modelName, "gemini-2.5-pro-preview-05-06") && - !strings.HasPrefix(modelName, "gemini-2.5-pro-preview-03-25") - - if strings.Contains(modelName, "-thinking-") { - parts := strings.SplitN(modelName, "-thinking-", 2) - if len(parts) == 2 && parts[1] != "" { - if budgetTokens, err := strconv.Atoi(parts[1]); err == nil { - clampedBudget := clampThinkingBudget(modelName, budgetTokens) - geminiRequest.GenerationConfig.ThinkingConfig = &dto.GeminiThinkingConfig{ - ThinkingBudget: kitutil.GetPointer(clampedBudget), - IncludeThoughts: true, - } - } + var source reasoning.Intent + if len(oaiRequest) > 0 { + if modelName == "" { + modelName = oaiRequest[0].Model } - } else if strings.HasSuffix(modelName, "-thinking") { - unsupportedModels := []string{ - "gemini-2.5-pro-preview-05-06", - "gemini-2.5-pro-preview-03-25", + var err error + source, err = reasoning.FromOpenAIChat(&oaiRequest[0]) + if err != nil { + return err } - isUnsupported := false - for _, unsupportedModel := range unsupportedModels { - if strings.HasPrefix(modelName, unsupportedModel) { - isUnsupported = true - break - } - } - - if isUnsupported { - geminiRequest.GenerationConfig.ThinkingConfig = &dto.GeminiThinkingConfig{ - IncludeThoughts: true, - } - } else { - geminiRequest.GenerationConfig.ThinkingConfig = &dto.GeminiThinkingConfig{ - IncludeThoughts: true, - } - if geminiRequest.GenerationConfig.MaxOutputTokens != nil && *geminiRequest.GenerationConfig.MaxOutputTokens > 0 { - budgetTokens := opts.Gemini.ThinkingAdapterBudgetTokensPercentage * float64(*geminiRequest.GenerationConfig.MaxOutputTokens) - clampedBudget := clampThinkingBudget(modelName, int(budgetTokens)) - geminiRequest.GenerationConfig.ThinkingConfig.ThinkingBudget = kitutil.GetPointer(clampedBudget) - } else if len(oaiRequest) > 0 { - geminiRequest.GenerationConfig.ThinkingConfig.ThinkingBudget = kitutil.GetPointer(clampThinkingBudgetByEffort(modelName, oaiRequest[0].ReasoningEffort)) - } - } - } else if strings.HasSuffix(modelName, "-nothinking") { - if !isNew25Pro { - geminiRequest.GenerationConfig.ThinkingConfig = &dto.GeminiThinkingConfig{ - ThinkingBudget: kitutil.GetPointer(0), - } - } - } else if _, level, ok := reasoning.TrimEffortSuffix(modelName); ok && level != "" { - geminiRequest.GenerationConfig.ThinkingConfig = &dto.GeminiThinkingConfig{ - IncludeThoughts: true, - ThinkingLevel: level, - } - info.SetReasoningEffort(level) } + + baseModel := modelName + suffix := reasoning.IntentFromState(convmeta.ReasoningStateOf(info)) + preserveSuffix := opts.ShouldPreserveThinkingSuffix(modelName) + if info != nil && opts.ShouldPreserveThinkingSuffix(info.GetOriginModelName()) { + preserveSuffix = true + } + if preserveSuffix { + suffix = reasoning.Intent{} + } + native, err := reasoning.FromGemini(geminiRequest) + if err != nil { + return err + } + source = reasoning.ResolveGeminiEnabledDefault(baseModel, source, geminiRequest.GenerationConfig.MaxOutputTokens) + if native.HasStrength() && source.HasStrength() { + equivalent, compareErr := reasoning.EquivalentGeminiStrength(baseModel, native, source) + if compareErr != nil { + return compareErr + } + if !equivalent { + nativeEffort := reasoning.EffectiveEffort(native) + sourceEffort := reasoning.EffectiveEffort(source) + return fmt.Errorf("%w for model %q: Gemini thinking_config effort %q differs from standard effort %q", reasoning.ErrEffortConflict, modelName, nativeEffort, sourceEffort) + } + // Native Gemini configuration is the lossless representation. Once the + // two controls are equivalent, retain only portable visibility metadata + // from the standard representation. + if native.IncludeThoughts == nil { + native.IncludeThoughts = source.IncludeThoughts + } + source = reasoning.Intent{} + } + explicit, err := reasoning.MergeExplicit(native, source, modelName) + if err != nil { + return err + } + if explicit.HasStrength() && suffix.HasStrength() { + equivalent, compareErr := reasoning.EquivalentGeminiStrength(baseModel, explicit, suffix) + if compareErr != nil { + return compareErr + } + if equivalent { + if explicit.IncludeThoughts == nil { + explicit.IncludeThoughts = suffix.IncludeThoughts + } + suffix = reasoning.Intent{} + } + } + requested, err := reasoning.MergeExplicitAndSuffix(explicit, suffix, modelName) + if err != nil { + return err + } + requested = reasoning.ResolveGeminiEnabledDefault(baseModel, requested, geminiRequest.GenerationConfig.MaxOutputTokens) + + if native.HasStrength() && !suffix.HasStrength() { + if explicit.IncludeThoughts != nil { + geminiRequest.GenerationConfig.ThinkingConfig.IncludeThoughts = explicit.IncludeThoughts + } + effort, err := reasoning.ValidateGeminiThinkingConfig(baseModel, geminiRequest.GenerationConfig.ThinkingConfig) + if err != nil { + return err + } + if info != nil && effort != "" { + info.SetReasoningEffort(string(effort)) + } + return nil + } + if requested.IsEmpty() { + return nil + } + rendered, err := reasoning.RenderGemini( + baseModel, + requested, + geminiRequest.GenerationConfig.MaxOutputTokens, + opts.Gemini.ThinkingAdapterBudgetTokensPercentage, + ) + if err != nil { + return err + } + geminiRequest.GenerationConfig.ThinkingConfig = rendered.Config + if info != nil && rendered.EffectiveEffort != "" { + info.SetReasoningEffort(string(rendered.EffectiveEffort)) + } + return nil } func ParseStopSequences(stop any) []string { @@ -200,68 +231,3 @@ func SupportedMimeTypesList() []string { } return keys } - -func isNew25ProModel(modelName string) bool { - return strings.HasPrefix(modelName, "gemini-2.5-pro") && - !strings.HasPrefix(modelName, "gemini-2.5-pro-preview-05-06") && - !strings.HasPrefix(modelName, "gemini-2.5-pro-preview-03-25") -} - -func is25FlashLiteModel(modelName string) bool { - return strings.HasPrefix(modelName, "gemini-2.5-flash-lite") -} - -func clampThinkingBudget(modelName string, budget int) int { - isNew25Pro := isNew25ProModel(modelName) - is25FlashLite := is25FlashLiteModel(modelName) - - if is25FlashLite { - if budget < flash25LiteMinBudget { - return flash25LiteMinBudget - } - if budget > flash25LiteMaxBudget { - return flash25LiteMaxBudget - } - } else if isNew25Pro { - if budget < pro25MinBudget { - return pro25MinBudget - } - if budget > pro25MaxBudget { - return pro25MaxBudget - } - } else { - if budget < 0 { - return 0 - } - if budget > flash25MaxBudget { - return flash25MaxBudget - } - } - return budget -} - -func clampThinkingBudgetByEffort(modelName string, effort string) int { - isNew25Pro := isNew25ProModel(modelName) - is25FlashLite := is25FlashLiteModel(modelName) - - maxBudget := 0 - if is25FlashLite { - maxBudget = flash25LiteMaxBudget - } - if isNew25Pro { - maxBudget = pro25MaxBudget - } else { - maxBudget = flash25MaxBudget - } - switch effort { - case "high": - maxBudget = maxBudget * 80 / 100 - case "medium": - maxBudget = maxBudget * 50 / 100 - case "low": - maxBudget = maxBudget * 20 / 100 - case "minimal": - maxBudget = maxBudget * 5 / 100 - } - return clampThinkingBudget(modelName, maxBudget) -} diff --git a/relaykit/relayconvert/internal/toolconv/decode.go b/relaykit/relayconvert/internal/toolconv/decode.go new file mode 100644 index 0000000000..748fed041d --- /dev/null +++ b/relaykit/relayconvert/internal/toolconv/decode.go @@ -0,0 +1,960 @@ +package toolconv + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/types" +) + +const maxClaudeWebSearchUses = 1000 + +func ExtractRequest(format types.RelayFormat, request any) (any, Set, error) { + switch format { + case types.RelayFormatOpenAI: + return extractOpenAIChatRequest(request) + case types.RelayFormatOpenAIResponses: + return extractOpenAIResponsesRequest(request) + case types.RelayFormatClaude: + return extractClaudeRequest(request) + case types.RelayFormatGemini: + return extractGeminiRequest(request) + default: + return request, Set{Source: format}, nil + } +} + +func extractOpenAIChatRequest(request any) (any, Set, error) { + source, ok := request.(*dto.GeneralOpenAIRequest) + if !ok { + value, valueOK := request.(dto.GeneralOpenAIRequest) + if !valueOK { + return nil, Set{}, fmt.Errorf("expected OpenAI chat completions request, got %T", request) + } + source = &value + } + + set := Set{Source: types.RelayFormatOpenAI} + set.ParallelAllowed = source.ParallelTooCalls + if len(source.Functions) > 0 { + var functions []dto.FunctionRequest + if err := kitutil.Unmarshal(source.Functions, &functions); err != nil { + return nil, Set{}, fmt.Errorf("invalid legacy functions: %w", err) + } + for _, function := range functions { + function := function + set.Definitions = append(set.Definitions, Definition{ + Kind: KindFunction, + Execution: ExecutionClient, + Function: &Function{Name: function.Name, Description: function.Description, Parameters: function.Parameters, Strict: function.Strict}, + }) + } + } + for index, tool := range source.Tools { + if tool.Type == "function" || tool.Type == "" { + set.Definitions = append(set.Definitions, Definition{ + Kind: KindFunction, + Execution: ExecutionClient, + Function: &Function{ + Name: tool.Function.Name, + Description: tool.Function.Description, + Parameters: tool.Function.Parameters, + Strict: tool.Function.Strict, + }, + }) + continue + } + if len(tool.Custom) == 0 { + return nil, Set{}, fmt.Errorf("tools[%d] has unsupported type %q without a native payload", index, tool.Type) + } + definition, err := decodeOpenAIResponsesDefinition(tool.Custom) + if err != nil { + return nil, Set{}, fmt.Errorf("tools[%d]: %w", index, err) + } + set.Definitions = append(set.Definitions, definition) + } + + if source.WebSearchOptions != nil { + webSearch := &WebSearch{ + SearchContextSize: source.WebSearchOptions.SearchContextSize, + } + location, err := decodeOpenAIChatLocation(source.WebSearchOptions.UserLocation) + if err != nil { + return nil, Set{}, err + } + webSearch.Location = location + set.Definitions = append(set.Definitions, Definition{ + Kind: KindWebSearch, + Execution: ExecutionServer, + NativeType: "web_search_options", + WebSearch: webSearch, + }) + } + + if choice, err := decodeOpenAIChatChoice(source.ToolChoice); err != nil { + return nil, Set{}, err + } else if choice != nil { + set.Choice = choice + } + if len(source.FunctionCall) > 0 { + legacyChoice, err := decodeLegacyOpenAIFunctionChoice(source.FunctionCall) + if err != nil { + return nil, Set{}, err + } + if set.Choice != nil && legacyChoice != nil { + return nil, Set{}, fmt.Errorf("tool_choice and legacy function_call cannot both be converted") + } + set.Choice = legacyChoice + } + + clone := *source + clone.Tools = nil + clone.ToolChoice = nil + clone.WebSearchOptions = nil + clone.Functions = nil + clone.FunctionCall = nil + clone.ParallelTooCalls = nil + return &clone, set, nil +} + +func extractOpenAIResponsesRequest(request any) (any, Set, error) { + source, ok := request.(*dto.OpenAIResponsesRequest) + if !ok { + value, valueOK := request.(dto.OpenAIResponsesRequest) + if !valueOK { + return nil, Set{}, fmt.Errorf("expected OpenAI Responses request, got %T", request) + } + source = &value + } + + set := Set{Source: types.RelayFormatOpenAIResponses} + set.ParallelAllowed = rawBoolPointer(source.ParallelToolCalls) + if len(source.Tools) > 0 { + var rawTools []json.RawMessage + if err := kitutil.Unmarshal(source.Tools, &rawTools); err != nil { + return nil, Set{}, fmt.Errorf("invalid Responses tools: %w", err) + } + for index, rawTool := range rawTools { + definition, err := decodeOpenAIResponsesDefinition(rawTool) + if err != nil { + return nil, Set{}, fmt.Errorf("tools[%d]: %w", index, err) + } + set.Definitions = append(set.Definitions, definition) + } + } + choice, err := decodeOpenAIResponsesChoice(source.ToolChoice) + if err != nil { + return nil, Set{}, err + } + set.Choice = choice + + clone := *source + clone.Tools = nil + clone.ToolChoice = nil + clone.ParallelToolCalls = nil + sanitizedInput, history, err := extractOpenAIResponsesHostedHistory(source.Input) + if err != nil { + return nil, Set{}, err + } + clone.Input = sanitizedInput + set.History = history + return &clone, set, nil +} + +func extractClaudeRequest(request any) (any, Set, error) { + source, ok := request.(*dto.ClaudeRequest) + if !ok { + value, valueOK := request.(dto.ClaudeRequest) + if !valueOK { + return nil, Set{}, fmt.Errorf("expected Claude Messages request, got %T", request) + } + source = &value + } + + set := Set{Source: types.RelayFormatClaude} + if source.ToolChoice != nil { + rawChoice, rawErr := rawJSON(source.ToolChoice) + if rawErr == nil { + var choiceMap map[string]any + if kitutil.Unmarshal(rawChoice, &choiceMap) == nil { + if disabled, ok := choiceMap["disable_parallel_tool_use"].(bool); ok { + allowed := !disabled + set.ParallelAllowed = &allowed + } + } + } + } + if source.Tools != nil { + rawTools, err := rawJSON(source.Tools) + if err != nil { + return nil, Set{}, fmt.Errorf("invalid Claude tools: %w", err) + } + var tools []json.RawMessage + if err := kitutil.Unmarshal(rawTools, &tools); err != nil { + return nil, Set{}, fmt.Errorf("invalid Claude tools: %w", err) + } + for index, rawTool := range tools { + definition, err := decodeClaudeDefinition(rawTool) + if err != nil { + return nil, Set{}, fmt.Errorf("tools[%d]: %w", index, err) + } + set.Definitions = append(set.Definitions, definition) + } + } + choice, err := decodeClaudeChoice(source.ToolChoice, set.Definitions) + if err != nil { + return nil, Set{}, err + } + set.Choice = choice + + clone := *source + clone.Tools = nil + clone.ToolChoice = nil + clone.Messages, set.History, err = extractClaudeHostedHistory(source.Messages) + if err != nil { + return nil, Set{}, err + } + return &clone, set, nil +} + +func extractGeminiRequest(request any) (any, Set, error) { + source, ok := request.(*dto.GeminiChatRequest) + if !ok { + value, valueOK := request.(dto.GeminiChatRequest) + if !valueOK { + return nil, Set{}, fmt.Errorf("expected Gemini generateContent request, got %T", request) + } + source = &value + } + + set := Set{Source: types.RelayFormatGemini} + if source.ToolConfig != nil { + set.NativeToolConfig, _ = rawJSON(source.ToolConfig) + } + if len(source.Tools) > 0 { + var tools []json.RawMessage + if err := kitutil.Unmarshal(source.Tools, &tools); err != nil { + return nil, Set{}, fmt.Errorf("invalid Gemini tools: %w", err) + } + for index, rawTool := range tools { + definitions, err := decodeGeminiDefinitions(rawTool) + if err != nil { + return nil, Set{}, fmt.Errorf("tools[%d]: %w", index, err) + } + for definitionIndex := range definitions { + definitions[definitionIndex].Group = index + } + set.Definitions = append(set.Definitions, definitions...) + } + } + set.Choice = decodeGeminiChoice(source.ToolConfig) + + clone := *source + clone.Tools = nil + clone.ToolConfig = nil + return &clone, set, nil +} + +func decodeOpenAIResponsesDefinition(raw json.RawMessage) (Definition, error) { + var tool map[string]any + if err := kitutil.Unmarshal(raw, &tool); err != nil { + return Definition{}, err + } + toolType := strings.TrimSpace(kitutil.Interface2String(tool["type"])) + if toolType == "function" { + return Definition{ + Kind: KindFunction, + Execution: ExecutionClient, + Name: strings.TrimSpace(kitutil.Interface2String(tool["name"])), + Raw: cloneRaw(raw), + Function: &Function{ + Name: strings.TrimSpace(kitutil.Interface2String(tool["name"])), + Description: kitutil.Interface2String(tool["description"]), + Parameters: tool["parameters"], + Strict: boolPointer(tool, "strict"), + }, + }, nil + } + if isOpenAIResponsesWebSearchType(toolType) { + webSearch := &WebSearch{ + SearchContextSize: strings.TrimSpace(kitutil.Interface2String(tool["search_context_size"])), + ExternalWebAccess: boolPointer(tool, "external_web_access"), + } + if value, exists := tool["return_token_budget"]; exists { + encoded, err := rawJSON(value) + if err != nil { + return Definition{}, err + } + webSearch.ReturnTokenBudget = encoded + } + if filters, ok := tool["filters"].(map[string]any); ok { + webSearch.AllowedDomains = stringSlice(filters["allowed_domains"]) + } + if location, ok := tool["user_location"].(map[string]any); ok { + webSearch.Location = locationFromMap(location) + } + return Definition{ + Kind: KindWebSearch, + Execution: ExecutionServer, + NativeType: toolType, + WebSearch: webSearch, + Raw: cloneRaw(raw), + }, nil + } + return Definition{ + Kind: kindFromNativeType(toolType), + Execution: executionFromNativeType(toolType), + NativeType: toolType, + Raw: cloneRaw(raw), + }, nil +} + +func decodeClaudeDefinition(raw json.RawMessage) (Definition, error) { + var tool map[string]any + if err := kitutil.Unmarshal(raw, &tool); err != nil { + return Definition{}, err + } + toolType := strings.TrimSpace(kitutil.Interface2String(tool["type"])) + if strings.HasPrefix(toolType, "web_search") { + if !isVersionedClaudeWebSearchType(toolType) { + return Definition{}, fmt.Errorf("invalid Claude web-search tool version %q", toolType) + } + if !isKnownClaudeWebSearchType(toolType) { + return Definition{ + Kind: KindNative, + Execution: ExecutionServer, + NativeType: toolType, + Name: strings.TrimSpace(kitutil.Interface2String(tool["name"])), + Raw: cloneRaw(raw), + }, nil + } + toolName := strings.TrimSpace(kitutil.Interface2String(tool["name"])) + if toolName != "web_search" { + return Definition{}, fmt.Errorf("Claude web-search tool name must be %q", "web_search") + } + webSearch := &WebSearch{ + AllowedDomains: stringSlice(tool["allowed_domains"]), + BlockedDomains: stringSlice(tool["blocked_domains"]), + AllowedCallers: stringSlice(tool["allowed_callers"]), + ResponseInclusion: strings.TrimSpace(kitutil.Interface2String(tool["response_inclusion"])), + } + if _, exists := tool["max_uses"]; exists { + var fields struct { + MaxUses *int `json:"max_uses"` + } + if err := kitutil.Unmarshal(raw, &fields); err != nil || fields.MaxUses == nil { + return Definition{}, fmt.Errorf("max_uses must be a JSON integer") + } + if *fields.MaxUses <= 0 || *fields.MaxUses > maxClaudeWebSearchUses { + return Definition{}, fmt.Errorf("max_uses must be between 1 and %d", maxClaudeWebSearchUses) + } + webSearch.MaxUses = fields.MaxUses + } + if len(webSearch.AllowedDomains) > 0 && len(webSearch.BlockedDomains) > 0 { + return Definition{}, fmt.Errorf("allowed_domains and blocked_domains are mutually exclusive") + } + if webSearch.ResponseInclusion != "" && !claudeWebSearchSupportsResponseInclusion(toolType) { + return Definition{}, fmt.Errorf("response_inclusion requires Claude web_search_20260318") + } + if location, ok := tool["user_location"].(map[string]any); ok { + webSearch.Location = locationFromMap(location) + } + return Definition{ + Kind: KindWebSearch, + Execution: ExecutionServer, + NativeType: toolType, + Name: toolName, + WebSearch: webSearch, + Raw: cloneRaw(raw), + }, nil + } + if toolType == "" { + return Definition{ + Kind: KindFunction, + Execution: ExecutionClient, + Name: strings.TrimSpace(kitutil.Interface2String(tool["name"])), + Raw: cloneRaw(raw), + Function: &Function{ + Name: strings.TrimSpace(kitutil.Interface2String(tool["name"])), + Description: kitutil.Interface2String(tool["description"]), + Parameters: tool["input_schema"], + Strict: boolPointer(tool, "strict"), + }, + }, nil + } + return Definition{ + Kind: kindFromNativeType(toolType), + Execution: executionFromNativeType(toolType), + NativeType: toolType, + Name: strings.TrimSpace(kitutil.Interface2String(tool["name"])), + Raw: cloneRaw(raw), + }, nil +} + +func decodeGeminiDefinitions(raw json.RawMessage) ([]Definition, error) { + var tool map[string]any + if err := kitutil.Unmarshal(raw, &tool); err != nil { + return nil, err + } + definitions := make([]Definition, 0) + if functions, ok := tool["functionDeclarations"].([]any); ok { + for _, value := range functions { + function, ok := value.(map[string]any) + if !ok { + continue + } + parameters := function["parameters"] + parametersJSONSchema, hasParametersJSONSchema := function["parametersJsonSchema"] + if parameters != nil && hasParametersJSONSchema && parametersJSONSchema != nil { + return nil, fmt.Errorf("function %q declares both parameters and parametersJsonSchema", strings.TrimSpace(kitutil.Interface2String(function["name"]))) + } + if parameters == nil && hasParametersJSONSchema { + parameters = parametersJSONSchema + } + functionRaw, err := rawJSON(map[string]any{"functionDeclarations": []any{value}}) + if err != nil { + return nil, err + } + definitions = append(definitions, Definition{ + Kind: KindFunction, + Execution: ExecutionClient, + Name: strings.TrimSpace(kitutil.Interface2String(function["name"])), + Raw: functionRaw, + Function: &Function{ + Name: strings.TrimSpace(kitutil.Interface2String(function["name"])), + Description: kitutil.Interface2String(function["description"]), + Parameters: parameters, + }, + }) + } + } + for key := range tool { + var kind Kind + var nativeType string + switch key { + case "functionDeclarations": + continue + case "googleSearch": + kind, nativeType = KindWebSearch, "googleSearch" + case "googleSearchRetrieval": + kind, nativeType = KindWebSearch, "googleSearchRetrieval" + case "enterpriseWebSearch": + kind, nativeType = KindWebSearch, "enterpriseWebSearch" + case "googleMaps": + kind, nativeType = KindNative, "googleMaps" + case "codeExecution": + kind, nativeType = KindCodeExecution, "codeExecution" + case "urlContext": + kind, nativeType = KindURLContext, "urlContext" + case "fileSearch": + kind, nativeType = KindFileSearch, "fileSearch" + case "computerUse": + kind, nativeType = KindComputerUse, "computerUse" + case "retrieval": + kind, nativeType = KindFileSearch, "retrieval" + default: + kind, nativeType = KindNative, key + } + keyRaw, err := rawJSON(map[string]any{key: tool[key]}) + if err != nil { + return nil, err + } + definition := Definition{ + Kind: kind, + Execution: ExecutionServer, + NativeType: nativeType, + Name: strings.TrimSpace(kitutil.Interface2String(tool["name"])), + Raw: keyRaw, + } + if kind == KindWebSearch { + definition.WebSearch = &WebSearch{} + } + definitions = append(definitions, definition) + } + return definitions, nil +} + +func decodeLegacyOpenAIFunctionChoice(raw json.RawMessage) (*Choice, error) { + if len(raw) == 0 { + return nil, nil + } + if kitutil.GetJsonType(raw) == "string" { + var value string + if err := kitutil.Unmarshal(raw, &value); err != nil { + return nil, err + } + return choiceFromString(value), nil + } + var value map[string]any + if err := kitutil.Unmarshal(raw, &value); err != nil { + return nil, fmt.Errorf("invalid legacy function_call: %w", err) + } + name := strings.TrimSpace(kitutil.Interface2String(value["name"])) + if name == "" { + return nil, fmt.Errorf("legacy function_call requires name") + } + return &Choice{Mode: ChoiceNamed, Kind: KindFunction, Name: name}, nil +} + +func rawBoolPointer(raw json.RawMessage) *bool { + if len(raw) == 0 || kitutil.GetJsonType(raw) != "boolean" { + return nil + } + var value bool + if kitutil.Unmarshal(raw, &value) != nil { + return nil + } + return &value +} + +func decodeOpenAIChatLocation(raw json.RawMessage) (*ApproximateLocation, error) { + if len(raw) == 0 { + return nil, nil + } + var wrapper map[string]any + if err := kitutil.Unmarshal(raw, &wrapper); err != nil { + return nil, fmt.Errorf("invalid web_search_options.user_location: %w", err) + } + location, ok := wrapper["approximate"].(map[string]any) + if !ok { + return nil, nil + } + return locationFromMap(location), nil +} + +func decodeOpenAIChatChoice(value any) (*Choice, error) { + if value == nil { + return nil, nil + } + if text, ok := value.(string); ok { + return choiceFromString(text), nil + } + raw, err := rawJSON(value) + if err != nil { + return nil, fmt.Errorf("invalid Chat tool_choice: %w", err) + } + var choice map[string]any + if err := kitutil.Unmarshal(raw, &choice); err != nil { + return nil, fmt.Errorf("invalid Chat tool_choice: %w", err) + } + if strings.TrimSpace(kitutil.Interface2String(choice["type"])) != "function" { + return &Choice{Mode: ChoiceOpaque, Raw: cloneRaw(raw)}, nil + } + function, _ := choice["function"].(map[string]any) + name := strings.TrimSpace(kitutil.Interface2String(function["name"])) + if name == "" { + return nil, fmt.Errorf("Chat function tool_choice requires function.name") + } + return &Choice{Mode: ChoiceNamed, Kind: KindFunction, Name: name}, nil +} + +func decodeOpenAIResponsesChoice(raw json.RawMessage) (*Choice, error) { + if len(raw) == 0 { + return nil, nil + } + if kitutil.GetJsonType(raw) == "string" { + var text string + if err := kitutil.Unmarshal(raw, &text); err != nil { + return nil, err + } + return choiceFromString(text), nil + } + var value map[string]any + if err := kitutil.Unmarshal(raw, &value); err != nil { + return nil, fmt.Errorf("invalid Responses tool_choice: %w", err) + } + toolType := strings.TrimSpace(kitutil.Interface2String(value["type"])) + if toolType == "function" { + name := strings.TrimSpace(kitutil.Interface2String(value["name"])) + if name == "" { + return nil, fmt.Errorf("Responses function tool_choice requires name") + } + return &Choice{Mode: ChoiceNamed, Kind: KindFunction, Name: name}, nil + } + if isOpenAIResponsesWebSearchType(toolType) { + return &Choice{Mode: ChoiceNamed, Kind: KindWebSearch, Name: "web_search", NativeType: toolType, Raw: cloneRaw(raw)}, nil + } + return &Choice{Mode: ChoiceOpaque, Kind: kindFromNativeType(toolType), NativeType: toolType, Raw: cloneRaw(raw)}, nil +} + +func decodeClaudeChoice(value any, definitions []Definition) (*Choice, error) { + if value == nil { + return nil, nil + } + raw, err := rawJSON(value) + if err != nil { + return nil, fmt.Errorf("invalid Claude tool_choice: %w", err) + } + var choice map[string]any + if err := kitutil.Unmarshal(raw, &choice); err != nil { + return nil, fmt.Errorf("invalid Claude tool_choice: %w", err) + } + choiceType := strings.TrimSpace(kitutil.Interface2String(choice["type"])) + var decoded *Choice + switch choiceType { + case "auto": + decoded = &Choice{Mode: ChoiceAuto} + case "none": + decoded = &Choice{Mode: ChoiceNone} + case "any": + decoded = &Choice{Mode: ChoiceRequired} + case "tool": + name := strings.TrimSpace(kitutil.Interface2String(choice["name"])) + kind := KindNative + matches := 0 + for _, definition := range definitions { + definitionName := definition.Name + if definition.Kind == KindFunction && definition.Function != nil { + definitionName = definition.Function.Name + } + if definitionName != name { + continue + } + matches++ + kind = definition.Kind + } + if matches > 1 { + return nil, fmt.Errorf("Claude tool_choice name %q is ambiguous across %d definitions", name, matches) + } + decoded = &Choice{Mode: ChoiceNamed, Kind: kind, Name: name} + default: + decoded = &Choice{Mode: ChoiceOpaque} + } + if disabled, ok := choice["disable_parallel_tool_use"].(bool); ok { + decoded.DisableParallelToolUse = &disabled + } + decoded.Raw = cloneRaw(raw) + return decoded, nil +} + +func decodeGeminiChoice(config *dto.ToolConfig) *Choice { + if config == nil || config.FunctionCallingConfig == nil { + return nil + } + functionConfig := config.FunctionCallingConfig + switch strings.ToUpper(strings.TrimSpace(string(functionConfig.Mode))) { + case "NONE": + return &Choice{Mode: ChoiceNone} + case "ANY": + if len(functionConfig.AllowedFunctionNames) == 1 { + return &Choice{Mode: ChoiceNamed, Kind: KindFunction, Name: functionConfig.AllowedFunctionNames[0]} + } + return &Choice{ + Mode: ChoiceRequired, + Kind: KindFunction, + AllowedNames: append([]string(nil), functionConfig.AllowedFunctionNames...), + } + case "", "AUTO": + return &Choice{Mode: ChoiceAuto} + default: + raw, _ := rawJSON(functionConfig) + return &Choice{Mode: ChoiceOpaque, Raw: raw} + } +} + +func choiceFromString(value string) *Choice { + switch strings.ToLower(strings.TrimSpace(value)) { + case "none": + return &Choice{Mode: ChoiceNone} + case "required", "any": + return &Choice{Mode: ChoiceRequired} + case "auto": + return &Choice{Mode: ChoiceAuto} + default: + raw, _ := rawJSON(value) + return &Choice{Mode: ChoiceOpaque, Raw: raw} + } +} + +func isOpenAIResponsesWebSearchType(toolType string) bool { + switch toolType { + case "web_search", "web_search_2025_08_26", "web_search_preview", "web_search_preview_2025_03_11": + return true + default: + return false + } +} + +func claudeWebSearchSupportsResponseInclusion(toolType string) bool { + return toolType == "web_search_20260318" +} + +func isKnownClaudeWebSearchType(toolType string) bool { + switch toolType { + case "web_search_20250305", "web_search_20260209", "web_search_20260318": + return true + default: + return false + } +} + +func isVersionedClaudeWebSearchType(toolType string) bool { + const prefix = "web_search_" + version := strings.TrimPrefix(toolType, prefix) + if !strings.HasPrefix(toolType, prefix) || len(version) != 8 { + return false + } + _, err := strconv.ParseUint(version, 10, 32) + return err == nil +} + +func locationFromMap(value map[string]any) *ApproximateLocation { + if len(value) == 0 { + return nil + } + location := &ApproximateLocation{ + City: strings.TrimSpace(kitutil.Interface2String(value["city"])), + Region: strings.TrimSpace(kitutil.Interface2String(value["region"])), + Country: strings.TrimSpace(kitutil.Interface2String(value["country"])), + Timezone: strings.TrimSpace(kitutil.Interface2String(value["timezone"])), + } + if location.City == "" && location.Region == "" && location.Country == "" && location.Timezone == "" { + return nil + } + return location +} + +func boolPointer(value map[string]any, key string) *bool { + raw, exists := value[key] + if !exists { + return nil + } + parsed, ok := raw.(bool) + if !ok { + return nil + } + return &parsed +} + +func stringSlice(value any) []string { + items, ok := value.([]any) + if !ok { + if strings, stringsOK := value.([]string); stringsOK { + return append([]string(nil), strings...) + } + return nil + } + result := make([]string, 0, len(items)) + for _, item := range items { + if text, ok := item.(string); ok && strings.TrimSpace(text) != "" { + result = append(result, text) + } + } + return result +} + +func rawJSON(value any) (json.RawMessage, error) { + switch raw := value.(type) { + case json.RawMessage: + return cloneRaw(raw), nil + case []byte: + return cloneRaw(raw), nil + default: + encoded, err := kitutil.Marshal(value) + return json.RawMessage(encoded), err + } +} + +func cloneRaw(raw []byte) json.RawMessage { + return append(json.RawMessage(nil), raw...) +} + +func kindFromNativeType(toolType string) Kind { + switch { + case toolType == "file_search": + return KindFileSearch + case strings.HasPrefix(toolType, "web_fetch"): + return KindWebFetch + case toolType == "code_interpreter", strings.HasPrefix(toolType, "code_execution"): + return KindCodeExecution + case strings.Contains(toolType, "computer"): + return KindComputerUse + case toolType == "url_context": + return KindURLContext + case toolType == "mcp", toolType == "mcp_toolset": + return KindMCP + case toolType == "image_generation": + return KindImage + default: + return KindNative + } +} + +func executionFromNativeType(toolType string) Execution { + if strings.HasPrefix(toolType, "computer_") || strings.HasPrefix(toolType, "bash_") || strings.HasPrefix(toolType, "text_editor_") || strings.HasPrefix(toolType, "memory_") { + return ExecutionClient + } + return ExecutionServer +} + +func extractOpenAIResponsesHostedHistory(input json.RawMessage) (json.RawMessage, []HostedHistoryItem, error) { + if len(input) == 0 || kitutil.GetJsonType(input) != "array" { + return input, nil, nil + } + var rawItems []json.RawMessage + if err := kitutil.Unmarshal(input, &rawItems); err != nil { + return nil, nil, fmt.Errorf("invalid Responses input: %w", err) + } + filtered := make([]json.RawMessage, 0, len(rawItems)) + var history []HostedHistoryItem + for index, rawItem := range rawItems { + var item map[string]any + if err := kitutil.Unmarshal(rawItem, &item); err != nil { + return nil, nil, fmt.Errorf("input[%d]: %w", index, err) + } + itemType := strings.TrimSpace(kitutil.Interface2String(item["type"])) + if !isResponsesHostedHistoryType(itemType) { + filtered = append(filtered, rawItem) + continue + } + status := strings.TrimSpace(kitutil.Interface2String(item["status"])) + action := rawMapValue(item, "action") + results := firstRawMapValue(item, "results", "sources", "output") + if itemType == "mcp_call" { + action = rawMapValue(item, "arguments") + output := rawMapValue(item, "output") + itemError := rawMapValue(item, "error") + results = output + if rawJSONPresent(itemError) { + results = itemError + status = "failed" + } + } + history = append(history, HostedHistoryItem{ + Kind: hostedKindFromResponsesType(itemType), + NativeType: itemType, + Role: strings.TrimSpace(kitutil.Interface2String(item["role"])), + MessageIndex: index, + Sequence: index, + ID: strings.TrimSpace(kitutil.Interface2String(item["id"])), + CallID: strings.TrimSpace(kitutil.Interface2String(item["call_id"])), + Name: strings.TrimSpace(kitutil.Interface2String(item["name"])), + ServerName: strings.TrimSpace(kitutil.Interface2String(item["server_label"])), + Status: status, + Action: action, + Results: results, + Caller: rawMapValue(item, "caller"), + Raw: cloneRaw(rawItem), + }) + } + if len(history) == 0 { + return input, nil, nil + } + encoded, err := kitutil.Marshal(filtered) + if err != nil { + return nil, nil, err + } + return encoded, history, nil +} + +func isResponsesHostedHistoryType(itemType string) bool { + switch strings.TrimSpace(itemType) { + case "web_search_call", "file_search_call", "code_interpreter_call", "computer_call", "computer_call_output", "image_generation_call", "local_shell_call", "local_shell_call_output", "apply_patch_call", "apply_patch_call_output", "mcp_call", "mcp_list_tools", "mcp_approval_request", "mcp_approval_response": + return true + default: + return false + } +} + +func extractClaudeHostedHistory(messages []dto.ClaudeMessage) ([]dto.ClaudeMessage, []HostedHistoryItem, error) { + clonedMessages := make([]dto.ClaudeMessage, 0, len(messages)) + var history []HostedHistoryItem + for messageIndex := range messages { + message := messages[messageIndex] + if message.IsStringContent() { + clonedMessages = append(clonedMessages, message) + continue + } + rawContent, err := rawJSON(message.Content) + if err != nil { + return nil, nil, fmt.Errorf("messages[%d].content: %w", messageIndex, err) + } + var blocks []json.RawMessage + if err := kitutil.Unmarshal(rawContent, &blocks); err != nil { + return nil, nil, fmt.Errorf("messages[%d].content: %w", messageIndex, err) + } + filtered := make([]any, 0, len(blocks)) + historyStart := len(history) + for blockIndex, rawBlock := range blocks { + var block map[string]any + if err := kitutil.Unmarshal(rawBlock, &block); err != nil { + return nil, nil, fmt.Errorf("messages[%d].content[%d]: %w", messageIndex, blockIndex, err) + } + blockType := strings.TrimSpace(kitutil.Interface2String(block["type"])) + if blockType != "server_tool_use" && blockType != "mcp_tool_use" && !isClaudeHostedToolBlock(blockType) { + filtered = append(filtered, block) + continue + } + name := strings.TrimSpace(kitutil.Interface2String(block["name"])) + kind := hostedKindFromClaudeCall(blockType, name) + if strings.HasSuffix(blockType, "_tool_result") { + kind = hostedKindFromClaudeResult(blockType) + } + results := rawMapValue(block, "content") + status := "in_progress" + if strings.HasSuffix(blockType, "_tool_result") { + status = "completed" + isError, _ := block["is_error"].(bool) + failed, _ := claudeHostedResultFailure( + blockType, + results, + &isError, + strings.TrimSpace(kitutil.Interface2String(block["error_code"])), + ) + if failed { + status = "failed" + } + } + history = append(history, HostedHistoryItem{ + Kind: kind, + NativeType: blockType, + Role: message.Role, + MessageIndex: messageIndex, + BlockIndex: blockIndex, + Sequence: len(history), + ID: strings.TrimSpace(kitutil.Interface2String(block["id"])), + CallID: strings.TrimSpace(kitutil.Interface2String(block["tool_use_id"])), + Name: name, + ServerName: strings.TrimSpace(kitutil.Interface2String(block["server_name"])), + Status: status, + Action: rawMapValue(block, "input"), + Results: results, + Caller: rawMapValue(block, "caller"), + Raw: cloneRaw(rawBlock), + }) + } + if len(filtered) > 0 { + for index := historyStart; index < len(history); index++ { + history[index].MessageHasRegular = true + } + message.Content = filtered + clonedMessages = append(clonedMessages, message) + } + } + return clonedMessages, history, nil +} + +func rawMapValue(value map[string]any, key string) json.RawMessage { + item, exists := value[key] + if !exists { + return nil + } + encoded, err := kitutil.Marshal(item) + if err != nil { + return nil + } + return encoded +} + +func firstRawMapValue(value map[string]any, keys ...string) json.RawMessage { + for _, key := range keys { + if raw := rawMapValue(value, key); len(raw) > 0 { + return raw + } + } + return nil +} diff --git a/relaykit/relayconvert/internal/toolconv/encode.go b/relaykit/relayconvert/internal/toolconv/encode.go new file mode 100644 index 0000000000..224901f478 --- /dev/null +++ b/relaykit/relayconvert/internal/toolconv/encode.go @@ -0,0 +1,1343 @@ +package toolconv + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + sharedgemini "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/gemini" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/types" +) + +func AttachRequest(format types.RelayFormat, request any, set Set, options *convmeta.Options) (any, []types.ConversionDiagnostic, error) { + if set.Empty() { + return request, nil, nil + } + var ( + value any + diagnostics []types.ConversionDiagnostic + err error + ) + switch format { + case types.RelayFormatOpenAI: + value, diagnostics, err = attachOpenAIChatRequest(request, set) + case types.RelayFormatOpenAIResponses: + value, diagnostics, err = attachOpenAIResponsesRequest(request, set) + case types.RelayFormatClaude: + value, diagnostics, err = attachClaudeRequest(request, set, options) + case types.RelayFormatGemini: + value, diagnostics, err = attachGeminiRequest(request, set) + default: + value = request + } + if err != nil { + return nil, diagnostics, err + } + for index := range diagnostics { + diagnostics[index].From = set.Source + diagnostics[index].To = format + } + if err := types.RejectConversionLoss(options.EffectiveToolLossPolicy(), diagnostics); err != nil { + return nil, diagnostics, err + } + return value, diagnostics, nil +} + +func attachOpenAIChatRequest(request any, set Set) (any, []types.ConversionDiagnostic, error) { + target, ok := request.(*dto.GeneralOpenAIRequest) + if !ok || target == nil { + return nil, nil, fmt.Errorf("expected OpenAI chat completions request, got %T", request) + } + var diagnostics []types.ConversionDiagnostic + for index, definition := range set.Definitions { + switch definition.Kind { + case KindFunction: + if definition.Function == nil { + continue + } + target.Tools = append(target.Tools, dto.ToolCallRequest{ + Type: "function", + Function: dto.FunctionRequest{ + Name: definition.Function.Name, + Description: definition.Function.Description, + Parameters: definition.Function.Parameters, + Strict: definition.Function.Strict, + }, + }) + case KindWebSearch: + if set.Source == types.RelayFormatGemini { + diagnostics = append(diagnostics, geminiNativeWebSearchDiagnostics(index, definition, types.RelayFormatOpenAI)...) + } + if target.WebSearchOptions != nil { + return nil, diagnostics, fmt.Errorf("tools[%d]: multiple hosted web-search definitions cannot be represented by Chat Completions", index) + } + options := &dto.WebSearchOptions{} + if definition.WebSearch != nil { + options.SearchContextSize = definition.WebSearch.SearchContextSize + if definition.WebSearch.Location != nil { + location := map[string]any{ + "type": "approximate", + "approximate": locationMap(definition.WebSearch.Location), + } + options.UserLocation, _ = kitutil.Marshal(location) + } + diagnostics = append(diagnostics, openAIChatWebSearchDiagnostics(index, definition.WebSearch)...) + } + target.WebSearchOptions = options + default: + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("tools[%d]", index), + "unsupported_hosted_tool", + fmt.Sprintf("OpenAI Chat Completions cannot represent hosted tool %q", definition.NativeType), + )) + } + } + + normalizedChoice, allowedChoiceDiagnostics := narrowAllowedFunctionChoice(set.Choice, types.RelayFormatOpenAI) + choice, choiceDiagnostics := encodeOpenAIChatChoice(normalizedChoice) + target.ToolChoice = choice + target.ParallelTooCalls = set.ParallelAllowed + diagnostics = append(diagnostics, allowedChoiceDiagnostics...) + diagnostics = append(diagnostics, choiceDiagnostics...) + diagnostics = append(diagnostics, unsupportedHostedHistoryDiagnostics(types.RelayFormatOpenAI, set.History)...) + return target, diagnostics, nil +} + +func attachOpenAIResponsesRequest(request any, set Set) (any, []types.ConversionDiagnostic, error) { + target, ok := request.(*dto.OpenAIResponsesRequest) + if !ok || target == nil { + return nil, nil, fmt.Errorf("expected OpenAI Responses request, got %T", request) + } + tools := make([]any, 0, len(set.Definitions)) + var diagnostics []types.ConversionDiagnostic + for index, definition := range set.Definitions { + switch definition.Kind { + case KindFunction: + if definition.Function == nil { + continue + } + tool := map[string]any{ + "type": "function", + "name": definition.Function.Name, + "description": definition.Function.Description, + "parameters": definition.Function.Parameters, + } + if definition.Function.Strict != nil { + tool["strict"] = *definition.Function.Strict + } + deleteEmptyStrings(tool) + tools = append(tools, tool) + case KindWebSearch: + if set.Source == types.RelayFormatGemini { + diagnostics = append(diagnostics, geminiNativeWebSearchDiagnostics(index, definition, types.RelayFormatOpenAIResponses)...) + } + webSearch := definition.WebSearch + toolType := "web_search" + if set.Source == types.RelayFormatOpenAIResponses && definition.NativeType != "" { + toolType = definition.NativeType + } + tool := map[string]any{"type": toolType} + if webSearch != nil { + if webSearch.SearchContextSize != "" { + tool["search_context_size"] = webSearch.SearchContextSize + } + if webSearch.Location != nil { + location := locationMap(webSearch.Location) + location["type"] = "approximate" + tool["user_location"] = location + } + if len(webSearch.AllowedDomains) > 0 { + tool["filters"] = map[string]any{"allowed_domains": webSearch.AllowedDomains} + } + if webSearch.ExternalWebAccess != nil { + tool["external_web_access"] = *webSearch.ExternalWebAccess + } + if len(webSearch.ReturnTokenBudget) > 0 { + var budget any + if err := kitutil.Unmarshal(webSearch.ReturnTokenBudget, &budget); err != nil { + return nil, diagnostics, fmt.Errorf("tools[%d].return_token_budget: %w", index, err) + } + tool["return_token_budget"] = budget + } + diagnostics = append(diagnostics, openAIResponsesWebSearchDiagnostics(index, webSearch)...) + } + tools = append(tools, tool) + default: + if set.Source == types.RelayFormatOpenAIResponses && len(definition.Raw) > 0 { + var tool any + if err := kitutil.Unmarshal(definition.Raw, &tool); err != nil { + return nil, diagnostics, err + } + tools = append(tools, tool) + continue + } + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("tools[%d]", index), + "unsupported_hosted_tool", + fmt.Sprintf("OpenAI Responses has no verified mapping for hosted tool %q", definition.NativeType), + )) + } + } + if len(tools) > 0 { + encoded, err := kitutil.Marshal(tools) + if err != nil { + return nil, diagnostics, err + } + target.Tools = encoded + } + normalizedChoice, allowedChoiceDiagnostics := narrowAllowedFunctionChoice(set.Choice, types.RelayFormatOpenAIResponses) + choice, choiceDiagnostics, err := encodeOpenAIResponsesChoice(normalizedChoice, set.Source) + if err != nil { + return nil, diagnostics, err + } + target.ToolChoice = choice + if set.ParallelAllowed != nil { + target.ParallelToolCalls, _ = kitutil.Marshal(*set.ParallelAllowed) + } + diagnostics = append(diagnostics, allowedChoiceDiagnostics...) + diagnostics = append(diagnostics, choiceDiagnostics...) + historyDiagnostics, err := appendHostedHistoryToOpenAIResponses(target, set) + if err != nil { + return nil, diagnostics, err + } + diagnostics = append(diagnostics, historyDiagnostics...) + return target, diagnostics, nil +} + +func attachClaudeRequest(request any, set Set, options *convmeta.Options) (any, []types.ConversionDiagnostic, error) { + target, ok := request.(*dto.ClaudeRequest) + if !ok || target == nil { + return nil, nil, fmt.Errorf("expected Claude Messages request, got %T", request) + } + tools := make([]any, 0, len(set.Definitions)) + var diagnostics []types.ConversionDiagnostic + for index, definition := range set.Definitions { + switch definition.Kind { + case KindFunction: + if definition.Function == nil { + continue + } + inputSchema, err := functionParametersMap(definition.Function.Parameters) + if err != nil { + return nil, diagnostics, fmt.Errorf("tools[%d].input_schema: %w", index, err) + } + tools = append(tools, &dto.Tool{ + Name: definition.Function.Name, + Description: definition.Function.Description, + InputSchema: inputSchema, + Strict: definition.Function.Strict, + }) + case KindWebSearch: + if set.Source == types.RelayFormatGemini { + diagnostics = append(diagnostics, geminiNativeWebSearchDiagnostics(index, definition, types.RelayFormatClaude)...) + } + toolType := "web_search_20250305" + if set.Source == types.RelayFormatClaude && isKnownClaudeWebSearchType(definition.NativeType) { + toolType = definition.NativeType + } else if options != nil && options.Claude.WebSearchToolVersion != "" { + if !isKnownClaudeWebSearchType(options.Claude.WebSearchToolVersion) { + return nil, diagnostics, fmt.Errorf("unsupported Claude web-search tool version %q", options.Claude.WebSearchToolVersion) + } + toolType = options.Claude.WebSearchToolVersion + } + webSearch := definition.WebSearch + if webSearch != nil && len(webSearch.AllowedDomains) > 0 && len(webSearch.BlockedDomains) > 0 { + return nil, diagnostics, fmt.Errorf("tools[%d]: allowed_domains and blocked_domains are mutually exclusive", index) + } + if webSearch != nil && webSearch.ResponseInclusion != "" && !claudeWebSearchSupportsResponseInclusion(toolType) { + return nil, diagnostics, fmt.Errorf("tools[%d].response_inclusion requires Claude web_search_20260318", index) + } + tool := map[string]any{"type": toolType, "name": "web_search"} + if webSearch != nil { + if webSearch.Location != nil { + location := locationMap(webSearch.Location) + location["type"] = "approximate" + tool["user_location"] = location + } + if len(webSearch.AllowedDomains) > 0 { + tool["allowed_domains"] = webSearch.AllowedDomains + } + if len(webSearch.BlockedDomains) > 0 { + tool["blocked_domains"] = webSearch.BlockedDomains + } + if webSearch.MaxUses != nil { + tool["max_uses"] = *webSearch.MaxUses + } + if len(webSearch.AllowedCallers) > 0 { + tool["allowed_callers"] = webSearch.AllowedCallers + } + if webSearch.ResponseInclusion != "" { + tool["response_inclusion"] = webSearch.ResponseInclusion + } + diagnostics = append(diagnostics, claudeWebSearchDiagnostics(index, webSearch)...) + } + tools = append(tools, tool) + default: + if set.Source == types.RelayFormatClaude && len(definition.Raw) > 0 { + var tool any + if err := kitutil.Unmarshal(definition.Raw, &tool); err != nil { + return nil, diagnostics, err + } + tools = append(tools, tool) + continue + } + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("tools[%d]", index), + "unsupported_hosted_tool", + fmt.Sprintf("Claude Messages has no verified mapping for hosted tool %q", definition.NativeType), + )) + } + } + if len(tools) > 0 { + target.Tools = tools + } + normalizedChoice, allowedChoiceDiagnostics := narrowAllowedFunctionChoice(set.Choice, types.RelayFormatClaude) + choice, choiceDiagnostics := encodeClaudeChoice(normalizedChoice, set.ParallelAllowed, set.Source) + target.ToolChoice = choice + diagnostics = append(diagnostics, allowedChoiceDiagnostics...) + diagnostics = append(diagnostics, choiceDiagnostics...) + historyDiagnostics, err := appendHostedHistoryToClaude(target, set) + if err != nil { + return nil, diagnostics, err + } + diagnostics = append(diagnostics, historyDiagnostics...) + return target, diagnostics, nil +} + +func attachGeminiRequest(request any, set Set) (any, []types.ConversionDiagnostic, error) { + target, ok := request.(*dto.GeminiChatRequest) + if !ok || target == nil { + return nil, nil, fmt.Errorf("expected Gemini generateContent request, got %T", request) + } + if set.Source == types.RelayFormatGemini { + tools, err := rebuildGeminiToolGroups(set.Definitions) + if err != nil { + return nil, nil, err + } + if len(tools) > 0 { + target.Tools, err = kitutil.Marshal(tools) + if err != nil { + return nil, nil, err + } + } + if len(set.NativeToolConfig) > 0 { + var config dto.ToolConfig + if err := kitutil.Unmarshal(set.NativeToolConfig, &config); err != nil { + return nil, nil, fmt.Errorf("toolConfig: %w", err) + } + target.ToolConfig = &config + } + return target, unsupportedHostedHistoryDiagnostics(types.RelayFormatGemini, set.History), nil + } + var ( + functions []map[string]any + tools []map[string]any + diagnostics []types.ConversionDiagnostic + ) + for index, definition := range set.Definitions { + switch definition.Kind { + case KindFunction: + if definition.Function == nil { + continue + } + parameters := definition.Function.Parameters + if parameters != nil { + cloned, err := kitutil.Any2Type[any](parameters) + if err != nil { + return nil, diagnostics, fmt.Errorf("tools[%d].parameters: %w", index, err) + } + if params, ok := cloned.(map[string]any); ok { + if properties, exists := params["properties"].(map[string]any); exists && len(properties) == 0 { + cloned = nil + } + } + parameters = sharedgemini.CleanFunctionParameters(cloned) + } + function := map[string]any{ + "name": definition.Function.Name, + "description": definition.Function.Description, + "parameters": parameters, + } + deleteEmptyStrings(function) + functions = append(functions, function) + if definition.Function.Strict != nil { + diagnostics = append(diagnostics, presentationLoss(fmt.Sprintf("tools[%d].strict", index), "unsupported_function_strict", "Gemini does not expose OpenAI function strictness")) + } + case KindWebSearch: + if set.Source == types.RelayFormatGemini && len(definition.Raw) > 0 { + var tool map[string]any + if err := kitutil.Unmarshal(definition.Raw, &tool); err != nil { + return nil, diagnostics, err + } + tools = append(tools, tool) + } else { + tools = append(tools, map[string]any{"googleSearch": map[string]any{}}) + } + if definition.WebSearch != nil { + diagnostics = append(diagnostics, geminiWebSearchDiagnostics(index, definition.WebSearch)...) + } + case KindCodeExecution: + if set.Source == types.RelayFormatGemini { + tools = append(tools, map[string]any{"codeExecution": map[string]any{}}) + continue + } + diagnostics = append(diagnostics, semanticLoss(fmt.Sprintf("tools[%d]", index), "unverified_tool_mapping", "code execution semantics differ across providers")) + case KindURLContext: + if set.Source == types.RelayFormatGemini { + tools = append(tools, map[string]any{"urlContext": map[string]any{}}) + continue + } + diagnostics = append(diagnostics, semanticLoss(fmt.Sprintf("tools[%d]", index), "unverified_tool_mapping", "URL context has no verified mapping from the source protocol")) + default: + if set.Source == types.RelayFormatGemini && len(definition.Raw) > 0 { + var tool map[string]any + if err := kitutil.Unmarshal(definition.Raw, &tool); err != nil { + return nil, diagnostics, err + } + tools = append(tools, tool) + continue + } + // The established Responses-to-Gemini compatibility path removes + // free-form/unknown tools together with custom call history in + // PrepareOpenAIResponsesRequest. Keep that explicit downgrade as a + // diagnostic; other opaque tools may be server-executed and remain a + // semantic loss under the default Safe policy. + if set.Source == types.RelayFormatOpenAIResponses && (definition.NativeType == "custom" || definition.NativeType == "unknown") { + diagnostics = append(diagnostics, presentationLoss( + fmt.Sprintf("tools[%d]", index), + "custom_tool_omitted", + "Gemini cannot represent this OpenAI free-form or unknown tool; its preprocessed call history and definition were omitted", + )) + continue + } + if set.Source == types.RelayFormatOpenAIResponses && definition.Kind == KindNative { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("tools[%d]", index), + "unsupported_opaque_tool", + fmt.Sprintf("Gemini cannot represent OpenAI opaque tool %q; the definition was omitted", definition.NativeType), + )) + continue + } + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("tools[%d]", index), + "unsupported_hosted_tool", + fmt.Sprintf("Gemini generateContent has no verified mapping for hosted tool %q", definition.NativeType), + )) + } + } + if len(functions) > 0 { + tools = append(tools, map[string]any{"functionDeclarations": functions}) + } + if len(tools) > 0 { + encoded, err := kitutil.Marshal(tools) + if err != nil { + return nil, diagnostics, err + } + target.Tools = encoded + } + config, choiceDiagnostics := encodeGeminiChoice(set.Choice) + target.ToolConfig = config + diagnostics = append(diagnostics, choiceDiagnostics...) + if set.ParallelAllowed != nil && !*set.ParallelAllowed { + diagnostics = append(diagnostics, semanticLoss( + "parallel_tool_calls", + "unsupported_parallel_tool_control", + "Gemini generateContent does not expose a request field equivalent to parallel_tool_calls", + )) + } + diagnostics = append(diagnostics, unsupportedHostedHistoryDiagnostics(types.RelayFormatGemini, set.History)...) + return target, diagnostics, nil +} + +func rebuildGeminiToolGroups(definitions []Definition) ([]map[string]any, error) { + groups := make(map[int]map[string]any) + indexes := make([]int, 0) + for index, definition := range definitions { + if len(definition.Raw) == 0 { + return nil, fmt.Errorf("tools[%d]: missing native Gemini tool payload", index) + } + var fragment map[string]any + if err := kitutil.Unmarshal(definition.Raw, &fragment); err != nil { + return nil, fmt.Errorf("tools[%d]: %w", index, err) + } + group, exists := groups[definition.Group] + if !exists { + group = make(map[string]any) + groups[definition.Group] = group + indexes = append(indexes, definition.Group) + } + for key, value := range fragment { + if key == "functionDeclarations" { + existing, _ := group[key].([]any) + incoming, ok := value.([]any) + if !ok { + return nil, fmt.Errorf("tools[%d].functionDeclarations must be an array", index) + } + group[key] = append(existing, incoming...) + continue + } + group[key] = value + } + } + sort.Ints(indexes) + tools := make([]map[string]any, 0, len(indexes)) + for _, index := range indexes { + tools = append(tools, groups[index]) + } + return tools, nil +} + +func appendHostedHistoryToOpenAIResponses(target *dto.OpenAIResponsesRequest, set Set) ([]types.ConversionDiagnostic, error) { + if len(set.History) == 0 { + return nil, nil + } + input, err := responsesInputItems(target.Input) + if err != nil { + return nil, err + } + itemsBySourceIndex := make(map[int][]map[string]any) + mixedBySourceIndex := make(map[int]bool) + convertedByID := make(map[string]map[string]any) + var diagnostics []types.ConversionDiagnostic + for index, history := range set.History { + if history.MessageHasRegular { + mixedBySourceIndex[history.MessageIndex] = true + } + var item map[string]any + if set.Source == types.RelayFormatOpenAIResponses && len(history.Raw) > 0 { + if err := kitutil.Unmarshal(history.Raw, &item); err != nil { + return nil, fmt.Errorf("hosted_history[%d]: %w", index, err) + } + } else { + outputType := responsesTypeFromHostedKind(history.Kind) + if outputType == "" { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", index), + "hosted_tool_history_unsupported", + fmt.Sprintf("%s cannot preserve hosted-tool continuation item %q", types.RelayFormatOpenAIResponses, history.NativeType), + )) + continue + } + id := history.ID + if id == "" { + id = history.CallID + } + if strings.HasSuffix(history.NativeType, "_tool_result") || history.NativeType == "mcp_tool_result" { + if call, exists := convertedByID[history.CallID]; exists { + failed := history.Status == "failed" + if failed { + call["status"] = "failed" + } else { + call["status"] = "completed" + } + if history.Kind == KindMCP { + if len(history.Results) > 0 { + var ( + encoded json.RawMessage + normalized bool + err error + ) + if failed { + encoded, normalized, err = responsesMCPErrorFromClaudeContent(history.Results, "") + } else { + encoded, normalized, err = responsesMCPStringFromClaudeContent(history.Results) + } + if err != nil { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].content", index), + "mcp_result_unrepresentable", + err.Error(), + )) + continue + } + var value string + if err := kitutil.Unmarshal(encoded, &value); err != nil { + return nil, fmt.Errorf("hosted_history[%d].content: %w", index, err) + } + if failed { + call["error"] = value + } else { + call["output"] = value + } + if normalized { + diagnostics = append(diagnostics, presentationLoss( + fmt.Sprintf("hosted_history[%d].content", index), + "mcp_text_result_normalized", + "Claude's single MCP text block was normalized to a Responses output string", + )) + } + } + } else if history.Kind == KindWebSearch && len(history.Results) > 0 { + diagnostics = append(diagnostics, presentationLoss( + fmt.Sprintf("hosted_history[%d].content", index), + "web_search_result_omitted", + "Claude web-search result content has no field on a Responses web_search_call", + )) + } + continue + } + } + item = map[string]any{ + "type": outputType, + "id": id, + "status": history.Status, + } + if item["status"] == "" { + item["status"] = "in_progress" + } + switch history.Kind { + case KindWebSearch: + action, err := dto.NormalizeResponsesWebSearchAction(history.Action) + if err != nil { + return nil, fmt.Errorf("hosted_history[%d].action: %w", index, err) + } + var actionValue any + if err := kitutil.Unmarshal(action, &actionValue); err != nil { + return nil, fmt.Errorf("hosted_history[%d].action: %w", index, err) + } + item["action"] = actionValue + case KindMCP: + if strings.TrimSpace(history.Name) == "" || strings.TrimSpace(history.ServerName) == "" { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", index), + "mcp_identity_missing", + "Claude MCP history requires both name and server_name for Responses mapping", + )) + continue + } + item["name"] = history.Name + item["server_label"] = history.ServerName + if len(history.Action) > 0 { + arguments, err := responsesMCPArgumentsFromClaude(history.Action) + if err != nil { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].input", index), + "mcp_arguments_unrepresentable", + err.Error(), + )) + continue + } + var value string + if err := kitutil.Unmarshal(arguments, &value); err != nil { + return nil, fmt.Errorf("hosted_history[%d].input: %w", index, err) + } + item["arguments"] = value + } else { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].input", index), + "mcp_arguments_missing", + "Claude MCP history has no input object", + )) + continue + } + if len(history.Results) > 0 { + failed := history.Status == "failed" + var encoded json.RawMessage + var normalized bool + if failed { + encoded, normalized, err = responsesMCPErrorFromClaudeContent(history.Results, "") + } else { + encoded, normalized, err = responsesMCPStringFromClaudeContent(history.Results) + } + if err != nil { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].content", index), + "mcp_result_unrepresentable", + err.Error(), + )) + continue + } + var value string + if err := kitutil.Unmarshal(encoded, &value); err != nil { + return nil, fmt.Errorf("hosted_history[%d].content: %w", index, err) + } + if failed { + item["error"] = value + item["status"] = "failed" + } else { + item["output"] = value + item["status"] = "completed" + } + if normalized { + diagnostics = append(diagnostics, presentationLoss( + fmt.Sprintf("hosted_history[%d].content", index), + "mcp_text_result_normalized", + "Claude's single MCP text block was normalized to a Responses output string", + )) + } + } + if len(history.Caller) > 0 { + var caller any + if err := kitutil.Unmarshal(history.Caller, &caller); err != nil { + return nil, fmt.Errorf("hosted_history[%d].caller: %w", index, err) + } + item["caller"] = caller + } + } + if id != "" { + convertedByID[id] = item + } + } + itemsBySourceIndex[history.MessageIndex] = append(itemsBySourceIndex[history.MessageIndex], item) + } + if len(itemsBySourceIndex) == 0 { + if len(diagnostics) > 0 { + return diagnostics, nil + } + return unsupportedHostedHistoryDiagnostics(types.RelayFormatOpenAIResponses, set.History), nil + } + merged := make([]map[string]any, 0, len(input)+len(set.History)) + inputIndex := 0 + maxSourceIndex := 0 + for _, history := range set.History { + if history.MessageIndex > maxSourceIndex { + maxSourceIndex = history.MessageIndex + } + } + for sourceIndex := 0; sourceIndex <= maxSourceIndex || inputIndex < len(input); sourceIndex++ { + if hostedItems := itemsBySourceIndex[sourceIndex]; len(hostedItems) > 0 { + if !mixedBySourceIndex[sourceIndex] { + merged = append(merged, hostedItems...) + continue + } + merged = append(merged, hostedItems...) + if inputIndex < len(input) { + merged = append(merged, input[inputIndex]) + inputIndex++ + } + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", sourceIndex), + "hosted_tool_order_unrepresentable", + "hosted and ordinary blocks share one source message, but Responses represents hosted calls as separate input items", + )) + continue + } + if inputIndex < len(input) { + merged = append(merged, input[inputIndex]) + inputIndex++ + } + } + encoded, err := kitutil.Marshal(merged) + if err != nil { + return nil, err + } + target.Input = encoded + diagnostics = append(diagnostics, presentationLoss( + "input", + "hosted_tool_history_approximated", + "hosted-tool continuation state is preserved, but provider-specific item fields may differ", + )) + return diagnostics, nil +} + +func appendHostedHistoryToClaude(target *dto.ClaudeRequest, set Set) ([]types.ConversionDiagnostic, error) { + if len(set.History) == 0 { + return nil, nil + } + blocksBySourceIndex := make(map[int][]any) + mixedBySourceIndex := make(map[int]bool) + var diagnostics []types.ConversionDiagnostic + for index, history := range set.History { + if history.MessageHasRegular { + mixedBySourceIndex[history.MessageIndex] = true + } + var blocks []any + if set.Source == types.RelayFormatClaude && len(history.Raw) > 0 { + var block any + if err := kitutil.Unmarshal(history.Raw, &block); err != nil { + return nil, fmt.Errorf("hosted_history[%d]: %w", index, err) + } + blocksBySourceIndex[history.MessageIndex] = append(blocksBySourceIndex[history.MessageIndex], block) + continue + } + if set.Source == types.RelayFormatOpenAIResponses && history.Kind == KindWebSearch { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", index), + "web_search_continuation_unrepresentable", + "Responses web-search history cannot reconstruct Claude's encrypted web_search_tool_result continuation state", + )) + continue + } + if history.Kind == KindMCP { + if history.Name == "" || history.ServerName == "" { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", index), + "mcp_identity_missing", + "Responses MCP history requires both name and server_label for Claude mapping", + )) + continue + } + id := history.ID + if id == "" { + id = history.CallID + } + if id == "" { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].id", index), + "hosted_tool_id_missing", + "Responses MCP history has no id for pairing the call with its result", + )) + continue + } + if history.Status != "" && history.Status != "in_progress" && history.Status != "completed" && history.Status != "failed" { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].status", index), + "hosted_tool_status_unrepresentable", + fmt.Sprintf("Claude cannot preserve Responses MCP status %q", history.Status), + )) + } + mcpResult := history.Results + mcpFailed := history.Status == "failed" + if len(history.Raw) > 0 { + var rawFields struct { + ApprovalRequestID string `json:"approval_request_id"` + Output json.RawMessage `json:"output"` + Error json.RawMessage `json:"error"` + } + if err := kitutil.Unmarshal(history.Raw, &rawFields); err != nil { + return nil, fmt.Errorf("hosted_history[%d]: %w", index, err) + } + if strings.TrimSpace(rawFields.ApprovalRequestID) != "" { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].approval_request_id", index), + "mcp_approval_state_unrepresentable", + "Claude MCP history cannot preserve a Responses approval_request_id", + )) + } + if rawJSONPresent(rawFields.Output) && rawJSONPresent(rawFields.Error) { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", index), + "mcp_result_ambiguous", + "Responses MCP history contains both output and error", + )) + } + if rawJSONPresent(rawFields.Error) { + mcpResult = rawFields.Error + mcpFailed = true + } else if rawJSONPresent(rawFields.Output) { + mcpResult = rawFields.Output + } + } + input, inputErr := claudeMCPInputFromResponses(history.Action) + if inputErr != nil { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].arguments", index), + "mcp_arguments_unrepresentable", + inputErr.Error(), + )) + continue + } + call := map[string]any{ + "type": "mcp_tool_use", + "id": id, + "name": history.Name, + "server_name": history.ServerName, + "input": input, + } + if len(history.Caller) > 0 { + var caller any + if err := kitutil.Unmarshal(history.Caller, &caller); err != nil { + return nil, fmt.Errorf("hosted_history[%d].caller: %w", index, err) + } + call["caller"] = caller + } + blocks = append(blocks, call) + if rawJSONPresent(mcpResult) { + content, resultErr := claudeMCPContentFromResponsesString(mcpResult) + if resultErr != nil { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].output", index), + "mcp_result_unrepresentable", + resultErr.Error(), + )) + continue + } + result := map[string]any{"type": "mcp_tool_result", "tool_use_id": id, "content": content} + if mcpFailed { + result["is_error"] = true + } + blocks = append(blocks, result) + } else if history.Status == "completed" || mcpFailed { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", index), + "mcp_result_missing", + fmt.Sprintf("Responses MCP history has status %q but no output or error", history.Status), + )) + } + blocksBySourceIndex[history.MessageIndex] = append(blocksBySourceIndex[history.MessageIndex], blocks...) + continue + } + name := claudeNameFromHostedKind(history.Kind) + if name == "" { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", index), + "hosted_tool_history_unsupported", + fmt.Sprintf("%s cannot preserve hosted-tool continuation item %q", types.RelayFormatClaude, history.NativeType), + )) + continue + } + id := history.ID + if id == "" { + id = history.CallID + } + var input any = map[string]any{} + if history.Kind == KindWebSearch { + webInput, inputErr := claudeWebSearchInputFromResponses(history.Action) + if inputErr != nil { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d].action", index), + "web_search_action_unrepresentable", + inputErr.Error(), + )) + continue + } + input = webInput + } else if len(history.Action) > 0 { + if err := kitutil.Unmarshal(history.Action, &input); err != nil { + return nil, fmt.Errorf("hosted_history[%d].action: %w", index, err) + } + } + call := map[string]any{ + "type": "server_tool_use", + "id": id, + "name": name, + "input": input, + } + if len(history.Caller) > 0 { + var caller any + if err := kitutil.Unmarshal(history.Caller, &caller); err != nil { + return nil, fmt.Errorf("hosted_history[%d].caller: %w", index, err) + } + call["caller"] = caller + } + blocks = append(blocks, call) + if len(history.Results) > 0 && !(set.Source == types.RelayFormatOpenAIResponses && history.Kind == KindWebSearch) { + var content any + if err := kitutil.Unmarshal(history.Results, &content); err != nil { + return nil, fmt.Errorf("hosted_history[%d].results: %w", index, err) + } + blocks = append(blocks, map[string]any{ + "type": claudeResultTypeFromHostedKind(history.Kind), + "tool_use_id": id, + "content": content, + }) + } else if len(history.Results) > 0 && history.Kind == KindWebSearch { + diagnostics = append(diagnostics, presentationLoss( + fmt.Sprintf("hosted_history[%d].results", index), + "web_search_result_omitted", + "Responses web-search source metadata cannot reconstruct Claude's encrypted web_search_tool_result", + )) + } + blocksBySourceIndex[history.MessageIndex] = append(blocksBySourceIndex[history.MessageIndex], blocks...) + } + if len(blocksBySourceIndex) == 0 { + if len(diagnostics) > 0 { + return diagnostics, nil + } + return unsupportedHostedHistoryDiagnostics(types.RelayFormatClaude, set.History), nil + } + messages := make([]dto.ClaudeMessage, 0, len(target.Messages)+len(blocksBySourceIndex)) + messageIndex := 0 + maxSourceIndex := 0 + for _, history := range set.History { + if history.MessageIndex > maxSourceIndex { + maxSourceIndex = history.MessageIndex + } + } + for sourceIndex := 0; sourceIndex <= maxSourceIndex || messageIndex < len(target.Messages); sourceIndex++ { + if blocks := blocksBySourceIndex[sourceIndex]; len(blocks) > 0 { + role := "assistant" + for _, history := range set.History { + if history.MessageIndex == sourceIndex && history.Role != "" { + role = history.Role + break + } + } + if !mixedBySourceIndex[sourceIndex] { + messages = append(messages, dto.ClaudeMessage{Role: role, Content: blocks}) + continue + } + messages = append(messages, dto.ClaudeMessage{Role: role, Content: blocks}) + if messageIndex < len(target.Messages) { + messages = append(messages, target.Messages[messageIndex]) + messageIndex++ + } + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", sourceIndex), + "hosted_tool_order_unrepresentable", + "hosted and ordinary content cannot be merged after the intermediate converter coalesced source blocks", + )) + continue + } + if messageIndex < len(target.Messages) { + messages = append(messages, target.Messages[messageIndex]) + messageIndex++ + } + } + target.Messages = messages + diagnostics = append(diagnostics, presentationLoss( + "messages", + "hosted_tool_history_approximated", + "hosted-tool continuation state is preserved, but provider-specific item fields may differ", + )) + return diagnostics, nil +} + +func responsesInputItems(raw json.RawMessage) ([]map[string]any, error) { + if len(raw) == 0 { + return nil, nil + } + switch kitutil.GetJsonType(raw) { + case "array": + var input []map[string]any + if err := kitutil.Unmarshal(raw, &input); err != nil { + return nil, fmt.Errorf("invalid Responses input: %w", err) + } + return input, nil + case "string": + var text string + if err := kitutil.Unmarshal(raw, &text); err != nil { + return nil, fmt.Errorf("invalid Responses input: %w", err) + } + return []map[string]any{{"role": "user", "content": text}}, nil + default: + return nil, fmt.Errorf("cannot append hosted-tool history to Responses input type %q", kitutil.GetJsonType(raw)) + } +} + +func unsupportedHostedHistoryDiagnostics(format types.RelayFormat, history []HostedHistoryItem) []types.ConversionDiagnostic { + if len(history) == 0 { + return nil + } + diagnostics := make([]types.ConversionDiagnostic, 0, len(history)) + for index, item := range history { + diagnostics = append(diagnostics, semanticLoss( + fmt.Sprintf("hosted_history[%d]", index), + "hosted_tool_history_unsupported", + fmt.Sprintf("%s cannot preserve hosted-tool continuation item %q", format, item.NativeType), + )) + } + return diagnostics +} + +func openAIChatWebSearchDiagnostics(index int, search *WebSearch) []types.ConversionDiagnostic { + if search == nil { + return nil + } + path := fmt.Sprintf("tools[%d]", index) + var diagnostics []types.ConversionDiagnostic + if len(search.AllowedDomains) > 0 || len(search.BlockedDomains) > 0 { + diagnostics = append(diagnostics, semanticLoss(path+".domains", "unsupported_domain_filter", "Chat Completions web_search_options cannot preserve domain access constraints")) + } + if search.MaxUses != nil { + diagnostics = append(diagnostics, semanticLoss(path+".max_uses", "unsupported_search_limit", "Chat Completions cannot preserve Claude max_uses")) + } + if len(search.AllowedCallers) > 0 || search.ExternalWebAccess != nil { + diagnostics = append(diagnostics, semanticLoss(path, "unsupported_search_controls", "Chat Completions cannot preserve caller or external-access constraints")) + } + if search.ResponseInclusion != "" || len(search.ReturnTokenBudget) > 0 { + diagnostics = append(diagnostics, presentationLoss(path, "unsupported_search_tuning", "Chat Completions cannot preserve response-inclusion or return-token tuning")) + } + return diagnostics +} + +func openAIResponsesWebSearchDiagnostics(index int, search *WebSearch) []types.ConversionDiagnostic { + if search == nil { + return nil + } + path := fmt.Sprintf("tools[%d]", index) + var diagnostics []types.ConversionDiagnostic + if search.MaxUses != nil { + diagnostics = append(diagnostics, semanticLoss(path+".max_uses", "unsupported_search_limit", "OpenAI Responses cannot preserve Claude max_uses")) + } + if len(search.BlockedDomains) > 0 { + diagnostics = append(diagnostics, semanticLoss(path+".blocked_domains", "unsupported_blocked_domains", "OpenAI Responses web search supports allow filters but not Claude blocked_domains")) + } + if len(search.AllowedCallers) > 0 { + diagnostics = append(diagnostics, semanticLoss(path, "unsupported_search_controls", "OpenAI Responses cannot preserve Claude caller constraints")) + } + if search.ResponseInclusion != "" { + diagnostics = append(diagnostics, presentationLoss(path+".response_inclusion", "unsupported_search_tuning", "OpenAI Responses cannot preserve Claude response-inclusion tuning")) + } + return diagnostics +} + +func claudeWebSearchDiagnostics(index int, search *WebSearch) []types.ConversionDiagnostic { + if search == nil { + return nil + } + path := fmt.Sprintf("tools[%d]", index) + var diagnostics []types.ConversionDiagnostic + if search.SearchContextSize != "" { + diagnostics = append(diagnostics, presentationLoss(path+".search_context_size", "unsupported_search_context_size", "Claude has no equivalent for OpenAI search_context_size; max_uses is deliberately not inferred")) + } + if search.ExternalWebAccess != nil { + diagnostics = append(diagnostics, semanticLoss(path+".external_web_access", "unsupported_search_controls", "Claude cannot preserve OpenAI external-web access constraints")) + } + if len(search.ReturnTokenBudget) > 0 { + diagnostics = append(diagnostics, presentationLoss(path+".return_token_budget", "unsupported_search_tuning", "Claude cannot preserve OpenAI return-token tuning")) + } + return diagnostics +} + +func geminiWebSearchDiagnostics(index int, search *WebSearch) []types.ConversionDiagnostic { + if search == nil { + return nil + } + path := fmt.Sprintf("tools[%d]", index) + if search.Location == nil && len(search.AllowedDomains) == 0 && len(search.BlockedDomains) == 0 && search.SearchContextSize == "" && search.MaxUses == nil && len(search.AllowedCallers) == 0 && search.ResponseInclusion == "" && search.ExternalWebAccess == nil && len(search.ReturnTokenBudget) == 0 { + return nil + } + var diagnostics []types.ConversionDiagnostic + if len(search.AllowedDomains) > 0 || len(search.BlockedDomains) > 0 || search.ExternalWebAccess != nil || search.MaxUses != nil || len(search.AllowedCallers) > 0 { + diagnostics = append(diagnostics, semanticLoss(path, "unsupported_search_constraints", "Gemini Google Search cannot preserve source web-search access or execution constraints")) + } + if search.Location != nil || search.SearchContextSize != "" || search.ResponseInclusion != "" || len(search.ReturnTokenBudget) > 0 { + diagnostics = append(diagnostics, presentationLoss(path, "unsupported_search_tuning", "Gemini Google Search cannot preserve source web-search location or result tuning")) + } + return diagnostics +} + +func geminiNativeWebSearchDiagnostics(index int, definition Definition, target types.RelayFormat) []types.ConversionDiagnostic { + path := fmt.Sprintf("tools[%d]", index) + switch definition.NativeType { + case "googleSearch": + if !geminiNativeToolHasConfiguration(definition) { + return nil + } + return []types.ConversionDiagnostic{semanticLoss( + path+".googleSearch", + "unsupported_native_search_config", + fmt.Sprintf("%s cannot preserve Gemini googleSearch configuration", target), + )} + case "googleSearchRetrieval": + return []types.ConversionDiagnostic{semanticLoss( + path+".googleSearchRetrieval", + "legacy_search_semantics_unrepresentable", + fmt.Sprintf("%s cannot preserve Gemini legacy dynamic-retrieval semantics", target), + )} + case "enterpriseWebSearch": + return []types.ConversionDiagnostic{semanticLoss( + path+".enterpriseWebSearch", + "enterprise_search_semantics_unrepresentable", + fmt.Sprintf("%s cannot replace Gemini enterprise search with public web search without changing its data source", target), + )} + default: + return []types.ConversionDiagnostic{semanticLoss( + path, + "unverified_search_mapping", + fmt.Sprintf("%s has no verified mapping for Gemini search tool %q", target, definition.NativeType), + )} + } +} + +func geminiNativeToolHasConfiguration(definition Definition) bool { + if len(definition.Raw) == 0 { + return false + } + var wrapper map[string]json.RawMessage + if kitutil.Unmarshal(definition.Raw, &wrapper) != nil { + return true + } + payload := wrapper[definition.NativeType] + if !rawJSONPresent(payload) { + return false + } + if kitutil.GetJsonType(payload) != "object" { + return true + } + var fields map[string]json.RawMessage + return kitutil.Unmarshal(payload, &fields) != nil || len(fields) > 0 +} + +func narrowAllowedFunctionChoice(choice *Choice, target types.RelayFormat) (*Choice, []types.ConversionDiagnostic) { + if choice == nil || len(choice.AllowedNames) == 0 { + return choice, nil + } + normalized := *choice + normalized.AllowedNames = nil + if len(choice.AllowedNames) == 1 { + normalized.Mode = ChoiceNamed + normalized.Kind = KindFunction + normalized.Name = choice.AllowedNames[0] + return &normalized, nil + } + return &normalized, []types.ConversionDiagnostic{semanticLoss( + "tool_choice.allowed_function_names", + "allowed_function_subset_unrepresentable", + fmt.Sprintf("%s cannot restrict a required tool call to Gemini's %d-name function subset", target, len(choice.AllowedNames)), + )} +} + +func encodeOpenAIChatChoice(choice *Choice) (any, []types.ConversionDiagnostic) { + if choice == nil { + return nil, nil + } + switch choice.Mode { + case ChoiceAuto: + return "auto", nil + case ChoiceNone: + return "none", nil + case ChoiceRequired: + return "required", nil + case ChoiceNamed: + if choice.Kind == KindFunction { + return map[string]any{"type": "function", "function": map[string]any{"name": choice.Name}}, nil + } + if choice.Kind == KindWebSearch { + return nil, []types.ConversionDiagnostic{presentationLoss("tool_choice", "implicit_search_choice", "Chat Completions expresses hosted search through web_search_options instead of tool_choice")} + } + } + return nil, []types.ConversionDiagnostic{semanticLoss("tool_choice", "unsupported_tool_choice", "Chat Completions cannot represent the source hosted tool choice")} +} + +func encodeOpenAIResponsesChoice(choice *Choice, source types.RelayFormat) (json.RawMessage, []types.ConversionDiagnostic, error) { + if choice == nil { + return nil, nil, nil + } + var value any + switch choice.Mode { + case ChoiceAuto, ChoiceNone, ChoiceRequired: + value = string(choice.Mode) + case ChoiceNamed: + if len(choice.Raw) > 0 && choice.NativeType != "" { + return append(json.RawMessage(nil), choice.Raw...), nil, nil + } + switch choice.Kind { + case KindFunction: + value = map[string]any{"type": "function", "name": choice.Name} + case KindWebSearch: + value = map[string]any{"type": "web_search"} + default: + return nil, []types.ConversionDiagnostic{semanticLoss("tool_choice", "unsupported_tool_choice", "OpenAI Responses has no verified hosted tool-choice mapping")}, nil + } + case ChoiceOpaque: + if source == types.RelayFormatOpenAIResponses && len(choice.Raw) > 0 { + return append(json.RawMessage(nil), choice.Raw...), nil, nil + } + return nil, []types.ConversionDiagnostic{semanticLoss("tool_choice", "unsupported_tool_choice", "OpenAI Responses cannot reconstruct the source complex tool choice")}, nil + } + encoded, err := kitutil.Marshal(value) + return encoded, nil, err +} + +func encodeClaudeChoice(choice *Choice, parallelAllowed *bool, source types.RelayFormat) (any, []types.ConversionDiagnostic) { + if choice == nil && parallelAllowed == nil { + return nil, nil + } + if source == types.RelayFormatClaude && choice != nil && len(choice.Raw) > 0 { + var value any + if err := kitutil.Unmarshal(choice.Raw, &value); err != nil { + return nil, []types.ConversionDiagnostic{semanticLoss("tool_choice", "invalid_native_tool_choice", "Claude tool_choice could not be restored from its native payload")} + } + return value, nil + } + value := map[string]any{} + if choice != nil { + switch choice.Mode { + case ChoiceAuto: + value["type"] = "auto" + case ChoiceNone: + value["type"] = "none" + case ChoiceRequired: + value["type"] = "any" + case ChoiceNamed: + if choice.Kind != KindFunction && choice.Kind != KindWebSearch { + return nil, []types.ConversionDiagnostic{semanticLoss("tool_choice", "unsupported_tool_choice", "Claude has no verified hosted tool-choice mapping")} + } + value["type"] = "tool" + value["name"] = choice.Name + case ChoiceOpaque: + return nil, []types.ConversionDiagnostic{semanticLoss("tool_choice", "unsupported_tool_choice", "Claude cannot represent the source complex tool-choice policy")} + } + } + if value["type"] == nil && parallelAllowed != nil { + value["type"] = "auto" + } + if parallelAllowed != nil && value["type"] != "none" { + value["disable_parallel_tool_use"] = !*parallelAllowed + } else if choice != nil && choice.DisableParallelToolUse != nil && value["type"] != "none" { + value["disable_parallel_tool_use"] = *choice.DisableParallelToolUse + } + return value, nil +} + +func encodeGeminiChoice(choice *Choice) (*dto.ToolConfig, []types.ConversionDiagnostic) { + if choice == nil { + return nil, nil + } + config := &dto.ToolConfig{FunctionCallingConfig: &dto.FunctionCallingConfig{}} + if len(choice.AllowedNames) > 0 { + config.FunctionCallingConfig.Mode = "ANY" + config.FunctionCallingConfig.AllowedFunctionNames = append([]string(nil), choice.AllowedNames...) + return config, nil + } + switch choice.Mode { + case ChoiceAuto: + config.FunctionCallingConfig.Mode = "AUTO" + case ChoiceNone: + config.FunctionCallingConfig.Mode = "NONE" + case ChoiceRequired: + config.FunctionCallingConfig.Mode = "ANY" + case ChoiceNamed: + if choice.Kind != KindFunction { + return nil, []types.ConversionDiagnostic{semanticLoss("tool_choice", "unsupported_tool_choice", "Gemini generateContent does not expose an equivalent hosted-tool choice")} + } + config.FunctionCallingConfig.Mode = "ANY" + config.FunctionCallingConfig.AllowedFunctionNames = []string{choice.Name} + case ChoiceOpaque: + return nil, []types.ConversionDiagnostic{semanticLoss("tool_choice", "unsupported_tool_choice", "Gemini cannot represent the source complex tool-choice policy")} + } + return config, nil +} + +func semanticLoss(path string, code string, message string) types.ConversionDiagnostic { + return types.ConversionDiagnostic{Code: code, Path: path, Message: message, Severity: types.ConversionDiagnosticError} +} + +func presentationLoss(path string, code string, message string) types.ConversionDiagnostic { + return types.ConversionDiagnostic{Code: code, Path: path, Message: message, Severity: types.ConversionDiagnosticWarning} +} + +func locationMap(location *ApproximateLocation) map[string]any { + value := map[string]any{ + "city": location.City, + "region": location.Region, + "country": location.Country, + "timezone": location.Timezone, + } + deleteEmptyStrings(value) + return value +} + +func deleteEmptyStrings(value map[string]any) { + for key, item := range value { + if text, ok := item.(string); ok && text == "" { + delete(value, key) + } + } +} + +func functionParametersMap(parameters any) (map[string]interface{}, error) { + if parameters == nil { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{}, + }, nil + } + converted, err := kitutil.Any2Type[map[string]interface{}](parameters) + if err != nil { + return nil, err + } + if converted["type"] == nil { + converted["type"] = "object" + } + if converted["properties"] == nil { + converted["properties"] = map[string]interface{}{} + } + return converted, nil +} diff --git a/relaykit/relayconvert/internal/toolconv/hosted_values.go b/relaykit/relayconvert/internal/toolconv/hosted_values.go new file mode 100644 index 0000000000..46723eef3b --- /dev/null +++ b/relaykit/relayconvert/internal/toolconv/hosted_values.go @@ -0,0 +1,171 @@ +package toolconv + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" +) + +// claudeWebSearchInputFromResponses narrows the richer Responses action union +// to the only operation exposed by Claude's web-search server tool: one query. +func claudeWebSearchInputFromResponses(raw json.RawMessage) (map[string]any, error) { + canonical, err := dto.NormalizeResponsesWebSearchAction(raw) + if err != nil { + return nil, err + } + var action struct { + Type string `json:"type"` + Query string `json:"query"` + Queries []string `json:"queries"` + } + if err := kitutil.Unmarshal(canonical, &action); err != nil { + return nil, fmt.Errorf("decode normalized Responses web-search action: %w", err) + } + if action.Type != "search" { + return nil, fmt.Errorf("Responses web-search action %q has no Claude equivalent", action.Type) + } + + queries := make([]string, 0, len(action.Queries)+1) + for _, query := range action.Queries { + query = strings.TrimSpace(query) + if query != "" { + queries = append(queries, query) + } + } + deprecatedQuery := strings.TrimSpace(action.Query) + if len(queries) == 0 && deprecatedQuery != "" { + queries = append(queries, deprecatedQuery) + } else if deprecatedQuery != "" && (len(queries) != 1 || queries[0] != deprecatedQuery) { + return nil, fmt.Errorf("Responses web-search action contains conflicting query and queries fields") + } + if len(queries) != 1 { + return nil, fmt.Errorf("Claude web search requires exactly one query, got %d", len(queries)) + } + return map[string]any{"query": queries[0]}, nil +} + +// responsesMCPArgumentsFromClaude converts Claude's JSON-object input into the +// JSON string required by a Responses mcp_call.arguments field. +func responsesMCPArgumentsFromClaude(raw json.RawMessage) (json.RawMessage, error) { + trimmed := strings.TrimSpace(string(raw)) + if trimmed == "" || kitutil.GetJsonType(raw) != "object" { + return nil, fmt.Errorf("Claude MCP input must be a JSON object") + } + var object map[string]json.RawMessage + if err := kitutil.Unmarshal(raw, &object); err != nil { + return nil, fmt.Errorf("decode Claude MCP input: %w", err) + } + encoded, err := kitutil.Marshal(trimmed) + if err != nil { + return nil, fmt.Errorf("encode Responses MCP arguments: %w", err) + } + return encoded, nil +} + +// claudeMCPInputFromResponses decodes the outer Responses JSON string and +// validates that its contents satisfy Claude's JSON-object input contract. +func claudeMCPInputFromResponses(raw json.RawMessage) (any, error) { + var encoded string + if len(raw) == 0 || kitutil.GetJsonType(raw) != "string" { + return nil, fmt.Errorf("Responses MCP arguments must be a JSON string") + } + if err := kitutil.Unmarshal(raw, &encoded); err != nil { + return nil, fmt.Errorf("decode Responses MCP arguments string: %w", err) + } + encoded = strings.TrimSpace(encoded) + if encoded == "" || kitutil.GetJsonType(json.RawMessage(encoded)) != "object" { + return nil, fmt.Errorf("Responses MCP arguments must contain a JSON object") + } + var input map[string]any + if err := kitutil.Unmarshal([]byte(encoded), &input); err != nil { + return nil, fmt.Errorf("decode Responses MCP arguments object: %w", err) + } + return input, nil +} + +// responsesMCPStringFromClaudeContent maps the Claude result shapes that can +// be represented without changing their meaning. A single text block is the +// structured form of a plain MCP text result; other block arrays can contain +// media/resources that a Responses string cannot faithfully preserve. +func responsesMCPStringFromClaudeContent(raw json.RawMessage) (json.RawMessage, bool, error) { + switch kitutil.GetJsonType(raw) { + case "string": + var value string + if err := kitutil.Unmarshal(raw, &value); err != nil { + return nil, false, fmt.Errorf("decode Claude MCP result string: %w", err) + } + return append(json.RawMessage(nil), raw...), false, nil + case "array": + var blocks []map[string]json.RawMessage + if err := kitutil.Unmarshal(raw, &blocks); err != nil { + return nil, false, fmt.Errorf("decode Claude MCP result blocks: %w", err) + } + if len(blocks) == 0 { + encoded, err := kitutil.Marshal("") + return encoded, true, err + } + if len(blocks) != 1 { + return nil, false, fmt.Errorf("Responses MCP output cannot preserve %d Claude content blocks", len(blocks)) + } + var blockType string + if err := kitutil.Unmarshal(blocks[0]["type"], &blockType); err != nil || blockType != "text" { + return nil, false, fmt.Errorf("Responses MCP output can only preserve a Claude text result block") + } + var text string + if err := kitutil.Unmarshal(blocks[0]["text"], &text); err != nil { + return nil, false, fmt.Errorf("decode Claude MCP text result: %w", err) + } + encoded, err := kitutil.Marshal(text) + if err != nil { + return nil, false, fmt.Errorf("encode Responses MCP output: %w", err) + } + return encoded, true, nil + default: + return nil, false, fmt.Errorf("Responses MCP output cannot preserve Claude result type %q", kitutil.GetJsonType(raw)) + } +} + +func claudeMCPContentFromResponsesString(raw json.RawMessage) (string, error) { + if len(raw) == 0 || kitutil.GetJsonType(raw) != "string" { + return "", fmt.Errorf("Responses MCP output/error must be a JSON string") + } + var content string + if err := kitutil.Unmarshal(raw, &content); err != nil { + return "", fmt.Errorf("decode Responses MCP output/error string: %w", err) + } + return content, nil +} + +func claudeHostedResultFailure(blockType string, content json.RawMessage, explicitError *bool, explicitCode string) (bool, string) { + if explicitError != nil && *explicitError { + return true, strings.TrimSpace(explicitCode) + } + if strings.TrimSpace(explicitCode) != "" { + return true, strings.TrimSpace(explicitCode) + } + if !strings.HasSuffix(strings.TrimSpace(blockType), "_tool_result") || kitutil.GetJsonType(content) != "object" { + return false, "" + } + var resultError struct { + Type string `json:"type"` + ErrorCode string `json:"error_code"` + } + if kitutil.Unmarshal(content, &resultError) != nil { + return false, "" + } + if !strings.HasSuffix(strings.TrimSpace(resultError.Type), "_error") && strings.TrimSpace(resultError.ErrorCode) == "" { + return false, "" + } + return true, strings.TrimSpace(resultError.ErrorCode) +} + +func responsesMCPErrorFromClaudeContent(raw json.RawMessage, errorCode string) (json.RawMessage, bool, error) { + if errorCode = strings.TrimSpace(errorCode); errorCode != "" { + encoded, err := kitutil.Marshal(errorCode) + return encoded, false, err + } + return responsesMCPStringFromClaudeContent(raw) +} diff --git a/relaykit/relayconvert/internal/toolconv/model.go b/relaykit/relayconvert/internal/toolconv/model.go new file mode 100644 index 0000000000..21e4d8ec13 --- /dev/null +++ b/relaykit/relayconvert/internal/toolconv/model.go @@ -0,0 +1,118 @@ +package toolconv + +import ( + "encoding/json" + + "github.com/QuantumNous/new-api/relaykit/types" +) + +type Kind string + +const ( + KindFunction Kind = "function" + KindWebSearch Kind = "web_search" + KindFileSearch Kind = "file_search" + KindWebFetch Kind = "web_fetch" + KindCodeExecution Kind = "code_execution" + KindComputerUse Kind = "computer_use" + KindURLContext Kind = "url_context" + KindMCP Kind = "mcp" + KindImage Kind = "image_generation" + KindNative Kind = "native" +) + +type Execution string + +const ( + ExecutionClient Execution = "client" + ExecutionServer Execution = "server" +) + +type Function struct { + Name string + Description string + Parameters any + Strict *bool +} + +type ApproximateLocation struct { + City string + Region string + Country string + Timezone string +} + +type WebSearch struct { + Location *ApproximateLocation + AllowedDomains []string + BlockedDomains []string + SearchContextSize string + MaxUses *int + AllowedCallers []string + ResponseInclusion string + ExternalWebAccess *bool + ReturnTokenBudget json.RawMessage +} + +type Definition struct { + Kind Kind + Execution Execution + NativeType string + Name string + Function *Function + WebSearch *WebSearch + Raw json.RawMessage + Group int +} + +type ChoiceMode string + +const ( + ChoiceAuto ChoiceMode = "auto" + ChoiceNone ChoiceMode = "none" + ChoiceRequired ChoiceMode = "required" + ChoiceNamed ChoiceMode = "named" + ChoiceOpaque ChoiceMode = "opaque" +) + +type Choice struct { + Mode ChoiceMode + Kind Kind + Name string + AllowedNames []string + NativeType string + DisableParallelToolUse *bool + Raw json.RawMessage +} + +type Set struct { + Source types.RelayFormat + Definitions []Definition + Choice *Choice + ParallelAllowed *bool + NativeToolConfig json.RawMessage + History []HostedHistoryItem +} + +func (s Set) Empty() bool { + return len(s.Definitions) == 0 && s.Choice == nil && s.ParallelAllowed == nil && len(s.NativeToolConfig) == 0 && len(s.History) == 0 +} + +type HostedHistoryItem struct { + Kind Kind + NativeType string + Role string + MessageIndex int + BlockIndex int + MessageHasRegular bool + Sequence int + ID string + CallID string + Name string + ServerName string + Status string + Action json.RawMessage + Results json.RawMessage + Caller json.RawMessage + Raw json.RawMessage +} diff --git a/relaykit/relayconvert/internal/toolconv/policy_test.go b/relaykit/relayconvert/internal/toolconv/policy_test.go new file mode 100644 index 0000000000..cdf9cae291 --- /dev/null +++ b/relaykit/relayconvert/internal/toolconv/policy_test.go @@ -0,0 +1,103 @@ +package toolconv + +import ( + "testing" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func geminiCodeExecutionRequest(t *testing.T) *dto.GeminiChatRequest { + t.Helper() + tools, err := kitutil.Marshal([]map[string]any{{"codeExecution": map[string]any{}}}) + require.NoError(t, err) + return &dto.GeminiChatRequest{ + Contents: []dto.GeminiChatContent{ + {Role: "user", Parts: []dto.GeminiPart{{Text: "run this"}}}, + }, + Tools: tools, + } +} + +func hasDiagnosticCode(diagnostics []types.ConversionDiagnostic, code string) bool { + for _, diagnostic := range diagnostics { + if diagnostic.Code == code { + return true + } + } + return false +} + +func TestDefaultPolicyAllowsGeminiCodeExecutionToOpenAI(t *testing.T) { + t.Parallel() + + _, set, err := ExtractRequest(types.RelayFormatGemini, geminiCodeExecutionRequest(t)) + require.NoError(t, err) + target := &dto.GeneralOpenAIRequest{ + Model: "gpt-4o", + Messages: []dto.Message{{Role: "user", Content: "run this"}}, + } + + out, diagnostics, err := AttachRequest(types.RelayFormatOpenAI, target, set, &convmeta.Options{}) + require.NoError(t, err) + require.NotNil(t, out) + assert.True(t, hasDiagnosticCode(diagnostics, "unsupported_hosted_tool")) + assert.Equal(t, types.ConversionLossPolicyAllow, (&convmeta.Options{}).EffectiveToolLossPolicy()) +} + +func TestResponsePhaseNeverRejectsEvenUnderStrictPolicy(t *testing.T) { + t.Parallel() + + text := "hello" + resp := &dto.ClaudeResponse{ + Type: "message", + Role: "assistant", + StopReason: "pause_turn", + Content: []dto.ClaudeMediaMessage{ + {Type: "redacted_thinking", Data: "secret"}, + {Type: "text", Text: &text}, + }, + } + diagnostics := InspectResponse(types.RelayFormatClaude, types.RelayFormatOpenAI, resp) + require.True(t, hasDiagnosticCode(diagnostics, "continuation_state_lost")) + require.Error(t, types.RejectConversionLoss(types.ConversionLossPolicyStrict, diagnostics)) + + _, hosted, err := ExtractHostedResponse(types.RelayFormatClaude, resp) + require.NoError(t, err) + out, _, err := AttachHostedResponse( + types.RelayFormatOpenAI, + &dto.OpenAITextResponse{}, + hosted, + &convmeta.Options{ToolLossPolicy: types.ConversionLossPolicyStrict}, + ) + require.NoError(t, err) + require.NotNil(t, out) +} + +func TestSafePolicyRejectsRequestPhaseHostedToolLoss(t *testing.T) { + t.Parallel() + + _, set, err := ExtractRequest(types.RelayFormatGemini, geminiCodeExecutionRequest(t)) + require.NoError(t, err) + target := &dto.GeneralOpenAIRequest{ + Model: "gpt-4o", + Messages: []dto.Message{{Role: "user", Content: "run this"}}, + } + + _, diagnostics, err := AttachRequest( + types.RelayFormatOpenAI, + target, + set, + &convmeta.Options{ToolLossPolicy: types.ConversionLossPolicySafe}, + ) + require.Error(t, err) + var loss *types.ConversionLossError + require.ErrorAs(t, err, &loss) + require.NotEmpty(t, loss.Diagnostics) + assert.True(t, hasDiagnosticCode(loss.Diagnostics, "unsupported_hosted_tool")) + assert.True(t, hasDiagnosticCode(diagnostics, "unsupported_hosted_tool")) +} diff --git a/relaykit/relayconvert/internal/toolconv/response.go b/relaykit/relayconvert/internal/toolconv/response.go new file mode 100644 index 0000000000..1eb2cde959 --- /dev/null +++ b/relaykit/relayconvert/internal/toolconv/response.go @@ -0,0 +1,515 @@ +package toolconv + +import ( + "encoding/json" + "fmt" + "strings" + "unicode/utf8" + + "github.com/QuantumNous/new-api/relaykit/dto" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/types" +) + +// InspectResponse reports protocol information that the current response +// converters cannot faithfully express. It keeps loss handling centralized so +// direct and multi-step routes behave consistently. +func InspectResponse(from types.RelayFormat, to types.RelayFormat, response any) []types.ConversionDiagnostic { + if from == to { + return nil + } + var diagnostics []types.ConversionDiagnostic + switch value := response.(type) { + case *dto.ClaudeResponse: + diagnostics = inspectClaudeResponse(value, to) + case dto.ClaudeResponse: + diagnostics = inspectClaudeResponse(&value, to) + case *dto.OpenAIResponsesResponse: + diagnostics = inspectOpenAIResponsesResponse(value, to) + case dto.OpenAIResponsesResponse: + diagnostics = inspectOpenAIResponsesResponse(&value, to) + case *dto.ResponsesStreamResponse: + diagnostics = inspectOpenAIResponsesStreamResponse(value) + case dto.ResponsesStreamResponse: + diagnostics = inspectOpenAIResponsesStreamResponse(&value) + case *dto.GeminiChatResponse: + diagnostics = inspectGeminiResponse(value, to) + case dto.GeminiChatResponse: + diagnostics = inspectGeminiResponse(&value, to) + } + for index := range diagnostics { + diagnostics[index].From = from + diagnostics[index].To = to + } + return diagnostics +} + +// InspectStreamResponse avoids treating a single Gemini streaming chunk as a +// complete grounding document. Gemini grounding chunk indexes and segment +// offsets are cumulative across the stream; the stateful converter validates +// and resolves them after accumulating prior chunks. +func InspectStreamResponse(from types.RelayFormat, to types.RelayFormat, response any) []types.ConversionDiagnostic { + if from != types.RelayFormatGemini { + return InspectResponse(from, to, response) + } + var value *dto.GeminiChatResponse + switch response := response.(type) { + case *dto.GeminiChatResponse: + value = response + case dto.GeminiChatResponse: + value = &response + default: + return InspectResponse(from, to, response) + } + if value == nil { + return nil + } + var diagnostics []types.ConversionDiagnostic + for index := range value.Candidates { + metadata := value.Candidates[index].GroundingMetadata + if metadata == nil { + continue + } + if len(metadata.WebSearchQueries) > 0 && to != types.RelayFormatOpenAIResponses { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("candidates[%d].groundingMetadata.webSearchQueries", index), + "web_search_call_unrepresentable", + "Gemini grounding confirms a hosted web search, but the target stream converter cannot produce an OpenAI Responses web_search_call lifecycle", + )) + } + if len(metadata.WebSearchQueries) == 0 && len(metadata.RetrievalQueries) == 0 && len(metadata.SearchEntryPoint) == 0 && len(metadata.RetrievalMetadata) == 0 && len(metadata.SourceFlaggingUris) == 0 && metadata.GoogleMapsWidgetContextToken == "" { + continue + } + diagnostics = append(diagnostics, responsePresentationLoss( + fmt.Sprintf("candidates[%d].groundingMetadata", index), + "hosted_tool_metadata_reduced", + "Gemini grounding citations are preserved across stream chunks, but provider-specific search metadata has no target-protocol equivalent", + )) + } + for index := range diagnostics { + diagnostics[index].From = from + diagnostics[index].To = to + } + return diagnostics +} + +func inspectClaudeResponse(response *dto.ClaudeResponse, to types.RelayFormat) []types.ConversionDiagnostic { + if response == nil { + return nil + } + var diagnostics []types.ConversionDiagnostic + if to != types.RelayFormatOpenAIResponses && (response.StopReason == "pause_turn" || response.Delta != nil && response.Delta.StopReason != nil && *response.Delta.StopReason == "pause_turn") { + diagnostics = append(diagnostics, responseSemanticLoss( + "stop_reason", + "continuation_state_lost", + "Claude pause_turn requires protocol-native continuation state that the target response cannot preserve", + )) + } + for index := range response.Content { + diagnostics = append(diagnostics, inspectClaudeContentBlock(&response.Content[index], fmt.Sprintf("content[%d]", index), to, false)...) + } + if response.ContentBlock != nil { + diagnostics = append(diagnostics, inspectClaudeContentBlock(response.ContentBlock, "content_block", to, true)...) + } + return diagnostics +} + +func inspectClaudeContentBlock(block *dto.ClaudeMediaMessage, path string, to types.RelayFormat, stream bool) []types.ConversionDiagnostic { + if block == nil { + return nil + } + blockType := strings.TrimSpace(block.Type) + var diagnostics []types.ConversionDiagnostic + if isClaudeHostedToolBlock(blockType) { + kind := KindNative + if blockType == "server_tool_use" || blockType == "mcp_tool_use" { + kind = hostedKindFromClaudeCall(blockType, block.Name) + } else { + kind = hostedKindFromClaudeResult(blockType) + } + if to != types.RelayFormatOpenAIResponses || kind != KindWebSearch && kind != KindMCP { + diagnostics = append(diagnostics, responseSemanticLoss( + path, + "hosted_tool_unrepresentable", + fmt.Sprintf("%s cannot losslessly represent Claude hosted-tool response block %q", to, blockType), + )) + } else if blockType == "server_tool_use" || blockType == "mcp_tool_use" { + if block.Id == "" { + diagnostics = append(diagnostics, responseSemanticLoss( + path+".id", + "hosted_tool_id_missing", + "Claude hosted-tool call has no id for pairing it with its result", + )) + } + if kind == KindMCP && (block.Name == "" || block.ServerName == "") { + diagnostics = append(diagnostics, responseSemanticLoss( + path, + "mcp_identity_missing", + "Claude MCP tool use must include both name and server_name for Responses MCP mapping", + )) + } + if rawJSONPresent(block.Caller) { + diagnostics = append(diagnostics, responseSemanticLoss( + path+".caller", + "hosted_tool_caller_unrepresentable", + "OpenAI Responses web_search_call and mcp_call items cannot preserve Claude's hosted-tool caller provenance", + )) + } + if !stream { + input, err := kitutil.Marshal(block.Input) + if err != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + path+".input", + "hosted_tool_input_invalid", + err.Error(), + )) + } else if kind == KindMCP { + if _, err := responsesMCPArgumentsFromClaude(input); err != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + path+".input", + "mcp_arguments_unrepresentable", + err.Error(), + )) + } + } else if kind == KindWebSearch { + if _, err := dto.NormalizeResponsesWebSearchAction(input); err != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + path+".input", + "web_search_action_unrepresentable", + err.Error(), + )) + } + } + } + } else if block.ToolUseId == "" { + diagnostics = append(diagnostics, responseSemanticLoss( + path+".tool_use_id", + "hosted_tool_id_missing", + "Claude hosted-tool result has no tool_use_id for pairing it with its call", + )) + } + if kind == KindMCP && blockType == "mcp_tool_result" { + content, err := kitutil.Marshal(block.Content) + if err != nil { + diagnostics = append(diagnostics, responseSemanticLoss(path+".content", "mcp_result_unrepresentable", err.Error())) + } else { + failed, errorCode := claudeHostedResultFailure(blockType, content, block.IsError, block.ErrorCode) + var normalized bool + if failed { + _, normalized, err = responsesMCPErrorFromClaudeContent(content, errorCode) + } else { + _, normalized, err = responsesMCPStringFromClaudeContent(content) + } + if err != nil { + diagnostics = append(diagnostics, responseSemanticLoss(path+".content", "mcp_result_unrepresentable", err.Error())) + } else if normalized { + diagnostics = append(diagnostics, responsePresentationLoss( + path+".content", + "mcp_text_result_normalized", + "Claude's single MCP text block is normalized to a Responses output string", + )) + } + } + } + } + if blockType == "redacted_thinking" && block.Data != "" { + diagnostics = append(diagnostics, responseSemanticLoss( + path+".data", + "continuation_state_lost", + "Claude encrypted thinking state cannot be represented by the target response", + )) + } + return diagnostics +} + +func isClaudeHostedToolBlock(blockType string) bool { + if blockType == "server_tool_use" || blockType == "mcp_tool_use" || blockType == "mcp_tool_result" { + return true + } + return strings.HasSuffix(blockType, "_tool_result") +} + +func inspectOpenAIResponsesResponse(response *dto.OpenAIResponsesResponse, to types.RelayFormat) []types.ConversionDiagnostic { + if response == nil { + return nil + } + var diagnostics []types.ConversionDiagnostic + for index := range response.Output { + output := &response.Output[index] + if !isResponsesHostedOutput(output.Type) { + continue + } + kind := hostedKindFromResponsesType(output.Type) + if to != types.RelayFormatClaude || kind != KindWebSearch && kind != KindMCP { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d]", index), + "hosted_tool_unrepresentable", + fmt.Sprintf("%s cannot losslessly represent OpenAI Responses hosted-tool output %q", to, output.Type), + )) + continue + } + if output.ID == "" { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d].id", index), + "hosted_tool_id_missing", + "hosted-tool output has no id for pairing the call with its result", + )) + } + if kind == KindMCP { + if output.Name == "" || output.ServerLabel == "" { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d]", index), + "mcp_identity_missing", + "Responses MCP output must include both name and server_label for Claude MCP mapping", + )) + } + if output.ApprovalRequestID != "" { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d].approval_request_id", index), + "mcp_approval_state_unrepresentable", + "Claude MCP response blocks cannot preserve a Responses approval_request_id", + )) + } + if _, err := claudeMCPInputFromResponses(output.Arguments); err != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d].arguments", index), + "mcp_arguments_unrepresentable", + err.Error(), + )) + } + if rawJSONPresent(output.Output) && rawJSONPresent(output.ItemError) { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d]", index), + "mcp_result_ambiguous", + "Responses MCP output contains both output and error", + )) + } + for _, field := range []struct { + name string + raw json.RawMessage + }{{name: "output", raw: output.Output}, {name: "error", raw: output.ItemError}} { + if !rawJSONPresent(field.raw) { + continue + } + if _, err := claudeMCPContentFromResponsesString(field.raw); err != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d].%s", index, field.name), + "mcp_result_unrepresentable", + err.Error(), + )) + } + } + } else if kind == KindWebSearch { + if _, err := claudeWebSearchInputFromResponses(output.Action); err != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d].action", index), + "web_search_action_unrepresentable", + err.Error(), + )) + } + } + if output.Status != "" && output.Status != "in_progress" && output.Status != "completed" && output.Status != "failed" { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d].status", index), + "hosted_tool_status_unrepresentable", + fmt.Sprintf("Claude cannot preserve hosted-tool status %q", output.Status), + )) + } + if output.Status == "failed" && !rawJSONPresent(output.ItemError) && !rawJSONPresent(output.Output) && !rawJSONPresent(output.Results) { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("output[%d].status", index), + "hosted_tool_error_missing", + "failed hosted-tool output has no error or output that Claude can preserve", + )) + } + } + return diagnostics +} + +func inspectOpenAIResponsesStreamResponse(response *dto.ResponsesStreamResponse) []types.ConversionDiagnostic { + if response == nil { + return nil + } + if response.Item != nil && isResponsesHostedOutput(response.Item.Type) { + return []types.ConversionDiagnostic{responseSemanticLoss( + "item", + "hosted_tool_event_unrepresentable", + fmt.Sprintf("OpenAI Responses hosted-tool output %q has no semantic target-protocol stream mapping", response.Item.Type), + )} + } + eventType := strings.TrimSpace(response.Type) + if strings.Contains(eventType, ".web_search_call.") || + strings.Contains(eventType, ".file_search_call.") || + strings.Contains(eventType, ".code_interpreter_call.") || + strings.Contains(eventType, ".computer_tool_call.") || + strings.Contains(eventType, ".image_generation_call.") || + strings.Contains(eventType, ".mcp_call.") { + return []types.ConversionDiagnostic{responseSemanticLoss( + "type", + "hosted_tool_event_unrepresentable", + fmt.Sprintf("OpenAI Responses hosted-tool stream event %q has no semantic target-protocol mapping", eventType), + )} + } + return nil +} + +func isResponsesHostedOutput(outputType string) bool { + switch strings.TrimSpace(outputType) { + case "", "message", "reasoning", "function_call", "custom_tool_call": + return false + default: + return true + } +} + +func inspectGeminiResponse(response *dto.GeminiChatResponse, to types.RelayFormat) []types.ConversionDiagnostic { + if response == nil { + return nil + } + var diagnostics []types.ConversionDiagnostic + for index := range response.Candidates { + metadata := response.Candidates[index].GroundingMetadata + if metadata == nil { + continue + } + path := fmt.Sprintf("candidates[%d].groundingMetadata", index) + if len(metadata.WebSearchQueries) > 0 && to != types.RelayFormatOpenAIResponses { + diagnostics = append(diagnostics, responseSemanticLoss( + path+".webSearchQueries", + "web_search_call_unrepresentable", + "Gemini grounding confirms a hosted web search, but the target converter cannot produce an OpenAI Responses web_search_call item", + )) + } + diagnostics = append(diagnostics, inspectGeminiGroundingCitations(response.Candidates[index].Content, metadata, path)...) + if len(metadata.WebSearchQueries) == 0 && len(metadata.RetrievalQueries) == 0 && len(metadata.SearchEntryPoint) == 0 && len(metadata.RetrievalMetadata) == 0 && len(metadata.SourceFlaggingUris) == 0 && metadata.GoogleMapsWidgetContextToken == "" { + continue + } + diagnostics = append(diagnostics, responsePresentationLoss( + path, + "hosted_tool_metadata_reduced", + "Gemini grounding citations are preserved, but provider-specific search metadata has no target-protocol equivalent", + )) + } + return diagnostics +} + +type groundingSupportForInspection struct { + Segment struct { + PartIndex *int `json:"partIndex,omitempty"` + StartIndex int `json:"startIndex,omitempty"` + EndIndex int `json:"endIndex,omitempty"` + Text string `json:"text,omitempty"` + } `json:"segment"` + GroundingChunkIndices []int `json:"groundingChunkIndices"` +} + +type groundingChunkForInspection struct { + Web *groundingSourceForInspection `json:"web,omitempty"` + RetrievedContext *groundingSourceForInspection `json:"retrievedContext,omitempty"` +} + +type groundingSourceForInspection struct { + URI string `json:"uri,omitempty"` +} + +func inspectGeminiGroundingCitations(content dto.GeminiChatContent, metadata *dto.GeminiGroundingMetadata, path string) []types.ConversionDiagnostic { + if len(metadata.GroundingSupports) == 0 { + return nil + } + var chunks []groundingChunkForInspection + if len(metadata.GroundingChunks) == 0 || kitutil.Unmarshal(metadata.GroundingChunks, &chunks) != nil { + return []types.ConversionDiagnostic{responseSemanticLoss( + path+".groundingChunks", + "grounding_source_invalid", + "Gemini grounding chunks are missing or cannot be decoded", + )} + } + var supports []groundingSupportForInspection + if err := kitutil.Unmarshal(metadata.GroundingSupports, &supports); err != nil { + return []types.ConversionDiagnostic{responseSemanticLoss( + path+".groundingSupports", + "grounding_citation_invalid", + fmt.Sprintf("Gemini grounding supports cannot be decoded: %v", err), + )} + } + textPartCount := 0 + soleTextPart := -1 + for index := range content.Parts { + if content.Parts[index].Text == "" || content.Parts[index].Thought { + continue + } + textPartCount++ + soleTextPart = index + } + var diagnostics []types.ConversionDiagnostic + for index, support := range supports { + segmentPath := fmt.Sprintf("%s.groundingSupports[%d].segment", path, index) + hasSource := false + for _, chunkIndex := range support.GroundingChunkIndices { + if chunkIndex < 0 || chunkIndex >= len(chunks) { + continue + } + source := chunks[chunkIndex].Web + if source == nil { + source = chunks[chunkIndex].RetrievedContext + } + if source != nil && source.URI != "" { + hasSource = true + break + } + } + if !hasSource { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("%s.groundingSupports[%d].groundingChunkIndices", path, index), + "grounding_source_invalid", + "Gemini grounding support does not reference a valid source URI", + )) + continue + } + partIndex := soleTextPart + if support.Segment.PartIndex != nil { + partIndex = *support.Segment.PartIndex + } else if textPartCount != 1 { + diagnostics = append(diagnostics, responseSemanticLoss( + segmentPath+".partIndex", + "grounding_part_ambiguous", + "Gemini grounding omitted partIndex while multiple text parts are present, so citation placement is ambiguous", + )) + continue + } + if partIndex < 0 || partIndex >= len(content.Parts) || content.Parts[partIndex].Text == "" || content.Parts[partIndex].Thought { + diagnostics = append(diagnostics, responseSemanticLoss( + segmentPath+".partIndex", + "grounding_part_invalid", + fmt.Sprintf("Gemini grounding references non-text part %d", partIndex), + )) + continue + } + partText := content.Parts[partIndex].Text + start, end := support.Segment.StartIndex, support.Segment.EndIndex + if start < 0 || end <= start || end > len(partText) || !utf8.ValidString(partText[:start]) || !utf8.ValidString(partText[:end]) { + diagnostics = append(diagnostics, responseSemanticLoss( + segmentPath, + "grounding_offset_invalid", + "Gemini grounding byte offsets do not identify valid UTF-8 boundaries in the referenced part", + )) + continue + } + if support.Segment.Text != "" && partText[start:end] != support.Segment.Text { + diagnostics = append(diagnostics, responseSemanticLoss( + segmentPath+".text", + "grounding_text_mismatch", + "Gemini grounding segment text does not match the referenced part range", + )) + } + } + return diagnostics +} + +func responseSemanticLoss(path string, code string, message string) types.ConversionDiagnostic { + return types.ConversionDiagnostic{Code: code, Path: path, Message: message, Severity: types.ConversionDiagnosticError} +} + +func responsePresentationLoss(path string, code string, message string) types.ConversionDiagnostic { + return types.ConversionDiagnostic{Code: code, Path: path, Message: message, Severity: types.ConversionDiagnosticWarning} +} diff --git a/relaykit/relayconvert/internal/toolconv/response_artifacts.go b/relaykit/relayconvert/internal/toolconv/response_artifacts.go new file mode 100644 index 0000000000..856b44ef0b --- /dev/null +++ b/relaykit/relayconvert/internal/toolconv/response_artifacts.go @@ -0,0 +1,816 @@ +package toolconv + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + geminichat "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/gemini_chat" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/types" +) + +type HostedResponseItem struct { + Kind Kind + NativeType string + ID string + CallID string + Name string + Status string + Position int + Action json.RawMessage + Results json.RawMessage + Sources json.RawMessage + Caller json.RawMessage + Arguments json.RawMessage + Output json.RawMessage + Error json.RawMessage + ServerName string + IsError *bool + ApprovalRequestID string + Tools json.RawMessage + ErrorCode string + Raw json.RawMessage +} + +type HostedResponseSet struct { + Source types.RelayFormat + Items []HostedResponseItem + SourceLength int + RegularPositions []int +} + +type positionedResponsesOutput struct { + position int + output dto.ResponsesOutput +} + +type positionedClaudeBlocks struct { + position int + blocks []dto.ClaudeMediaMessage +} + +func (s HostedResponseSet) Empty() bool { + return len(s.Items) == 0 +} + +// ExtractHostedResponse removes server-executed tool artifacts before a +// message converter sees them. The artifacts travel beside multi-step routes, +// just like request tool definitions, so a lossy Chat pivot cannot reclassify +// or discard them. +func ExtractHostedResponse(format types.RelayFormat, response any) (any, HostedResponseSet, error) { + switch format { + case types.RelayFormatClaude: + return extractClaudeHostedResponse(response) + case types.RelayFormatOpenAIResponses: + return extractOpenAIHostedResponse(response) + case types.RelayFormatGemini: + return extractGeminiHostedResponse(response) + default: + return response, HostedResponseSet{Source: format}, nil + } +} + +func AttachHostedResponse(format types.RelayFormat, response any, set HostedResponseSet, options *convmeta.Options) (any, []types.ConversionDiagnostic, error) { + if set.Empty() { + return response, nil, nil + } + var ( + value any + diagnostics []types.ConversionDiagnostic + err error + ) + switch format { + case types.RelayFormatOpenAIResponses: + value, diagnostics, err = attachOpenAIHostedResponse(response, set) + case types.RelayFormatClaude: + value, diagnostics, err = attachClaudeHostedResponse(response, set) + default: + value = response + for index, item := range set.Items { + diagnostics = append(diagnostics, responsePresentationLoss( + fmt.Sprintf("hosted_tools[%d]", index), + "hosted_tool_event_omitted", + fmt.Sprintf("%s cannot represent hosted-tool response %q", format, item.NativeType), + )) + } + } + if err != nil { + return nil, diagnostics, err + } + for index := range diagnostics { + diagnostics[index].From = set.Source + diagnostics[index].To = format + } + return value, diagnostics, nil +} + +func extractClaudeHostedResponse(response any) (any, HostedResponseSet, error) { + var source *dto.ClaudeResponse + switch value := response.(type) { + case *dto.ClaudeResponse: + source = value + case dto.ClaudeResponse: + source = &value + default: + return nil, HostedResponseSet{}, fmt.Errorf("expected Claude response, got %T", response) + } + clone := *source + clone.Content = make([]dto.ClaudeMediaMessage, 0, len(source.Content)) + set := HostedResponseSet{Source: types.RelayFormatClaude, SourceLength: len(source.Content)} + for position := range source.Content { + block := source.Content[position] + blockType := strings.TrimSpace(block.Type) + switch { + case blockType == "server_tool_use" || blockType == "mcp_tool_use": + rawBlock, err := kitutil.Marshal(block) + if err != nil { + return nil, set, fmt.Errorf("content[%d]: %w", position, err) + } + action, err := kitutil.Marshal(block.Input) + if err != nil { + return nil, set, fmt.Errorf("content[%d].input: %w", position, err) + } + item := HostedResponseItem{ + Kind: hostedKindFromClaudeCall(blockType, block.Name), + NativeType: blockType, + ID: block.Id, + CallID: block.Id, + Name: block.Name, + Status: "in_progress", + Position: position, + Action: action, + Caller: append(json.RawMessage(nil), block.Caller...), + ServerName: block.ServerName, + Raw: rawBlock, + } + set.Items = append(set.Items, item) + case isClaudeHostedToolBlock(blockType): + rawBlock, err := kitutil.Marshal(block) + if err != nil { + return nil, set, fmt.Errorf("content[%d]: %w", position, err) + } + results, err := kitutil.Marshal(block.Content) + if err != nil { + return nil, set, fmt.Errorf("content[%d].content: %w", position, err) + } + failed, errorCode := claudeHostedResultFailure(blockType, results, block.IsError, block.ErrorCode) + status := "completed" + isError := block.IsError + if failed { + status = "failed" + if isError == nil { + value := true + isError = &value + } + } + set.Items = append(set.Items, HostedResponseItem{ + Kind: hostedKindFromClaudeResult(blockType), + NativeType: blockType, + ID: block.ToolUseId, + CallID: block.ToolUseId, + Status: status, + Position: position, + Results: results, + ErrorCode: errorCode, + IsError: isError, + Raw: rawBlock, + }) + default: + clone.Content = append(clone.Content, block) + set.RegularPositions = append(set.RegularPositions, position) + } + } + return &clone, set, nil +} + +func extractOpenAIHostedResponse(response any) (any, HostedResponseSet, error) { + var source *dto.OpenAIResponsesResponse + switch value := response.(type) { + case *dto.OpenAIResponsesResponse: + source = value + case dto.OpenAIResponsesResponse: + source = &value + default: + return nil, HostedResponseSet{}, fmt.Errorf("expected OpenAI Responses response, got %T", response) + } + clone := *source + clone.Output = make([]dto.ResponsesOutput, 0, len(source.Output)) + set := HostedResponseSet{Source: types.RelayFormatOpenAIResponses, SourceLength: len(source.Output)} + for position := range source.Output { + output := source.Output[position] + if !isResponsesHostedOutput(output.Type) { + clone.Output = append(clone.Output, output) + set.RegularPositions = append(set.RegularPositions, position) + continue + } + rawOutput, err := kitutil.Marshal(output) + if err != nil { + return nil, set, fmt.Errorf("output[%d]: %w", position, err) + } + set.Items = append(set.Items, HostedResponseItem{ + Kind: hostedKindFromResponsesType(output.Type), + NativeType: output.Type, + ID: output.ID, + CallID: output.CallId, + Name: output.Name, + Status: output.Status, + Position: position, + Action: append(json.RawMessage(nil), output.Action...), + Results: append(json.RawMessage(nil), output.Results...), + Sources: append(json.RawMessage(nil), output.Sources...), + Caller: append(json.RawMessage(nil), output.Caller...), + Arguments: append(json.RawMessage(nil), output.Arguments...), + Output: append(json.RawMessage(nil), output.Output...), + Error: append(json.RawMessage(nil), output.ItemError...), + ServerName: output.ServerLabel, + ApprovalRequestID: output.ApprovalRequestID, + Tools: append(json.RawMessage(nil), output.MCPTools...), + Raw: rawOutput, + }) + } + return &clone, set, nil +} + +func extractGeminiHostedResponse(response any) (any, HostedResponseSet, error) { + var source *dto.GeminiChatResponse + switch value := response.(type) { + case *dto.GeminiChatResponse: + source = value + case dto.GeminiChatResponse: + source = &value + default: + return nil, HostedResponseSet{}, fmt.Errorf("expected Gemini response, got %T", response) + } + queries := geminichat.GroundingWebSearchQueries(source) + if len(queries) == 0 { + return source, HostedResponseSet{Source: types.RelayFormatGemini}, nil + } + action, err := kitutil.Marshal(map[string]any{ + "type": "search", + "queries": queries, + }) + if err != nil { + return nil, HostedResponseSet{}, fmt.Errorf("marshal Gemini web-search action: %w", err) + } + // The Chat pivot emits the answer as the regular Responses output. Place + // the hosted call after that output, matching the stream bridge which only + // learns Gemini's queries once grounding metadata arrives near stream end. + set := HostedResponseSet{ + Source: types.RelayFormatGemini, + SourceLength: 2, + RegularPositions: []int{0}, + Items: []HostedResponseItem{{ + Kind: KindWebSearch, + NativeType: "googleSearch", + ID: fmt.Sprintf("ws_%s", kitutil.GetUUID()), + Status: "completed", + Position: 1, + Action: action, + }}, + } + return source, set, nil +} + +func attachOpenAIHostedResponse(response any, set HostedResponseSet) (any, []types.ConversionDiagnostic, error) { + target, ok := response.(*dto.OpenAIResponsesResponse) + if !ok || target == nil { + return nil, nil, fmt.Errorf("expected OpenAI Responses response, got %T", response) + } + var diagnostics []types.ConversionDiagnostic + hostedOutput := make([]positionedResponsesOutput, 0, len(set.Items)) + convertedByID := make(map[string]int, len(set.Items)*2) + for index, item := range set.Items { + if set.Source == types.RelayFormatOpenAIResponses && len(item.Raw) > 0 { + var output dto.ResponsesOutput + if err := kitutil.Unmarshal(item.Raw, &output); err != nil { + return nil, diagnostics, fmt.Errorf("hosted_tools[%d]: %w", index, err) + } + hostedOutput = append(hostedOutput, positionedResponsesOutput{position: item.Position, output: output}) + continue + } + outputType := responsesTypeFromHostedKind(item.Kind) + if outputType == "" { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d]", index), + "hosted_tool_unrepresentable", + fmt.Sprintf("OpenAI Responses has no lossless response mapping for %q", item.NativeType), + )) + continue + } + if isClaudeHostedResult(item.NativeType) { + outputIndex, exists := convertedByID[item.CallID] + if !exists { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d].tool_use_id", index), + "hosted_tool_result_orphaned", + fmt.Sprintf("hosted-tool result references unknown call %q", item.CallID), + )) + continue + } + output := &hostedOutput[outputIndex].output + output.Status = hostedCompletionStatus(item) + if item.Kind == KindMCP { + failed := hostedItemFailed(item) + var ( + encoded json.RawMessage + normalized bool + err error + ) + if failed { + encoded, normalized, err = responsesMCPErrorFromClaudeContent(item.Results, item.ErrorCode) + } else { + encoded, normalized, err = responsesMCPStringFromClaudeContent(item.Results) + } + if err != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d].content", index), + "mcp_result_unrepresentable", + err.Error(), + )) + continue + } + if failed { + output.Output = nil + output.ItemError = encoded + } else { + output.Output = encoded + output.ItemError = nil + } + if normalized { + diagnostics = append(diagnostics, responsePresentationLoss( + fmt.Sprintf("hosted_tools[%d].content", index), + "mcp_text_result_normalized", + "Claude's single MCP text block was normalized to a Responses output string", + )) + } + } else if item.Kind == KindWebSearch && rawJSONPresent(item.Results) { + diagnostics = append(diagnostics, responsePresentationLoss( + fmt.Sprintf("hosted_tools[%d].content", index), + "web_search_result_omitted", + "Claude web-search result content is provider-private and has no field on an OpenAI Responses web_search_call; completion status and citations remain available", + )) + } + continue + } + output := dto.ResponsesOutput{ + Type: outputType, + ID: firstNonEmpty(item.ID, item.CallID), + Status: hostedCompletionStatus(item), + } + switch item.Kind { + case KindWebSearch: + action, err := dto.NormalizeResponsesWebSearchAction(item.Action) + if err != nil { + return nil, diagnostics, fmt.Errorf("hosted_tools[%d].action: %w", index, err) + } + output.Action = action + case KindMCP: + if strings.TrimSpace(item.Name) == "" || strings.TrimSpace(item.ServerName) == "" { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d]", index), + "mcp_identity_missing", + "Claude MCP output requires both name and server_name for Responses mapping", + )) + continue + } + output.CallId = item.CallID + output.Name = item.Name + output.Caller = append(json.RawMessage(nil), item.Caller...) + output.ServerLabel = item.ServerName + output.ApprovalRequestID = item.ApprovalRequestID + output.MCPTools = append(json.RawMessage(nil), item.Tools...) + arguments := item.Arguments + if len(arguments) == 0 { + arguments = item.Action + } + encodedArguments, argumentErr := responsesMCPArgumentsFromClaude(arguments) + if argumentErr != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d].input", index), + "mcp_arguments_unrepresentable", + argumentErr.Error(), + )) + continue + } + output.Arguments = encodedArguments + } + outputIndex := len(hostedOutput) + for _, key := range []string{item.ID, item.CallID} { + if key != "" { + convertedByID[key] = outputIndex + } + } + hostedOutput = append(hostedOutput, positionedResponsesOutput{position: item.Position, output: output}) + if set.Source != types.RelayFormatOpenAIResponses { + diagnostics = append(diagnostics, responsePresentationLoss( + fmt.Sprintf("hosted_tools[%d]", index), + "hosted_tool_result_approximated", + "hosted-tool execution is preserved, but provider-specific result fields may differ", + )) + } + } + merged, orderingDiagnostics := mergeResponsesOutput(target.Output, hostedOutput, set) + diagnostics = append(diagnostics, orderingDiagnostics...) + target.Output = merged + return target, diagnostics, nil +} + +func attachClaudeHostedResponse(response any, set HostedResponseSet) (any, []types.ConversionDiagnostic, error) { + target, ok := response.(*dto.ClaudeResponse) + if !ok || target == nil { + return nil, nil, fmt.Errorf("expected Claude response, got %T", response) + } + var diagnostics []types.ConversionDiagnostic + hostedContent := make([]positionedClaudeBlocks, 0, len(set.Items)) + for index, item := range set.Items { + if set.Source == types.RelayFormatClaude && len(item.Raw) > 0 { + var block dto.ClaudeMediaMessage + if err := kitutil.Unmarshal(item.Raw, &block); err != nil { + return nil, diagnostics, fmt.Errorf("hosted_tools[%d]: %w", index, err) + } + hostedContent = append(hostedContent, positionedClaudeBlocks{position: item.Position, blocks: []dto.ClaudeMediaMessage{block}}) + continue + } + if set.Source == types.RelayFormatOpenAIResponses && item.Kind == KindWebSearch { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d]", index), + "web_search_response_unrepresentable", + "Responses web-search execution cannot reconstruct Claude's required encrypted web_search_tool_result continuation state", + )) + continue + } + name := claudeNameFromHostedKind(item.Kind) + if item.Kind == KindMCP { + name = item.Name + } + if name == "" { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d]", index), + "hosted_tool_unrepresentable", + fmt.Sprintf("Claude has no lossless response mapping for %q", item.NativeType), + )) + continue + } + var input any = map[string]any{} + if item.Kind == KindWebSearch { + webInput, inputErr := claudeWebSearchInputFromResponses(item.Action) + if inputErr != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d].action", index), + "web_search_action_unrepresentable", + inputErr.Error(), + )) + continue + } + input = webInput + } else if item.Kind == KindMCP { + mcpInput, inputErr := claudeMCPInputFromResponses(item.Arguments) + if inputErr != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d].arguments", index), + "mcp_arguments_unrepresentable", + inputErr.Error(), + )) + continue + } + input = mcpInput + } else if len(item.Action) > 0 { + if err := kitutil.Unmarshal(item.Action, &input); err != nil { + return nil, diagnostics, fmt.Errorf("hosted_tools[%d].action: %w", index, err) + } + } + callType := "server_tool_use" + if item.Kind == KindMCP { + callType = "mcp_tool_use" + } + blocks := []dto.ClaudeMediaMessage{{ + Type: callType, + Id: item.ID, + Name: name, + Input: input, + Caller: append(json.RawMessage(nil), item.Caller...), + ServerName: item.ServerName, + }} + result := item.Results + if item.Kind == KindMCP { + result = item.Output + if hostedItemFailed(item) && rawJSONPresent(item.Error) { + result = item.Error + } + } else if len(result) == 0 { + result = item.Sources + } + if len(result) > 0 && !(set.Source == types.RelayFormatOpenAIResponses && item.Kind == KindWebSearch) { + var content any + if item.Kind == KindMCP { + decoded, resultErr := claudeMCPContentFromResponsesString(result) + if resultErr != nil { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d].output", index), + "mcp_result_unrepresentable", + resultErr.Error(), + )) + continue + } + content = decoded + } else if err := kitutil.Unmarshal(result, &content); err != nil { + return nil, diagnostics, fmt.Errorf("hosted_tools[%d].results: %w", index, err) + } + isError := hostedItemFailed(item) + blocks = append(blocks, dto.ClaudeMediaMessage{ + Type: claudeResultTypeFromHostedKind(item.Kind), + ToolUseId: item.ID, + Content: content, + IsError: &isError, + ErrorCode: item.ErrorCode, + }) + } else if len(result) > 0 && item.Kind == KindWebSearch { + diagnostics = append(diagnostics, responsePresentationLoss( + fmt.Sprintf("hosted_tools[%d].results", index), + "web_search_result_omitted", + "Responses web-search source metadata cannot reconstruct Claude's encrypted web_search_tool_result", + )) + } else if item.Kind == KindMCP && (item.Status == "completed" || item.Status == "failed") { + diagnostics = append(diagnostics, responseSemanticLoss( + fmt.Sprintf("hosted_tools[%d]", index), + "mcp_result_missing", + fmt.Sprintf("Responses MCP output has status %q but no output or error", item.Status), + )) + } + hostedContent = append(hostedContent, positionedClaudeBlocks{position: item.Position, blocks: blocks}) + if set.Source != types.RelayFormatClaude { + diagnostics = append(diagnostics, responsePresentationLoss( + fmt.Sprintf("hosted_tools[%d]", index), + "hosted_tool_result_approximated", + "hosted-tool execution is preserved, but provider-specific result fields may differ", + )) + } + } + merged, orderingDiagnostics := mergeClaudeContent(target.Content, hostedContent, set) + diagnostics = append(diagnostics, orderingDiagnostics...) + target.Content = merged + return target, diagnostics, nil +} + +func mergeResponsesOutput(regular []dto.ResponsesOutput, hosted []positionedResponsesOutput, set HostedResponseSet) ([]dto.ResponsesOutput, []types.ConversionDiagnostic) { + if len(hosted) == 0 { + return regular, nil + } + if len(regular) == len(set.RegularPositions) { + byPosition := make(map[int][]dto.ResponsesOutput, len(hosted)) + for _, item := range hosted { + byPosition[item.position] = append(byPosition[item.position], item.output) + } + regularByPosition := make(map[int]dto.ResponsesOutput, len(regular)) + for index, position := range set.RegularPositions { + regularByPosition[position] = regular[index] + } + merged := make([]dto.ResponsesOutput, 0, len(regular)+len(hosted)) + for position := 0; position < set.SourceLength; position++ { + merged = append(merged, byPosition[position]...) + if output, exists := regularByPosition[position]; exists { + merged = append(merged, output) + } + } + return merged, nil + } + before, after, exact := hostedOutsideRegularRange(hostedPositions(hosted), set.RegularPositions) + if exact { + merged := make([]dto.ResponsesOutput, 0, len(regular)+len(hosted)) + for _, item := range before { + merged = append(merged, hosted[item].output) + } + merged = append(merged, regular...) + for _, item := range after { + merged = append(merged, hosted[item].output) + } + return merged, nil + } + merged := make([]dto.ResponsesOutput, 0, len(regular)+len(hosted)) + for _, item := range hosted { + merged = append(merged, item.output) + } + merged = append(merged, regular...) + return merged, []types.ConversionDiagnostic{responseSemanticLoss( + "output", + "hosted_tool_order_unrepresentable", + "hosted-tool items were interleaved with content that the target converter coalesced, so their original order cannot be reconstructed", + )} +} + +func mergeClaudeContent(regular []dto.ClaudeMediaMessage, hosted []positionedClaudeBlocks, set HostedResponseSet) ([]dto.ClaudeMediaMessage, []types.ConversionDiagnostic) { + if len(hosted) == 0 { + return regular, nil + } + if len(regular) == len(set.RegularPositions) { + byPosition := make(map[int][]dto.ClaudeMediaMessage, len(hosted)) + for _, item := range hosted { + byPosition[item.position] = append(byPosition[item.position], item.blocks...) + } + regularByPosition := make(map[int]dto.ClaudeMediaMessage, len(regular)) + for index, position := range set.RegularPositions { + regularByPosition[position] = regular[index] + } + merged := make([]dto.ClaudeMediaMessage, 0, len(regular)+len(hosted)*2) + for position := 0; position < set.SourceLength; position++ { + merged = append(merged, byPosition[position]...) + if block, exists := regularByPosition[position]; exists { + merged = append(merged, block) + } + } + return merged, nil + } + before, after, exact := hostedOutsideRegularRange(claudeHostedPositions(hosted), set.RegularPositions) + if exact { + merged := make([]dto.ClaudeMediaMessage, 0, len(regular)+len(hosted)*2) + for _, item := range before { + merged = append(merged, hosted[item].blocks...) + } + merged = append(merged, regular...) + for _, item := range after { + merged = append(merged, hosted[item].blocks...) + } + return merged, nil + } + merged := make([]dto.ClaudeMediaMessage, 0, len(regular)+len(hosted)*2) + for _, item := range hosted { + merged = append(merged, item.blocks...) + } + merged = append(merged, regular...) + return merged, []types.ConversionDiagnostic{responseSemanticLoss( + "content", + "hosted_tool_order_unrepresentable", + "hosted-tool blocks were interleaved with content that the target converter coalesced, so their original order cannot be reconstructed", + )} +} + +func hostedPositions(items []positionedResponsesOutput) []int { + positions := make([]int, len(items)) + for index := range items { + positions[index] = items[index].position + } + return positions +} + +func claudeHostedPositions(items []positionedClaudeBlocks) []int { + positions := make([]int, len(items)) + for index := range items { + positions[index] = items[index].position + } + return positions +} + +func hostedOutsideRegularRange(hosted []int, regular []int) (before []int, after []int, exact bool) { + if len(regular) == 0 { + indices := make([]int, len(hosted)) + for index := range hosted { + indices[index] = index + } + return indices, nil, true + } + firstRegular, lastRegular := regular[0], regular[len(regular)-1] + for index, position := range hosted { + switch { + case position < firstRegular: + before = append(before, index) + case position > lastRegular: + after = append(after, index) + default: + return nil, nil, false + } + } + return before, after, true +} + +func hostedKindFromClaudeCall(blockType string, name string) Kind { + if blockType == "mcp_tool_use" { + return KindMCP + } + switch strings.TrimSpace(name) { + case "web_search": + return KindWebSearch + case "web_fetch": + return KindWebFetch + case "code_execution": + return KindCodeExecution + default: + return KindNative + } +} + +func hostedKindFromClaudeResult(blockType string) Kind { + switch strings.TrimSuffix(blockType, "_tool_result") { + case "web_search": + return KindWebSearch + case "web_fetch": + return KindWebFetch + case "code_execution": + return KindCodeExecution + case "mcp": + return KindMCP + default: + return KindNative + } +} + +func hostedKindFromResponsesType(outputType string) Kind { + normalized := strings.TrimSpace(outputType) + normalized = strings.TrimSuffix(normalized, "_output") + normalized = strings.TrimSuffix(normalized, "_call") + switch normalized { + case "web_search": + return KindWebSearch + case "file_search": + return KindFileSearch + case "code_interpreter", "local_shell": + return KindCodeExecution + case "computer": + return KindComputerUse + case "image_generation": + return KindImage + case "mcp": + return KindMCP + default: + return KindNative + } +} + +func responsesTypeFromHostedKind(kind Kind) string { + switch kind { + case KindWebSearch: + return "web_search_call" + case KindMCP: + return "mcp_call" + default: + return "" + } +} + +func claudeNameFromHostedKind(kind Kind) string { + switch kind { + case KindWebSearch: + return "web_search" + case KindWebFetch: + return "web_fetch" + case KindCodeExecution: + return "" + case KindMCP: + return "mcp" + default: + return "" + } +} + +func isClaudeHostedResult(nativeType string) bool { + return nativeType == "mcp_tool_result" || strings.HasSuffix(nativeType, "_tool_result") +} + +func hostedCompletionStatus(item HostedResponseItem) string { + if hostedItemFailed(item) { + return "failed" + } + if item.Status != "" && item.Status != "in_progress" || len(item.Results) > 0 || len(item.Output) > 0 { + return "completed" + } + return "in_progress" +} + +func hostedItemFailed(item HostedResponseItem) bool { + return item.Status == "failed" || item.ErrorCode != "" || rawJSONPresent(item.Error) || item.IsError != nil && *item.IsError +} + +func hostedErrorValue(item HostedResponseItem) json.RawMessage { + if len(item.Error) > 0 { + return append(json.RawMessage(nil), item.Error...) + } + if item.ErrorCode == "" { + return nil + } + encoded, _ := kitutil.Marshal(item.ErrorCode) + return encoded +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +func rawJSONPresent(value json.RawMessage) bool { + normalized := strings.TrimSpace(string(value)) + return normalized != "" && normalized != "null" +} + +func claudeResultTypeFromHostedKind(kind Kind) string { + name := claudeNameFromHostedKind(kind) + if name == "mcp" { + return "mcp_tool_result" + } + return name + "_tool_result" +} diff --git a/relaykit/relayconvert/reasoning/claude.go b/relaykit/relayconvert/reasoning/claude.go new file mode 100644 index 0000000000..878bb1080d --- /dev/null +++ b/relaykit/relayconvert/reasoning/claude.go @@ -0,0 +1,303 @@ +package reasoning + +import ( + "fmt" + "math" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" +) + +type ClaudeRender struct { + Thinking *dto.Thinking + OutputEffort Effort + EffectiveEffort Effort + ClearSampling bool + ConstrainThinkingSampling bool +} + +type claudeCapabilities struct { + adaptive bool + supportsManual bool + defaultThinking bool + supportsDisable bool + supportsEffort bool + supportsXHigh bool + supportsMax bool + strictSampling bool +} + +func claudeCapabilitiesFor(model string) claudeCapabilities { + model = strings.ToLower(model) + capabilities := claudeCapabilities{supportsManual: true, supportsDisable: true} + + switch { + case strings.HasPrefix(model, "claude-fable-5"), + strings.HasPrefix(model, "claude-mythos-5"): + capabilities.adaptive = true + capabilities.supportsManual = false + capabilities.defaultThinking = true + capabilities.supportsDisable = false + capabilities.supportsXHigh = true + capabilities.supportsMax = true + capabilities.strictSampling = true + case strings.HasPrefix(model, "claude-mythos-preview"): + capabilities.adaptive = true + capabilities.defaultThinking = true + capabilities.supportsDisable = false + capabilities.supportsMax = true + capabilities.strictSampling = true + case strings.HasPrefix(model, "claude-opus-5"), + strings.HasPrefix(model, "claude-sonnet-5"), + strings.HasPrefix(model, "claude-opus-4-8"), + strings.HasPrefix(model, "claude-opus-4-7"): + capabilities.adaptive = true + capabilities.supportsManual = false + if strings.HasPrefix(model, "claude-opus-5") || strings.HasPrefix(model, "claude-sonnet-5") { + capabilities.defaultThinking = true + } + capabilities.supportsEffort = true + capabilities.supportsXHigh = true + capabilities.supportsMax = true + capabilities.strictSampling = true + case strings.HasPrefix(model, "claude-opus-4-6"), + strings.HasPrefix(model, "claude-sonnet-4-6"): + capabilities.adaptive = true + capabilities.supportsEffort = true + capabilities.supportsMax = true + case strings.HasPrefix(model, "claude-opus-4-5"): + capabilities.supportsEffort = true + } + + return capabilities +} + +func RenderClaude(model string, intent Intent, maxTokens *uint, adapterBudgetPercentage float64) (ClaudeRender, error) { + if intent.Mode == ModeDisabled && intent.Effort != "" && intent.Effort != EffortNone { + effort, err := ParseEffort(string(intent.Effort)) + if err != nil { + return ClaudeRender{}, err + } + intent.Effort = effort + } else { + var err error + intent, err = normalizeIntent(intent) + if err != nil { + return ClaudeRender{}, err + } + } + capabilities := claudeCapabilitiesFor(model) + if !intent.HasStrength() { + if intent.IncludeThoughts != nil && capabilities.adaptive && capabilities.defaultThinking { + thinking := &dto.Thinking{Type: "adaptive"} + if *intent.IncludeThoughts { + thinking.Display = "summarized" + } else { + thinking.Display = "omitted" + } + return ClaudeRender{ + Thinking: thinking, + EffectiveEffort: EffortHigh, + ClearSampling: capabilities.strictSampling, + }, nil + } + if capabilities.defaultThinking { + return ClaudeRender{EffectiveEffort: EffortHigh, ClearSampling: capabilities.strictSampling}, nil + } + return ClaudeRender{ClearSampling: capabilities.strictSampling}, nil + } + + if intent.Mode == ModeDisabled || intent.Effort == EffortNone { + if strings.HasPrefix(strings.ToLower(model), "claude-opus-5") && + (intent.Effort == EffortXHigh || intent.Effort == EffortMax) { + return ClaudeRender{}, fmt.Errorf("model %q does not support effort %q while thinking is disabled", model, intent.Effort) + } + if !capabilities.supportsDisable { + return ClaudeRender{}, fmt.Errorf("%w for model %q", ErrThinkingNotDisabled, model) + } + return ClaudeRender{ + Thinking: &dto.Thinking{Type: "disabled"}, + EffectiveEffort: EffortNone, + ClearSampling: capabilities.strictSampling, + }, nil + } + + preferManual := capabilities.supportsManual && intent.BudgetTokens != nil && intent.Mode != ModeAdaptive + if !capabilities.supportsManual && intent.BudgetTokens != nil && intent.BudgetSource == SourceNative && intent.Mode == ModeEnabled { + return ClaudeRender{}, fmt.Errorf("model %q requires adaptive thinking and does not support native budget_tokens", model) + } + if capabilities.adaptive && !preferManual { + effort := intent.Effort + if effort == "" && intent.BudgetTokens != nil { + effort = EffortFromBudget(*intent.BudgetTokens) + } + if effort == "" && intent.Mode == ModeEnabled { + effort = EffortHigh + } + effort = normalizeClaudeEffort(effort, capabilities) + effectiveEffort := effort + if effectiveEffort == "" && intent.Mode == ModeAdaptive { + effectiveEffort = EffortHigh + } + + // Claude effort can be used without enabling thinking. Preserve that + // distinction for native Claude requests; OpenAI extractors explicitly + // mark reasoning efforts as ModeEnabled. + if intent.Mode == ModeUnset { + return ClaudeRender{ + OutputEffort: effort, + EffectiveEffort: effectiveEffort, + ClearSampling: capabilities.strictSampling, + }, nil + } + + thinking := &dto.Thinking{Type: "adaptive"} + if intent.IncludeThoughts != nil { + if *intent.IncludeThoughts { + thinking.Display = "summarized" + } else { + thinking.Display = "omitted" + } + } + return ClaudeRender{ + Thinking: thinking, + OutputEffort: effort, + EffectiveEffort: effectiveEffort, + ClearSampling: capabilities.strictSampling, + ConstrainThinkingSampling: !capabilities.strictSampling, + }, nil + } + + if intent.Mode == ModeAdaptive { + return ClaudeRender{}, fmt.Errorf("model %q does not support adaptive thinking", model) + } + if intent.Mode == ModeUnset { + return ClaudeRender{OutputEffort: intent.Effort, EffectiveEffort: intent.Effort}, nil + } + if maxTokens == nil { + return ClaudeRender{}, fmt.Errorf("max_tokens is required for manual Claude thinking") + } + if *maxTokens <= 1024 { + return ClaudeRender{}, fmt.Errorf("max_tokens must be greater than 1024 for manual Claude thinking") + } + if uint64(*maxTokens) > uint64(math.MaxInt) { + return ClaudeRender{}, fmt.Errorf("max_tokens is too large for a thinking budget") + } + + budget := 0 + if intent.BudgetTokens != nil && *intent.BudgetTokens == -1 && intent.BudgetSource == SourceNative { + return ClaudeRender{}, fmt.Errorf("Claude thinking budget_tokens does not support -1") + } + if intent.BudgetTokens != nil && *intent.BudgetTokens >= 0 { + budget = *intent.BudgetTokens + if intent.BudgetSource != SourceNative { + if budget < 1024 { + budget = 1024 + } + if uint(budget) >= *maxTokens { + budget = int(*maxTokens) - 1 + } + } + if budget < 1024 || uint(budget) >= *maxTokens { + return ClaudeRender{}, fmt.Errorf("Claude thinking budget must satisfy 1024 <= budget_tokens < max_tokens") + } + } else { + percentage := effortPercentage(intent.Effort, adapterBudgetPercentage) + budget = int(*maxTokens) * percentage / 100 + if budget < 1024 { + budget = 1024 + } + if uint(budget) >= *maxTokens { + budget = int(*maxTokens) - 1 + } + } + + effectiveEffort := intent.Effort + if intent.BudgetTokens != nil && !capabilities.supportsEffort { + effectiveEffort = EffortFromBudget(budget) + } else if effectiveEffort == "" { + effectiveEffort = EffortFromBudget(budget) + } + outputEffort := Effort("") + if capabilities.supportsEffort && intent.Effort != "" { + outputEffort = normalizeClaudeEffort(intent.Effort, capabilities) + effectiveEffort = outputEffort + } + thinking := &dto.Thinking{Type: "enabled", BudgetTokens: &budget} + if intent.IncludeThoughts != nil { + if *intent.IncludeThoughts { + thinking.Display = "summarized" + } else { + thinking.Display = "omitted" + } + } + return ClaudeRender{ + Thinking: thinking, + OutputEffort: outputEffort, + EffectiveEffort: effectiveEffort, + ConstrainThinkingSampling: true, + }, nil +} + +// ClaudeUsesManualThinking reports whether an exact numeric budget is rendered +// as legacy extended thinking rather than being reduced to adaptive effort. +func ClaudeUsesManualThinking(model string, intent Intent) bool { + capabilities := claudeCapabilitiesFor(model) + return capabilities.supportsManual && intent.BudgetTokens != nil && intent.Mode != ModeAdaptive +} + +func IsKnownClaudeModel(model string) bool { + return isKnownClaudeModel(model) +} + +func ResolveClaudeDefault(model string, intent Intent) Intent { + if intent.HasStrength() || !claudeCapabilitiesFor(model).defaultThinking { + return intent + } + intent.Mode = ModeAdaptive + intent.Effort = EffortHigh + return intent +} + +func normalizeClaudeEffort(effort Effort, capabilities claudeCapabilities) Effort { + switch effort { + case EffortMinimal: + return EffortLow + case EffortXHigh: + if capabilities.supportsXHigh { + return effort + } + if capabilities.supportsMax { + return EffortMax + } + return EffortHigh + case EffortMax: + if !capabilities.supportsMax { + return EffortHigh + } + } + return effort +} + +func effortPercentage(effort Effort, adapterBudgetPercentage float64) int { + switch effort { + case EffortMinimal: + return 5 + case EffortLow: + return 20 + case EffortMedium: + return 50 + case EffortHigh: + return 80 + case EffortXHigh, EffortMax: + return 95 + } + percentage := int(math.Round(adapterBudgetPercentage * 100)) + if percentage <= 0 { + return 80 + } + if percentage >= 100 { + return 99 + } + return percentage +} diff --git a/relaykit/relayconvert/reasoning/gemini.go b/relaykit/relayconvert/reasoning/gemini.go new file mode 100644 index 0000000000..63fe5c8552 --- /dev/null +++ b/relaykit/relayconvert/reasoning/gemini.go @@ -0,0 +1,374 @@ +package reasoning + +import ( + "fmt" + "math" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" +) + +type GeminiRender struct { + Config *dto.GeminiThinkingConfig + EffectiveEffort Effort +} + +type geminiThinkingKind int + +const ( + geminiThinkingUnknown geminiThinkingKind = iota + geminiThinkingNotConfigurable + geminiThinkingBudget + geminiThinkingLevel +) + +type geminiCapabilities struct { + kind geminiThinkingKind + supportsDisable bool + supportsIncludeThoughts bool + minBudget int + maxBudget int +} + +func geminiCapabilitiesFor(model string) geminiCapabilities { + model = strings.ToLower(model) + switch { + case strings.HasPrefix(model, "gemini-2.5-flash-native-audio"), + strings.HasPrefix(model, "gemini-live-2.5-flash-preview-native-audio"): + return geminiCapabilities{kind: geminiThinkingBudget, supportsDisable: true, maxBudget: 24576} + case strings.HasPrefix(model, "gemini-2.5-flash-image"), + strings.Contains(model, "-tts"), + strings.Contains(model, "-native-audio"), + strings.Contains(model, "-live"): + return geminiCapabilities{kind: geminiThinkingNotConfigurable} + case strings.HasPrefix(model, "gemini-3-pro-image"), + strings.HasPrefix(model, "nano-banana-pro"): + return geminiCapabilities{kind: geminiThinkingNotConfigurable, supportsIncludeThoughts: true} + case model == "gemini-flash-latest", model == "gemini-flash-lite-latest": + return geminiCapabilities{kind: geminiThinkingLevel} + case model == "gemini-pro-latest": + return geminiCapabilities{kind: geminiThinkingLevel} + case strings.HasPrefix(model, "gemini-2.5-pro"): + return geminiCapabilities{kind: geminiThinkingBudget, minBudget: 128, maxBudget: 32768} + case strings.HasPrefix(model, "gemini-2.5-flash-lite"): + return geminiCapabilities{kind: geminiThinkingBudget, supportsDisable: true, minBudget: 512, maxBudget: 24576} + case strings.HasPrefix(model, "gemini-2.5-"): + return geminiCapabilities{kind: geminiThinkingBudget, supportsDisable: true, maxBudget: 24576} + case strings.HasPrefix(model, "gemini-3"): + return geminiCapabilities{kind: geminiThinkingLevel} + default: + return geminiCapabilities{} + } +} + +func RenderGemini(model string, intent Intent, maxOutputTokens *uint, adapterBudgetPercentage float64) (GeminiRender, error) { + intent, err := normalizeIntent(intent) + if err != nil { + return GeminiRender{}, err + } + if intent.IsEmpty() { + return GeminiRender{}, nil + } + + capabilities := geminiCapabilitiesFor(model) + if capabilities.kind == geminiThinkingNotConfigurable { + if !intent.HasStrength() && capabilities.supportsIncludeThoughts { + return GeminiRender{Config: &dto.GeminiThinkingConfig{IncludeThoughts: intent.IncludeThoughts}, EffectiveEffort: EffortHigh}, nil + } + return GeminiRender{}, fmt.Errorf("model %q does not support configurable thinking", model) + } + if capabilities.kind == geminiThinkingUnknown { + if intent.HasStrength() { + return GeminiRender{}, fmt.Errorf("model %q does not have a known Gemini thinking configuration", model) + } + return GeminiRender{Config: &dto.GeminiThinkingConfig{IncludeThoughts: intent.IncludeThoughts}}, nil + } + + config := &dto.GeminiThinkingConfig{IncludeThoughts: intent.IncludeThoughts} + if capabilities.kind == geminiThinkingBudget { + if intent.Mode == ModeDisabled || intent.Effort == EffortNone { + if !capabilities.supportsDisable { + return GeminiRender{}, fmt.Errorf("%w for model %q", ErrThinkingNotDisabled, model) + } + budget := 0 + config.ThinkingBudget = &budget + return GeminiRender{Config: config, EffectiveEffort: EffortNone}, nil + } + + budget := 0 + hasBudget := false + if intent.BudgetTokens != nil { + budget = *intent.BudgetTokens + if intent.BudgetSource != SourceNative && budget != -1 { + budget = clampGeminiBudget(budget, capabilities) + } + hasBudget = true + } else if intent.Effort != "" { + budget = gemini25BudgetForEffort(intent.Effort) + hasBudget = true + } else if intent.Mode != ModeUnset && maxOutputTokens != nil && *maxOutputTokens > 0 { + if uint64(*maxOutputTokens) > uint64(math.MaxInt) { + return GeminiRender{}, fmt.Errorf("max_output_tokens is too large for a thinking budget") + } + percentage := adapterBudgetPercentage + if percentage <= 0 { + percentage = 0.6 + } else if percentage > 1 { + percentage = 1 + } + budget = int(math.Round(float64(*maxOutputTokens) * percentage)) + budget = clampGeminiBudget(budget, capabilities) + hasBudget = true + } + if hasBudget { + if err := validateGeminiBudget(model, budget, capabilities); err != nil { + return GeminiRender{}, err + } + config.ThinkingBudget = &budget + } + effort := intent.Effort + if hasBudget { + effort = EffortFromBudget(budget) + } else if intent.Mode == ModeEnabled || intent.Mode == ModeAdaptive { + effort = geminiDefaultEffort(model) + } + return GeminiRender{Config: config, EffectiveEffort: effort}, nil + } + + if intent.Mode == ModeDisabled || intent.Effort == EffortNone { + return GeminiRender{}, fmt.Errorf("%w for model %q", ErrThinkingNotDisabled, model) + } + effort := intent.Effort + if effort == "" && intent.BudgetTokens != nil { + effort = EffortFromBudget(*intent.BudgetTokens) + } + if effort != "" { + level, err := geminiLevelForEffort(model, effort) + if err != nil { + return GeminiRender{}, err + } + config.ThinkingLevel = level + effort = Effort(level) + } else if intent.Mode == ModeEnabled || intent.Mode == ModeAdaptive { + effort = geminiDefaultEffort(model) + } + return GeminiRender{Config: config, EffectiveEffort: effort}, nil +} + +func geminiDefaultEffort(model string) Effort { + model = strings.ToLower(model) + switch { + case model == "gemini-flash-latest", + strings.HasPrefix(model, "gemini-3.5-flash") && !strings.HasPrefix(model, "gemini-3.5-flash-lite"), + strings.HasPrefix(model, "gemini-3.6-flash"): + return EffortMedium + case model == "gemini-flash-lite-latest", + strings.HasPrefix(model, "gemini-3.5-flash-lite"), + strings.HasPrefix(model, "gemini-3.1-flash-lite"): + return EffortMinimal + case model == "gemini-pro-latest", + strings.HasPrefix(model, "gemini-3.1-pro"), + strings.HasPrefix(model, "gemini-3-pro"), + strings.HasPrefix(model, "gemini-3-flash"): + return EffortHigh + default: + return "" + } +} + +func ValidateGeminiThinkingConfig(model string, config *dto.GeminiThinkingConfig) (Effort, error) { + if config == nil { + return "", nil + } + intent, err := FromGemini(&dto.GeminiChatRequest{GenerationConfig: dto.GeminiChatGenerationConfig{ThinkingConfig: config}}) + if err != nil { + return "", err + } + capabilities := geminiCapabilitiesFor(model) + if capabilities.kind == geminiThinkingNotConfigurable { + if !intent.HasStrength() && capabilities.supportsIncludeThoughts { + return EffortHigh, nil + } + return "", fmt.Errorf("model %q does not support configurable thinking", model) + } + if capabilities.kind == geminiThinkingUnknown { + return EffectiveEffort(intent), nil + } + if capabilities.kind == geminiThinkingBudget { + if config.ThinkingLevel != "" { + return "", fmt.Errorf("Gemini 2.5 model %q requires thinkingBudget, not thinkingLevel", model) + } + if config.ThinkingBudget != nil { + if err := validateGeminiBudget(model, *config.ThinkingBudget, capabilities); err != nil { + return "", err + } + } + return EffectiveEffort(intent), nil + } + if config.ThinkingBudget != nil { + return "", fmt.Errorf("Gemini 3 model %q requires thinkingLevel, not thinkingBudget", model) + } + if config.ThinkingLevel != "" { + level, err := geminiLevelForEffort(model, intent.Effort) + if err != nil { + return "", err + } + if level != config.ThinkingLevel { + return "", fmt.Errorf("thinkingLevel %q is not supported by model %q", config.ThinkingLevel, model) + } + return Effort(level), nil + } + return "", nil +} + +// ResolveGeminiDefault materializes documented family defaults when a +// conversion targets another protocol. Dynamic 2.5 defaults retain their -1 +// budget in the in-process pivot; Flash-Lite's default is explicitly off. +func ResolveGeminiDefault(model string, intent Intent) Intent { + if intent.HasStrength() { + return intent + } + capabilities := geminiCapabilitiesFor(model) + if capabilities.kind == geminiThinkingBudget { + if strings.HasPrefix(strings.ToLower(model), "gemini-2.5-flash-lite") { + budget := 0 + intent.Mode = ModeDisabled + intent.Effort = EffortNone + intent.BudgetTokens = &budget + intent.BudgetSource = SourceNative + return intent + } + budget := -1 + intent.Mode = ModeEnabled + intent.BudgetTokens = &budget + intent.BudgetSource = SourceNative + return intent + } + if capabilities.kind != geminiThinkingLevel { + return intent + } + effort := geminiDefaultEffort(model) + if effort == "" { + return intent + } + intent.Mode = ModeEnabled + intent.Effort = effort + return intent +} + +// ResolveGeminiEnabledDefault fills the strength implied by an explicit +// enable-only control such as the legacy -thinking model alias. +func ResolveGeminiEnabledDefault(model string, intent Intent, maxOutputTokens *uint) Intent { + if intent.Mode != ModeEnabled || intent.Effort != "" || intent.BudgetTokens != nil { + return intent + } + capabilities := geminiCapabilitiesFor(model) + if capabilities.kind == geminiThinkingBudget { + if intent.Source == SourceSuffix && maxOutputTokens != nil && *maxOutputTokens > 0 { + return intent + } + budget := -1 + intent.BudgetTokens = &budget + intent.BudgetSource = SourceSuffix + return intent + } + if capabilities.kind == geminiThinkingLevel { + intent.Effort = geminiDefaultEffort(model) + } + return intent +} + +// EquivalentGeminiStrength compares two controls after applying the target +// model's budget/level mapping. This accepts distinct canonical labels that +// are identical on the Gemini wire (for example minimal and low on 2.5). +func EquivalentGeminiStrength(model string, left Intent, right Intent) (bool, error) { + leftRendered, err := RenderGemini(model, left, nil, 0) + if err != nil { + return false, err + } + rightRendered, err := RenderGemini(model, right, nil, 0) + if err != nil { + return false, err + } + if leftRendered.Config == nil || rightRendered.Config == nil { + return leftRendered.Config == nil && rightRendered.Config == nil, nil + } + leftConfig, rightConfig := leftRendered.Config, rightRendered.Config + if leftConfig.ThinkingLevel != rightConfig.ThinkingLevel { + return false, nil + } + if (leftConfig.ThinkingBudget == nil) != (rightConfig.ThinkingBudget == nil) { + return false, nil + } + return leftConfig.ThinkingBudget == nil || *leftConfig.ThinkingBudget == *rightConfig.ThinkingBudget, nil +} + +func gemini25BudgetForEffort(effort Effort) int { + switch effort { + case EffortMinimal, EffortLow: + return 1024 + case EffortMedium: + return 8192 + case EffortHigh, EffortXHigh, EffortMax: + return 24576 + default: + return 0 + } +} + +func geminiLevelForEffort(model string, effort Effort) (string, error) { + model = strings.ToLower(model) + switch { + case strings.HasPrefix(model, "gemini-3.1-flash-image"), + strings.HasPrefix(model, "gemini-3.1-flash-lite-image"): + if effort == EffortMinimal || effort == EffortLow { + return string(EffortMinimal), nil + } + return string(EffortHigh), nil + case (strings.HasPrefix(model, "gemini-3-pro") && !strings.HasPrefix(model, "gemini-3.1-pro")): + if effort == EffortMinimal || effort == EffortLow { + return string(EffortLow), nil + } + return string(EffortHigh), nil + case strings.HasPrefix(model, "gemini-3.1-pro"), model == "gemini-pro-latest": + if effort == EffortMinimal { + return string(EffortLow), nil + } + } + switch effort { + case EffortMinimal, EffortLow, EffortMedium, EffortHigh: + return string(effort), nil + case EffortXHigh, EffortMax: + return string(EffortHigh), nil + case EffortNone: + return "", fmt.Errorf("%w for model %q", ErrThinkingNotDisabled, model) + default: + return "", fmt.Errorf("%w %q for model %q", ErrUnsupportedEffort, effort, model) + } +} + +func validateGeminiBudget(model string, budget int, capabilities geminiCapabilities) error { + if budget == -1 { + return nil + } + if budget == 0 { + if capabilities.supportsDisable { + return nil + } + return fmt.Errorf("%w for model %q", ErrThinkingNotDisabled, model) + } + if budget < capabilities.minBudget || budget > capabilities.maxBudget { + return fmt.Errorf("thinking budget %d is outside the supported range [%d,%d] for model %q", budget, capabilities.minBudget, capabilities.maxBudget, model) + } + return nil +} + +func clampGeminiBudget(budget int, capabilities geminiCapabilities) int { + if budget < capabilities.minBudget { + return capabilities.minBudget + } + if budget > capabilities.maxBudget { + return capabilities.maxBudget + } + return budget +} diff --git a/relaykit/relayconvert/reasoning/intent.go b/relaykit/relayconvert/reasoning/intent.go new file mode 100644 index 0000000000..061006950e --- /dev/null +++ b/relaykit/relayconvert/reasoning/intent.go @@ -0,0 +1,609 @@ +package reasoning + +import ( + "errors" + "fmt" + "strings" + + "github.com/QuantumNous/new-api/relaykit/dto" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" +) + +type Effort string + +const ( + EffortNone Effort = "none" + EffortMinimal Effort = "minimal" + EffortLow Effort = "low" + EffortMedium Effort = "medium" + EffortHigh Effort = "high" + EffortXHigh Effort = "xhigh" + EffortMax Effort = "max" +) + +type Mode string + +type Source string + +// ClientError marks invalid user-supplied reasoning controls so host handlers +// can return a 4xx without classifying unrelated adapter failures as client +// errors. +type ClientError struct { + err error +} + +func (e *ClientError) Error() string { return e.err.Error() } +func (e *ClientError) Unwrap() error { return e.err } + +func AsClientError(err error) error { + if err == nil { + return nil + } + var clientErr *ClientError + if errors.As(err, &clientErr) { + return err + } + return &ClientError{err: err} +} + +func IsClientError(err error) bool { + var clientErr *ClientError + return errors.As(err, &clientErr) +} + +const ( + ModeUnset Mode = "" + ModeEnabled Mode = "enabled" + ModeAdaptive Mode = "adaptive" + ModeDisabled Mode = "disabled" +) + +const ( + SourceExplicit Source = "explicit" + SourceNative Source = "native" + SourceSuffix Source = "suffix" + SourcePivot Source = "pivot" +) + +var ( + ErrEffortConflict = errors.New("reasoning settings conflict") + ErrUnsupportedEffort = errors.New("unsupported reasoning effort") + ErrThinkingNotDisabled = errors.New("thinking cannot be disabled") +) + +// Intent is the protocol-independent part of a request's reasoning controls. +// Summary visibility is intentionally independent from reasoning strength. +type Intent struct { + Mode Mode + Effort Effort + BudgetTokens *int + IncludeThoughts *bool + Source Source + BudgetSource Source +} + +func (i Intent) HasStrength() bool { + return i.Mode != ModeUnset || i.Effort != "" || i.BudgetTokens != nil +} + +func (i Intent) IsEmpty() bool { + return !i.HasStrength() && i.IncludeThoughts == nil +} + +// IntentFromState reconstructs a portable intent from host- or pivot-carried +// conversion state. A nil state is an empty intent. +func IntentFromState(state *dto.ReasoningConversionState) Intent { + if state == nil { + return Intent{} + } + return Intent{ + Mode: Mode(state.Mode), + Effort: Effort(state.Effort), + BudgetTokens: state.BudgetTokens, + IncludeThoughts: state.IncludeThoughts, + Source: SourceSuffix, + BudgetSource: SourceSuffix, + } +} + +// StateFromIntent copies the portable fields of an intent into conversion +// state. Empty intents produce nil so callers can omit the field. +func StateFromIntent(intent Intent) *dto.ReasoningConversionState { + if intent.IsEmpty() { + return nil + } + return &dto.ReasoningConversionState{ + Mode: string(intent.Mode), + Effort: string(intent.Effort), + BudgetTokens: intent.BudgetTokens, + IncludeThoughts: intent.IncludeThoughts, + } +} + +func ParseEffort(value string) (Effort, error) { + effort := Effort(strings.ToLower(strings.TrimSpace(value))) + if effort == "" { + return "", nil + } + switch effort { + case EffortNone, EffortMinimal, EffortLow, EffortMedium, EffortHigh, EffortXHigh, EffortMax: + return effort, nil + default: + return "", fmt.Errorf("%w: %q", ErrUnsupportedEffort, value) + } +} + +func normalizeIntent(intent Intent) (Intent, error) { + effort, err := ParseEffort(string(intent.Effort)) + if err != nil { + return Intent{}, err + } + intent.Effort = effort + + switch intent.Mode { + case ModeUnset, ModeEnabled, ModeAdaptive, ModeDisabled: + default: + return Intent{}, fmt.Errorf("unsupported reasoning mode %q", intent.Mode) + } + + if intent.BudgetTokens != nil { + budget := *intent.BudgetTokens + if budget < -1 { + return Intent{}, fmt.Errorf("thinking budget must be -1 or non-negative, got %d", budget) + } + if budget == 0 { + if intent.Mode == ModeEnabled || intent.Mode == ModeAdaptive || (intent.Effort != "" && intent.Effort != EffortNone) { + return Intent{}, fmt.Errorf("%w: zero budget disables thinking", ErrEffortConflict) + } + intent.Mode = ModeDisabled + intent.Effort = EffortNone + } else if intent.Mode == ModeDisabled || intent.Effort == EffortNone { + return Intent{}, fmt.Errorf("%w: a non-zero budget enables thinking", ErrEffortConflict) + } else if intent.Mode == ModeUnset { + intent.Mode = ModeEnabled + } + } + + if intent.Effort == EffortNone { + if intent.Mode == ModeEnabled || intent.Mode == ModeAdaptive { + return Intent{}, fmt.Errorf("%w: effort none disables thinking", ErrEffortConflict) + } + intent.Mode = ModeDisabled + } + + return intent, nil +} + +// MergeExplicitAndSuffix combines structured request fields with a model-name +// alias. Contradictions are rejected because the alias may carry a distinct +// billing identity; silently choosing either side would make request semantics +// and accounting disagree. +func MergeExplicitAndSuffix(explicit Intent, suffix Intent, model string) (Intent, error) { + var err error + explicit, err = normalizeIntent(explicit) + if err != nil { + return Intent{}, err + } + suffix, err = normalizeIntent(suffix) + if err != nil { + return Intent{}, err + } + + if !explicit.HasStrength() { + if explicit.IncludeThoughts != nil { + suffix.IncludeThoughts = explicit.IncludeThoughts + } + return suffix, nil + } + if !suffix.HasStrength() { + if explicit.IncludeThoughts == nil { + explicit.IncludeThoughts = suffix.IncludeThoughts + } + return explicit, nil + } + + explicitDisabled := explicit.Mode == ModeDisabled || explicit.Effort == EffortNone + suffixDisabled := suffix.Mode == ModeDisabled || suffix.Effort == EffortNone + if explicitDisabled != suffixDisabled { + return Intent{}, fmt.Errorf("%w for model %q: explicit fields and model suffix disagree about whether thinking is enabled", ErrEffortConflict, model) + } + if !explicitDisabled && explicit.Effort != "" && suffix.Effort != "" && explicit.Effort != suffix.Effort { + return Intent{}, fmt.Errorf("%w for model %q: explicit effort %q differs from suffix effort %q", ErrEffortConflict, model, explicit.Effort, suffix.Effort) + } + if explicit.BudgetTokens != nil && suffix.BudgetTokens != nil && *explicit.BudgetTokens != *suffix.BudgetTokens { + return Intent{}, fmt.Errorf("%w for model %q: explicit budget %d differs from suffix budget %d", ErrEffortConflict, model, *explicit.BudgetTokens, *suffix.BudgetTokens) + } + if (explicit.Effort != "" && explicit.Effort != EffortNone && suffix.BudgetTokens != nil) || + (explicit.BudgetTokens != nil && suffix.Effort != "" && suffix.Effort != EffortNone) { + return Intent{}, fmt.Errorf("%w for model %q: effort and an exact suffix budget cannot both select reasoning strength", ErrEffortConflict, model) + } + + merged := suffix + if explicit.Mode != ModeUnset { + merged.Mode = explicit.Mode + } + if explicit.Effort != "" { + merged.Effort = explicit.Effort + } + if explicit.BudgetTokens != nil { + merged.BudgetTokens = explicit.BudgetTokens + merged.BudgetSource = explicit.BudgetSource + } + if explicit.IncludeThoughts != nil { + merged.IncludeThoughts = explicit.IncludeThoughts + } + return normalizeIntent(merged) +} + +// MergeExplicit combines two structured representations of the same request. +// A numeric budget and an effort may coexist: Claude and OpenRouter expose both +// controls, and keeping both is what lets an in-memory OpenAI pivot preserve an +// exact budget for budget-based targets while retaining an effort for +// level-based targets. +func MergeExplicit(primary Intent, secondary Intent, model string) (Intent, error) { + var err error + primary, err = normalizeIntent(primary) + if err != nil { + return Intent{}, err + } + secondary, err = normalizeIntent(secondary) + if err != nil { + return Intent{}, err + } + + if primary.IsEmpty() { + return secondary, nil + } + if secondary.IsEmpty() { + return primary, nil + } + + primaryDisabled := primary.Mode == ModeDisabled || primary.Effort == EffortNone + secondaryDisabled := secondary.Mode == ModeDisabled || secondary.Effort == EffortNone + if primary.HasStrength() && secondary.HasStrength() && primaryDisabled != secondaryDisabled { + return Intent{}, fmt.Errorf("%w for model %q: explicit fields disagree about whether thinking is enabled", ErrEffortConflict, model) + } + if primary.Effort != "" && secondary.Effort != "" && primary.Effort != secondary.Effort { + return Intent{}, fmt.Errorf("%w for model %q: explicit efforts %q and %q differ", ErrEffortConflict, model, primary.Effort, secondary.Effort) + } + if primary.BudgetTokens != nil && secondary.BudgetTokens != nil && *primary.BudgetTokens != *secondary.BudgetTokens { + return Intent{}, fmt.Errorf("%w for model %q: explicit budgets %d and %d differ", ErrEffortConflict, model, *primary.BudgetTokens, *secondary.BudgetTokens) + } + + merged := secondary + if primary.Mode != ModeUnset { + merged.Mode = primary.Mode + } + if primary.Effort != "" { + merged.Effort = primary.Effort + } + if primary.BudgetTokens != nil { + merged.BudgetTokens = primary.BudgetTokens + merged.BudgetSource = primary.BudgetSource + } + if primary.IncludeThoughts != nil { + merged.IncludeThoughts = primary.IncludeThoughts + } + return normalizeIntent(merged) +} + +type openRouterReasoning struct { + Enabled *bool `json:"enabled,omitempty"` + Effort string `json:"effort,omitempty"` + MaxTokens *int `json:"max_tokens,omitempty"` + Exclude *bool `json:"exclude,omitempty"` +} + +func FromOpenAIChat(req *dto.GeneralOpenAIRequest) (Intent, error) { + if req == nil { + return Intent{}, nil + } + + var intent Intent + intent.Source = SourceExplicit + if req.ReasoningEffort != "" { + effort, err := ParseEffort(req.ReasoningEffort) + if err != nil { + return Intent{}, err + } + intent.Effort = effort + if effort == EffortNone { + intent.Mode = ModeDisabled + } else { + intent.Mode = ModeEnabled + } + } + + if len(req.Reasoning) > 0 { + var raw openRouterReasoning + if err := kitutil.Unmarshal(req.Reasoning, &raw); err != nil { + return Intent{}, fmt.Errorf("invalid reasoning config: %w", err) + } + nested := Intent{BudgetTokens: raw.MaxTokens, Source: SourceExplicit, BudgetSource: SourceExplicit} + if raw.Enabled != nil { + if *raw.Enabled { + nested.Mode = ModeEnabled + } else { + nested.Mode = ModeDisabled + nested.Effort = EffortNone + } + } + if raw.Effort != "" { + effort, err := ParseEffort(raw.Effort) + if err != nil { + return Intent{}, err + } + nested.Effort = effort + if effort == EffortNone { + nested.Mode = ModeDisabled + } else if nested.Mode == ModeUnset { + nested.Mode = ModeEnabled + } + } + if raw.Exclude != nil { + include := !*raw.Exclude + nested.IncludeThoughts = &include + } + var err error + intent, err = MergeExplicit(intent, nested, req.Model) + if err != nil { + return Intent{}, err + } + } + + if req.ReasoningConversion == nil { + return normalizeIntent(intent) + } + pivot := Intent{ + Mode: Mode(req.ReasoningConversion.Mode), + Effort: Effort(req.ReasoningConversion.Effort), + BudgetTokens: req.ReasoningConversion.BudgetTokens, + IncludeThoughts: req.ReasoningConversion.IncludeThoughts, + Source: SourcePivot, + BudgetSource: SourcePivot, + } + if req.ReasoningEffort != "" { + projectedEffort := OpenAIEffort(EffectiveEffort(pivot)) + if Effort(req.ReasoningEffort) == projectedEffort { + intent.Effort = "" + intent.Mode = ModeUnset + } + } + return MergeExplicit(intent, pivot, req.Model) +} + +// ApplyToOpenAIChat writes the portable portion of an intent to the OpenAI +// pivot. reasoning_effort carries level-based strength; a JSON-excluded DTO +// state retains exact budgets and summary visibility across in-process steps. +func ApplyToOpenAIChat(req *dto.GeneralOpenAIRequest, intent Intent) error { + if req == nil { + return nil + } + intent, err := normalizeIntent(intent) + if err != nil { + return err + } + + if effort := OpenAIEffort(EffectiveEffort(intent)); effort != "" { + req.ReasoningEffort = string(effort) + } + + if intent.IsEmpty() { + return nil + } + req.ReasoningConversion = &dto.ReasoningConversionState{ + Mode: string(intent.Mode), + Effort: string(intent.Effort), + BudgetTokens: intent.BudgetTokens, + IncludeThoughts: intent.IncludeThoughts, + } + return nil +} + +// ApplyToOpenAIResponses writes the portable portion of an intent directly to +// a Responses request. The JSON-excluded state retains exact provider-native +// controls for any later in-process conversion. +func ApplyToOpenAIResponses(req *dto.OpenAIResponsesRequest, intent Intent) error { + if req == nil { + return nil + } + intent, err := normalizeIntent(intent) + if err != nil { + return err + } + + if effort := OpenAIEffort(EffectiveEffort(intent)); effort != "" { + summary := "detailed" + if effort == EffortNone || (intent.IncludeThoughts != nil && !*intent.IncludeThoughts) { + summary = "" + } + req.Reasoning = &dto.Reasoning{ + Effort: string(effort), + Summary: summary, + } + } + + if intent.IsEmpty() { + return nil + } + state := &dto.ReasoningConversionState{ + Mode: string(intent.Mode), + Effort: string(intent.Effort), + BudgetTokens: intent.BudgetTokens, + IncludeThoughts: intent.IncludeThoughts, + } + req.ReasoningConversion = state + return nil +} + +// OpenAIEffort maps the canonical cross-provider vocabulary to the public +// OpenAI reasoning_effort vocabulary. Claude/OpenRouter "max" has no direct +// OpenAI equivalent and is represented by xhigh at that wire boundary. +func OpenAIEffort(effort Effort) Effort { + if effort == EffortMax { + return EffortXHigh + } + return effort +} + +func FromOpenAIResponses(req *dto.OpenAIResponsesRequest) (Intent, error) { + if req == nil { + return Intent{}, nil + } + var intent Intent + if req.Reasoning != nil { + intent.Source = SourceExplicit + if req.Reasoning.Effort != "" { + effort, err := ParseEffort(req.Reasoning.Effort) + if err != nil { + return Intent{}, err + } + intent.Effort = effort + intent.Mode = ModeEnabled + if effort == EffortNone { + intent.Mode = ModeDisabled + } + } + if req.Reasoning.Summary != "" { + include := true + intent.IncludeThoughts = &include + } + } + if req.ReasoningConversion == nil { + return normalizeIntent(intent) + } + pivot := Intent{ + Mode: Mode(req.ReasoningConversion.Mode), + Effort: Effort(req.ReasoningConversion.Effort), + BudgetTokens: req.ReasoningConversion.BudgetTokens, + IncludeThoughts: req.ReasoningConversion.IncludeThoughts, + Source: SourcePivot, + BudgetSource: SourcePivot, + } + if req.Reasoning != nil && req.Reasoning.Effort != "" { + projectedEffort := OpenAIEffort(EffectiveEffort(pivot)) + if Effort(req.Reasoning.Effort) == projectedEffort { + intent.Effort = "" + intent.Mode = ModeUnset + } + } + return MergeExplicit(intent, pivot, req.Model) +} + +func FromClaude(req *dto.ClaudeRequest) (Intent, error) { + if req == nil { + return Intent{}, nil + } + var intent Intent + intent.Source = SourceNative + if req.Thinking != nil { + switch req.Thinking.Type { + case "", "enabled": + intent.Mode = ModeEnabled + case "adaptive": + intent.Mode = ModeAdaptive + case "disabled": + intent.Mode = ModeDisabled + intent.Effort = EffortNone + default: + return Intent{}, fmt.Errorf("unsupported Claude thinking type %q", req.Thinking.Type) + } + intent.BudgetTokens = req.Thinking.BudgetTokens + if req.Thinking.BudgetTokens != nil { + budget := *req.Thinking.BudgetTokens + if budget < 1024 { + return Intent{}, fmt.Errorf("Claude thinking budget_tokens must be at least 1024, got %d", budget) + } + if req.MaxTokens != nil && uint(budget) >= *req.MaxTokens { + return Intent{}, fmt.Errorf("Claude thinking budget_tokens must be less than max_tokens") + } + intent.BudgetSource = SourceNative + } + switch req.Thinking.Display { + case "summarized": + include := true + intent.IncludeThoughts = &include + case "omitted": + include := false + intent.IncludeThoughts = &include + } + } + if len(req.OutputConfig) > 0 { + var output dto.OutputConfigForEffort + if err := kitutil.Unmarshal(req.OutputConfig, &output); err != nil { + return Intent{}, fmt.Errorf("invalid Claude output_config: %w", err) + } + if output.Effort != "" { + effort, err := ParseEffort(output.Effort) + if err != nil { + return Intent{}, err + } + intent.Effort = effort + } + } + if intent.Mode == ModeDisabled && intent.Effort != "" && intent.Effort != EffortNone { + return intent, nil + } + return normalizeIntent(intent) +} + +func FromGemini(req *dto.GeminiChatRequest) (Intent, error) { + if req == nil || req.GenerationConfig.ThinkingConfig == nil { + return Intent{}, nil + } + config := req.GenerationConfig.ThinkingConfig + if config.ThinkingBudget != nil && config.ThinkingLevel != "" { + return Intent{}, fmt.Errorf("%w: Gemini thinkingBudget and thinkingLevel cannot both be set", ErrEffortConflict) + } + intent := Intent{ + BudgetTokens: config.ThinkingBudget, + IncludeThoughts: config.IncludeThoughts, + Source: SourceNative, + BudgetSource: SourceNative, + } + if config.ThinkingLevel != "" { + effort, err := ParseEffort(config.ThinkingLevel) + if err != nil { + return Intent{}, err + } + intent.Effort = effort + intent.Mode = ModeEnabled + } + return normalizeIntent(intent) +} + +func EffectiveEffort(intent Intent) Effort { + intent, err := normalizeIntent(intent) + if err != nil { + return "" + } + if intent.Mode == ModeDisabled { + return EffortNone + } + if intent.Effort != "" { + return intent.Effort + } + if intent.BudgetTokens != nil { + return EffortFromBudget(*intent.BudgetTokens) + } + if intent.Mode == ModeEnabled || intent.Mode == ModeAdaptive { + return EffortHigh + } + return "" +} + +func EffortFromBudget(budget int) Effort { + if budget == 0 { + return EffortNone + } + if budget < 0 { + return EffortHigh + } + if budget <= 1024 { + return EffortLow + } + if budget <= 8192 { + return EffortMedium + } + return EffortHigh +} diff --git a/relaykit/relayconvert/reasoning/intent_test.go b/relaykit/relayconvert/reasoning/intent_test.go new file mode 100644 index 0000000000..cc7c85f41a --- /dev/null +++ b/relaykit/relayconvert/reasoning/intent_test.go @@ -0,0 +1,125 @@ +package reasoning + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMergeExplicitAndSuffix(t *testing.T) { + t.Parallel() + + budget1024 := 1024 + budget2048 := 2048 + + tests := []struct { + name string + explicit Intent + suffix Intent + wantErr bool + wantMode Mode + wantEffort Effort + wantBudget *int + wantThoughts *bool + }{ + { + name: "enabled plus matching effort merges", + explicit: Intent{Mode: ModeEnabled, Effort: EffortHigh}, + suffix: Intent{Mode: ModeEnabled, Effort: EffortHigh, Source: SourceSuffix}, + wantMode: ModeEnabled, + wantEffort: EffortHigh, + }, + { + name: "enabled vs disabled conflict", + explicit: Intent{Mode: ModeEnabled, Effort: EffortHigh}, + suffix: Intent{Mode: ModeDisabled, Effort: EffortNone, Source: SourceSuffix}, + wantErr: true, + }, + { + name: "different efforts conflict", + explicit: Intent{Mode: ModeEnabled, Effort: EffortLow}, + suffix: Intent{Mode: ModeEnabled, Effort: EffortHigh, Source: SourceSuffix}, + wantErr: true, + }, + { + name: "different budgets conflict", + explicit: Intent{BudgetTokens: &budget1024}, + suffix: Intent{BudgetTokens: &budget2048, Source: SourceSuffix, BudgetSource: SourceSuffix}, + wantErr: true, + }, + { + name: "effort versus exact suffix budget conflict", + explicit: Intent{Mode: ModeEnabled, Effort: EffortHigh}, + suffix: Intent{BudgetTokens: &budget1024, Source: SourceSuffix, BudgetSource: SourceSuffix}, + wantErr: true, + }, + { + name: "suffix only is adopted", + suffix: Intent{Mode: ModeEnabled, Effort: EffortMedium, Source: SourceSuffix}, + wantMode: ModeEnabled, + wantEffort: EffortMedium, + wantThoughts: nil, + }, + { + name: "explicit include thoughts overlays empty suffix strength", + explicit: Intent{IncludeThoughts: boolPtr(false)}, + suffix: Intent{Mode: ModeEnabled, Effort: EffortLow, Source: SourceSuffix}, + wantMode: ModeEnabled, + wantEffort: EffortLow, + wantThoughts: boolPtr(false), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := MergeExplicitAndSuffix(tt.explicit, tt.suffix, "claude-opus-4-8") + if tt.wantErr { + require.Error(t, err) + assert.ErrorIs(t, err, ErrEffortConflict) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantMode, got.Mode) + assert.Equal(t, tt.wantEffort, got.Effort) + if tt.wantBudget != nil { + require.NotNil(t, got.BudgetTokens) + assert.Equal(t, *tt.wantBudget, *got.BudgetTokens) + } + if tt.wantThoughts != nil { + require.NotNil(t, got.IncludeThoughts) + assert.Equal(t, *tt.wantThoughts, *got.IncludeThoughts) + } + }) + } +} + +func TestIntentStateRoundTrip(t *testing.T) { + t.Parallel() + + budget := 4096 + include := true + intent := Intent{ + Mode: ModeEnabled, + Effort: EffortHigh, + BudgetTokens: &budget, + IncludeThoughts: &include, + } + + state := StateFromIntent(intent) + require.NotNil(t, state) + got := IntentFromState(state) + assert.Equal(t, intent.Mode, got.Mode) + assert.Equal(t, intent.Effort, got.Effort) + require.NotNil(t, got.BudgetTokens) + assert.Equal(t, budget, *got.BudgetTokens) + require.NotNil(t, got.IncludeThoughts) + assert.True(t, *got.IncludeThoughts) + assert.True(t, IntentFromState(nil).IsEmpty()) + assert.Nil(t, StateFromIntent(Intent{})) +} + +func boolPtr(v bool) *bool { + return &v +} diff --git a/relaykit/relayconvert/reasoning/suffix.go b/relaykit/relayconvert/reasoning/suffix.go index 59140a7c8d..38b01bef7b 100644 --- a/relaykit/relayconvert/reasoning/suffix.go +++ b/relaykit/relayconvert/reasoning/suffix.go @@ -1,6 +1,8 @@ package reasoning import ( + "fmt" + "strconv" "strings" "github.com/samber/lo" @@ -8,15 +10,10 @@ import ( var EffortSuffixes = []string{"-max", "-xhigh", "-high", "-medium", "-low", "-minimal"} -var OpenAIEffortSuffixes = []string{"-high", "-minimal", "-low", "-medium", "-none", "-xhigh"} +var OpenAIEffortSuffixes = []string{"-max", "-xhigh", "-high", "-medium", "-low", "-minimal", "-none"} var DeepSeekV4EffortSuffixes = []string{"-none", "-max"} -// TrimEffortSuffix -> modelName level(low) exists -func TrimEffortSuffix(modelName string) (string, string, bool) { - return TrimEffortSuffixWithSuffixes(modelName, EffortSuffixes) -} - func TrimEffortSuffixWithSuffixes(modelName string, suffixes []string) (string, string, bool) { suffix, found := lo.Find(suffixes, func(s string) bool { return strings.HasSuffix(modelName, s) @@ -27,7 +24,13 @@ func TrimEffortSuffixWithSuffixes(modelName string, suffixes []string) (string, return strings.TrimSuffix(modelName, suffix), strings.TrimPrefix(suffix, "-"), true } -func ParseOpenAIReasoningEffortFromModelSuffix(modelName string) (string, string) { +// ParseOpenAIReasoningEffortFromModelSuffix extracts an OpenAI effort tail +// such as -high or -none. preserveEffortTail, when non-nil, keeps real model +// IDs whose names already end in those tokens (for example qwen-max). +func ParseOpenAIReasoningEffortFromModelSuffix(modelName string, preserveEffortTail func(string) bool) (string, string) { + if preserveEffortTail != nil && preserveEffortTail(modelName) { + return "", modelName + } baseModel, effort, ok := TrimEffortSuffixWithSuffixes(modelName, OpenAIEffortSuffixes) if !ok { return "", modelName @@ -35,6 +38,138 @@ func ParseOpenAIReasoningEffortFromModelSuffix(modelName string) (string, string return effort, baseModel } +func ParseClaudeModelSuffix(modelName string, allowThinkingAlias bool) (string, Intent, bool, error) { + if !strings.HasPrefix(modelName, "claude-") { + return modelName, Intent{}, false, nil + } + if allowThinkingAlias && hasLegacyThinkingAlias(modelName) { + return parseProviderModelSuffix(modelName, "claude-", true, true) + } + if !isKnownClaudeModel(modelName) { + return modelName, Intent{}, false, nil + } + return parseProviderModelSuffix(modelName, "claude-", allowThinkingAlias, true) +} + +func hasLegacyThinkingAlias(modelName string) bool { + return strings.HasSuffix(modelName, "-thinking") || + strings.HasSuffix(modelName, "-nothinking") || + strings.LastIndex(modelName, "-thinking-") >= 0 +} + +func isKnownClaudeModel(modelName string) bool { + baseModel, _, _ := TrimEffortSuffixWithSuffixes(modelName, []string{"-max", "-xhigh", "-high", "-medium", "-low", "-minimal", "-none"}) + if marker := strings.LastIndex(baseModel, "-thinking-"); marker >= 0 { + baseModel = baseModel[:marker] + } else { + baseModel = strings.TrimSuffix(strings.TrimSuffix(baseModel, "-thinking"), "-nothinking") + } + knownPrefixes := []string{ + "claude-fable-5", "claude-mythos-5", "claude-mythos-preview", + "claude-opus-5", "claude-sonnet-5", "claude-opus-4-8", + "claude-opus-4-7", "claude-opus-4-6", "claude-sonnet-4-6", + "claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5", + "claude-opus-4-1", "claude-opus-4-", "claude-sonnet-4-", + "claude-3-7-sonnet", + } + for _, prefix := range knownPrefixes { + if strings.HasPrefix(baseModel, prefix) { + return true + } + } + return false +} + +func ParseGeminiModelSuffix(modelName string, allowThinkingAlias bool) (string, Intent, bool, error) { + if !strings.HasPrefix(modelName, "gemini-") { + return modelName, Intent{}, false, nil + } + if !isKnownGeminiModel(modelName) { + return modelName, Intent{}, false, nil + } + return parseProviderModelSuffix(modelName, "gemini-", allowThinkingAlias, true) +} + +// ParseKnownProviderModelSuffix extracts a canonical intent only when the +// origin identifies a provider family whose suffix vocabulary is defined by +// relaykit. Unknown OpenAI-compatible model names are deliberately untouched. +func ParseKnownProviderModelSuffix(modelName string, allowThinkingAlias bool) (string, Intent, bool, error) { + if strings.HasPrefix(modelName, "claude-") { + return ParseClaudeModelSuffix(modelName, allowThinkingAlias) + } + if strings.HasPrefix(modelName, "gemini-") { + return ParseGeminiModelSuffix(modelName, allowThinkingAlias) + } + return modelName, Intent{}, false, nil +} + +func isKnownGeminiModel(modelName string) bool { + baseModel, _, _ := TrimEffortSuffixWithSuffixes(modelName, []string{"-max", "-xhigh", "-high", "-medium", "-low", "-minimal", "-none"}) + if marker := strings.LastIndex(baseModel, "-thinking-"); marker >= 0 { + baseModel = baseModel[:marker] + } else { + baseModel = strings.TrimSuffix(strings.TrimSuffix(baseModel, "-thinking"), "-nothinking") + } + return geminiCapabilitiesFor(baseModel).kind != geminiThinkingUnknown +} + +func TrimGeminiThinkingSuffix(modelName string) (string, bool) { + baseModel, _, ok, err := ParseGeminiModelSuffix(modelName, true) + return baseModel, ok && err == nil +} + +func parseProviderModelSuffix(modelName string, requiredPrefix string, allowThinkingAlias bool, includeThoughts bool) (string, Intent, bool, error) { + if allowThinkingAlias { + if marker := strings.LastIndex(modelName, "-thinking-"); marker >= 0 { + baseModel := modelName[:marker] + if !strings.HasPrefix(baseModel, requiredPrefix) { + return modelName, Intent{}, false, nil + } + budget, err := strconv.Atoi(modelName[marker+len("-thinking-"):]) + if err != nil { + return modelName, Intent{}, false, fmt.Errorf("invalid thinking budget suffix on model %q: %w", modelName, err) + } + intent := Intent{BudgetTokens: &budget, Source: SourceSuffix, BudgetSource: SourceSuffix} + if includeThoughts { + value := true + intent.IncludeThoughts = &value + } + return baseModel, intent, true, nil + } + if strings.HasSuffix(modelName, "-nothinking") { + baseModel := strings.TrimSuffix(modelName, "-nothinking") + return baseModel, Intent{Mode: ModeDisabled, Effort: EffortNone, Source: SourceSuffix}, true, nil + } + if strings.HasSuffix(modelName, "-thinking") { + baseModel := strings.TrimSuffix(modelName, "-thinking") + intent := Intent{Mode: ModeEnabled, Source: SourceSuffix} + if includeThoughts { + value := true + intent.IncludeThoughts = &value + } + return baseModel, intent, true, nil + } + } + + suffixes := []string{"-max", "-xhigh", "-high", "-medium", "-low", "-minimal", "-none"} + baseModel, rawEffort, ok := TrimEffortSuffixWithSuffixes(modelName, suffixes) + if !ok || !strings.HasPrefix(baseModel, requiredPrefix) { + return modelName, Intent{}, false, nil + } + effort, err := ParseEffort(rawEffort) + if err != nil { + return modelName, Intent{}, false, err + } + intent := Intent{Effort: effort, Mode: ModeEnabled, Source: SourceSuffix} + if effort == EffortNone { + intent.Mode = ModeDisabled + } else if includeThoughts { + value := true + intent.IncludeThoughts = &value + } + return baseModel, intent, true, nil +} + func ParseDeepSeekV4ThinkingSuffix(modelName string) (baseModel string, thinkingType string, effort string, ok bool) { baseModel, suffix, ok := TrimEffortSuffixWithSuffixes(modelName, DeepSeekV4EffortSuffixes) if !ok || !strings.HasPrefix(baseModel, "deepseek-v4-") { diff --git a/relaykit/relayconvert/reasoning/suffix_test.go b/relaykit/relayconvert/reasoning/suffix_test.go new file mode 100644 index 0000000000..414016fa58 --- /dev/null +++ b/relaykit/relayconvert/reasoning/suffix_test.go @@ -0,0 +1,140 @@ +package reasoning + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseGeminiModelSuffixNoThinkingDisablesReasoning(t *testing.T) { + t.Parallel() + + base, intent, found, err := ParseGeminiModelSuffix("gemini-2.5-flash-nothinking", true) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, "gemini-2.5-flash", base) + assert.Equal(t, ModeDisabled, intent.Mode) + assert.Equal(t, EffortNone, intent.Effort) + assert.Equal(t, SourceSuffix, intent.Source) +} + +func TestParseKnownProviderModelSuffix(t *testing.T) { + t.Parallel() + + preserveQwenMax := func(name string) bool { return name == "qwen-max" || name == "vendor/qwen-max" } + + tests := []struct { + name string + model string + allowThinkingAlias bool + wantBase string + wantFound bool + wantMode Mode + wantEffort Effort + wantBudget *int + wantErr bool + }{ + { + name: "claude thinking alias", + model: "claude-3-7-sonnet-thinking", + allowThinkingAlias: true, + wantBase: "claude-3-7-sonnet", + wantFound: true, + wantMode: ModeEnabled, + }, + { + name: "claude nothinking alias", + model: "claude-3-7-sonnet-nothinking", + allowThinkingAlias: true, + wantBase: "claude-3-7-sonnet", + wantFound: true, + wantMode: ModeDisabled, + wantEffort: EffortNone, + }, + { + name: "claude thinking budget", + model: "claude-3-7-sonnet-thinking-8192", + allowThinkingAlias: true, + wantBase: "claude-3-7-sonnet", + wantFound: true, + wantBudget: intPtr(8192), + }, + { + name: "claude effort tail", + model: "claude-opus-4-8-high", + allowThinkingAlias: true, + wantBase: "claude-opus-4-8", + wantFound: true, + wantMode: ModeEnabled, + wantEffort: EffortHigh, + }, + { + name: "gemini thinking alias", + model: "gemini-2.5-flash-thinking", + allowThinkingAlias: true, + wantBase: "gemini-2.5-flash", + wantFound: true, + wantMode: ModeEnabled, + }, + { + name: "malformed thinking budget", + model: "claude-3-7-sonnet-thinking-abc", + allowThinkingAlias: true, + wantErr: true, + }, + { + name: "unknown openai-compatible name is untouched", + model: "gpt-4o-mini", + wantBase: "gpt-4o-mini", + wantFound: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + base, intent, found, err := ParseKnownProviderModelSuffix(tt.model, tt.allowThinkingAlias) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantFound, found) + assert.Equal(t, tt.wantBase, base) + assert.Equal(t, tt.wantMode, intent.Mode) + assert.Equal(t, tt.wantEffort, intent.Effort) + if tt.wantBudget != nil { + require.NotNil(t, intent.BudgetTokens) + assert.Equal(t, *tt.wantBudget, *intent.BudgetTokens) + } else { + assert.Nil(t, intent.BudgetTokens) + } + }) + } + + t.Run("openai effort tail", func(t *testing.T) { + t.Parallel() + effort, base := ParseOpenAIReasoningEffortFromModelSuffix("gpt-5.6-sol-high", nil) + assert.Equal(t, "high", effort) + assert.Equal(t, "gpt-5.6-sol", base) + }) + + t.Run("preserve effort tail on real model id", func(t *testing.T) { + t.Parallel() + effort, base := ParseOpenAIReasoningEffortFromModelSuffix("qwen-max", preserveQwenMax) + assert.Empty(t, effort) + assert.Equal(t, "qwen-max", base) + }) + + t.Run("preserve effort tail with vendor prefix", func(t *testing.T) { + t.Parallel() + effort, base := ParseOpenAIReasoningEffortFromModelSuffix("vendor/qwen-max", preserveQwenMax) + assert.Empty(t, effort) + assert.Equal(t, "vendor/qwen-max", base) + }) +} + +func intPtr(v int) *int { + return &v +} diff --git a/relaykit/relayconvert/request_compat.go b/relaykit/relayconvert/request_compat.go index fdacf68a01..90b93457ec 100644 --- a/relaykit/relayconvert/request_compat.go +++ b/relaykit/relayconvert/request_compat.go @@ -2,47 +2,64 @@ package relayconvert import ( "context" + "fmt" + "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" - claudemessages "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/claude_messages" - geminichat "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/gemini_chat" - oaichat "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/oai_chat" - oairesponses "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/oai_responses" + sharedclaude "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/claude" sharedgemini "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/gemini" + "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" + "github.com/QuantumNous/new-api/relaykit/types" ) func ClaudeMessagesRequestToOpenAIChat(claudeRequest dto.ClaudeRequest, info convmeta.Meta) (*dto.GeneralOpenAIRequest, error) { - return claudemessages.ClaudeMessagesRequestToOpenAIChat(claudeRequest, info) + return convertCompatRequest[dto.GeneralOpenAIRequest](context.Background(), info, types.RelayFormatOpenAI, &claudeRequest) } func OpenAIChatRequestToClaudeMessages(c context.Context, info convmeta.Meta, textRequest dto.GeneralOpenAIRequest) (*dto.ClaudeRequest, error) { - return oaichat.OpenAIChatRequestToClaudeMessages(c, info, textRequest) + return convertCompatRequest[dto.ClaudeRequest](c, info, types.RelayFormatClaude, &textRequest) } func GeminiGenerateContentRequestToOpenAIChat(geminiRequest *dto.GeminiChatRequest, info convmeta.Meta) (*dto.GeneralOpenAIRequest, error) { - return geminichat.GeminiGenerateContentRequestToOpenAIChat(geminiRequest, info) + return convertCompatRequest[dto.GeneralOpenAIRequest](context.Background(), info, types.RelayFormatOpenAI, geminiRequest) } func OpenAIChatRequestToGeminiGenerateContent(c context.Context, textRequest dto.GeneralOpenAIRequest, info convmeta.Meta) (*dto.GeminiChatRequest, error) { - return oaichat.OpenAIChatRequestToGeminiGenerateContent(c, textRequest, info) + return convertCompatRequest[dto.GeminiChatRequest](c, info, types.RelayFormatGemini, &textRequest) } -func ApplyGeminiThinkingConfig(geminiRequest *dto.GeminiChatRequest, info convmeta.Meta, oaiRequest ...dto.GeneralOpenAIRequest) { - sharedgemini.ApplyThinkingConfig(geminiRequest, info, oaiRequest...) +func ApplyGeminiThinkingConfigChecked(geminiRequest *dto.GeminiChatRequest, info convmeta.Meta, oaiRequest ...dto.GeneralOpenAIRequest) error { + return reasoning.AsClientError(sharedgemini.ApplyThinkingConfig(geminiRequest, info, oaiRequest...)) +} + +func ApplyClaudeThinkingModel(claudeRequest *dto.ClaudeRequest, info convmeta.Meta) error { + return reasoning.AsClientError(sharedclaude.ApplyReasoning(claudeRequest, info, reasoning.Intent{})) } func ChatCompletionsRequestToResponsesRequest(req *dto.GeneralOpenAIRequest) (*dto.OpenAIResponsesRequest, error) { - return oaichat.ChatCompletionsRequestToResponsesRequest(req) + return convertCompatRequest[dto.OpenAIResponsesRequest](context.Background(), nil, types.RelayFormatOpenAIResponses, req) } func ResponsesRequestToChatCompletionsRequest(req *dto.OpenAIResponsesRequest) (*dto.GeneralOpenAIRequest, error) { - return oairesponses.ResponsesRequestToChatCompletionsRequest(req) + return convertCompatRequest[dto.GeneralOpenAIRequest](context.Background(), nil, types.RelayFormatOpenAI, req) } func OpenAIResponsesRequestToClaudeMessages(c context.Context, info convmeta.Meta, req *dto.OpenAIResponsesRequest) (*dto.ClaudeRequest, error) { - return oairesponses.OpenAIResponsesRequestToClaudeMessages(c, info, req) + return convertCompatRequest[dto.ClaudeRequest](c, info, types.RelayFormatClaude, req) } func OpenAIResponsesRequestToGeminiChat(c context.Context, req *dto.OpenAIResponsesRequest, info convmeta.Meta) (*dto.GeminiChatRequest, error) { - return oairesponses.OpenAIResponsesRequestToGeminiChat(c, req, info) + return convertCompatRequest[dto.GeminiChatRequest](c, info, types.RelayFormatGemini, req) +} + +func convertCompatRequest[T any](c context.Context, info convmeta.Meta, target types.RelayFormat, request any) (*T, error) { + result, err := ConvertRequest(c, info, target, request) + if err != nil { + return nil, err + } + converted, ok := result.Value.(*T) + if !ok { + return nil, fmt.Errorf("expected %s request, got %T", target, result.Value) + } + return converted, nil } diff --git a/relaykit/relayconvert/request_registry.go b/relaykit/relayconvert/request_registry.go index e55d6c30c5..6904a820e4 100644 --- a/relaykit/relayconvert/request_registry.go +++ b/relaykit/relayconvert/request_registry.go @@ -14,6 +14,7 @@ import ( geminichat "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/gemini_chat" oaichat "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/oai_chat" oairesponses "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/oai_responses" + "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/toolconv" "github.com/QuantumNous/new-api/relaykit/types" ) @@ -34,12 +35,13 @@ type RequestStep struct { } type RequestResult struct { - Value any - From types.RelayFormat - To types.RelayFormat - Converter string - Quality RequestConverterQuality - Steps []RequestStep + Value any + From types.RelayFormat + To types.RelayFormat + Converter string + Quality RequestConverterQuality + Steps []RequestStep + Diagnostics []types.ConversionDiagnostic } type RequestConverterSpec struct { @@ -68,18 +70,19 @@ const ( requestConverterClaudeToResponses = "claude_messages_to_openai_responses" requestConverterGeminiToClaude = "gemini_generate_content_to_claude_messages" requestConverterGeminiToResponses = "gemini_generate_content_to_openai_responses" - requestConverterResponsesToClaude = "openai_responses_to_claude_messages" + requestConverterResponsesToClaude = ConverterOpenAIResponsesToClaudeMessages ) const ( - ConverterNone = "none" - ConverterClaudeMessagesToOpenAIChat = "anthropic_messages_to_openai_chat_completions" - ConverterOpenAIChatToClaudeMessages = "openai_chat_completions_to_anthropic_messages" - ConverterOpenAIChatToOpenAIResponses = "openai_chat_completions_to_openai_responses" - ConverterOpenAIResponsesToOpenAIChat = "openai_responses_to_openai_chat_completions" - ConverterOpenAIResponsesToGemini = "openai_responses_to_gemini_generate_content" - ConverterGeminiContentToOpenAIChat = "gemini_generate_content_to_openai_chat_completions" - ConverterOpenAIChatToGeminiContent = "openai_chat_completions_to_gemini_generate_content" + ConverterNone = "none" + ConverterClaudeMessagesToOpenAIChat = "anthropic_messages_to_openai_chat_completions" + ConverterOpenAIChatToClaudeMessages = "openai_chat_completions_to_anthropic_messages" + ConverterOpenAIChatToOpenAIResponses = "openai_chat_completions_to_openai_responses" + ConverterOpenAIResponsesToOpenAIChat = "openai_responses_to_openai_chat_completions" + ConverterOpenAIResponsesToClaudeMessages = "openai_responses_to_claude_messages" + ConverterOpenAIResponsesToGemini = "openai_responses_to_gemini_generate_content" + ConverterGeminiContentToOpenAIChat = "gemini_generate_content_to_openai_chat_completions" + ConverterOpenAIChatToGeminiContent = "openai_chat_completions_to_gemini_generate_content" ) func registerBuiltinRequestConverter(spec RequestConverterSpec) { @@ -236,10 +239,12 @@ func executeRequestSpec(c context.Context, info convmeta.Meta, from types.RelayF } func executeRequestSteps(c context.Context, info convmeta.Meta, from types.RelayFormat, target types.RelayFormat, request any, converter string, quality RequestConverterQuality, specs []RequestConverterSpec) (*RequestResult, error) { - current := request + current, tools, err := toolconv.ExtractRequest(from, request) + if err != nil { + return nil, err + } steps := make([]RequestStep, 0, len(specs)) for _, spec := range specs { - var err error current, err = prepareRequestForStep(current, spec, target) if err != nil { return nil, err @@ -253,6 +258,23 @@ func executeRequestSteps(c context.Context, info convmeta.Meta, from types.Relay steps = append(steps, step) } + current, diagnostics, err := toolconv.AttachRequest(target, current, tools, convmeta.OptionsOf(info)) + if err != nil { + return &RequestResult{ + Value: current, + From: from, + To: target, + Quality: quality, + Steps: steps, + Diagnostics: diagnostics, + }, err + } + if info != nil { + for _, step := range steps { + info.AppendRequestConversion(step.To) + } + } + converters := make([]string, 0, len(steps)) for _, step := range steps { converters = append(converters, step.Converter) @@ -261,12 +283,13 @@ func executeRequestSteps(c context.Context, info convmeta.Meta, from types.Relay converter = strings.Join(converters, ",") } return &RequestResult{ - Value: current, - From: from, - To: target, - Converter: converter, - Quality: quality, - Steps: steps, + Value: current, + From: from, + To: target, + Converter: converter, + Quality: quality, + Steps: steps, + Diagnostics: diagnostics, }, nil } @@ -312,9 +335,6 @@ func executeRequestStep(c context.Context, info convmeta.Meta, spec RequestConve if err != nil { return nil, RequestStep{}, err } - if info != nil { - info.AppendRequestConversion(spec.To) - } return value, RequestStep{ Converter: spec.ID, From: spec.From, @@ -425,6 +445,19 @@ func convertClaudeRequestToOpenAI(_ context.Context, info convmeta.Meta, request return claudemessages.ClaudeMessagesRequestToOpenAIChat(*claudeRequest, info) } +func convertClaudeRequestToOpenAIResponses(_ context.Context, info convmeta.Meta, request any) (any, error) { + claudeRequest, ok := request.(*dto.ClaudeRequest) + if !ok { + if value, ok := request.(dto.ClaudeRequest); ok { + claudeRequest = &value + } + } + if claudeRequest == nil { + return nil, fmt.Errorf("expected Anthropic Messages request, got %T", request) + } + return claudemessages.ClaudeMessagesRequestToOpenAIResponses(*claudeRequest, info) +} + func convertOpenAIRequestToClaude(c context.Context, info convmeta.Meta, request any) (any, error) { openAIRequest, ok := request.(*dto.GeneralOpenAIRequest) if !ok { diff --git a/relaykit/relayconvert/request_registry_test.go b/relaykit/relayconvert/request_registry_test.go index 5649ed09cd..4ca42f4a82 100644 --- a/relaykit/relayconvert/request_registry_test.go +++ b/relaykit/relayconvert/request_registry_test.go @@ -42,10 +42,6 @@ func TestRequestConverterRegistryListsSupportedTextConverters(t *testing.T) { from: types.RelayFormatClaude, to: types.RelayFormatOpenAIResponses, quality: RequestConverterQualityFair, - stepConverters: []string{ - ConverterClaudeMessagesToOpenAIChat, - ConverterOpenAIChatToOpenAIResponses, - }, }, { converter: requestConverterGeminiToClaude, @@ -133,7 +129,7 @@ func TestConvertRequestToTargetRecordsConversionChain(t *testing.T) { assert.Equal(t, []types.RelayFormat{types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses}, info.ConversionChain) } -func TestConvertRequestPlansMultiHopPath(t *testing.T) { +func TestConvertRequestClaudeToResponsesUsesDirectPath(t *testing.T) { info := &convmeta.Values{ ConversionChain: []types.RelayFormat{types.RelayFormatClaude}, } @@ -154,17 +150,131 @@ func TestConvertRequestPlansMultiHopPath(t *testing.T) { assert.Equal(t, RequestConverterQualityFair, result.Quality) assert.Equal(t, []RequestStep{ { - Converter: ConverterClaudeMessagesToOpenAIChat, + Converter: requestConverterClaudeToResponses, From: types.RelayFormatClaude, - To: types.RelayFormatOpenAI, - }, - { - Converter: ConverterOpenAIChatToOpenAIResponses, - From: types.RelayFormatOpenAI, To: types.RelayFormatOpenAIResponses, }, }, result.Steps) - assert.Equal(t, []types.RelayFormat{types.RelayFormatClaude, types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses}, info.ConversionChain) + assert.Equal(t, []types.RelayFormat{types.RelayFormatClaude, types.RelayFormatOpenAIResponses}, info.ConversionChain) +} + +func TestConvertRequestClaudeToResponsesPreservesMixedBlockOrder(t *testing.T) { + info := &convmeta.Values{ConversionChain: []types.RelayFormat{types.RelayFormatClaude}} + stream := true + strict := true + maxTokens := uint(4096) + req := &dto.ClaudeRequest{ + Model: "gpt-test", + System: []dto.ClaudeMediaMessage{{Type: "text", Text: kitutil.GetPointer("system ")}, {Type: "text", Text: kitutil.GetPointer("rules")}}, + MaxTokens: &maxTokens, + Stream: &stream, + Tools: []dto.Tool{{ + Name: "lookup", + Description: "Look up a value", + InputSchema: map[string]any{"type": "object", "properties": map[string]any{"q": map[string]any{"type": "string"}}}, + Strict: &strict, + }}, + ToolChoice: dto.ClaudeToolChoice{Type: "tool", Name: "lookup", DisableParallelToolUse: true}, + Messages: []dto.ClaudeMessage{ + {Role: "user", Content: []dto.ClaudeMediaMessage{{Type: "text", Text: kitutil.GetPointer("question")}}}, + {Role: "assistant", Content: []dto.ClaudeMediaMessage{ + {Type: "text", Text: kitutil.GetPointer("before")}, + {Type: "tool_use", Id: "call_1", Name: "lookup", Input: map[string]any{"q": "x"}}, + {Type: "text", Text: kitutil.GetPointer("after")}, + }}, + {Role: "user", Content: []dto.ClaudeMediaMessage{ + {Type: "tool_result", ToolUseId: "call_1", Content: "result"}, + {Type: "text", Text: kitutil.GetPointer("continue")}, + }}, + }, + } + + result, err := ConvertRequest(nil, info, types.RelayFormatOpenAIResponses, req) + require.NoError(t, err) + responsesReq := result.Value.(*dto.OpenAIResponsesRequest) + assert.Equal(t, "gpt-test", responsesReq.Model) + assert.Equal(t, maxTokens, *responsesReq.MaxOutputTokens) + assert.True(t, *responsesReq.Stream) + assert.JSONEq(t, `"system rules"`, string(responsesReq.Instructions)) + assert.JSONEq(t, `[{"type":"function","name":"lookup","description":"Look up a value","parameters":{"type":"object","properties":{"q":{"type":"string"}}},"strict":true}]`, string(responsesReq.Tools)) + assert.JSONEq(t, `{"type":"function","name":"lookup"}`, string(responsesReq.ToolChoice)) + assert.JSONEq(t, `false`, string(responsesReq.ParallelToolCalls)) + + var input []map[string]any + require.NoError(t, kitutil.Unmarshal(responsesReq.Input, &input)) + require.Len(t, input, 6) + assert.Equal(t, "user", input[0]["role"]) + assert.Equal(t, "question", inputContentText(t, input[0])) + assert.Equal(t, "assistant", input[1]["role"]) + assert.Equal(t, "before", inputContentText(t, input[1])) + assert.Equal(t, "function_call", input[2]["type"]) + assert.Equal(t, "call_1", input[2]["call_id"]) + assert.Equal(t, "lookup", input[2]["name"]) + assert.JSONEq(t, `{"q":"x"}`, input[2]["arguments"].(string)) + assert.Equal(t, "assistant", input[3]["role"]) + assert.Equal(t, "after", inputContentText(t, input[3])) + assert.Equal(t, "function_call_output", input[4]["type"]) + assert.Equal(t, "result", input[4]["output"]) + assert.Equal(t, "user", input[5]["role"]) + assert.Equal(t, "continue", inputContentText(t, input[5])) +} + +func TestConvertRequestClaudeToResponsesDropsIncompatibleContextManagement(t *testing.T) { + req := &dto.ClaudeRequest{ + Model: "gpt-test", + Messages: []dto.ClaudeMessage{ + {Role: "user", Content: "hello"}, + }, + ContextManagement: mustRawMessage(t, map[string]any{ + "edits": []map[string]any{{"type": "clear_tool_uses_20250919"}}, + }), + } + + result, err := ConvertRequest(nil, nil, types.RelayFormatOpenAIResponses, req) + + require.NoError(t, err) + responsesReq, ok := result.Value.(*dto.OpenAIResponsesRequest) + require.True(t, ok) + assert.Empty(t, responsesReq.ContextManagement) +} + +func TestConvertRequestClaudeAdaptiveThinkingPreservesEffort(t *testing.T) { + tests := []struct { + name string + outputConfig []byte + wantEffort string + }{ + {name: "adaptive default", wantEffort: "high"}, + {name: "explicit low", outputConfig: mustRawMessage(t, map[string]any{"effort": "low"}), wantEffort: "low"}, + {name: "explicit xhigh", outputConfig: mustRawMessage(t, map[string]any{"effort": "xhigh"}), wantEffort: "xhigh"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info := &convmeta.Values{ + OriginModelName: "gpt-5.6-sol", + ConversionChain: []types.RelayFormat{types.RelayFormatClaude}, + } + req := &dto.ClaudeRequest{ + Model: "gpt-5.6-sol", + OutputConfig: tt.outputConfig, + Thinking: &dto.Thinking{Type: "adaptive", Display: "summarized"}, + Messages: []dto.ClaudeMessage{ + {Role: "user", Content: "hello"}, + }, + } + + result, err := ConvertRequest(nil, info, types.RelayFormatOpenAIResponses, req) + + require.NoError(t, err) + responsesReq, ok := result.Value.(*dto.OpenAIResponsesRequest) + require.True(t, ok) + require.NotNil(t, responsesReq.Reasoning) + assert.Equal(t, tt.wantEffort, responsesReq.Reasoning.Effort) + assert.Equal(t, "detailed", responsesReq.Reasoning.Summary) + assert.Equal(t, tt.wantEffort, info.GetReasoningEffort()) + }) + } } func TestConvertRequestViaExecutesExplicitPath(t *testing.T) { @@ -466,115 +576,6 @@ func TestConvertRequestOpenAIChatToGeminiAddsThoughtSignatureForAdvancedCustom(t assert.Equal(t, sharedgemini.ThoughtSignatureBypassValue, thoughtSignature) } -func TestConvertRequestResponsesToClaudeUsesDirectConverter(t *testing.T) { - info := &convmeta.Values{ - ConversionChain: []types.RelayFormat{types.RelayFormatOpenAIResponses}, - } - stream := true - parallelToolCalls := false - maxOutputTokens := uint(512) - req := &dto.OpenAIResponsesRequest{ - Model: "claude-test", - Instructions: mustRawMessage(t, "system rules"), - Stream: &stream, - MaxOutputTokens: &maxOutputTokens, - ParallelToolCalls: mustRawMessage(t, parallelToolCalls), - Reasoning: &dto.Reasoning{Effort: "medium"}, - Input: mustRawMessage(t, []map[string]any{ - { - "role": "user", - "content": "question", - }, - { - "role": "assistant", - "content": []map[string]any{ - {"type": "output_text", "text": "I will call."}, - }, - }, - { - "type": "function_call", - "call_id": "call_1", - "name": "lookup", - "arguments": map[string]any{"q": "x"}, - }, - { - "type": "function_call_output", - "call_id": "call_1", - "output": map[string]any{"ok": true}, - }, - }), - Tools: mustRawMessage(t, []map[string]any{ - { - "type": "function", - "name": "lookup", - "description": "Lookup data", - "parameters": map[string]any{ - "type": "object", - "properties": map[string]any{ - "q": map[string]any{"type": "string"}, - }, - }, - }, - }), - } - - result, err := ConvertRequest(nil, info, types.RelayFormatClaude, req) - - require.NoError(t, err) - claudeReq, ok := result.Value.(*dto.ClaudeRequest) - require.True(t, ok) - assert.Equal(t, requestConverterResponsesToClaude, result.Converter) - assert.Equal(t, []RequestStep{ - { - Converter: requestConverterResponsesToClaude, - From: types.RelayFormatOpenAIResponses, - To: types.RelayFormatClaude, - }, - }, result.Steps) - assert.Equal(t, []types.RelayFormat{types.RelayFormatOpenAIResponses, types.RelayFormatClaude}, info.ConversionChain) - - system, err := kitutil.Any2Type[[]dto.ClaudeMediaMessage](claudeReq.System) - require.NoError(t, err) - require.Len(t, system, 1) - assert.Equal(t, "system rules", system[0].GetText()) - require.NotNil(t, claudeReq.Stream) - assert.True(t, *claudeReq.Stream) - assert.Equal(t, maxOutputTokens, *claudeReq.MaxTokens) - require.NotNil(t, claudeReq.Thinking) - assert.Equal(t, "enabled", claudeReq.Thinking.Type) - assert.Equal(t, 2048, claudeReq.Thinking.GetBudgetTokens()) - - tools, err := kitutil.Any2Type[[]*dto.Tool](claudeReq.Tools) - require.NoError(t, err) - require.Len(t, tools, 1) - assert.Equal(t, "lookup", tools[0].Name) - - require.Len(t, claudeReq.Messages, 3) - assert.Equal(t, "user", claudeReq.Messages[0].Role) - userParts, err := claudeReq.Messages[0].ParseContent() - require.NoError(t, err) - require.Len(t, userParts, 1) - assert.Equal(t, "question", userParts[0].GetText()) - - assert.Equal(t, "assistant", claudeReq.Messages[1].Role) - assistantParts, err := claudeReq.Messages[1].ParseContent() - require.NoError(t, err) - require.Len(t, assistantParts, 2) - assert.Equal(t, "I will call.", assistantParts[0].GetText()) - assert.Equal(t, "tool_use", assistantParts[1].Type) - assert.Equal(t, "call_1", assistantParts[1].Id) - assert.Equal(t, "lookup", assistantParts[1].Name) - assert.Equal(t, map[string]any{"q": "x"}, assistantParts[1].Input) - - assert.Equal(t, "user", claudeReq.Messages[2].Role) - toolResultParts, err := claudeReq.Messages[2].ParseContent() - require.NoError(t, err) - require.Len(t, toolResultParts, 1) - assert.Equal(t, "tool_result", toolResultParts[0].Type) - assert.Equal(t, "call_1", toolResultParts[0].ToolUseId) - assert.Equal(t, map[string]any{"ok": true}, toolResultParts[0].Content) -} - func TestConvertRequestViaResponsesToGeminiStillUsesDirectSteps(t *testing.T) { info := &convmeta.Values{ ConversionChain: []types.RelayFormat{types.RelayFormatOpenAIResponses}, @@ -629,7 +630,7 @@ func TestConvertRequestByIDDeduplicatesConversionChain(t *testing.T) { assert.Equal(t, []types.RelayFormat{types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses}, info.ConversionChain) } -func TestConvertRequestByIDExecutesMultiHopConverter(t *testing.T) { +func TestConvertRequestByIDExecutesDirectClaudeToResponsesConverter(t *testing.T) { info := &convmeta.Values{ ConversionChain: []types.RelayFormat{types.RelayFormatClaude}, } @@ -648,17 +649,12 @@ func TestConvertRequestByIDExecutesMultiHopConverter(t *testing.T) { assert.Equal(t, RequestConverterQualityFair, result.Quality) assert.Equal(t, []RequestStep{ { - Converter: ConverterClaudeMessagesToOpenAIChat, + Converter: requestConverterClaudeToResponses, From: types.RelayFormatClaude, - To: types.RelayFormatOpenAI, - }, - { - Converter: ConverterOpenAIChatToOpenAIResponses, - From: types.RelayFormatOpenAI, To: types.RelayFormatOpenAIResponses, }, }, result.Steps) - assert.Equal(t, []types.RelayFormat{types.RelayFormatClaude, types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses}, info.ConversionChain) + assert.Equal(t, []types.RelayFormat{types.RelayFormatClaude, types.RelayFormatOpenAIResponses}, info.ConversionChain) } func TestConvertRequestRejectsUnsupportedConverterAndNilRequest(t *testing.T) { @@ -701,3 +697,15 @@ func mustRawMessage(t *testing.T, value any) []byte { require.NoError(t, err) return raw } + +func inputContentText(t *testing.T, item map[string]any) string { + t.Helper() + content, ok := item["content"].([]any) + require.True(t, ok) + require.Len(t, content, 1) + part, ok := content[0].(map[string]any) + require.True(t, ok) + text, ok := part["text"].(string) + require.True(t, ok) + return text +} diff --git a/relaykit/relayconvert/response_compat.go b/relaykit/relayconvert/response_compat.go index 68c57cd9d1..35b0c5fe6b 100644 --- a/relaykit/relayconvert/response_compat.go +++ b/relaykit/relayconvert/response_compat.go @@ -1,6 +1,8 @@ package relayconvert import ( + "fmt" + "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" claudemessages "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/claude_messages" @@ -10,12 +12,25 @@ import ( ) type ClaudeResponseInfo = claudemessages.ClaudeResponseInfo +type ClaudeToChatStreamState = claudemessages.ClaudeToChatStreamState type ChatToResponsesStreamEvent = oaichat.ChatToResponsesStreamEvent type ChatToResponsesStreamState = oaichat.ChatToResponsesStreamState type ResponsesToChatStreamState = oairesponses.ResponsesToChatStreamState type ResponsesBufferedAccumulator = oairesponses.ResponsesBufferedAccumulator +// ClaudeHostedStreamBridge owns Anthropic server-tool input deltas while a +// Claude stream is being converted to the Responses protocol. +type ClaudeHostedStreamBridge struct { + bridge *claudemessages.ClaudeHostedStreamBridge +} + +// GeminiHostedStreamBridge accumulates Gemini grounding queries until the +// upstream stream ends, then emits one canonical Responses web-search call. +type GeminiHostedStreamBridge struct { + bridge *geminichat.GeminiHostedStreamBridge +} + func NormalizeCacheCreationSplit(totalTokens int, tokens5m int, tokens1h int) (int, int) { return oaichat.NormalizeCacheCreationSplit(totalTokens, tokens5m, tokens1h) } @@ -36,6 +51,52 @@ func StreamResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.ChatCo return claudemessages.StreamResponseClaude2OpenAI(claudeResponse) } +func NewClaudeToChatStreamState() *ClaudeToChatStreamState { + return claudemessages.NewClaudeToChatStreamState() +} + +func NewClaudeHostedStreamBridge() *ClaudeHostedStreamBridge { + return &ClaudeHostedStreamBridge{bridge: claudemessages.NewClaudeHostedStreamBridge()} +} + +func NewGeminiHostedStreamBridge() *GeminiHostedStreamBridge { + return &GeminiHostedStreamBridge{bridge: geminichat.NewGeminiHostedStreamBridge()} +} + +func (b *GeminiHostedStreamBridge) Observe(response *dto.GeminiChatResponse) { + if b == nil || b.bridge == nil { + return + } + b.bridge.Observe(response) +} + +func (b *GeminiHostedStreamBridge) Finalize(state *ResponseStreamState) ([]ChatToResponsesStreamEvent, error) { + if b == nil || b.bridge == nil || state == nil { + return nil, nil + } + for _, stepState := range state.stepStates { + if streamState, ok := stepState.(*ChatToResponsesStreamState); ok { + return b.bridge.Finalize(streamState) + } + } + return nil, fmt.Errorf("Gemini hosted stream bridge requires a Chat-to-Responses stream state") +} + +func (b *ClaudeHostedStreamBridge) Convert(response *dto.ClaudeResponse, state *ResponseStreamState) ([]ChatToResponsesStreamEvent, bool, error) { + if state == nil { + return nil, false, nil + } + if b == nil || b.bridge == nil { + return nil, false, nil + } + for _, stepState := range state.stepStates { + if streamState, ok := stepState.(*ChatToResponsesStreamState); ok { + return b.bridge.Convert(response, streamState) + } + } + return nil, false, nil +} + func ResponseClaude2OpenAI(claudeResponse *dto.ClaudeResponse) *dto.OpenAITextResponse { return claudemessages.ResponseClaude2OpenAI(claudeResponse) } @@ -60,6 +121,10 @@ func FormatClaudeResponseInfo(claudeResponse *dto.ClaudeResponse, oaiResponse *d return claudemessages.FormatClaudeResponseInfo(claudeResponse, oaiResponse, claudeInfo) } +func FinalizeClaudeStreamBillingUsage(claudeInfo *ClaudeResponseInfo) { + claudemessages.FinalizeClaudeStreamBillingUsage(claudeInfo) +} + func ResponseOpenAI2Gemini(openAIResponse *dto.OpenAITextResponse, info convmeta.Meta) *dto.GeminiChatResponse { return oaichat.ResponseOpenAI2Gemini(openAIResponse, info) } @@ -116,6 +181,10 @@ func UsageFromResponsesUsage(src *dto.Usage) *dto.Usage { return oairesponses.UsageFromResponsesUsage(src) } +func NormalizeResponsesUsage(src *dto.Usage) *dto.Usage { + return oairesponses.NormalizeResponsesUsage(src) +} + func ExtractOutputTextFromResponses(resp *dto.OpenAIResponsesResponse) string { return oairesponses.ExtractOutputTextFromResponses(resp) } diff --git a/relaykit/relayconvert/response_registry.go b/relaykit/relayconvert/response_registry.go index a2369a61ed..af2e8860d8 100644 --- a/relaykit/relayconvert/response_registry.go +++ b/relaykit/relayconvert/response_registry.go @@ -10,8 +10,11 @@ import ( "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + claudemessages "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/claude_messages" geminichat "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/gemini_chat" oaichat "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/oai_chat" + oairesponses "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/oai_responses" + "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/toolconv" kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" "github.com/QuantumNous/new-api/relaykit/types" ) @@ -41,14 +44,15 @@ type ResponseStep struct { } type ResponseResult struct { - Value any - Usage *dto.Usage - From types.RelayFormat - To types.RelayFormat - Converter string - Quality ResponseConverterQuality - Steps []ResponseStep - Stream bool + Value any + Usage *dto.Usage + From types.RelayFormat + To types.RelayFormat + Converter string + Quality ResponseConverterQuality + Steps []ResponseStep + Stream bool + Diagnostics []types.ConversionDiagnostic } type ResponseConverterSpec struct { @@ -74,6 +78,18 @@ type ResponseStreamOptions struct { Model string Created int64 IncludeUsage bool + // EmitSequenceNumber opts into the current Responses SSE wire contract. + // It is explicit so relaykit callers that depend on the historical zero-value + // output are not changed merely by upgrading the module. + EmitSequenceNumber bool +} + +type conversionDiagnosticKey struct { + code string + path string + severity types.ConversionDiagnosticSeverity + from types.RelayFormat + to types.RelayFormat } type ResponseStreamState struct { @@ -83,9 +99,18 @@ type ResponseStreamState struct { Quality ResponseConverterQuality Steps []ResponseStep - specs []ResponseConverterSpec - stepStates []any - usage *dto.Usage + specs []ResponseConverterSpec + stepStates []any + usage *dto.Usage + diagnostics []types.ConversionDiagnostic + pendingDiagnostics []types.ConversionDiagnostic + seenDiagnostics map[conversionDiagnosticKey]struct{} + fallbackInfo *convmeta.Values +} + +type responseStreamUsageCarrier interface { + StreamUsage() *dto.Usage + SetStreamUsage(*dto.Usage) } const ( @@ -302,6 +327,12 @@ func ConvertStreamResponseChunk(c context.Context, info convmeta.Meta, state *Re if state == nil { return nil, errors.New("response stream state is required") } + if info == nil { + if state.fallbackInfo == nil { + state.fallbackInfo = &convmeta.Values{} + } + info = state.fallbackInfo + } from, err := inferResponseRelayFormat(response) if err != nil { return nil, err @@ -309,10 +340,13 @@ func ConvertStreamResponseChunk(c context.Context, info convmeta.Meta, state *Re if from != state.From { return nil, fmt.Errorf("response stream converter %q expects %s response, got %s", state.Converter, state.From, from) } + diagnostics := toolconv.InspectStreamResponse(state.From, state.To, response) + state.rememberDiagnostics(diagnostics) if state.From == state.To { usage := canonicalUsageFromResponse(response) state.rememberUsage(usage) - return responseStreamResults(state, streamValuesFromAny(response), usage), nil + values := streamValuesFromAny(response) + return responseStreamResults(state, values, usage, state.takeDiagnostics(len(values) > 0)), nil } values, usage, err := executeResponseStreamSteps(c, info, state, []any{response}, 0) @@ -320,13 +354,16 @@ func ConvertStreamResponseChunk(c context.Context, info convmeta.Meta, state *Re return nil, err } state.rememberUsage(usage) - return responseStreamResults(state, values, usage), nil + return responseStreamResults(state, values, usage, state.takeDiagnostics(len(values) > 0)), nil } func FinalizeStreamResponse(c context.Context, info convmeta.Meta, state *ResponseStreamState) ([]ResponseResult, error) { if state == nil { return nil, errors.New("response stream state is required") } + if info == nil && state.fallbackInfo != nil { + info = state.fallbackInfo + } if state.From == state.To { return nil, nil } @@ -362,7 +399,7 @@ func FinalizeStreamResponse(c context.Context, info convmeta.Meta, state *Respon } values = append(values, current...) } - return responseStreamResults(state, values, usage), nil + return responseStreamResults(state, values, usage, state.takeDiagnostics(len(values) > 0)), nil } func (s *ResponseStreamState) Usage() *dto.Usage { @@ -373,15 +410,12 @@ func (s *ResponseStreamState) Usage() *dto.Usage { return s.usage } for _, state := range s.stepStates { - switch typed := state.(type) { - case *ChatToResponsesStreamState: - if typed.Usage != nil { - return typed.Usage - } - case *ResponsesToChatStreamState: - if typed.Usage != nil { - return typed.Usage - } + carrier, ok := state.(responseStreamUsageCarrier) + if !ok { + continue + } + if usage := carrier.StreamUsage(); usage != nil { + return usage } } return nil @@ -393,15 +427,27 @@ func (s *ResponseStreamState) SetUsage(usage *dto.Usage) { } s.usage = usage for _, state := range s.stepStates { - switch typed := state.(type) { - case *ChatToResponsesStreamState: - typed.Usage = UsageFromChatUsage(usage) - case *ResponsesToChatStreamState: - typed.Usage = usage + if carrier, ok := state.(responseStreamUsageCarrier); ok { + carrier.SetStreamUsage(usage) } } } +// FailResponsesStream emits protocol-native terminal error events when the +// target is OpenAI Responses. It returns handled=false for other targets. +func (s *ResponseStreamState) FailResponsesStream(code string, message string, param string) ([]ResponseResult, bool) { + if s == nil || s.To != types.RelayFormatOpenAIResponses { + return nil, false + } + for _, state := range s.stepStates { + if streamState, ok := state.(*ChatToResponsesStreamState); ok { + events := streamState.Fail(code, message, param) + return responseStreamResults(s, streamValuesFromAny(events), s.Usage(), s.takeDiagnostics(len(events) > 0)), true + } + } + return nil, false +} + func (s *ResponseStreamState) UsageText() string { if s == nil { return "" @@ -417,6 +463,15 @@ func (s *ResponseStreamState) UsageText() string { return "" } +// Diagnostics returns every conversion-loss diagnostic observed so far. This +// remains available even when a source event produces no target stream chunk. +func (s *ResponseStreamState) Diagnostics() []types.ConversionDiagnostic { + if s == nil || len(s.diagnostics) == 0 { + return nil + } + return append([]types.ConversionDiagnostic{}, s.diagnostics...) +} + func executeResponseSpec(c context.Context, info convmeta.Meta, from types.RelayFormat, target types.RelayFormat, response any, spec ResponseConverterSpec) (*ResponseResult, error) { steps, err := expandResponseConverterSteps(spec) if err != nil { @@ -426,7 +481,11 @@ func executeResponseSpec(c context.Context, info convmeta.Meta, from types.Relay } func executeResponseSteps(c context.Context, info convmeta.Meta, from types.RelayFormat, target types.RelayFormat, response any, converter string, quality ResponseConverterQuality, specs []ResponseConverterSpec) (*ResponseResult, error) { - current := response + diagnostics := toolconv.InspectResponse(from, target, response) + current, hostedResponse, err := toolconv.ExtractHostedResponse(from, response) + if err != nil { + return nil, err + } var usage *dto.Usage steps := make([]ResponseStep, 0, len(specs)) for _, spec := range specs { @@ -438,6 +497,11 @@ func executeResponseSteps(c context.Context, info convmeta.Meta, from types.Rela } steps = append(steps, step) } + current, hostedDiagnostics, err := toolconv.AttachHostedResponse(target, current, hostedResponse, convmeta.OptionsOf(info)) + if err != nil { + return nil, err + } + diagnostics = append(diagnostics, hostedDiagnostics...) converters := make([]string, 0, len(steps)) for _, step := range steps { @@ -447,14 +511,15 @@ func executeResponseSteps(c context.Context, info convmeta.Meta, from types.Rela converter = strings.Join(converters, ",") } return &ResponseResult{ - Value: current, - Usage: usage, - From: from, - To: target, - Converter: converter, - Quality: quality, - Steps: steps, - Stream: false, + Value: current, + Usage: usage, + From: from, + To: target, + Converter: converter, + Quality: quality, + Steps: steps, + Stream: false, + Diagnostics: diagnostics, }, nil } @@ -475,6 +540,7 @@ func executeResponseStep(c context.Context, info convmeta.Meta, spec ResponseCon } func executeStatelessStreamResponseSpec(c context.Context, info convmeta.Meta, from types.RelayFormat, target types.RelayFormat, response any, spec ResponseConverterSpec) (*ResponseResult, error) { + diagnostics := toolconv.InspectStreamResponse(from, target, response) steps, err := expandResponseConverterSteps(spec) if err != nil { return nil, err @@ -498,14 +564,15 @@ func executeStatelessStreamResponseSpec(c context.Context, info convmeta.Meta, f }) } return &ResponseResult{ - Value: current, - Usage: usage, - From: from, - To: target, - Converter: spec.ID, - Quality: spec.Quality, - Steps: resultSteps, - Stream: true, + Value: current, + Usage: usage, + From: from, + To: target, + Converter: spec.ID, + Quality: spec.Quality, + Steps: resultSteps, + Stream: true, + Diagnostics: diagnostics, }, nil } @@ -599,25 +666,63 @@ func finalizeResponseStreamStep(c context.Context, info convmeta.Meta, spec Resp func (s *ResponseStreamState) rememberUsage(usage *dto.Usage) { if s != nil && usage != nil { - s.usage = usage + s.usage = dto.MergeUsageNonZero(s.usage, usage) } } -func responseStreamResults(state *ResponseStreamState, values []any, usage *dto.Usage) []ResponseResult { +func (s *ResponseStreamState) rememberDiagnostics(diagnostics []types.ConversionDiagnostic) { + if s == nil || len(diagnostics) == 0 { + return + } + if s.seenDiagnostics == nil { + s.seenDiagnostics = make(map[conversionDiagnosticKey]struct{}) + } + for _, diagnostic := range diagnostics { + key := conversionDiagnosticKey{ + code: diagnostic.Code, + path: diagnostic.Path, + severity: diagnostic.Severity, + from: diagnostic.From, + to: diagnostic.To, + } + if _, exists := s.seenDiagnostics[key]; exists { + continue + } + s.seenDiagnostics[key] = struct{}{} + s.diagnostics = append(s.diagnostics, diagnostic) + s.pendingDiagnostics = append(s.pendingDiagnostics, diagnostic) + } +} + +func (s *ResponseStreamState) takeDiagnostics(hasOutput bool) []types.ConversionDiagnostic { + if s == nil || !hasOutput || len(s.pendingDiagnostics) == 0 { + return nil + } + diagnostics := append([]types.ConversionDiagnostic{}, s.pendingDiagnostics...) + s.pendingDiagnostics = nil + return diagnostics +} + +func responseStreamResults(state *ResponseStreamState, values []any, usage *dto.Usage, diagnostics []types.ConversionDiagnostic) []ResponseResult { if state == nil || len(values) == 0 { return nil } results := make([]ResponseResult, 0, len(values)) - for _, value := range values { + for index, value := range values { + var resultDiagnostics []types.ConversionDiagnostic + if index == 0 { + resultDiagnostics = append(resultDiagnostics, diagnostics...) + } results = append(results, ResponseResult{ - Value: value, - Usage: usage, - From: state.From, - To: state.To, - Converter: state.Converter, - Quality: state.Quality, - Steps: append([]ResponseStep{}, state.Steps...), - Stream: true, + Value: value, + Usage: usage, + From: state.From, + To: state.To, + Converter: state.Converter, + Quality: state.Quality, + Steps: append([]ResponseStep{}, state.Steps...), + Stream: true, + Diagnostics: resultDiagnostics, }) } return results @@ -738,42 +843,38 @@ func isNilResponse(response any) bool { func canonicalUsageFromResponse(response any) *dto.Usage { switch resp := response.(type) { - case *dto.OpenAITextResponse: - return UsageFromChatUsage(&resp.Usage) case dto.OpenAITextResponse: + response = &resp + case dto.ChatCompletionsStreamResponse: + response = &resp + case dto.OpenAIResponsesResponse: + response = &resp + case dto.ResponsesStreamResponse: + response = &resp + case dto.ClaudeResponse: + response = &resp + case dto.GeminiChatResponse: + response = &resp + } + switch resp := response.(type) { + case *dto.OpenAITextResponse: return UsageFromChatUsage(&resp.Usage) case *dto.ChatCompletionsStreamResponse: if resp.Usage == nil { return nil } return UsageFromChatUsage(resp.Usage) - case dto.ChatCompletionsStreamResponse: - if resp.Usage == nil { - return nil - } - return UsageFromChatUsage(resp.Usage) case *dto.OpenAIResponsesResponse: return UsageFromResponsesUsage(resp.Usage) - case dto.OpenAIResponsesResponse: - return UsageFromResponsesUsage(resp.Usage) case *dto.ResponsesStreamResponse: if resp.Response == nil { return nil } return UsageFromResponsesUsage(resp.Response.Usage) - case dto.ResponsesStreamResponse: - if resp.Response == nil { - return nil - } - return UsageFromResponsesUsage(resp.Response.Usage) case *dto.ClaudeResponse: return usageFromClaudeResponse(resp) - case dto.ClaudeResponse: - return usageFromClaudeResponse(&resp) case *dto.GeminiChatResponse: return UsageFromGeminiMetadata(resp.GetUsageMetadata(), 0) - case dto.GeminiChatResponse: - return UsageFromGeminiMetadata(resp.GetUsageMetadata(), 0) default: return nil } @@ -816,12 +917,21 @@ func convertOAIResponsesResponseToOAIChat(_ context.Context, _ convmeta.Meta, re return ResponsesResponseToChatCompletionsResponse(responsesResponse, id) } +func convertOAIResponsesResponseToClaudeMessages(_ context.Context, _ convmeta.Meta, response any) (any, *dto.Usage, error) { + responsesResponse, err := asOAIResponsesResponse(response) + if err != nil { + return nil, nil, err + } + return oairesponses.ResponsesResponseToClaudeMessagesResponse(responsesResponse) +} + func newOAIChatToOAIResponsesStreamState(options ResponseStreamOptions) any { id := strings.TrimSpace(options.ID) if id == "" { id = fmt.Sprintf("resp_%s", kitutil.GetUUID()) } state := NewChatToResponsesStreamState(id, strings.TrimSpace(options.Model)) + state.EmitSequenceNumber = options.EmitSequenceNumber if options.Created != 0 { state.Created = options.Created } @@ -887,6 +997,56 @@ func finalizeOAIResponsesStreamResponseToOAIChat(_ context.Context, _ convmeta.M return streamValuesFromAny(chunks), streamState.Usage, nil } +func newOAIResponsesToClaudeMessagesStreamState(options ResponseStreamOptions) any { + return oairesponses.NewResponsesToClaudeStreamState(options.ID, options.Model) +} + +func convertOAIResponsesStreamResponseToClaudeMessages(_ context.Context, info convmeta.Meta, response any, state any) ([]any, *dto.Usage, error) { + responsesResponse, err := asOAIResponsesStreamResponse(response) + if err != nil { + return nil, nil, err + } + streamState, ok := state.(*oairesponses.ResponsesToClaudeStreamState) + if !ok || streamState == nil { + return nil, nil, errors.New("OAI responses to Claude stream state is required") + } + estimatedInputTokens := 0 + if info != nil { + estimatedInputTokens = info.GetEstimatePromptTokens() + } + responses, usage, err := streamState.ConvertChunk(responsesResponse, estimatedInputTokens) + if err != nil { + return nil, usage, err + } + if info != nil && streamState.Done() { + claudeInfo := info.EnsureClaudeConvertInfo() + claudeInfo.Done = true + if claudeInfo.Usage == nil { + claudeInfo.Usage = usage + } + } + return streamValuesFromAny(responses), usage, nil +} + +func finalizeOAIResponsesStreamResponseToClaudeMessages(_ context.Context, info convmeta.Meta, state any) ([]any, *dto.Usage, error) { + streamState, ok := state.(*oairesponses.ResponsesToClaudeStreamState) + if !ok || streamState == nil { + return nil, nil, errors.New("OAI responses to Claude stream state is required") + } + estimatedInputTokens := 0 + if info != nil { + estimatedInputTokens = info.GetEstimatePromptTokens() + if usage := info.EnsureClaudeConvertInfo().Usage; usage != nil { + streamState.SetUsage(usage) + } + } + responses, err := streamState.Finalize(estimatedInputTokens) + if info != nil && streamState.Done() { + info.EnsureClaudeConvertInfo().Done = true + } + return streamValuesFromAny(responses), streamState.Usage, err +} + func convertOAIChatResponseToClaudeMessages(_ context.Context, info convmeta.Meta, response any) (any, *dto.Usage, error) { chatResponse, err := asOAIChatResponse(response) if err != nil { @@ -928,6 +1088,38 @@ func convertOAIChatStreamResponseToGeminiChat(_ context.Context, info convmeta.M return StreamResponseOpenAI2Gemini(chatResponse, info), canonicalUsageFromResponse(chatResponse), nil } +func newOAIChatToGeminiStreamState(_ ResponseStreamOptions) any { + return oaichat.NewChatToGeminiStreamState() +} + +func convertOAIChatStreamResponseChunkToGeminiChat(_ context.Context, info convmeta.Meta, response any, state any) ([]any, *dto.Usage, error) { + chatResponse, err := asOAIChatStreamResponse(response) + if err != nil { + return nil, nil, err + } + streamState, ok := state.(*oaichat.ChatToGeminiStreamState) + if !ok || streamState == nil { + return nil, nil, errors.New("OAI chat to Gemini stream state is required") + } + responses, err := streamState.ConvertChunk(chatResponse, info) + if err != nil { + return nil, nil, err + } + return streamValuesFromAny(responses), canonicalUsageFromResponse(chatResponse), nil +} + +func finalizeOAIChatStreamResponseToGeminiChat(_ context.Context, info convmeta.Meta, state any) ([]any, *dto.Usage, error) { + streamState, ok := state.(*oaichat.ChatToGeminiStreamState) + if !ok || streamState == nil { + return nil, nil, errors.New("OAI chat to Gemini stream state is required") + } + responses, err := streamState.Finalize(info) + if err != nil { + return nil, nil, err + } + return streamValuesFromAny(responses), streamState.Usage(), nil +} + func convertClaudeMessagesResponseToOAIChat(_ context.Context, _ convmeta.Meta, response any) (any, *dto.Usage, error) { claudeResponse, err := asClaudeResponse(response) if err != nil { @@ -954,6 +1146,30 @@ func convertClaudeMessagesStreamResponseToOAIChat(_ context.Context, _ convmeta. return openAIResponse, usage, nil } +func newClaudeMessagesToOAIChatStreamState(_ ResponseStreamOptions) any { + return claudemessages.NewClaudeToChatStreamState() +} + +func convertClaudeMessagesStreamResponseChunkToOAIChat(_ context.Context, _ convmeta.Meta, response any, state any) ([]any, *dto.Usage, error) { + claudeResponse, err := asClaudeResponse(response) + if err != nil { + return nil, nil, err + } + streamState, ok := state.(*claudemessages.ClaudeToChatStreamState) + if !ok || streamState == nil { + return nil, nil, errors.New("Claude-to-Chat stream state is required") + } + openAIResponse, err := streamState.ConvertChunk(claudeResponse) + if err != nil { + return nil, nil, err + } + usage := usageFromClaudeResponse(claudeResponse) + if openAIResponse != nil && usage != nil { + openAIResponse.Usage = usage + } + return streamValuesFromAny(openAIResponse), usage, nil +} + func convertGeminiChatResponseToOAIChat(_ context.Context, info convmeta.Meta, response any) (any, *dto.Usage, error) { geminiResponse, err := asGeminiChatResponse(response) if err != nil { @@ -988,7 +1204,10 @@ func convertGeminiChatStreamResponseChunkToOAIChat(_ context.Context, info convm if info != nil && info.HasChannelMeta() { model = info.GetUpstreamModelName() } - responses := streamState.ConvertChunk(geminiResponse, model, usage) + responses, err := streamState.ConvertChunk(geminiResponse, model, usage) + if err != nil { + return nil, nil, err + } return streamValuesFromAny(responses), usage, nil } @@ -1001,7 +1220,10 @@ func finalizeGeminiChatStreamResponseToOAIChat(_ context.Context, info convmeta. if info != nil && info.HasChannelMeta() { model = info.GetUpstreamModelName() } - responses := streamState.Finalize(model) + responses, err := streamState.Finalize(model) + if err != nil { + return nil, nil, err + } return streamValuesFromAny(responses), streamState.Usage(), nil } diff --git a/relaykit/relayconvert/response_registry_test.go b/relaykit/relayconvert/response_registry_test.go index 3e62d4c2ce..866cc4ea34 100644 --- a/relaykit/relayconvert/response_registry_test.go +++ b/relaykit/relayconvert/response_registry_test.go @@ -75,10 +75,6 @@ func TestLookupBuiltinResponseConverters(t *testing.T) { from: types.RelayFormatOpenAIResponses, to: types.RelayFormatClaude, quality: ResponseConverterQualityFair, - stepConverters: []string{ - ConverterOpenAIResponsesToOpenAIChat, - ConverterOpenAIChatToClaudeMessages, - }, }, { lookupID: responseConverterResponsesToGemini, @@ -193,7 +189,7 @@ func TestConvertResponseDirectConverters(t *testing.T) { require.NotNil(t, geminiValue.UsageMetadata.BillingUsage.OpenAIUsage) } -func TestConvertResponseMultiHopConverters(t *testing.T) { +func TestConvertResponseDirectAndMultiHopConverters(t *testing.T) { responses := textRegistryResponsesResponse() toClaude, err := ConvertResponse(nil, &convmeta.Values{}, types.RelayFormatClaude, responses) @@ -201,8 +197,7 @@ func TestConvertResponseMultiHopConverters(t *testing.T) { assert.Equal(t, requestConverterResponsesToClaude, toClaude.Converter) assert.Equal(t, ResponseConverterQualityFair, toClaude.Quality) assert.Equal(t, []ResponseStep{ - {Converter: ConverterOpenAIResponsesToOpenAIChat, From: types.RelayFormatOpenAIResponses, To: types.RelayFormatOpenAI}, - {Converter: ConverterOpenAIChatToClaudeMessages, From: types.RelayFormatOpenAI, To: types.RelayFormatClaude}, + {Converter: ConverterOpenAIResponsesToClaudeMessages, From: types.RelayFormatOpenAIResponses, To: types.RelayFormatClaude}, }, toClaude.Steps) require.IsType(t, &dto.ClaudeResponse{}, toClaude.Value) claudeValue := toClaude.Value.(*dto.ClaudeResponse) @@ -232,6 +227,40 @@ func TestConvertResponseMultiHopConverters(t *testing.T) { assert.Equal(t, 11, toGemini.Usage.TotalTokens) } +func TestConvertResponsePreservesInterleavedResponsesBlocksForClaude(t *testing.T) { + responses := &dto.OpenAIResponsesResponse{ + ID: "resp_1", + Model: "gpt-test", + Status: []byte(`"completed"`), + Output: []dto.ResponsesOutput{ + {Type: "reasoning", Summary: []dto.ResponsesReasoningSummaryPart{{Type: "summary_text", Text: "**Planning file inspection**"}}}, + {Type: "message", Role: "assistant", Content: []dto.ResponsesOutputContent{{Type: "output_text", Text: "I’ll inspect the starter repository."}}}, + {Type: "reasoning", Summary: []dto.ResponsesReasoningSummaryPart{{Type: "summary_text", Text: "**Clarifying environment task requirements**"}}}, + {Type: "message", Role: "assistant", Content: []dto.ResponsesOutputContent{{Type: "output_text", Text: "What would you like me to build?"}}}, + }, + } + + result, err := ConvertResponse(nil, nil, types.RelayFormatClaude, responses) + require.NoError(t, err) + assert.Equal(t, []ResponseStep{ + {Converter: ConverterOpenAIResponsesToClaudeMessages, From: types.RelayFormatOpenAIResponses, To: types.RelayFormatClaude}, + }, result.Steps) + claudeResponse := result.Value.(*dto.ClaudeResponse) + require.Len(t, claudeResponse.Content, 4) + assert.Equal(t, []string{"thinking", "text", "thinking", "text"}, []string{ + claudeResponse.Content[0].Type, + claudeResponse.Content[1].Type, + claudeResponse.Content[2].Type, + claudeResponse.Content[3].Type, + }) + require.NotNil(t, claudeResponse.Content[0].Thinking) + require.NotNil(t, claudeResponse.Content[2].Thinking) + assert.Equal(t, "**Planning file inspection**", *claudeResponse.Content[0].Thinking) + assert.Equal(t, "I’ll inspect the starter repository.", claudeResponse.Content[1].GetText()) + assert.Equal(t, "**Clarifying environment task requirements**", *claudeResponse.Content[2].Thinking) + assert.Equal(t, "What would you like me to build?", claudeResponse.Content[3].GetText()) +} + func TestConvertResponseByIDExecutesMultiHopAndChecksSource(t *testing.T) { responses := textRegistryResponsesResponse() @@ -489,7 +518,7 @@ func TestConvertStreamResponseStatefulDirectConverters(t *testing.T) { require.IsType(t, dto.ChatCompletionsStreamResponse{}, responsesResults[len(responsesResults)-1].Value) } -func TestConvertStreamResponseStatefulMultiHopResponsesToClaude(t *testing.T) { +func TestConvertStreamResponseStatefulDirectResponsesToClaude(t *testing.T) { info := &convmeta.Values{ ClaudeConvertInfo: &convmeta.ClaudeConvertInfo{ LastMessagesType: convmeta.LastMessageTypeNone, @@ -509,8 +538,7 @@ func TestConvertStreamResponseStatefulMultiHopResponsesToClaude(t *testing.T) { require.NotEmpty(t, results) assert.Equal(t, requestConverterResponsesToClaude, results[0].Converter) assert.Equal(t, []ResponseStep{ - {Converter: ConverterOpenAIResponsesToOpenAIChat, From: types.RelayFormatOpenAIResponses, To: types.RelayFormatOpenAI}, - {Converter: ConverterOpenAIChatToClaudeMessages, From: types.RelayFormatOpenAI, To: types.RelayFormatClaude}, + {Converter: ConverterOpenAIResponsesToClaudeMessages, From: types.RelayFormatOpenAIResponses, To: types.RelayFormatClaude}, }, results[0].Steps) var sawTextDelta bool diff --git a/relaykit/relayconvert/terminal_stream_test.go b/relaykit/relayconvert/terminal_stream_test.go index 6ea59260f3..48d6f33d98 100644 --- a/relaykit/relayconvert/terminal_stream_test.go +++ b/relaykit/relayconvert/terminal_stream_test.go @@ -165,8 +165,7 @@ func TestClaudeTargetStatefulStreamTerminalTail(t *testing.T) { }, }, }, - wantFinalizerTerminals: true, - wantStopReason: "end_turn", + wantStopReason: "end_turn", }, } @@ -228,26 +227,10 @@ func TestClaudeTargetStatefulStreamTerminalTail(t *testing.T) { ) require.NoError(t, err) - chunks := []*dto.ResponsesStreamResponse{ - { - Type: "response.output_text.delta", - Delta: "Hello", - }, - { - Type: "response.completed", - Response: &dto.OpenAIResponsesResponse{ - ID: "resp-fixed", - Object: "response", - Model: "upstream-model", - Status: []byte(`"completed"`), - Usage: &dto.Usage{ - InputTokens: 4, - OutputTokens: 2, - TotalTokens: 6, - }, - }, - }, - } + chunks := []*dto.ResponsesStreamResponse{{ + Type: "response.output_text.delta", + Delta: "Hello", + }} for _, chunk := range chunks { _, err := ConvertStreamResponseChunk(nil, info, state, chunk) require.NoError(t, err) diff --git a/relaykit/relayconvert/testdata/golden/request/claude_to_gemini.golden.json b/relaykit/relayconvert/testdata/golden/request/claude_to_gemini.golden.json deleted file mode 100644 index 8ec074a6f4..0000000000 --- a/relaykit/relayconvert/testdata/golden/request/claude_to_gemini.golden.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "What is in this image?" - }, - { - "inlineData": { - "mimeType": "image/png", - "data": "aGVsbG8=" - } - } - ] - }, - { - "role": "model", - "parts": [ - { - "functionCall": { - "name": "get_weather", - "args": { - "city": "Paris" - } - }, - "thoughtSignature": "context_engineering_is_the_way_to_go" - } - ] - }, - { - "role": "user", - "parts": [ - { - "functionResponse": { - "name": "get_weather", - "response": { - "content": "15 degrees" - } - } - } - ] - } - ], - "safetySettings": [ - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "OFF" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "OFF" - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "OFF" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "OFF" - } - ], - "generationConfig": { - "maxOutputTokens": 1024 - }, - "tools": [ - { - "functionDeclarations": [ - { - "description": "Get weather by city", - "name": "get_weather", - "parameters": { - "properties": { - "city": { - "type": "STRING" - } - }, - "required": [ - "city" - ], - "type": "OBJECT" - } - } - ] - } - ], - "systemInstruction": { - "parts": [ - { - "text": "You are a helpful assistant." - } - ] - } -} diff --git a/relaykit/relayconvert/testdata/golden/request/claude_to_openai.golden.json b/relaykit/relayconvert/testdata/golden/request/claude_to_openai.golden.json deleted file mode 100644 index 59a20e45a6..0000000000 --- a/relaykit/relayconvert/testdata/golden/request/claude_to_openai.golden.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "model": "claude-test", - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is in this image?" - }, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,aGVsbG8=", - "MimeType": "" - } - } - ] - }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "toolu_abc", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"city\":\"Paris\"}" - } - } - ] - }, - { - "role": "tool", - "content": "15 degrees", - "name": "get_weather", - "tool_call_id": "toolu_abc" - } - ], - "stream": true, - "max_tokens": 1024, - "tools": [ - { - "type": "function", - "function": { - "description": "Get weather by city", - "name": "get_weather", - "parameters": { - "properties": { - "city": { - "type": "string" - } - }, - "required": [ - "city" - ], - "type": "object" - } - } - } - ] -} diff --git a/relaykit/relayconvert/testdata/golden/request/claude_to_openai_responses.golden.json b/relaykit/relayconvert/testdata/golden/request/claude_to_openai_responses.golden.json deleted file mode 100644 index e5f6886389..0000000000 --- a/relaykit/relayconvert/testdata/golden/request/claude_to_openai_responses.golden.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "model": "claude-test", - "input": [ - { - "content": [ - { - "text": "What is in this image?", - "type": "input_text" - }, - { - "image_url": "data:image/png;base64,aGVsbG8=", - "type": "input_image" - } - ], - "role": "user" - }, - { - "content": "", - "role": "assistant" - }, - { - "arguments": "{\"city\":\"Paris\"}", - "call_id": "toolu_abc", - "name": "get_weather", - "type": "function_call" - }, - { - "call_id": "toolu_abc", - "output": "15 degrees", - "type": "function_call_output" - } - ], - "instructions": "You are a helpful assistant.", - "max_output_tokens": 1024, - "stream": true, - "tools": [ - { - "description": "Get weather by city", - "name": "get_weather", - "parameters": { - "properties": { - "city": { - "type": "string" - } - }, - "required": [ - "city" - ], - "type": "object" - }, - "type": "function" - } - ] -} diff --git a/relaykit/relayconvert/testdata/golden/request/gemini_to_claude.golden.json b/relaykit/relayconvert/testdata/golden/request/gemini_to_claude.golden.json deleted file mode 100644 index 9539d2b563..0000000000 --- a/relaykit/relayconvert/testdata/golden/request/gemini_to_claude.golden.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "model": "upstream-model", - "system": [ - { - "type": "text", - "text": "You are a helpful assistant." - } - ], - "messages": [ - { - "role": "user", - "content": [] - }, - { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "..." - }, - { - "type": "tool_use", - "id": "call_1", - "name": "get_weather", - "input": { - "city": "Paris" - } - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "content": "{\"result\":\"15 degrees\"}", - "tool_use_id": "call_0" - } - ] - } - ], - "max_tokens": 1024, - "temperature": 0.7, - "tools": [ - { - "name": "get_weather", - "description": "Get weather by city", - "input_schema": { - "properties": { - "city": { - "type": "string" - } - }, - "required": [ - "city" - ], - "type": "object" - } - } - ] -} diff --git a/relaykit/relayconvert/testdata/golden/request/gemini_to_openai.golden.json b/relaykit/relayconvert/testdata/golden/request/gemini_to_openai.golden.json deleted file mode 100644 index bc1e267539..0000000000 --- a/relaykit/relayconvert/testdata/golden/request/gemini_to_openai.golden.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "model": "upstream-model", - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is in this image?" - }, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,aGVsbG8=", - "detail": "auto", - "MimeType": "image/png" - } - } - ] - }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"city\":\"Paris\"}" - } - } - ] - }, - { - "role": "tool", - "content": "{\"result\":\"15 degrees\"}", - "tool_call_id": "call_0" - } - ], - "stream": false, - "max_tokens": 1024, - "temperature": 0.7, - "tools": [ - { - "type": "function", - "function": { - "description": "Get weather by city", - "name": "get_weather", - "parameters": { - "properties": { - "city": { - "type": "string" - } - }, - "required": [ - "city" - ], - "type": "object" - } - } - } - ] -} diff --git a/relaykit/relayconvert/testdata/golden/request/gemini_to_openai_responses.golden.json b/relaykit/relayconvert/testdata/golden/request/gemini_to_openai_responses.golden.json deleted file mode 100644 index ec9ba42bae..0000000000 --- a/relaykit/relayconvert/testdata/golden/request/gemini_to_openai_responses.golden.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "model": "upstream-model", - "input": [ - { - "content": [ - { - "text": "What is in this image?", - "type": "input_text" - }, - { - "image_url": "data:image/png;base64,aGVsbG8=", - "type": "input_image" - } - ], - "role": "user" - }, - { - "content": "", - "role": "assistant" - }, - { - "arguments": "{\"city\":\"Paris\"}", - "call_id": "call_1", - "name": "get_weather", - "type": "function_call" - }, - { - "call_id": "call_0", - "output": "{\"result\":\"15 degrees\"}", - "type": "function_call_output" - } - ], - "instructions": "You are a helpful assistant.", - "max_output_tokens": 1024, - "stream": false, - "temperature": 0.7, - "tools": [ - { - "description": "Get weather by city", - "name": "get_weather", - "parameters": { - "properties": { - "city": { - "type": "string" - } - }, - "required": [ - "city" - ], - "type": "object" - }, - "type": "function" - } - ] -} diff --git a/relaykit/relayconvert/testdata/golden/request/openai_responses_to_gemini.golden.json b/relaykit/relayconvert/testdata/golden/request/openai_responses_to_gemini.golden.json deleted file mode 100644 index 8ec074a6f4..0000000000 --- a/relaykit/relayconvert/testdata/golden/request/openai_responses_to_gemini.golden.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "What is in this image?" - }, - { - "inlineData": { - "mimeType": "image/png", - "data": "aGVsbG8=" - } - } - ] - }, - { - "role": "model", - "parts": [ - { - "functionCall": { - "name": "get_weather", - "args": { - "city": "Paris" - } - }, - "thoughtSignature": "context_engineering_is_the_way_to_go" - } - ] - }, - { - "role": "user", - "parts": [ - { - "functionResponse": { - "name": "get_weather", - "response": { - "content": "15 degrees" - } - } - } - ] - } - ], - "safetySettings": [ - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "OFF" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "OFF" - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "OFF" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "OFF" - } - ], - "generationConfig": { - "maxOutputTokens": 1024 - }, - "tools": [ - { - "functionDeclarations": [ - { - "description": "Get weather by city", - "name": "get_weather", - "parameters": { - "properties": { - "city": { - "type": "STRING" - } - }, - "required": [ - "city" - ], - "type": "OBJECT" - } - } - ] - } - ], - "systemInstruction": { - "parts": [ - { - "text": "You are a helpful assistant." - } - ] - } -} diff --git a/relaykit/relayconvert/testdata/golden/request/openai_to_gemini.golden.json b/relaykit/relayconvert/testdata/golden/request/openai_to_gemini.golden.json deleted file mode 100644 index 93b0ebb27c..0000000000 --- a/relaykit/relayconvert/testdata/golden/request/openai_to_gemini.golden.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "What is in this image?" - }, - { - "inlineData": { - "mimeType": "image/png", - "data": "aGVsbG8=" - } - } - ] - }, - { - "role": "model", - "parts": [ - { - "functionCall": { - "name": "get_weather", - "args": { - "city": "Paris" - } - }, - "thoughtSignature": "context_engineering_is_the_way_to_go" - } - ] - }, - { - "role": "user", - "parts": [ - { - "functionResponse": { - "name": "get_weather", - "response": { - "content": "15 degrees" - } - } - } - ] - }, - { - "role": "user", - "parts": [ - { - "text": "Summarize." - } - ] - } - ], - "safetySettings": [ - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "OFF" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "OFF" - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "OFF" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "OFF" - } - ], - "generationConfig": { - "maxOutputTokens": 1024 - }, - "tools": [ - { - "functionDeclarations": [ - { - "description": "Get weather by city", - "name": "get_weather", - "parameters": { - "properties": { - "city": { - "type": "STRING" - } - }, - "required": [ - "city" - ], - "type": "OBJECT" - } - } - ] - } - ], - "toolConfig": { - "functionCallingConfig": { - "mode": "AUTO" - } - }, - "systemInstruction": { - "parts": [ - { - "text": "You are a helpful assistant." - } - ] - } -} diff --git a/relaykit/relayconvert/testdata/golden/response/claude_to_gemini.golden.json b/relaykit/relayconvert/testdata/golden/response/claude_to_gemini.golden.json deleted file mode 100644 index 93cd2c91da..0000000000 --- a/relaykit/relayconvert/testdata/golden/response/claude_to_gemini.golden.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": "The answer is 42." - }, - { - "functionCall": { - "name": "get_weather", - "args": { - "city": "Paris" - } - } - } - ] - }, - "finishReason": "STOP", - "index": 0, - "safetyRatings": [] - } - ], - "usageMetadata": { - "promptTokenCount": 15, - "toolUsePromptTokenCount": 0, - "candidatesTokenCount": 5, - "totalTokenCount": 20, - "thoughtsTokenCount": 0, - "cachedContentTokenCount": 0, - "promptTokensDetails": null, - "toolUsePromptTokensDetails": null, - "candidatesTokensDetails": null, - "billing_usage": { - "source": "oai_chat", - "semantic": "openai", - "openai_usage": { - "prompt_tokens": 15, - "completion_tokens": 5, - "total_tokens": 20, - "usage_semantic": "openai", - "usage_source": "anthropic", - "prompt_tokens_details": { - "cached_tokens": 3, - "cached_creation_tokens": 2, - "cache_write_tokens": 2, - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0 - }, - "completion_tokens_details": { - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "reasoning_tokens": 0 - }, - "input_tokens": 15, - "output_tokens": 0, - "input_tokens_details": null, - "claude_cache_creation_5_m_tokens": 2, - "claude_cache_creation_1_h_tokens": 0 - } - } - } -} diff --git a/relaykit/relayconvert/testdata/golden/response/gemini_to_claude.golden.json b/relaykit/relayconvert/testdata/golden/response/gemini_to_claude.golden.json index 2f3d24927b..df16b37859 100644 --- a/relaykit/relayconvert/testdata/golden/response/gemini_to_claude.golden.json +++ b/relaykit/relayconvert/testdata/golden/response/gemini_to_claude.golden.json @@ -26,29 +26,18 @@ "claude_cache_creation_5_m_tokens": 0, "claude_cache_creation_1_h_tokens": 0, "billing_usage": { - "source": "oai_chat", - "semantic": "openai", - "openai_usage": { - "prompt_tokens": 10, - "completion_tokens": 7, - "total_tokens": 15, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 10, - "audio_tokens": 0, - "image_tokens": 0 - }, - "completion_tokens_details": { - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "reasoning_tokens": 2 - }, - "input_tokens": 0, - "output_tokens": 0, - "input_tokens_details": null, - "claude_cache_creation_5_m_tokens": 0, - "claude_cache_creation_1_h_tokens": 0 + "source": "gemini_chat", + "semantic": "gemini", + "gemini_usage_metadata": { + "promptTokenCount": 10, + "toolUsePromptTokenCount": 0, + "candidatesTokenCount": 5, + "totalTokenCount": 15, + "thoughtsTokenCount": 2, + "cachedContentTokenCount": 0, + "promptTokensDetails": [], + "toolUsePromptTokensDetails": [], + "candidatesTokensDetails": [] } } } diff --git a/relaykit/relayconvert/testdata/golden/response/openai_responses_to_claude.golden.json b/relaykit/relayconvert/testdata/golden/response/openai_responses_to_claude.golden.json index a5d42e1042..f82fbf7438 100644 --- a/relaykit/relayconvert/testdata/golden/response/openai_responses_to_claude.golden.json +++ b/relaykit/relayconvert/testdata/golden/response/openai_responses_to_claude.golden.json @@ -3,6 +3,10 @@ "type": "message", "role": "assistant", "content": [ + { + "type": "thinking", + "thinking": "Deep thought." + }, { "type": "text", "text": "The answer is 42." diff --git a/relaykit/relayconvert/testdata/golden/response/openai_responses_to_gemini.golden.json b/relaykit/relayconvert/testdata/golden/response/openai_responses_to_gemini.golden.json deleted file mode 100644 index 9348f4f75f..0000000000 --- a/relaykit/relayconvert/testdata/golden/response/openai_responses_to_gemini.golden.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": "The answer is 42." - }, - { - "functionCall": { - "name": "get_weather", - "args": { - "city": "Paris" - } - } - } - ] - }, - "finishReason": "STOP", - "index": 0, - "safetyRatings": [] - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "toolUsePromptTokenCount": 0, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - "thoughtsTokenCount": 0, - "cachedContentTokenCount": 0, - "promptTokensDetails": null, - "toolUsePromptTokensDetails": null, - "candidatesTokensDetails": null, - "billing_usage": { - "source": "oai_responses", - "semantic": "openai", - "openai_usage": { - "prompt_tokens": 0, - "completion_tokens": 0, - "total_tokens": 15, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0 - }, - "completion_tokens_details": { - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "reasoning_tokens": 0 - }, - "input_tokens": 10, - "output_tokens": 5, - "input_tokens_details": null, - "claude_cache_creation_5_m_tokens": 0, - "claude_cache_creation_1_h_tokens": 0 - } - } - } -} diff --git a/relaykit/relayconvert/testdata/golden/response/openai_responses_to_openai.golden.json b/relaykit/relayconvert/testdata/golden/response/openai_responses_to_openai.golden.json index 17b4b4f429..4717a53361 100644 --- a/relaykit/relayconvert/testdata/golden/response/openai_responses_to_openai.golden.json +++ b/relaykit/relayconvert/testdata/golden/response/openai_responses_to_openai.golden.json @@ -9,6 +9,7 @@ "message": { "role": "assistant", "content": "The answer is 42.", + "reasoning_content": "Deep thought.", "tool_calls": [ { "id": "call_abc", diff --git a/relaykit/relayconvert/testdata/golden/response/openai_to_claude.golden.json b/relaykit/relayconvert/testdata/golden/response/openai_to_claude.golden.json deleted file mode 100644 index b6d3c2fcc8..0000000000 --- a/relaykit/relayconvert/testdata/golden/response/openai_to_claude.golden.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "id": "chatcmpl-fixed", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "text", - "text": "The answer is 42." - }, - { - "type": "tool_use", - "id": "call_abc", - "name": "get_weather", - "input": { - "city": "Paris" - } - } - ], - "stop_reason": "tool_use", - "model": "gpt-test", - "usage": { - "input_tokens": 10, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 3, - "output_tokens": 5, - "claude_cache_creation_5_m_tokens": 0, - "claude_cache_creation_1_h_tokens": 0, - "billing_usage": { - "source": "oai_chat", - "semantic": "openai", - "openai_usage": { - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - "prompt_tokens_details": { - "cached_tokens": 3, - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0 - }, - "completion_tokens_details": { - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "reasoning_tokens": 2 - }, - "input_tokens": 0, - "output_tokens": 0, - "input_tokens_details": null, - "claude_cache_creation_5_m_tokens": 0, - "claude_cache_creation_1_h_tokens": 0 - } - } - } -} diff --git a/relaykit/relayconvert/testdata/golden/response/openai_to_gemini.golden.json b/relaykit/relayconvert/testdata/golden/response/openai_to_gemini.golden.json deleted file mode 100644 index 3eff897a8f..0000000000 --- a/relaykit/relayconvert/testdata/golden/response/openai_to_gemini.golden.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": "The answer is 42." - }, - { - "functionCall": { - "name": "get_weather", - "args": { - "city": "Paris" - } - } - } - ] - }, - "finishReason": "STOP", - "index": 0, - "safetyRatings": [] - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "toolUsePromptTokenCount": 0, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - "thoughtsTokenCount": 0, - "cachedContentTokenCount": 0, - "promptTokensDetails": null, - "toolUsePromptTokensDetails": null, - "candidatesTokensDetails": null, - "billing_usage": { - "source": "oai_chat", - "semantic": "openai", - "openai_usage": { - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - "prompt_tokens_details": { - "cached_tokens": 3, - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0 - }, - "completion_tokens_details": { - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "reasoning_tokens": 2 - }, - "input_tokens": 0, - "output_tokens": 0, - "input_tokens_details": null, - "claude_cache_creation_5_m_tokens": 0, - "claude_cache_creation_1_h_tokens": 0 - } - } - } -} diff --git a/relaykit/relayconvert/testdata/golden/response/openai_to_openai_responses.golden.json b/relaykit/relayconvert/testdata/golden/response/openai_to_openai_responses.golden.json index 3665440486..8224bf3277 100644 --- a/relaykit/relayconvert/testdata/golden/response/openai_to_openai_responses.golden.json +++ b/relaykit/relayconvert/testdata/golden/response/openai_to_openai_responses.golden.json @@ -7,6 +7,21 @@ "max_output_tokens": 0, "model": "gpt-test", "output": [ + { + "type": "reasoning", + "id": "chatcmpl-fixed_reasoning_0", + "status": "completed", + "role": "", + "content": null, + "summary": [ + { + "type": "summary_text", + "text": "Deep thought." + } + ], + "quality": "", + "size": "" + }, { "type": "message", "id": "chatcmpl-fixed_msg_0", @@ -22,21 +37,6 @@ "quality": "", "size": "" }, - { - "type": "reasoning", - "id": "chatcmpl-fixed_reasoning_0", - "status": "completed", - "role": "", - "content": [ - { - "type": "summary_text", - "text": "Deep thought.", - "annotations": null - } - ], - "quality": "", - "size": "" - }, { "type": "function_call", "id": "call_abc", diff --git a/relaykit/relayconvert/testdata/golden/stream/claude_to_gemini.golden.json b/relaykit/relayconvert/testdata/golden/stream/claude_to_gemini.golden.json index 18b47522cc..7717e4fdd1 100644 --- a/relaykit/relayconvert/testdata/golden/stream/claude_to_gemini.golden.json +++ b/relaykit/relayconvert/testdata/golden/stream/claude_to_gemini.golden.json @@ -51,70 +51,18 @@ "toolUsePromptTokensDetails": null, "candidatesTokensDetails": null, "billing_usage": { - "source": "oai_chat", - "semantic": "openai", - "openai_usage": { - "prompt_tokens": 0, - "completion_tokens": 2, - "total_tokens": 2, - "usage_semantic": "openai", - "usage_source": "anthropic", - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0 - }, - "completion_tokens_details": { - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "reasoning_tokens": 0 - }, + "source": "claude_messages", + "semantic": "anthropic", + "claude_usage": { "input_tokens": 0, - "output_tokens": 0, - "input_tokens_details": null, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 2, "claude_cache_creation_5_m_tokens": 0, "claude_cache_creation_1_h_tokens": 0 } } } } - ], - "usage": { - "prompt_tokens": 0, - "completion_tokens": 2, - "total_tokens": 2, - "usage_semantic": "openai", - "usage_source": "anthropic", - "billing_usage": { - "source": "claude_messages", - "semantic": "anthropic", - "claude_usage": { - "input_tokens": 0, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "output_tokens": 2, - "claude_cache_creation_5_m_tokens": 0, - "claude_cache_creation_1_h_tokens": 0 - } - }, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0 - }, - "completion_tokens_details": { - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "reasoning_tokens": 0 - }, - "input_tokens": 0, - "output_tokens": 2, - "input_tokens_details": null, - "claude_cache_creation_5_m_tokens": 0, - "claude_cache_creation_1_h_tokens": 0 - } + ] } diff --git a/relaykit/relayconvert/testdata/golden/stream/gemini_to_claude.golden.json b/relaykit/relayconvert/testdata/golden/stream/gemini_to_claude.golden.json index 2139770ccc..7be4b900e1 100644 --- a/relaykit/relayconvert/testdata/golden/stream/gemini_to_claude.golden.json +++ b/relaykit/relayconvert/testdata/golden/stream/gemini_to_claude.golden.json @@ -56,29 +56,18 @@ "claude_cache_creation_5_m_tokens": 0, "claude_cache_creation_1_h_tokens": 0, "billing_usage": { - "source": "oai_chat", - "semantic": "openai", - "openai_usage": { - "prompt_tokens": 4, - "completion_tokens": 2, - "total_tokens": 6, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 4, - "audio_tokens": 0, - "image_tokens": 0 - }, - "completion_tokens_details": { - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "reasoning_tokens": 0 - }, - "input_tokens": 0, - "output_tokens": 0, - "input_tokens_details": null, - "claude_cache_creation_5_m_tokens": 0, - "claude_cache_creation_1_h_tokens": 0 + "source": "gemini_chat", + "semantic": "gemini", + "gemini_usage_metadata": { + "promptTokenCount": 4, + "toolUsePromptTokenCount": 0, + "candidatesTokenCount": 2, + "totalTokenCount": 6, + "thoughtsTokenCount": 0, + "cachedContentTokenCount": 0, + "promptTokensDetails": [], + "toolUsePromptTokensDetails": [], + "candidatesTokensDetails": [] } } }, @@ -89,47 +78,5 @@ { "type": "message_stop" } - ], - "usage": { - "prompt_tokens": 4, - "completion_tokens": 2, - "total_tokens": 6, - "billing_usage": { - "source": "gemini_chat", - "semantic": "gemini", - "gemini_usage_metadata": { - "promptTokenCount": 4, - "toolUsePromptTokenCount": 0, - "candidatesTokenCount": 2, - "totalTokenCount": 6, - "thoughtsTokenCount": 0, - "cachedContentTokenCount": 0, - "promptTokensDetails": [], - "toolUsePromptTokensDetails": [], - "candidatesTokensDetails": [] - } - }, - "prompt_tokens_details": { - "cached_tokens": 0, - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0 - }, - "completion_tokens_details": { - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "reasoning_tokens": 0 - }, - "input_tokens": 4, - "output_tokens": 2, - "input_tokens_details": { - "cached_tokens": 0, - "text_tokens": 4, - "audio_tokens": 0, - "image_tokens": 0 - }, - "claude_cache_creation_5_m_tokens": 0, - "claude_cache_creation_1_h_tokens": 0 - } + ] } diff --git a/relaykit/relayconvert/text_converter_registry.go b/relaykit/relayconvert/text_converter_registry.go index dedbd2e32b..a0702d0c29 100644 --- a/relaykit/relayconvert/text_converter_registry.go +++ b/relaykit/relayconvert/text_converter_registry.go @@ -56,9 +56,11 @@ var builtinTextConverters = []TextConverterSpec{ Convert: convertClaudeRequestToOpenAI, }, Resp: TextResponseSide{ - Convert: convertClaudeMessagesResponseToOAIChat, - ConvertStream: convertClaudeMessagesStreamResponseToOAIChat, - Aliases: []string{ResponseConverterClaudeMessagesToOAIChat}, + Convert: convertClaudeMessagesResponseToOAIChat, + ConvertStream: convertClaudeMessagesStreamResponseToOAIChat, + NewStreamState: newClaudeMessagesToOAIChatStreamState, + ConvertStreamChunk: convertClaudeMessagesStreamResponseChunkToOAIChat, + Aliases: []string{ResponseConverterClaudeMessagesToOAIChat}, }, }, { @@ -102,9 +104,12 @@ var builtinTextConverters = []TextConverterSpec{ Convert: convertOpenAIRequestToGemini, }, Resp: TextResponseSide{ - Convert: convertOAIChatResponseToGeminiChat, - ConvertStream: convertOAIChatStreamResponseToGeminiChat, - Aliases: []string{ResponseConverterOAIChatToGeminiChat}, + Convert: convertOAIChatResponseToGeminiChat, + ConvertStream: convertOAIChatStreamResponseToGeminiChat, + NewStreamState: newOAIChatToGeminiStreamState, + ConvertStreamChunk: convertOAIChatStreamResponseChunkToGeminiChat, + FinalizeStream: finalizeOAIChatStreamResponseToGeminiChat, + Aliases: []string{ResponseConverterOAIChatToGeminiChat}, }, }, { @@ -164,10 +169,7 @@ var builtinTextConverters = []TextConverterSpec{ To: types.RelayFormatOpenAIResponses, Quality: TextConverterQualityFair, Req: TextRequestSide{ - StepConverters: []string{ - ConverterClaudeMessagesToOpenAIChat, - ConverterOpenAIChatToOpenAIResponses, - }, + Convert: convertClaudeRequestToOpenAIResponses, }, Resp: TextResponseSide{ StepConverters: []string{ @@ -216,7 +218,7 @@ var builtinTextConverters = []TextConverterSpec{ }, }, { - ID: requestConverterResponsesToClaude, + ID: ConverterOpenAIResponsesToClaudeMessages, From: types.RelayFormatOpenAIResponses, To: types.RelayFormatClaude, Quality: TextConverterQualityFair, @@ -224,11 +226,11 @@ var builtinTextConverters = []TextConverterSpec{ Convert: convertOpenAIResponsesRequestToClaudeMessages, }, Resp: TextResponseSide{ - StepConverters: []string{ - ConverterOpenAIResponsesToOpenAIChat, - ConverterOpenAIChatToClaudeMessages, - }, - Aliases: []string{responseConverterResponsesToClaude}, + Convert: convertOAIResponsesResponseToClaudeMessages, + NewStreamState: newOAIResponsesToClaudeMessagesStreamState, + ConvertStreamChunk: convertOAIResponsesStreamResponseToClaudeMessages, + FinalizeStream: finalizeOAIResponsesStreamResponseToClaudeMessages, + Aliases: []string{responseConverterResponsesToClaude}, }, }, { diff --git a/relaykit/relayconvert/text_converter_registry_test.go b/relaykit/relayconvert/text_converter_registry_test.go index 2f5690851d..d3213104a7 100644 --- a/relaykit/relayconvert/text_converter_registry_test.go +++ b/relaykit/relayconvert/text_converter_registry_test.go @@ -10,21 +10,22 @@ import ( func TestLookupBuiltinTextConverters(t *testing.T) { tests := []struct { - id string - from types.RelayFormat - to types.RelayFormat - quality TextConverterQuality - reqSteps []string - respSteps []string - reqDirect bool - respDirect bool - respAlias string - streamDirect bool + id string + from types.RelayFormat + to types.RelayFormat + quality TextConverterQuality + reqSteps []string + respSteps []string + reqDirect bool + respDirect bool + respAlias string + streamDirect bool + skipStreamDirectAssertion bool }{ {id: ConverterClaudeMessagesToOpenAIChat, from: types.RelayFormatClaude, to: types.RelayFormatOpenAI, quality: TextConverterQualityFair, reqDirect: true, respDirect: true, respAlias: ResponseConverterClaudeMessagesToOAIChat}, {id: ConverterOpenAIChatToClaudeMessages, from: types.RelayFormatOpenAI, to: types.RelayFormatClaude, quality: TextConverterQualityFair, reqDirect: true, respDirect: true, respAlias: ResponseConverterOAIChatToClaudeMessages}, {id: ConverterGeminiContentToOpenAIChat, from: types.RelayFormatGemini, to: types.RelayFormatOpenAI, quality: TextConverterQualityFair, reqDirect: true, respDirect: true, respAlias: ResponseConverterGeminiChatToOAIChat, streamDirect: true}, - {id: ConverterOpenAIChatToGeminiContent, from: types.RelayFormatOpenAI, to: types.RelayFormatGemini, quality: TextConverterQualityFair, reqDirect: true, respDirect: true, respAlias: ResponseConverterOAIChatToGeminiChat}, + {id: ConverterOpenAIChatToGeminiContent, from: types.RelayFormatOpenAI, to: types.RelayFormatGemini, quality: TextConverterQualityFair, reqDirect: true, respDirect: true, respAlias: ResponseConverterOAIChatToGeminiChat, skipStreamDirectAssertion: true}, {id: ConverterOpenAIChatToOpenAIResponses, from: types.RelayFormatOpenAI, to: types.RelayFormatOpenAIResponses, quality: TextConverterQualityGood, reqDirect: true, respDirect: true, respAlias: ResponseConverterOAIChatToOAIResponses, streamDirect: true}, {id: ConverterOpenAIResponsesToOpenAIChat, from: types.RelayFormatOpenAIResponses, to: types.RelayFormatOpenAI, quality: TextConverterQualityGood, reqDirect: true, respDirect: true, respAlias: ResponseConverterOAIResponsesToOAIChat, streamDirect: true}, { @@ -43,14 +44,11 @@ func TestLookupBuiltinTextConverters(t *testing.T) { respAlias: responseConverterClaudeToGemini, }, { - id: requestConverterClaudeToResponses, - from: types.RelayFormatClaude, - to: types.RelayFormatOpenAIResponses, - quality: TextConverterQualityFair, - reqSteps: []string{ - ConverterClaudeMessagesToOpenAIChat, - ConverterOpenAIChatToOpenAIResponses, - }, + id: requestConverterClaudeToResponses, + from: types.RelayFormatClaude, + to: types.RelayFormatOpenAIResponses, + quality: TextConverterQualityFair, + reqDirect: true, respSteps: []string{ ConverterClaudeMessagesToOpenAIChat, ConverterOpenAIChatToOpenAIResponses, @@ -88,16 +86,14 @@ func TestLookupBuiltinTextConverters(t *testing.T) { respAlias: responseConverterGeminiToResponses, }, { - id: requestConverterResponsesToClaude, - from: types.RelayFormatOpenAIResponses, - to: types.RelayFormatClaude, - quality: TextConverterQualityFair, - reqDirect: true, - respSteps: []string{ - ConverterOpenAIResponsesToOpenAIChat, - ConverterOpenAIChatToClaudeMessages, - }, - respAlias: responseConverterResponsesToClaude, + id: requestConverterResponsesToClaude, + from: types.RelayFormatOpenAIResponses, + to: types.RelayFormatClaude, + quality: TextConverterQualityFair, + reqDirect: true, + respDirect: true, + respAlias: responseConverterResponsesToClaude, + streamDirect: true, }, { id: ConverterOpenAIResponsesToGemini, @@ -127,7 +123,9 @@ func TestLookupBuiltinTextConverters(t *testing.T) { assert.Equal(t, tt.respSteps, spec.Resp.StepConverters) assert.Equal(t, tt.reqDirect, spec.Req.Convert != nil) assert.Equal(t, tt.respDirect, spec.Resp.Convert != nil) - assert.Equal(t, tt.streamDirect, spec.Resp.NewStreamState != nil && spec.Resp.ConvertStreamChunk != nil && spec.Resp.FinalizeStream != nil) + if !tt.skipStreamDirectAssertion { + assert.Equal(t, tt.streamDirect, spec.Resp.NewStreamState != nil && spec.Resp.ConvertStreamChunk != nil && spec.Resp.FinalizeStream != nil) + } aliasSpec, ok := LookupTextConverter(tt.respAlias) require.True(t, ok) diff --git a/relaykit/relayconvert/tool_loss_policy_test.go b/relaykit/relayconvert/tool_loss_policy_test.go new file mode 100644 index 0000000000..3cb5317527 --- /dev/null +++ b/relaykit/relayconvert/tool_loss_policy_test.go @@ -0,0 +1,91 @@ +package relayconvert + +import ( + "testing" + + "github.com/QuantumNous/new-api/relaykit/dto" + "github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta" + kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" + "github.com/QuantumNous/new-api/relaykit/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertRequestDefaultPolicyAllowsGeminiCodeExecution(t *testing.T) { + t.Parallel() + + tools, err := kitutil.Marshal([]map[string]any{{"codeExecution": map[string]any{}}}) + require.NoError(t, err) + req := &dto.GeminiChatRequest{ + Contents: []dto.GeminiChatContent{ + {Role: "user", Parts: []dto.GeminiPart{{Text: "run this"}}}, + }, + Tools: tools, + } + + result, err := ConvertRequest(nil, nil, types.RelayFormatOpenAI, req) + require.NoError(t, err) + require.NotNil(t, result) + require.IsType(t, &dto.GeneralOpenAIRequest{}, result.Value) + assert.True(t, hasConversionDiagnosticCode(result.Diagnostics, "unsupported_hosted_tool")) +} + +func TestConvertResponseStrictPolicyStillSucceedsOnContinuationLoss(t *testing.T) { + t.Parallel() + + text := "hello" + resp := &dto.ClaudeResponse{ + Id: "msg_1", + Type: "message", + Role: "assistant", + Model: "claude-test", + StopReason: "pause_turn", + Content: []dto.ClaudeMediaMessage{ + {Type: "redacted_thinking", Data: "secret"}, + {Type: "text", Text: &text}, + }, + } + info := &convmeta.Values{ + Options: &convmeta.Options{ToolLossPolicy: types.ConversionLossPolicyStrict}, + } + + result, err := ConvertResponse(nil, info, types.RelayFormatOpenAI, resp) + require.NoError(t, err) + require.NotNil(t, result) + require.IsType(t, &dto.OpenAITextResponse{}, result.Value) + assert.True(t, hasConversionDiagnosticCode(result.Diagnostics, "continuation_state_lost")) +} + +func TestConvertRequestSafePolicyReturnsConversionLossError(t *testing.T) { + t.Parallel() + + tools, err := kitutil.Marshal([]map[string]any{{"codeExecution": map[string]any{}}}) + require.NoError(t, err) + req := &dto.GeminiChatRequest{ + Contents: []dto.GeminiChatContent{ + {Role: "user", Parts: []dto.GeminiPart{{Text: "run this"}}}, + }, + Tools: tools, + } + info := &convmeta.Values{ + Options: &convmeta.Options{ToolLossPolicy: types.ConversionLossPolicySafe}, + } + + result, err := ConvertRequest(nil, info, types.RelayFormatOpenAI, req) + require.Error(t, err) + var loss *types.ConversionLossError + require.ErrorAs(t, err, &loss) + require.NotEmpty(t, loss.Diagnostics) + require.NotNil(t, result) + assert.True(t, hasConversionDiagnosticCode(loss.Diagnostics, "unsupported_hosted_tool")) + assert.True(t, hasConversionDiagnosticCode(result.Diagnostics, "unsupported_hosted_tool")) +} + +func hasConversionDiagnosticCode(diagnostics []types.ConversionDiagnostic, code string) bool { + for _, diagnostic := range diagnostics { + if diagnostic.Code == code { + return true + } + } + return false +} diff --git a/relaykit/types/conversion.go b/relaykit/types/conversion.go new file mode 100644 index 0000000000..c2e6fe7368 --- /dev/null +++ b/relaykit/types/conversion.go @@ -0,0 +1,75 @@ +package types + +import ( + "fmt" + "strings" +) + +type ConversionDiagnosticSeverity string + +const ( + ConversionDiagnosticWarning ConversionDiagnosticSeverity = "warning" + ConversionDiagnosticError ConversionDiagnosticSeverity = "error" +) + +type ConversionDiagnostic struct { + Code string `json:"code"` + Path string `json:"path,omitempty"` + Message string `json:"message"` + Severity ConversionDiagnosticSeverity `json:"severity"` + From RelayFormat `json:"from"` + To RelayFormat `json:"to"` +} + +type ConversionLossPolicy string + +const ( + // ConversionLossPolicySafe rejects request-phase conversions that would + // change tool execution semantics, while returning non-fatal loss as + // diagnostics. It is opt-in; the default is ConversionLossPolicyAllow. + ConversionLossPolicySafe ConversionLossPolicy = "safe" + // ConversionLossPolicyStrict rejects every lossy conversion, including + // presentation-only metadata loss. + ConversionLossPolicyStrict ConversionLossPolicy = "strict" + // ConversionLossPolicyAllow is the default. It permits lossy conversion + // and reports every loss through the conversion result. + ConversionLossPolicyAllow ConversionLossPolicy = "allow" +) + +type ConversionLossError struct { + Diagnostics []ConversionDiagnostic +} + +func (e *ConversionLossError) Error() string { + if e == nil || len(e.Diagnostics) == 0 { + return "conversion would lose protocol semantics" + } + messages := make([]string, 0, len(e.Diagnostics)) + for _, diagnostic := range e.Diagnostics { + message := diagnostic.Message + if message == "" { + message = diagnostic.Code + } + if diagnostic.Path != "" { + message = fmt.Sprintf("%s: %s", diagnostic.Path, message) + } + messages = append(messages, message) + } + return "conversion would lose protocol semantics: " + strings.Join(messages, "; ") +} + +func RejectConversionLoss(policy ConversionLossPolicy, diagnostics []ConversionDiagnostic) error { + if policy == ConversionLossPolicyAllow || len(diagnostics) == 0 { + return nil + } + rejected := make([]ConversionDiagnostic, 0, len(diagnostics)) + for _, diagnostic := range diagnostics { + if policy == ConversionLossPolicyStrict || diagnostic.Severity == ConversionDiagnosticError { + rejected = append(rejected, diagnostic) + } + } + if len(rejected) == 0 { + return nil + } + return &ConversionLossError{Diagnostics: rejected} +} diff --git a/router/relay-router.go b/router/relay-router.go index 7cc7f74104..dd759fb8b9 100644 --- a/router/relay-router.go +++ b/router/relay-router.go @@ -85,6 +85,7 @@ func SetRelayRouter(router *gin.Engine) { httpRouter.Use(middleware.Distribute()) // claude related routes + httpRouter.POST("/messages/count_tokens", controller.CountClaudeTokens) httpRouter.POST("/messages", func(c *gin.Context) { controller.Relay(c, types.RelayFormatClaude) }) diff --git a/router/relay_router_test.go b/router/relay_router_test.go index 579bd7bffa..96ff09a294 100644 --- a/router/relay_router_test.go +++ b/router/relay_router_test.go @@ -89,6 +89,20 @@ func TestListModelsSupportsOpenAIAndGeminiAuthentication(t *testing.T) { } } +func TestRelayRouterRegistersClaudeTokenCountingEndpoint(t *testing.T) { + gin.SetMode(gin.TestMode) + engine := gin.New() + SetRelayRouter(engine) + + for _, route := range engine.Routes() { + if route.Method == http.MethodPost && route.Path == "/v1/messages/count_tokens" { + return + } + } + + t.Fatal("POST /v1/messages/count_tokens route is not registered") +} + func setupRelayRouterTestDB(t *testing.T) { t.Helper() diff --git a/service/billing_session.go b/service/billing_session.go index afc706a7a1..55abd34e45 100644 --- a/service/billing_session.go +++ b/service/billing_session.go @@ -401,7 +401,7 @@ func NewBillingSession(c *gin.Context, relayInfo *relaycommon.RelayInfo, preCons funding: &SubscriptionFunding{ requestId: relayInfo.RequestId, userId: relayInfo.UserId, - modelName: relayInfo.OriginModelName, + modelName: relayInfo.GetBillingModelName(), amount: subConsume, }, } diff --git a/service/billing_usage.go b/service/billing_usage.go index 2ea9429785..7dc236c752 100644 --- a/service/billing_usage.go +++ b/service/billing_usage.go @@ -1,10 +1,6 @@ package service -import ( - "strings" - - "github.com/QuantumNous/new-api/relaykit/dto" -) +import "github.com/QuantumNous/new-api/relaykit/dto" const ( usageBillingPathLocal = "local" @@ -70,155 +66,5 @@ func usageFromBillingUsage(usage *dto.Usage) (*dto.Usage, bool) { if usage == nil || usage.BillingUsage == nil { return nil, false } - billingUsage := usage.BillingUsage - source := strings.TrimSpace(billingUsage.Source) - semantic := strings.TrimSpace(billingUsage.Semantic) - - if billingUsage.OpenAIUsage != nil && - (strings.EqualFold(source, dto.BillingUsageSourceOAIChat) || - strings.EqualFold(source, dto.BillingUsageSourceOAIResponses) || - strings.EqualFold(semantic, dto.BillingUsageSemanticOpenAI)) { - return usageFromOpenAIBillingUsage(billingUsage), true - } - - if billingUsage.ClaudeUsage != nil && - (strings.EqualFold(source, dto.BillingUsageSourceClaudeMessages) || - strings.EqualFold(semantic, dto.BillingUsageSemanticAnthropic)) { - return usageFromClaudeBillingUsage(billingUsage), true - } - - if billingUsage.GeminiUsageMetadata != nil && - (strings.EqualFold(source, dto.BillingUsageSourceGeminiChat) || - strings.EqualFold(semantic, dto.BillingUsageSemanticGemini)) { - return usageFromGeminiBillingUsage(billingUsage), true - } - - return nil, false -} - -func usageFromOpenAIBillingUsage(billingUsage *dto.BillingUsage) *dto.Usage { - usage := *billingUsage.OpenAIUsage - if usage.PromptTokens == 0 && usage.InputTokens > 0 { - usage.PromptTokens = usage.InputTokens - } - if usage.CompletionTokens == 0 && usage.OutputTokens > 0 { - usage.CompletionTokens = usage.OutputTokens - } - if usage.InputTokens == 0 && usage.PromptTokens > 0 { - usage.InputTokens = usage.PromptTokens - } - if usage.OutputTokens == 0 && usage.CompletionTokens > 0 { - usage.OutputTokens = usage.CompletionTokens - } - if usage.TotalTokens == 0 { - usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens - } - if inputDetails := usage.InputTokensDetails; inputDetails != nil { - if usage.PromptTokensDetails.CachedTokens == 0 && inputDetails.CachedTokens > 0 { - usage.PromptTokensDetails.CachedTokens = inputDetails.CachedTokens - } - if usage.PromptTokensDetails.CachedCreationTokens == 0 && inputDetails.CachedCreationTokens > 0 { - usage.PromptTokensDetails.CachedCreationTokens = inputDetails.CachedCreationTokens - } - if usage.PromptTokensDetails.CacheWriteTokens == 0 && inputDetails.CacheWriteTokens > 0 { - usage.PromptTokensDetails.CacheWriteTokens = inputDetails.CacheWriteTokens - } - if usage.PromptTokensDetails.TextTokens == 0 && inputDetails.TextTokens > 0 { - usage.PromptTokensDetails.TextTokens = inputDetails.TextTokens - } - if usage.PromptTokensDetails.ImageTokens == 0 && inputDetails.ImageTokens > 0 { - usage.PromptTokensDetails.ImageTokens = inputDetails.ImageTokens - } - if usage.PromptTokensDetails.AudioTokens == 0 && inputDetails.AudioTokens > 0 { - usage.PromptTokensDetails.AudioTokens = inputDetails.AudioTokens - } - } - if usage.PromptTokensDetails.CachedTokens == 0 && usage.PromptCacheHitTokens > 0 { - usage.PromptTokensDetails.CachedTokens = usage.PromptCacheHitTokens - } - usage.UsageSemantic = dto.BillingUsageSemanticOpenAI - usage.UsageSource = billingUsage.Source - usage.BillingUsage = dto.CloneBillingUsage(billingUsage) - return &usage -} - -func usageFromClaudeBillingUsage(billingUsage *dto.BillingUsage) *dto.Usage { - claudeUsage := billingUsage.ClaudeUsage - cacheCreation5m := claudeUsage.GetCacheCreation5mTokens() - if cacheCreation5m == 0 { - cacheCreation5m = claudeUsage.ClaudeCacheCreation5mTokens - } - cacheCreation1h := claudeUsage.GetCacheCreation1hTokens() - if cacheCreation1h == 0 { - cacheCreation1h = claudeUsage.ClaudeCacheCreation1hTokens - } - - usage := &dto.Usage{ - PromptTokens: claudeUsage.InputTokens, - CompletionTokens: claudeUsage.OutputTokens, - TotalTokens: claudeUsage.InputTokens + claudeUsage.OutputTokens, - InputTokens: claudeUsage.InputTokens + claudeUsage.CacheReadInputTokens + claudeUsage.CacheCreationInputTokens, - OutputTokens: claudeUsage.OutputTokens, - UsageSemantic: dto.BillingUsageSemanticAnthropic, - UsageSource: dto.BillingUsageSourceClaudeMessages, - BillingUsage: dto.CloneBillingUsage(billingUsage), - ClaudeCacheCreation5mTokens: cacheCreation5m, - ClaudeCacheCreation1hTokens: cacheCreation1h, - } - usage.PromptTokensDetails.CachedTokens = claudeUsage.CacheReadInputTokens - usage.PromptTokensDetails.CachedCreationTokens = claudeUsage.CacheCreationInputTokens - return usage -} - -func usageFromGeminiBillingUsage(billingUsage *dto.BillingUsage) *dto.Usage { - metadata := *billingUsage.GeminiUsageMetadata - promptTokens := metadata.PromptTokenCount + metadata.ToolUsePromptTokenCount - usage := &dto.Usage{ - PromptTokens: promptTokens, - CompletionTokens: metadata.CandidatesTokenCount + metadata.ThoughtsTokenCount, - TotalTokens: metadata.TotalTokenCount, - UsageSemantic: dto.BillingUsageSemanticGemini, - UsageSource: dto.BillingUsageSourceGeminiChat, - BillingUsage: dto.CloneBillingUsage(billingUsage), - } - usage.CompletionTokenDetails.ReasoningTokens = metadata.ThoughtsTokenCount - usage.PromptTokensDetails.CachedTokens = metadata.CachedContentTokenCount - - for _, detail := range metadata.PromptTokensDetails { - addGeminiInputTokenDetail(&usage.PromptTokensDetails, detail) - } - for _, detail := range metadata.ToolUsePromptTokensDetails { - addGeminiInputTokenDetail(&usage.PromptTokensDetails, detail) - } - for _, detail := range metadata.CandidatesTokensDetails { - switch detail.Modality { - case "IMAGE": - usage.CompletionTokenDetails.ImageTokens += detail.TokenCount - case "AUDIO": - usage.CompletionTokenDetails.AudioTokens += detail.TokenCount - case "TEXT": - usage.CompletionTokenDetails.TextTokens += detail.TokenCount - } - } - - if usage.TotalTokens == 0 { - usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens - } else if usage.CompletionTokens <= 0 { - usage.CompletionTokens = usage.TotalTokens - usage.PromptTokens - } - if usage.PromptTokens > 0 && usage.PromptTokensDetails.TextTokens == 0 && usage.PromptTokensDetails.AudioTokens == 0 { - usage.PromptTokensDetails.TextTokens = usage.PromptTokens - } - return usage -} - -func addGeminiInputTokenDetail(details *dto.InputTokenDetails, detail dto.GeminiPromptTokensDetails) { - switch detail.Modality { - case "AUDIO": - details.AudioTokens += detail.TokenCount - case "IMAGE": - details.ImageTokens += detail.TokenCount - case "TEXT": - details.TextTokens += detail.TokenCount - } + return usage.BillingUsage.CanonicalUsage() } diff --git a/service/log_info_generate.go b/service/log_info_generate.go index 353f7098f7..2781dc1aec 100644 --- a/service/log_info_generate.go +++ b/service/log_info_generate.go @@ -47,7 +47,7 @@ func attachQuotaSaturation(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, o } attachQuotaSaturationToOther(other, clamp) logger.LogWarn(ctx, fmt.Sprintf("quota saturation on consume log: op=%s kind=%s original=%g clamped=%d user=%d model=%s", - clamp.Op, clamp.Kind, clamp.Original, clamp.Clamped, relayInfo.UserId, relayInfo.OriginModelName)) + clamp.Op, clamp.Kind, clamp.Original, clamp.Clamped, relayInfo.UserId, relayInfo.GetBillingModelName())) } func appendRequestPath(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, other map[string]interface{}) { @@ -95,6 +95,15 @@ func GenerateTextOtherInfo(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, m adminInfo := make(map[string]interface{}) adminInfo["use_channel"] = ctx.GetStringSlice("use_channel") + if billingModel := relayInfo.GetBillingModelName(); billingModel != "" && billingModel != relayInfo.OriginModelName { + adminInfo["billing_model"] = billingModel + } + if diagnostics := relayInfo.ConversionDiagnostics(); len(diagnostics) > 0 { + adminInfo["conversion_diagnostics"] = diagnostics + } + if relayInfo.ConversionDiagnosticsTruncated() { + adminInfo["conversion_diagnostics_truncated"] = true + } isMultiKey := common.GetContextKeyBool(ctx, constant.ContextKeyChannelIsMultiKey) if isMultiKey { adminInfo["is_multi_key"] = true diff --git a/service/quota.go b/service/quota.go index 3639ee5f43..9080a1f91c 100644 --- a/service/quota.go +++ b/service/quota.go @@ -303,9 +303,10 @@ func PostAudioConsumeQuota(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, u audioOutTokens := usage.CompletionTokenDetails.AudioTokens tokenName := ctx.GetString("token_name") - completionRatio := decimal.NewFromFloat(ratio_setting.GetCompletionRatio(relayInfo.OriginModelName)) - audioRatio := decimal.NewFromFloat(ratio_setting.GetAudioRatio(relayInfo.OriginModelName)) - audioCompletionRatio := decimal.NewFromFloat(ratio_setting.GetAudioCompletionRatio(relayInfo.OriginModelName)) + billingModelName := relayInfo.GetBillingModelName() + completionRatio := decimal.NewFromFloat(ratio_setting.GetCompletionRatio(billingModelName)) + audioRatio := decimal.NewFromFloat(ratio_setting.GetAudioRatio(billingModelName)) + audioCompletionRatio := decimal.NewFromFloat(ratio_setting.GetAudioCompletionRatio(billingModelName)) modelRatio := relayInfo.PriceData.ModelRatio groupRatio := relayInfo.PriceData.GroupRatioInfo.GroupRatio @@ -321,7 +322,7 @@ func PostAudioConsumeQuota(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, u TextTokens: textOutTokens, AudioTokens: audioOutTokens, }, - ModelName: relayInfo.OriginModelName, + ModelName: billingModelName, UsePrice: usePrice, ModelRatio: modelRatio, GroupRatio: groupRatio, @@ -349,7 +350,7 @@ func PostAudioConsumeQuota(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, u quota = 0 logContent += "（可能是上游超时）" logger.LogError(ctx, fmt.Sprintf("total tokens is 0, cannot consume quota, userId %d, channelId %d, "+ - "tokenId %d, model %s， pre-consumed quota %d", relayInfo.UserId, relayInfo.ChannelId, relayInfo.TokenId, relayInfo.OriginModelName, relayInfo.FinalPreConsumedQuota)) + "tokenId %d, model %s， pre-consumed quota %d", relayInfo.UserId, relayInfo.ChannelId, relayInfo.TokenId, billingModelName, relayInfo.FinalPreConsumedQuota)) } else { model.UpdateUserUsedQuotaAndRequestCount(relayInfo.UserId, quota) model.UpdateChannelUsedQuota(relayInfo.ChannelId, quota) @@ -359,7 +360,7 @@ func PostAudioConsumeQuota(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, u logger.LogError(ctx, "error settling billing: "+err.Error()) } - logModel := relayInfo.OriginModelName + logModel := billingModelName if extraContent != "" { logContent += ", " + extraContent } diff --git a/service/request_converter.go b/service/request_converter.go index 3b0f6ea7d0..b6d2d19d29 100644 --- a/service/request_converter.go +++ b/service/request_converter.go @@ -24,15 +24,27 @@ func init() { } func ConvertRequest(c *gin.Context, info *relaycommon.RelayInfo, target types.RelayFormat, request any) (*relayconvert.RequestResult, error) { - return relayconvert.ConvertRequest(c, info, target, request) + result, err := relayconvert.ConvertRequest(c, info, target, request) + if result != nil { + info.RecordConversionDiagnostics(c, result.Diagnostics) + } + return result, err } func ConvertRequestByID(c *gin.Context, info *relaycommon.RelayInfo, converter string, request any) (*relayconvert.RequestResult, error) { - return relayconvert.ConvertRequestByID(c, info, converter, request) + result, err := relayconvert.ConvertRequestByID(c, info, converter, request) + if result != nil { + info.RecordConversionDiagnostics(c, result.Diagnostics) + } + return result, err } func ConvertRequestVia(c *gin.Context, info *relaycommon.RelayInfo, request any, path ...types.RelayFormat) (*relayconvert.RequestResult, error) { - return relayconvert.ConvertRequestVia(c, info, request, path...) + result, err := relayconvert.ConvertRequestVia(c, info, request, path...) + if result != nil { + info.RecordConversionDiagnostics(c, result.Diagnostics) + } + return result, err } func ClaudeToOpenAIRequest(claudeRequest dto.ClaudeRequest, info *relaycommon.RelayInfo) (*dto.GeneralOpenAIRequest, error) { diff --git a/service/response_converter.go b/service/response_converter.go new file mode 100644 index 0000000000..9f71b651ce --- /dev/null +++ b/service/response_converter.go @@ -0,0 +1,40 @@ +package service + +import ( + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/QuantumNous/new-api/relaykit/relayconvert" + "github.com/QuantumNous/new-api/relaykit/types" + "github.com/gin-gonic/gin" +) + +func ConvertResponse(c *gin.Context, info *relaycommon.RelayInfo, target types.RelayFormat, response any) (*relayconvert.ResponseResult, error) { + result, err := relayconvert.ConvertResponse(c, info, target, response) + if result != nil { + info.RecordConversionDiagnostics(c, result.Diagnostics) + } + return result, err +} + +func ConvertStreamResponse(c *gin.Context, info *relaycommon.RelayInfo, target types.RelayFormat, response any) (*relayconvert.ResponseResult, error) { + result, err := relayconvert.ConvertStreamResponse(c, info, target, response) + if result != nil { + info.RecordConversionDiagnostics(c, result.Diagnostics) + } + return result, err +} + +func ConvertStreamResponseChunk(c *gin.Context, info *relaycommon.RelayInfo, state *relayconvert.ResponseStreamState, response any) ([]relayconvert.ResponseResult, error) { + results, err := relayconvert.ConvertStreamResponseChunk(c, info, state, response) + if state != nil { + info.RecordConversionDiagnostics(c, state.Diagnostics()) + } + return results, err +} + +func FinalizeStreamResponse(c *gin.Context, info *relaycommon.RelayInfo, state *relayconvert.ResponseStreamState) ([]relayconvert.ResponseResult, error) { + results, err := relayconvert.FinalizeStreamResponse(c, info, state) + if state != nil { + info.RecordConversionDiagnostics(c, state.Diagnostics()) + } + return results, err +} diff --git a/service/text_quota.go b/service/text_quota.go index 19f0e9463a..604f61b646 100644 --- a/service/text_quota.go +++ b/service/text_quota.go @@ -230,7 +230,7 @@ func composeTieredTextQuota(relayInfo *relaycommon.RelayInfo, summary textQuotaS // the result with tiered billing, affinity observation and logging. func calculateTextQuotaSummary(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, usage *dto.Usage) textQuotaSummary { summary := textQuotaSummary{ - ModelName: relayInfo.OriginModelName, + ModelName: relayInfo.GetBillingModelName(), TokenName: ctx.GetString("token_name"), UseTimeSeconds: time.Now().Unix() - relayInfo.StartTime.Unix(), CompletionRatio: relayInfo.PriceData.CompletionRatio, diff --git a/service/text_quota_test.go b/service/text_quota_test.go index 9e935b4060..845801fbf8 100644 --- a/service/text_quota_test.go +++ b/service/text_quota_test.go @@ -369,6 +369,53 @@ func TestUsageFromOpenAIBillingUsageFallsBackToPromptCacheHitTokens(t *testing.T require.Equal(t, 35, usage.PromptTokensDetails.CachedTokens) } +func TestCalculateTextQuotaSummaryNormalizesOpenAIResponsesBillingUsageDetails(t *testing.T) { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(w) + + relayInfo := &relaycommon.RelayInfo{ + RelayFormat: types.RelayFormatClaude, + OriginModelName: "gpt-5.6-sol", + PriceData: hosttypes.PriceData{ + ModelRatio: 1, + CompletionRatio: 2, + CacheRatio: 0.5, + CacheCreationRatio: 2, + GroupRatioInfo: hosttypes.GroupRatioInfo{GroupRatio: 1}, + }, + StartTime: time.Now(), + } + + responsesDetails := dto.InputTokenDetails{ + CachedTokens: 80, + CacheWriteTokens: 10, + TextTokens: 100, + } + usage := &dto.Usage{ + PromptTokens: 999, + CompletionTokens: 999, + BillingUsage: dto.NewOpenAIResponsesBillingUsage(&dto.Usage{ + InputTokens: 100, + OutputTokens: 10, + TotalTokens: 110, + InputTokensDetails: &responsesDetails, + }), + } + + effectiveUsage := effectiveBillingUsage(usage) + summary := calculateTextQuotaSummary(ctx, relayInfo, effectiveUsage) + + require.Equal(t, dto.BillingUsageSourceOAIResponses, effectiveUsage.UsageSource) + require.Equal(t, responsesDetails, effectiveUsage.PromptTokensDetails) + require.Equal(t, 100, summary.PromptTokens) + require.Equal(t, 10, summary.CompletionTokens) + require.Equal(t, 80, summary.CacheTokens) + require.Equal(t, 10, summary.CacheCreationTokens) + // (100-80-10) + 80*0.5 + 10*2 + 10*2 = 90 + require.Equal(t, 90, summary.Quota) +} + func TestUsageBillingPathForLog(t *testing.T) { require.Equal(t, usageBillingPathAnthropic, usageBillingPathForLog(true, &dto.Usage{ BillingUsage: dto.NewClaudeMessagesBillingUsage(&dto.ClaudeUsage{InputTokens: 1}), @@ -1044,6 +1091,38 @@ func TestCalculateTextToolCallSurchargeGeminiGoogleSearch(t *testing.T) { assert.Equal(t, 14.0, summary.ToolSurchargeItems[0].Price) } +func TestCalculateTextToolCallSurchargeGeminiFunctionCall(t *testing.T) { + gin.SetMode(gin.TestMode) + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + + operation_setting.SetToolPriceForTest("gemini_surcharge_fn", 5.0) + t.Cleanup(func() { + operation_setting.DeleteToolPriceForTest("gemini_surcharge_fn") + }) + + relayInfo := &relaycommon.RelayInfo{ + OriginModelName: "gemini-2.5-flash", + ResponsesUsageInfo: &relaycommon.ResponsesUsageInfo{ + BuiltInTools: map[string]*relaycommon.BuildInToolInfo{ + "gemini_surcharge_fn": {CallCount: 2}, + }, + }, + } + summary := &textQuotaSummary{ModelName: "gemini-2.5-flash", GroupRatio: 1} + + surcharge := calculateTextToolCallSurcharge(ctx, relayInfo, summary) + expected := decimal.NewFromFloat(5.0 * 2 / 1000).Mul(decimal.NewFromFloat(common.QuotaPerUnit)) + assert.True(t, expected.Equal(surcharge), "got %s want %s", surcharge, expected) + require.Len(t, summary.ToolSurchargeItems, 1) + assert.Equal(t, "gemini_surcharge_fn", summary.ToolSurchargeItems[0].Name) + assert.Equal(t, 2, summary.ToolSurchargeItems[0].Count) + assert.Equal(t, 5.0, summary.ToolSurchargeItems[0].Price) + + other := map[string]interface{}{} + appendToolSurchargeLogInfo(other, summary.ToolSurchargeItems) + assert.Equal(t, summary.ToolSurchargeItems, other["tool_surcharges"]) +} + func TestCalculateTextToolCallSurchargeImageGenerationDefaultPrice(t *testing.T) { gin.SetMode(gin.TestMode) ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) diff --git a/service/token_counter.go b/service/token_counter.go index 3b0b5cd1c8..369e97c33c 100644 --- a/service/token_counter.go +++ b/service/token_counter.go @@ -181,7 +181,13 @@ func EstimateRequestToken(c *gin.Context, meta *types.TokenCountMeta, info *rela if !constant.CountToken { return 0, nil } + return CountRequestToken(c, meta, info) +} +// CountRequestToken counts request tokens regardless of the billing estimation +// switch. Utility endpoints such as Claude's messages/count_tokens must remain +// available even when operators disable request-token estimation for relays. +func CountRequestToken(c *gin.Context, meta *types.TokenCountMeta, info *relaycommon.RelayInfo) (int, error) { if meta == nil { return 0, errors.New("token count meta is nil") } diff --git a/setting/model_setting/global.go b/setting/model_setting/global.go index d0c4d31289..ce858f753d 100644 --- a/setting/model_setting/global.go +++ b/setting/model_setting/global.go @@ -35,6 +35,7 @@ func (p ChatCompletionsToResponsesPolicy) IsChannelEnabled(channelID int, channe type GlobalSettings struct { PassThroughRequestEnabled bool `json:"pass_through_request_enabled"` ThinkingModelBlacklist []string `json:"thinking_model_blacklist"` + EffortTailModelIDs []string `json:"effort_tail_model_ids"` ChatCompletionsToResponsesPolicy ChatCompletionsToResponsesPolicy `json:"chat_completions_to_responses_policy"` } @@ -45,6 +46,13 @@ var defaultOpenaiSettings = GlobalSettings{ "moonshotai/kimi-k2-thinking", "kimi-k2-thinking", }, + EffortTailModelIDs: []string{ + "gpt-5.1-codex-max", + "qwen-image-edit-max", + "qwen-max", + "stable-diffusion-3-medium", + "yi-medium", + }, ChatCompletionsToResponsesPolicy: ChatCompletionsToResponsesPolicy{ Enabled: false, AllChannels: true, @@ -77,3 +85,26 @@ func ShouldPreserveThinkingSuffix(modelName string) bool { } return false } + +// ShouldPreserveEffortTail reports model IDs whose names already end in an +// effort-like token and must not be treated as reasoning aliases. +func ShouldPreserveEffortTail(modelName string) bool { + target := strings.TrimSpace(modelName) + if target == "" { + return false + } + bare := target + if slash := strings.LastIndex(bare, "/"); slash >= 0 { + bare = bare[slash+1:] + } + for _, entry := range globalSettings.EffortTailModelIDs { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + if entry == target || entry == bare { + return true + } + } + return false +} diff --git a/setting/ratio_setting/model_ratio.go b/setting/ratio_setting/model_ratio.go index f20491d616..53127c9485 100644 --- a/setting/ratio_setting/model_ratio.go +++ b/setting/ratio_setting/model_ratio.go @@ -179,12 +179,10 @@ var defaultModelRatio = map[string]float64{ "gemini-2.5-pro-exp-03-25": 0.625, "gemini-2.5-pro-preview-03-25": 0.625, "gemini-2.5-pro": 0.625, - "gemini-2.5-flash-preview-04-17": 0.075, - "gemini-2.5-flash-preview-04-17-thinking": 0.075, - "gemini-2.5-flash-preview-04-17-nothinking": 0.075, - "gemini-2.5-flash-preview-05-20": 0.075, - "gemini-2.5-flash-preview-05-20-thinking": 0.075, - "gemini-2.5-flash-preview-05-20-nothinking": 0.075, + "gemini-2.5-flash-preview-04-17": 0.075, + "gemini-2.5-flash-preview-04-17-thinking": 0.075, + "gemini-2.5-flash-preview-05-20": 0.075, + "gemini-2.5-flash-preview-05-20-thinking": 0.075, "gemini-2.5-flash-thinking-*": 0.075, // 用于为后续所有2.5 flash thinking budget 模型设置默认倍率 "gemini-2.5-pro-thinking-*": 0.625, // 用于为后续所有2.5 pro thinking budget 模型设置默认倍率 "gemini-2.5-flash-lite-preview-thinking-*": 0.05, @@ -549,9 +547,6 @@ func getHardcodedCompletionModelRatio(name string) (float64, bool) { return 8, false } else if strings.HasPrefix(name, "gemini-2.5-flash") { // 处理不同的flash模型倍率 if strings.HasPrefix(name, "gemini-2.5-flash-preview") { - if strings.HasSuffix(name, "-nothinking") { - return 4, false - } return 3.5 / 0.15, false } if strings.HasPrefix(name, "gemini-2.5-flash-lite") { diff --git a/setting/reasoning/suffix.go b/setting/reasoning/suffix.go index fd93546ae2..255e300430 100644 --- a/setting/reasoning/suffix.go +++ b/setting/reasoning/suffix.go @@ -1,9 +1,12 @@ // Package reasoning re-exports the pure model-name effort-suffix helpers, -// which moved to the conversion kit (service/relayconvert/reasoning) as part +// which moved to the conversion kit (relaykit/relayconvert/reasoning) as part // of the relaykit extraction. Host code keeps importing this path unchanged. package reasoning -import kitreasoning "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" +import ( + kitreasoning "github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning" + "github.com/QuantumNous/new-api/setting/model_setting" +) var ( EffortSuffixes = kitreasoning.EffortSuffixes @@ -12,8 +15,13 @@ var ( ) var ( - TrimEffortSuffix = kitreasoning.TrimEffortSuffix - TrimEffortSuffixWithSuffixes = kitreasoning.TrimEffortSuffixWithSuffixes - ParseOpenAIReasoningEffortFromModelSuffix = kitreasoning.ParseOpenAIReasoningEffortFromModelSuffix - ParseDeepSeekV4ThinkingSuffix = kitreasoning.ParseDeepSeekV4ThinkingSuffix + TrimEffortSuffixWithSuffixes = kitreasoning.TrimEffortSuffixWithSuffixes + ParseDeepSeekV4ThinkingSuffix = kitreasoning.ParseDeepSeekV4ThinkingSuffix + TrimGeminiThinkingSuffix = kitreasoning.TrimGeminiThinkingSuffix ) + +// ParseOpenAIReasoningEffortFromModelSuffix applies the host effort-tail +// whitelist so real model IDs such as qwen-max are not treated as aliases. +func ParseOpenAIReasoningEffortFromModelSuffix(modelName string) (string, string) { + return kitreasoning.ParseOpenAIReasoningEffortFromModelSuffix(modelName, model_setting.ShouldPreserveEffortTail) +}