ext/new-api
2
0
Fork 0
mirror of https://github.com/QuantumNous/new-api.git synced 2026-09-14 00:01:53 +00:00
Code Activity

feat: enhance text protocol conversion and advanced custom routing (#5825)

Browse Source 
* refactor: consolidate relay protocol converters

* refactor relayconvert text converters

* feat: refine relay converters and advanced custom routing

* refactor: enhance logging and add thought signature handling for Gemini requests

* refactor: enhance channel cache and pricing endpoint handling for advanced custom models

* feat: preserve billing usage semantics

* feat: add protocol-aware billing usage

* Delete useless files

* chore: update action versions in workflow files

* chore: update Docker action versions in workflow files

* fix: harden billing usage settlement and hot-path route matching

- estimate Gemini completion tokens locally when billable usageMetadata is
  prompt-only but output content was received (e.g. client aborts the stream
  before the final chunk), and rebuild the attached billing_usage as estimated
  so settlement does not bill zero output tokens
- guard NewClaudeMessagesBillingUsage against all-zero ClaudeUsage, matching
  the OpenAI/Gemini constructors, so a zero billing_usage cannot override a
  non-zero top-level usage during settlement
- cache compiled advanced-custom route model regexes; they run on the request
  hot path and were recompiled per request
- move the effectiveBillingUsage remap to PostTextConsumeQuota only, and
  document that calculateTextQuotaSummary expects remapped usage
- document the updatePricingLock -> channelSyncLock lock ordering that
  InitChannelCache/CacheUpdateChannel rely on, and the aux-struct pitfall in
  GeminiChatResponse.UnmarshalJSON
This commit is contained in:
Calcium-Ion
2026-07-11 20:44:12 +08:00
committed by GitHub
parent 1250fb2eb5
commit c36418c863
106 changed files with 13345 additions and 4307 deletions
Download Patch File Download Diff File Expand all files Collapse all files
relay/channel/gemini/adaptor.go
+13 -15
View File
@@ -9,7 +9,6 @@ import (
"github.com/QuantumNous/new-api/dto"
"github.com/QuantumNous/new-api/relay/channel"
"github.com/QuantumNous/new-api/relay/channel/openai"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/relay/constant"
"github.com/QuantumNous/new-api/service/relayconvert"
@@ -45,12 +44,15 @@ func (a *Adaptor) ConvertGeminiRequest(c *gin.Context, info *relaycommon.RelayIn
}
func (a *Adaptor) ConvertClaudeRequest(c *gin.Context, info *relaycommon.RelayInfo, req *dto.ClaudeRequest) (any, error) {
adaptor := openai.Adaptor{}
oaiReq, err := adaptor.ConvertClaudeRequest(c, info, req)
result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatGemini, req)
if err != nil {
return nil, err
}
return a.ConvertOpenAIRequest(c, info, oaiReq.(*dto.GeneralOpenAIRequest))
geminiRequest, ok := result.Value.(*dto.GeminiChatRequest)
if !ok {
return nil, fmt.Errorf("expected Gemini generateContent request, got %T", result.Value)
}
return geminiRequest, nil
}
func (a *Adaptor) ConvertAudioRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.AudioRequest) (io.Reader, error) {
@@ -181,13 +183,11 @@ func (a *Adaptor) ConvertOpenAIRequest(c *gin.Context, info *relaycommon.RelayIn
if request == nil {
return nil, errors.New("request is nil")
}
geminiRequest, err := CovertOpenAI2Gemini(c, *request, info)
result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatGemini, request)
if err != nil {
return nil, err
}
return geminiRequest, nil
return result.Value, nil
}
func (a *Adaptor) ConvertRerankRequest(c *gin.Context, relayMode int, request dto.RerankRequest) (any, error) {
@@ -239,17 +239,15 @@ func (a *Adaptor) ConvertEmbeddingRequest(c *gin.Context, info *relaycommon.Rela
}
func (a *Adaptor) ConvertOpenAIResponsesRequest(c *gin.Context, info *relaycommon.RelayInfo, request dto.OpenAIResponsesRequest) (any, error) {
request, err := preprocessGeminiOpenAIResponsesRequest(request)
result, err := relayconvert.ConvertRequest(c, info, types.RelayFormatGemini, &request)
if err != nil {
return nil, err
}
chatRequest, err := relayconvert.ResponsesRequestToChatCompletionsRequest(&request)
if err != nil {
return nil, err
geminiRequest, ok := result.Value.(*dto.GeminiChatRequest)
if !ok {
return nil, fmt.Errorf("expected Gemini generateContent request, got %T", result.Value)
}
return a.ConvertOpenAIRequest(c, info, chatRequest)
return geminiRequest, nil
}
func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (any, error) {
relay/channel/gemini/adaptor_responses.go
-99
View File
@@ -1,99 +0,0 @@
package gemini
import (
"strings"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/dto"
)
const (
geminiResponsesInputTypeCustomToolCall = "custom_tool_call"
geminiResponsesInputTypeCustomToolCallOutput = "custom_tool_call_output"
geminiResponsesInputTypeFunctionCallOutput = "function_call_output"
)
func preprocessGeminiOpenAIResponsesRequest(request dto.OpenAIResponsesRequest) (dto.OpenAIResponsesRequest, error) {
tools, err := filterGeminiResponsesTools(request.Tools)
if err != nil {
return request, err
}
request.Tools = tools
input, err := filterGeminiResponsesInput(request.Input)
if err != nil {
return request, err
}
request.Input = input
return request, nil
}
func filterGeminiResponsesTools(raw []byte) ([]byte, error) {
if !geminiRawJSONPresent(raw) || common.GetJsonType(raw) != "array" {
return raw, nil
}
var tools []map[string]any
if err := common.Unmarshal(raw, &tools); err != nil {
return nil, err
}
filtered := make([]map[string]any, 0, len(tools))
for _, tool := range tools {
if strings.TrimSpace(common.Interface2String(tool["type"])) != "function" {
// TODO: Support Responses custom/freeform tools when Gemini has a safe equivalent representation.
continue
}
filtered = append(filtered, tool)
}
if len(filtered) == 0 {
return nil, nil
}
return common.Marshal(filtered)
}
func filterGeminiResponsesInput(raw []byte) ([]byte, error) {
if !geminiRawJSONPresent(raw) || common.GetJsonType(raw) != "array" {
return raw, nil
}
var items []map[string]any
if err := common.Unmarshal(raw, &items); err != nil {
return nil, err
}
skippedCustomCallIDs := make(map[string]struct{})
for _, item := range items {
if strings.TrimSpace(common.Interface2String(item["type"])) != geminiResponsesInputTypeCustomToolCall {
continue
}
if callID := strings.TrimSpace(common.Interface2String(item["call_id"])); callID != "" {
skippedCustomCallIDs[callID] = struct{}{}
}
}
filtered := make([]map[string]any, 0, len(items))
for _, item := range items {
itemType := strings.TrimSpace(common.Interface2String(item["type"]))
switch itemType {
case geminiResponsesInputTypeCustomToolCall, geminiResponsesInputTypeCustomToolCallOutput:
// TODO: Support Responses custom/freeform tool calls once Gemini can preserve their semantics.
continue
case geminiResponsesInputTypeFunctionCallOutput:
if _, ok := skippedCustomCallIDs[strings.TrimSpace(common.Interface2String(item["call_id"]))]; ok {
continue
}
}
filtered = append(filtered, item)
}
return common.Marshal(filtered)
}
func geminiRawJSONPresent(raw []byte) bool {
if len(raw) == 0 {
return false
}
return common.GetJsonType(raw) != "null"
}
relay/channel/gemini/relay-gemini-native.go
+2 -2
View File
@@ -39,8 +39,8 @@ func GeminiTextGenerationHandler(c *gin.Context, info *relaycommon.RelayInfo, re
common.SetContextKey(c, constant.ContextKeyAdminRejectReason, fmt.Sprintf("gemini_block_reason=%s", *geminiResponse.PromptFeedback.BlockReason))
}
// 计算使用量（基于 UsageMetadata）
usage := buildUsageFromGeminiMetadata(geminiResponse.UsageMetadata, info.GetEstimatePromptTokens())
// 计算使用量（优先上游 UsageMetadata，缺失时本地估算并保留 Gemini 计费语义）
usage := buildUsageFromGeminiResponse(c, info, &geminiResponse)
service.IOCopyBytesGracefully(c, resp, responseBody)
relay/channel/gemini/relay-gemini.go
+95 -1356
View File
File diff suppressed because it is too large Load Diff
relay/channel/gemini/relay_gemini_usage_test.go
+183
View File
@@ -331,3 +331,186 @@ func TestGeminiTextGenerationHandlerUsesEstimatedPromptTokensWhenUsagePromptMiss
require.Equal(t, 100, usage.CompletionTokens)
require.Equal(t, 110, usage.TotalTokens)
}
func TestGeminiChatHandlerMissingUsageMetadataBuildsEstimatedBillingUsage(t *testing.T) {
t.Parallel()
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
info := &relaycommon.RelayInfo{
RelayFormat: types.RelayFormatGemini,
OriginModelName: "gemini-3-flash-preview",
ChannelMeta: &relaycommon.ChannelMeta{
UpstreamModelName: "gemini-3-flash-preview",
},
}
info.SetEstimatePromptTokens(20)
body := []byte(`{"candidates":[{"content":{"role":"model","parts":[{"text":"ok"}]}}]}`)
resp := &http.Response{
Body: io.NopCloser(bytes.NewReader(body)),
}
usage, newAPIError := GeminiChatHandler(c, info, resp)
require.Nil(t, newAPIError)
require.NotNil(t, usage)
require.Equal(t, 20, usage.PromptTokens)
require.NotNil(t, usage.BillingUsage)
require.True(t, usage.BillingUsage.Estimated)
require.Equal(t, dto.BillingUsageSourceGeminiChat, usage.BillingUsage.Source)
require.Equal(t, dto.BillingUsageSemanticGemini, usage.BillingUsage.Semantic)
require.NotNil(t, usage.BillingUsage.GeminiUsageMetadata)
require.Equal(t, usage.PromptTokens, usage.BillingUsage.GeminiUsageMetadata.PromptTokenCount)
require.Equal(t, usage.CompletionTokens, usage.BillingUsage.GeminiUsageMetadata.CandidatesTokenCount)
require.True(t, common.GetContextKeyBool(c, constant.ContextKeyLocalCountTokens))
}
func TestGeminiStreamHandlerPromptOnlyUsageMetadataEstimatesCompletionTokens(t *testing.T) {
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
oldStreamingTimeout := constant.StreamingTimeout
constant.StreamingTimeout = 300
t.Cleanup(func() {
constant.StreamingTimeout = oldStreamingTimeout
})
info := &relaycommon.RelayInfo{
OriginModelName: "gemini-3-flash-preview",
ChannelMeta: &relaycommon.ChannelMeta{
UpstreamModelName: "gemini-3-flash-preview",
},
}
info.SetEstimatePromptTokens(20)
// Simulates a client aborting the stream before the final chunk: text was
// streamed but the last observed usageMetadata only carries prompt tokens.
chunk := dto.GeminiChatResponse{
Candidates: []dto.GeminiChatCandidate{
{
Content: dto.GeminiChatContent{
Role: "model",
Parts: []dto.GeminiPart{
{Text: "partial streamed answer before disconnect"},
},
},
},
},
UsageMetadata: dto.GeminiUsageMetadata{
PromptTokenCount: 151,
TotalTokenCount: 151,
},
}
chunkData, err := common.Marshal(chunk)
require.NoError(t, err)
streamBody := []byte("data: " + string(chunkData) + "\n" + "data: [DONE]\n")
resp := &http.Response{
Body: io.NopCloser(bytes.NewReader(streamBody)),
}
usage, newAPIError := geminiStreamHandler(c, info, resp, func(_ string, _ *dto.GeminiChatResponse) bool {
return true
})
require.Nil(t, newAPIError)
require.NotNil(t, usage)
require.Equal(t, 151, usage.PromptTokens)
require.Greater(t, usage.CompletionTokens, 0)
require.Equal(t, usage.PromptTokens+usage.CompletionTokens, usage.TotalTokens)
require.NotNil(t, usage.BillingUsage)
require.True(t, usage.BillingUsage.Estimated)
require.NotNil(t, usage.BillingUsage.GeminiUsageMetadata)
require.Equal(t, usage.CompletionTokens, usage.BillingUsage.GeminiUsageMetadata.CandidatesTokenCount)
}
func TestGeminiChatHandlerPromptOnlyUsageMetadataEstimatesCompletionTokens(t *testing.T) {
t.Parallel()
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
info := &relaycommon.RelayInfo{
RelayFormat: types.RelayFormatGemini,
OriginModelName: "gemini-3-flash-preview",
ChannelMeta: &relaycommon.ChannelMeta{
UpstreamModelName: "gemini-3-flash-preview",
},
}
payload := dto.GeminiChatResponse{
Candidates: []dto.GeminiChatCandidate{
{
Content: dto.GeminiChatContent{
Role: "model",
Parts: []dto.GeminiPart{
{Text: "answer text without candidate token count"},
},
},
},
},
UsageMetadata: dto.GeminiUsageMetadata{
PromptTokenCount: 151,
TotalTokenCount: 151,
},
}
body, err := common.Marshal(payload)
require.NoError(t, err)
resp := &http.Response{
Body: io.NopCloser(bytes.NewReader(body)),
}
usage, newAPIError := GeminiChatHandler(c, info, resp)
require.Nil(t, newAPIError)
require.NotNil(t, usage)
require.Equal(t, 151, usage.PromptTokens)
require.Greater(t, usage.CompletionTokens, 0)
require.Equal(t, usage.PromptTokens+usage.CompletionTokens, usage.TotalTokens)
require.NotNil(t, usage.BillingUsage)
require.True(t, usage.BillingUsage.Estimated)
}
func TestGeminiStreamHandlerEmptyUsageMetadataBuildsEstimatedBillingUsage(t *testing.T) {
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
oldStreamingTimeout := constant.StreamingTimeout
constant.StreamingTimeout = 300
t.Cleanup(func() {
constant.StreamingTimeout = oldStreamingTimeout
})
info := &relaycommon.RelayInfo{
OriginModelName: "gemini-3-flash-preview",
ChannelMeta: &relaycommon.ChannelMeta{
UpstreamModelName: "gemini-3-flash-preview",
},
}
info.SetEstimatePromptTokens(20)
streamBody := []byte("data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"partial\"}]}}],\"usageMetadata\":{}}\n" + "data: [DONE]\n")
resp := &http.Response{
Body: io.NopCloser(bytes.NewReader(streamBody)),
}
usage, newAPIError := geminiStreamHandler(c, info, resp, func(_ string, _ *dto.GeminiChatResponse) bool {
return true
})
require.Nil(t, newAPIError)
require.NotNil(t, usage)
require.Equal(t, 20, usage.PromptTokens)
require.NotNil(t, usage.BillingUsage)
require.True(t, usage.BillingUsage.Estimated)
require.Equal(t, dto.BillingUsageSourceGeminiChat, usage.BillingUsage.Source)
require.NotNil(t, usage.BillingUsage.GeminiUsageMetadata)
require.Equal(t, usage.PromptTokens, usage.BillingUsage.GeminiUsageMetadata.PromptTokenCount)
require.Equal(t, usage.CompletionTokens, usage.BillingUsage.GeminiUsageMetadata.CandidatesTokenCount)
require.True(t, common.GetContextKeyBool(c, constant.ContextKeyLocalCountTokens))
}
relay/channel/gemini/relay_responses.go
+39 -12
View File
@@ -32,7 +32,7 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
if len(geminiResponse.Candidates) == 0 {
usage := buildUsageFromGeminiMetadata(geminiResponse.UsageMetadata, info.GetEstimatePromptTokens())
usage := buildUsageFromGeminiResponse(c, info, &geminiResponse)
if geminiResponse.PromptFeedback != nil && geminiResponse.PromptFeedback.BlockReason != nil {
common.SetContextKey(c, constant.ContextKeyAdminRejectReason, fmt.Sprintf("gemini_block_reason=%s", *geminiResponse.PromptFeedback.BlockReason))
return &usage, types.NewOpenAIError(
@@ -51,13 +51,21 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h
chatResp := responseGeminiChat2OpenAI(c, &geminiResponse)
chatResp.Model = info.UpstreamModelName
usage := buildUsageFromGeminiMetadata(geminiResponse.UsageMetadata, info.GetEstimatePromptTokens())
if responseID := helper.GetResponseID(c); responseID != "" {
chatResp.Id = responseID
}
usage := buildUsageFromGeminiResponse(c, info, &geminiResponse)
chatResp.Usage = usage
responsesResp, responsesUsage, err := service.ChatCompletionsResponseToResponsesResponse(chatResp, helper.GetResponseID(c))
convertResult, err := relayconvert.ConvertResponse(c, info, types.RelayFormatOpenAIResponses, chatResp)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
responsesResp, ok := convertResult.Value.(*dto.OpenAIResponsesResponse)
if !ok {
return nil, types.NewOpenAIError(fmt.Errorf("expected OpenAI responses response, got %T", convertResult.Value), types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
}
responsesUsage := convertResult.Usage
if responsesUsage == nil || responsesUsage.TotalTokens == 0 {
responsesResp.Usage = relayconvert.UsageFromChatUsage(&usage)
}
@@ -73,8 +81,14 @@ func GeminiResponsesHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *h
func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) {
responseID := helper.GetResponseID(c)
created := common.GetTimestamp()
state := relayconvert.NewChatToResponsesStreamState(responseID, info.UpstreamModelName)
state.Created = created
state, err := relayconvert.NewResponseStreamState(types.RelayFormatOpenAI, types.RelayFormatOpenAIResponses, relayconvert.ResponseStreamOptions{
ID: responseID,
Model: info.UpstreamModelName,
Created: created,
})
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
finishReason := constant.FinishReasonStop
toolCallIndexByChoice := make(map[int]map[string]int)
nextToolCallIndexByChoice := make(map[int]int)
@@ -90,12 +104,17 @@ func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, r
return true
}
sendChunk := func(chunk *dto.ChatCompletionsStreamResponse) bool {
events, err := relayconvert.ChatCompletionsStreamChunkToResponsesEvents(chunk, state)
results, err := relayconvert.ConvertStreamResponseChunk(c, info, state, chunk)
if err != nil {
streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
for _, event := range events {
for _, result := range results {
event, ok := result.Value.(relayconvert.ChatToResponsesStreamEvent)
if !ok {
streamErr = types.NewOpenAIError(fmt.Errorf("expected OAI responses stream event, got %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError)
return false
}
if !sendEvent(event) {
return false
}
@@ -103,7 +122,7 @@ func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, r
return true
}
usage, err := geminiStreamHandler(c, info, resp, func(data string, geminiResponse *dto.GeminiChatResponse) bool {
usage, streamAPIError := geminiStreamHandler(c, info, resp, func(data string, geminiResponse *dto.GeminiChatResponse) bool {
response, isStop := streamResponseGeminiChat2OpenAI(geminiResponse)
response.Id = responseID
response.Created = created
@@ -143,17 +162,25 @@ func GeminiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, r
}
return true
})
if err != nil {
return usage, err
if streamAPIError != nil {
return usage, streamAPIError
}
if streamErr != nil {
return nil, streamErr
}
if usage != nil {
state.Usage = relayconvert.UsageFromChatUsage(usage)
state.SetUsage(usage)
}
for _, event := range relayconvert.FinalizeChatCompletionsStreamToResponses(state) {
finalResults, err := relayconvert.FinalizeStreamResponse(c, info, state)
if err != nil {
return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
for _, result := range finalResults {
event, ok := result.Value.(relayconvert.ChatToResponsesStreamEvent)
if !ok {
return nil, types.NewOpenAIError(fmt.Errorf("expected OAI responses stream event, got %T", result.Value), types.ErrorCodeBadResponse, http.StatusInternalServerError)
}
if !sendEvent(event) {
return nil, streamErr
}