mirror of
https://github.com/QuantumNous/new-api.git
synced 2026-10-30 11:06:56 +00:00
CaIon
bb28f2ccad
Token estimates badly undercounted agent requests. Claude tool definitions, Responses function calls and their outputs, Chat assistant tool calls, and Gemini system instructions, function calls, function responses, and tools never reached the counter; Chat tool schemas were counted as Go map syntax and Claude tool results as escaped JSON. Measured against upstream usage on 54 real Claude Code and Codex conversation windows, native prompt estimates were 31-93% low, and the Claude and Gemini completion fallbacks about 90% low because streamed tool calls were not counted. All of these now reach the counter. The completion fallbacks count tool names and arguments, plus 40 tokens per Claude tool_use block. The estimator weights are refitted per tokenizer: Claude before Opus 4.7, Claude 4.7 and later (about 30% more tokens for the same text), GPT, and Gemini. Prompt estimates now land within 1% bias and 3-4% mean error for each family. Token counting moves into tokenkit, an independent module like relaykit, so other gateways can use it. It also replaces the flat 520-token image estimate with each provider's sizing rules, checked against upstream usage on sizes outside the test table: OpenAI patch and tile rules per model and detail level (gpt-5.5 and later default to original detail, and their high detail is billed as measured, not as documented), Claude's two resolution tiers, and Gemini 3 media resolution and Gemini 2.x tiles. Chat image_url parts without detail no longer default to high in ParseContent, so they estimate as the upstream default auto; no converter sent that value upstream. tiktoken-go moves to v0.8.1, about twice as fast on ordinary text. It requires Go 1.26, so the root module and tokenkit move to Go 1.26; relaykit stays on 1.25.1. In every version BPE cost grows with the square of a pre-tokenized piece, so 100,000 Chinese characters without punctuation took 21.6 s to count during pre-consume. Count now cuts single-class runs longer than 1024 bytes, which leaves ordinary text unchanged and brings that input down to 106 ms. The encoder cache no longer grows with client-supplied model names.
103 lines
2.6 KiB
Go
103 lines
2.6 KiB
Go
package tokenkit
|
|
|
|
import (
|
|
"strings"
|
|
"sync"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"github.com/tiktoken-go/tokenizer"
|
|
"github.com/tiktoken-go/tokenizer/codec"
|
|
)
|
|
|
|
var (
|
|
o200kBase = sync.OnceValue(func() tokenizer.Codec { return codec.NewO200kBase() })
|
|
cl100kBase = sync.OnceValue(func() tokenizer.Codec { return codec.NewCl100kBase() })
|
|
)
|
|
|
|
// maxRunBytes bounds a run of letters, whitespace, or symbols passed to the
|
|
// encoder in one piece. BPE cost grows with the square of a pre-tokenized
|
|
// piece: 100,000 Chinese characters without punctuation take about 20 s in one
|
|
// piece. Cutting longer runs keeps counting linear; ordinary text has no such
|
|
// runs, and a cut changes the count only inside the run (under 0.5% for CJK).
|
|
const maxRunBytes = 1024
|
|
|
|
// Count returns the token count of text for model. GPT and o-series models
|
|
// are counted with their tiktoken encoding; other models use Estimate.
|
|
func Count(model, text string) int {
|
|
if text == "" {
|
|
return 0
|
|
}
|
|
model = strings.ToLower(model)
|
|
if ModelFamily(model) != FamilyOpenAI {
|
|
return Estimate(model, text)
|
|
}
|
|
encoder := o200kBase()
|
|
if usesCl100k(model) {
|
|
encoder = cl100kBase()
|
|
}
|
|
|
|
total, start := 0, 0
|
|
runClass, runStart := runNone, 0
|
|
for i, r := range text {
|
|
class := runClassOf(r)
|
|
if class != runClass {
|
|
runClass, runStart = class, i
|
|
continue
|
|
}
|
|
if class != runNumber && i-runStart >= maxRunBytes {
|
|
n, _ := encoder.Count(text[start:i])
|
|
total += n
|
|
start, runStart = i, i
|
|
}
|
|
}
|
|
n, _ := encoder.Count(text[start:])
|
|
return total + n
|
|
}
|
|
|
|
type runClass int
|
|
|
|
const (
|
|
runNone runClass = iota
|
|
runLetter
|
|
runNumber
|
|
runSpace
|
|
runSymbol
|
|
)
|
|
|
|
// runClassOf groups runes the way tiktoken's pre-tokenizer joins them into
|
|
// pieces. Digits are split into groups of three by the encoder itself.
|
|
func runClassOf(r rune) runClass {
|
|
if r < utf8.RuneSelf {
|
|
switch {
|
|
case 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z':
|
|
return runLetter
|
|
case '0' <= r && r <= '9':
|
|
return runNumber
|
|
case r == ' ' || '\t' <= r && r <= '\r':
|
|
return runSpace
|
|
default:
|
|
return runSymbol
|
|
}
|
|
}
|
|
switch {
|
|
case unicode.IsLetter(r) || unicode.IsMark(r):
|
|
return runLetter
|
|
case unicode.IsNumber(r):
|
|
return runNumber
|
|
case unicode.IsSpace(r):
|
|
return runSpace
|
|
default:
|
|
return runSymbol
|
|
}
|
|
}
|
|
|
|
// usesCl100k reports whether an OpenAI model predates o200k_base: GPT-4 (but
|
|
// not GPT-4o, GPT-4.1, or GPT-4.5) and GPT-3.5. Every later model, including
|
|
// GPT-5, GPT-6, and the o-series, uses o200k_base.
|
|
func usesCl100k(model string) bool {
|
|
model = strings.TrimPrefix(model, "ft:")
|
|
return model == "gpt-4" || strings.HasPrefix(model, "gpt-4-") ||
|
|
strings.HasPrefix(model, "gpt-3.5") || strings.HasPrefix(model, "gpt-35")
|
|
}
|