lukaszraczylo/traefik-llmgateway
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/traefik-llmgateway.git synced 2026-10-16 05:37:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5 from lukaszraczylo/perf/stream-translation

Browse Source 
Perf/stream translation
This commit is contained in:
lukaszraczylo
2026-10-15 22:17:05 +01:00
committed by GitHub
parent 2608e01c50 77d5d1f9d0
commit e5fe48a9a6
15 changed files with 873 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr.yaml
+2 -2
View File
@@ -18,7 +18,7 @@ jobs:
pr-checks:
uses: lukaszraczylo/shared-actions/.github/workflows/go-pr.yaml@main
with:
go-version: "1.25.x"
go-version: "1.26.x"
coverage-threshold: 75
secrets: inherit
@@ -40,7 +40,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: "1.25.x"
go-version: "1.26.x"
cache: true
- name: yaegi-check
.github/workflows/release.yml
+1 -1
View File
@@ -24,5 +24,5 @@ jobs:
release:
uses: lukaszraczylo/shared-actions/.github/workflows/go-release.yaml@main
with:
go-version: "1.25.x"
go-version: "1.26.x"
secrets: inherit
Makefile
+5 -1
View File
@@ -40,8 +40,12 @@ yaegi-check:
# pruning rule and naming-bridge mechanism. Not run by CI (network
# fetch, and the generated table is committed); an operator re-runs this
# by hand when LiteLLM's own pricing drifts.
# PRICING_MAX_AGE_MONTHS drops entries older than N months (0 = keep all);
# PRICING_KEEP is a comma-separated list of bare ids exempt from that rule.
PRICING_MAX_AGE_MONTHS ?= 12
PRICING_KEEP ?=
pricing-sync:
cd tools/pricing-sync && GOWORK=off go run . $(CURDIR)
cd tools/pricing-sync && GOWORK=off go run . -max-age-months=$(PRICING_MAX_AGE_MONTHS) -keep=$(PRICING_KEEP) $(CURDIR)
gofmt -w pricing_data_gen.go
INTEGRATION_COMPOSE := integration/docker-compose.yml
README.md
+1
View File
@@ -380,6 +380,7 @@ config accepts them as YAML, which decodes to the same JSON shape.
| `admin` | `*AdminConfig` | `nil` (disabled) | Read-only admin dashboard — see [Admin](#admin). `nil` or `enabled: false` means the `/admin*` routes are not registered at all. |
| `metrics` | `*MetricsConfig` | `nil` (disabled) | Prometheus text-exposition endpoint — see [Metrics](#metrics). `nil` or `enabled: false` means the metrics route is not registered at all. |
| `modelAliases` | `map[string]string` | `{}` | Operator-defined alias id → target model id — see [Model aliases](#model-aliases). Omitted or empty means no aliases, byte-identical to a gateway built before this field existed. |
| `requireModelPrefix` | `bool` | `false` | When `true`, a model resolves only as `provider/model` or as a `modelAliases` id. A bare id returns 404 `unknown model` and `GET /v1/models` lists no bare ids. `false` keeps the sorted-first bare-id collision winner. |
| `modelMeta` | `map[string]*ModelMetaConfig` | `{}` | Per-model context-window and cost overrides, keyed by exact `provider/model` or a bare model id — see [Model metadata](#model-metadata). Omitted or empty means no overrides; every model's metadata still resolves through discovery and the built-in table. |
| `passthroughUnknown` | `bool` | `false` | `false`: a request matching none of the plugin's routes gets a 404 JSON envelope. `true`: it falls through to the router's own backing service. **Does not cover the three media routes** — see the upgrade note below. |
| `maxInFlightBodyRequests` | `int` | `0` (self-tuned) | Caps how many unified/media requests may concurrently be inside their own request-body read+decode step — see [Body-admission limit](#body-admission-limit). `0` self-tunes from the host's own `GOMAXPROCS`; a positive value pins an exact cap instead, clamped to a maximum of 10,000. |
bench_test.go
+226
View File
@@ -0,0 +1,226 @@
package traefikllmgateway
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
)
const benchStreamDeltas = 500
func benchOpenAIRequest() map[string]any {
return map[string]any{
"model": "bench-model",
"max_tokens": float64(1024),
"temperature": 0.7,
"messages": []any{
map[string]any{"role": "system", "content": "You are a helpful assistant."},
map[string]any{"role": "user", "content": "What is the weather in Paris and Berlin?"},
map[string]any{"role": "assistant", "content": "", "tool_calls": []any{
map[string]any{"id": "call_1", "type": "function", "function": map[string]any{"name": "get_weather", "arguments": `{"city":"Paris"}`}},
map[string]any{"id": "call_2", "type": "function", "function": map[string]any{"name": "get_weather", "arguments": `{"city":"Berlin"}`}},
}},
map[string]any{"role": "tool", "tool_call_id": "call_1", "content": "18C, cloudy"},
map[string]any{"role": "tool", "tool_call_id": "call_2", "content": "15C, rain"},
map[string]any{"role": "user", "content": "Summarise both in one sentence."},
},
"tools": []any{
map[string]any{"type": "function", "function": map[string]any{
"name": "get_weather",
"description": "Get the weather for a city",
"parameters": map[string]any{
"type": "object",
"properties": map[string]any{"city": map[string]any{"type": "string"}},
"required": []any{"city"},
},
}},
},
"tool_choice": "auto",
}
}
func benchAnthropicStream() []byte {
var sb strings.Builder
sb.WriteString("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01B\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"claude-opus-5\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n")
sb.WriteString("event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n")
for i := 0; i < benchStreamDeltas; i++ {
fmt.Fprintf(&sb, "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" token%d\"}}\n\n", i)
}
sb.WriteString("event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n")
sb.WriteString("event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"output_tokens\":500}}\n\n")
sb.WriteString("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n")
return []byte(sb.String())
}
func benchGeminiStream() []byte {
var sb strings.Builder
for i := 0; i < benchStreamDeltas; i++ {
fmt.Fprintf(&sb, "data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\" token%d\"}]}}],\"responseId\":\"resp_B\"}\n\n", i)
}
sb.WriteString("data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[]},\"finishReason\":\"STOP\"}],\"usageMetadata\":{\"promptTokenCount\":5,\"candidatesTokenCount\":500}}\n\n")
return []byte(sb.String())
}
func BenchmarkAnthropicRequestFromOpenAI(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if _, err := anthropicRequestFromOpenAI(benchOpenAIRequest()); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkGeminiRequestFromOpenAI(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if _, err := geminiRequestFromOpenAI(benchOpenAIRequest()); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkOpenAIResponseFromAnthropic(b *testing.B) {
body := []byte(`{"id":"msg_01XYZ","type":"message","role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."},{"type":"tool_use","id":"tu_1","name":"get_weather","input":{"city":"Paris"}}],"model":"claude-opus-5","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":8}}`)
b.ReportAllocs()
b.SetBytes(int64(len(body)))
for i := 0; i < b.N; i++ {
if _, _, err := openAIResponseFromAnthropic(body, "gw-model", 1734000000); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkOpenAIResponseFromGemini(b *testing.B) {
body := []byte(`{"candidates":[{"content":{"role":"model","parts":[{"text":"The capital of France is Paris."},{"functionCall":{"name":"get_weather","args":{"city":"Paris"}}}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":12,"candidatesTokenCount":8},"responseId":"resp_1"}`)
b.ReportAllocs()
b.SetBytes(int64(len(body)))
for i := 0; i < b.N; i++ {
if _, _, err := openAIResponseFromGemini(body, "gw-model", 1734000000); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkReadSSE(b *testing.B) {
stream := benchAnthropicStream()
b.ReportAllocs()
b.SetBytes(int64(len(stream)))
for i := 0; i < b.N; i++ {
if err := readSSE(bytes.NewReader(stream), func(sseEvent) error { return nil }); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkAnthropicStreamTranslate(b *testing.B) {
stream := benchAnthropicStream()
b.ReportAllocs()
b.SetBytes(int64(len(stream)))
for i := 0; i < b.N; i++ {
st := newAnthropicStreamState("gw-model", 1734000000)
err := readSSE(bytes.NewReader(stream), func(ev sseEvent) error {
_, terr := st.translate(ev)
return terr
})
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkGeminiStreamTranslate(b *testing.B) {
stream := benchGeminiStream()
b.ReportAllocs()
b.SetBytes(int64(len(stream)))
for i := 0; i < b.N; i++ {
st := newGeminiStreamState("gw-model", 1734000000)
err := readSSE(bytes.NewReader(stream), func(ev sseEvent) error {
_, terr := st.translate(ev)
return terr
})
if err != nil {
b.Fatal(err)
}
}
}
func newBenchGateway(b *testing.B, provider *ProviderConfig) http.Handler {
b.Helper()
devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0)
if err != nil {
b.Fatalf("open %s: %v", os.DevNull, err)
}
origStderr := os.Stderr
os.Stderr = devNull
b.Cleanup(func() {
os.Stderr = origStderr
_ = devNull.Close()
})
cfg := CreateConfig()
cfg.Providers = map[string]*ProviderConfig{"bench": provider}
cfg.Groups = map[string]*GroupConfig{"default": {}}
cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "alice", Group: "default", APIKey: "sk-alice", Limits: &LimitsConfig{}}}}
next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
h, err := New(context.Background(), next, cfg, "llmgw")
if err != nil {
b.Fatalf("New: %v", err)
}
return h
}
func benchChatRequest(payload []byte) *http.Request {
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(payload))
req.Header.Set("Authorization", "Bearer sk-alice")
req.Header.Set("Content-Type", "application/json")
return req
}
func BenchmarkUnifiedChatOpenAI(b *testing.B) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"x","object":"chat.completion","model":"bench-model","choices":[{"index":0,"message":{"role":"assistant","content":"hello"},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5}}`))
}))
b.Cleanup(upstream.Close)
h := newBenchGateway(b, &ProviderConfig{Type: "openai", BaseURL: upstream.URL, APIKey: "k", Models: []string{"bench-model"}})
payload := []byte(`{"model":"bench-model","messages":[{"role":"user","content":"hi"}]}`)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, benchChatRequest(payload))
if rec.Code != http.StatusOK {
b.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
}
}
}
func BenchmarkUnifiedChatAnthropicStream(b *testing.B) {
stream := benchAnthropicStream()
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
w.Header().Set("Content-Type", "text/event-stream")
_, _ = w.Write(stream)
}))
b.Cleanup(upstream.Close)
h := newBenchGateway(b, &ProviderConfig{Type: "anthropic", BaseURL: upstream.URL, APIKey: "k", Models: []string{"bench-model"}})
payload := []byte(`{"model":"bench-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
b.ReportAllocs()
b.SetBytes(int64(len(stream)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, benchChatRequest(payload))
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "[DONE]") {
b.Fatalf("status = %d, body tail = %.200s", rec.Code, rec.Body.String())
}
}
}
llmgateway.go
+5
View File
@@ -55,6 +55,11 @@ type Config struct {
// (validateModelAliases, registry.go).
ModelAliases map[string]string `json:"modelAliases,omitempty"`
Providers map[string]*ProviderConfig `json:"providers,omitempty"`
// RequireModelPrefix, when true, makes every model addressable only as
// "provider/model" or as a ModelAliases id: a bare discovered id is not
// resolved (404 unknown model) and is not listed by GET /v1/models. It
// removes the sorted-first bare-id collision winner. Defaults to false.
RequireModelPrefix bool `json:"requireModelPrefix,omitempty"`
// ModelMeta declares per-model metadata overrides (feature v0.23):
// context window size and per-token cost, keyed by an exact
// "provider/model" id or a bare model id (applying wherever that bare
pricing_data_gen.go
+174 -238
View File
@@ -4,247 +4,183 @@ package traefikllmgateway
// model_prices_and_context_window.json — DO NOT EDIT.
//
// Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
// Source digest: sha256:0e8d23239c27
// 236 entries kept (pruned from 3111 upstream entries — see
// Source digest: sha256:9f7b337871ed
// 172 entries kept (pruned from 4422 upstream entries — see
// tools/pricing-sync/main.go's own doc comment for the exact pruning
// rule and bare-id collision resolution).
//
// Regenerate with: make pricing-sync
var builtinModelMetaTable = map[string]builtinModelMeta{
"MiniMax-M2": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"MiniMax-M2.1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"MiniMax-M2.1-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000},
"MiniMax-M2.1-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000},
"MiniMax-M2.5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"MiniMax-M2.5-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000},
"MiniMax-M2.5-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000},
"MiniMax-M3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"chat-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000},
"chatgpt-4o-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-3-7-sonnet-20250219": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-3-haiku-20240307": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 1250000},
"claude-3-opus-20240229": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000},
"claude-4-opus-20250514": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000},
"claude-4-sonnet-20250514": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-fable-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000},
"claude-haiku-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000},
"claude-haiku-4-5-20251001": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000},
"claude-mythos-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000},
"claude-mythos-preview": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000},
"claude-opus-4-1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000},
"claude-opus-4-1-20250805": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000},
"claude-opus-4-20250514": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000},
"claude-opus-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-5-20251101": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-6-20260205": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-7": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-7-20260416": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-8": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-sonnet-4-20250514": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-sonnet-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-sonnet-4-5-20250929": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-sonnet-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-sonnet-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"daybreak-blue-latest": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000},
"daybreak-red-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000},
"deepseek-chat": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000},
"deepseek-coder": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 140000, OutputCostPerMTokMicroUSD: 280000},
"deepseek-r1": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 550000, OutputCostPerMTokMicroUSD: 2190000},
"deepseek-reasoner": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000},
"deepseek-v3": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 270000, OutputCostPerMTokMicroUSD: 1100000},
"deepseek-v3.2": {ContextTokens: 163840, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 400000},
"deepseek-v4-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 440000, OutputCostPerMTokMicroUSD: 1320000},
"deepseek-v4-flash-0731": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 400000},
"deepseek-v4-pro": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1320000, OutputCostPerMTokMicroUSD: 3960000},
"ft:gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000},
"ft:gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000},
"ft:gpt-3.5-turbo-0613": {ContextTokens: 4096, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000},
"ft:gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000},
"ft:gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000},
"ft:gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 12000000},
"ft:gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 3200000},
"ft:gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 800000},
"ft:gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000},
"ft:gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000},
"ft:gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"ft:o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 16000000},
"glm-5.1": {ContextTokens: 202745, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000},
"glm-5.2": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000},
"gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000},
"gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000},
"gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000},
"gpt-3.5-turbo-16k": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 4000000},
"gpt-4": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000},
"gpt-4-0125-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-4-0314": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000},
"gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000},
"gpt-4-1106-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-4-turbo": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-4-turbo-2024-04-09": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-4-turbo-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-4.1": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000},
"gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000},
"gpt-4.1-mini": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000},
"gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000},
"gpt-4.1-nano": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000},
"gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000},
"gpt-4o": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-4o-2024-05-13": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000},
"gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-4o-audio-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-4o-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-4o-audio-preview-2025-06-03": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-4o-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000},
"gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000},
"gpt-4o-mini-audio-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000},
"gpt-4o-mini-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000},
"gpt-4o-mini-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000},
"gpt-4o-mini-search-preview-2025-03-11": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000},
"gpt-4o-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-4o-search-preview-2025-03-11": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5-chat": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000},
"gpt-5-mini-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000},
"gpt-5-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000},
"gpt-5-nano-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000},
"gpt-5-search-api": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5-search-api-2025-10-14": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5.1": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5.1-2025-11-13": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5.1-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5.2": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000},
"gpt-5.2-2025-12-11": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000},
"gpt-5.2-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000},
"gpt-5.3-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000},
"gpt-5.4": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000},
"gpt-5.4-2026-03-05": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000},
"gpt-5.4-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000},
"gpt-5.4-mini-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000},
"gpt-5.4-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000},
"gpt-5.4-nano-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000},
"gpt-5.5": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-5.5-2026-04-23": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-5.6": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-5.6-cyber": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000},
"gpt-5.6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1200000},
"gpt-5.6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-5.6-terra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 12000000},
"gpt-audio": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-audio-1.5": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-audio-2025-08-28": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-audio-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000},
"gpt-audio-mini-2025-10-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000},
"gpt-audio-mini-2025-12-15": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000},
"gpt-oss-120b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 350000, OutputCostPerMTokMicroUSD: 750000},
"grok-2": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"grok-2-1212": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"grok-2-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"grok-2-vision": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"grok-2-vision-1212": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"grok-2-vision-latest": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"grok-3": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"grok-3-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"grok-3-fast-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"grok-3-fast-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"grok-3-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"grok-3-mini": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 500000},
"grok-3-mini-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 500000},
"grok-3-mini-fast": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 4000000},
"grok-3-mini-fast-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 4000000},
"grok-3-mini-fast-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 4000000},
"grok-3-mini-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 500000},
"grok-4": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"grok-4-0709": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"grok-4-1-fast": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000},
"grok-4-1-fast-non-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000},
"grok-4-1-fast-non-reasoning-latest": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000},
"grok-4-1-fast-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000},
"grok-4-1-fast-reasoning-latest": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000},
"grok-4-fast-non-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000},
"grok-4-fast-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000},
"grok-4-latest": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"grok-4.20-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-multi-agent-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-multi-agent-beta-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.3-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.5": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"grok-4.5-latest": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"grok-4.6": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"grok-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000},
"grok-build-0.1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000},
"grok-code-fast": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000},
"grok-code-fast-1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000},
"grok-code-fast-1-0825": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000},
"grok-vision-beta": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000},
"kimi-k2-0711-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000},
"kimi-k2-0905-preview": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000},
"kimi-k2-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000},
"kimi-k2-thinking-turbo": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 1150000, OutputCostPerMTokMicroUSD: 8000000},
"kimi-k2-turbo-preview": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 1150000, OutputCostPerMTokMicroUSD: 8000000},
"kimi-k2.5": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 3000000},
"kimi-k2.6": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000},
"kimi-k2.7-code": {ContextTokens: 229376, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000},
"kimi-k3": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"kimi-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000},
"kimi-latest-128k": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000},
"kimi-latest-32k": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000},
"kimi-latest-8k": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000},
"kimi-thinking-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000},
"llama-3.3-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 850000, OutputCostPerMTokMicroUSD: 1200000},
"llama3.1-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 600000},
"llama3.1-8b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 100000},
"moonshot-v1-128k": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000},
"moonshot-v1-128k-0430": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000},
"moonshot-v1-128k-vision-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000},
"moonshot-v1-32k": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000},
"moonshot-v1-32k-0430": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000},
"moonshot-v1-32k-vision-preview": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000},
"moonshot-v1-8k": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000},
"moonshot-v1-8k-0430": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000},
"moonshot-v1-8k-vision-preview": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000},
"moonshot-v1-auto": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000},
"o1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000},
"o1-2024-12-17": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000},
"o3": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000},
"o3-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000},
"o3-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000},
"o3-mini-2025-01-31": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000},
"o4-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000},
"o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000},
"qwen-3-32b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 800000},
"qwen-coder": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1500000},
"qwen-max": {ContextTokens: 30720, InputCostPerMTokMicroUSD: 1600000, OutputCostPerMTokMicroUSD: 6400000},
"qwen-plus": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000},
"qwen-plus-2025-01-25": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000},
"qwen-plus-2025-04-28": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000},
"qwen-plus-2025-07-14": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000},
"qwen-turbo": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000},
"qwen-turbo-2024-11-01": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000},
"qwen-turbo-2025-04-28": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000},
"qwen-turbo-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000},
"qwen3-next-80b-a3b-instruct": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000},
"qwen3-next-80b-a3b-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000},
"qwen3-vl-235b-a22b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000},
"qwen3-vl-235b-a22b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 4000000},
"qwen3-vl-32b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 640000},
"qwen3-vl-32b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 2870000},
"qwen3.7-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 7500000},
"qwen3.8-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"qwq-plus": {ContextTokens: 98304, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 2400000},
"text-embedding-3-large": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 130000, OutputCostPerMTokMicroUSD: 0},
"text-embedding-3-small": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 20000, OutputCostPerMTokMicroUSD: 0},
"text-embedding-ada-002": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0},
"text-embedding-ada-002-v2": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0},
"zai-glm-4.6": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2250000, OutputCostPerMTokMicroUSD: 2750000},
"zai-glm-4.7": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2250000, OutputCostPerMTokMicroUSD: 2750000},
"MiniMax-M2": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"MiniMax-M2.1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"MiniMax-M2.1-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000},
"MiniMax-M2.1-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000},
"MiniMax-M2.5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"MiniMax-M2.5-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000},
"MiniMax-M2.5-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000},
"MiniMax-M3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"chat-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000},
"claude-fable-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000},
"claude-fable-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000},
"claude-haiku-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000},
"claude-haiku-4-5-20251001": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000},
"claude-mythos-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000},
"claude-mythos-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000},
"claude-opus-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-5-20251101": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-6-20260205": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-7": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-7-20260416": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-4-8": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"claude-opus-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000},
"claude-sonnet-4-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-sonnet-4-5-20250929": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-sonnet-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"claude-sonnet-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"claude-sonnet-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"daybreak-blue-latest": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000},
"daybreak-red-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000},
"deepseek-chat": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000},
"deepseek-coder": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 140000, OutputCostPerMTokMicroUSD: 280000},
"deepseek-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"deepseek-r1": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 550000, OutputCostPerMTokMicroUSD: 2190000},
"deepseek-reasoner": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000},
"deepseek-v3": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 270000, OutputCostPerMTokMicroUSD: 1100000},
"deepseek-v3.2": {ContextTokens: 163840, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 400000},
"deepseek-v4-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"deepseek-v4-flash-0731": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 400000},
"deepseek-v4-flash-vision-exp": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000},
"deepseek-v4-pro": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1320000, OutputCostPerMTokMicroUSD: 3960000},
"ft:gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000},
"ft:gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000},
"ft:gpt-3.5-turbo-0613": {ContextTokens: 4096, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000},
"ft:gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000},
"ft:gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000},
"gemma-4-31b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 990000, OutputCostPerMTokMicroUSD: 1490000},
"glm-5.1": {ContextTokens: 202745, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000},
"glm-5.2": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000},
"gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000},
"gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000},
"gpt-3.5-turbo-16k": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 4000000},
"gpt-4": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000},
"gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000},
"gpt-4-1106-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-4-turbo": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-4.1": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000},
"gpt-4.1-mini": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000},
"gpt-4.1-nano": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000},
"gpt-4o": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-4o-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000},
"gpt-4o-mini-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000},
"gpt-4o-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000},
"gpt-5-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000},
"gpt-5-search-api": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5-search-api-2025-10-14": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5.1": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5.1-2025-11-13": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-5.2": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000},
"gpt-5.2-2025-12-11": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000},
"gpt-5.4": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000},
"gpt-5.4-2026-03-05": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000},
"gpt-5.4-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000},
"gpt-5.4-mini-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000},
"gpt-5.4-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000},
"gpt-5.4-nano-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000},
"gpt-5.5": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-5.5-2026-04-23": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000},
"gpt-5.5-cyber": {ContextTokens: 0, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000},
"gpt-5.6": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000},
"gpt-5.6-cyber": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000},
"gpt-5.6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1200000},
"gpt-5.6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000},
"gpt-5.6-terra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 12000000},
"gpt-6-astra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000},
"gpt-6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 500000},
"gpt-6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-audio": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-audio-1.5": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000},
"gpt-audio-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000},
"gpt-audio-mini-2025-12-15": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000},
"gpt-oss-120b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 350000, OutputCostPerMTokMicroUSD: 750000},
"gpt-rosalind-research": {ContextTokens: 0, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000},
"grok-4.20": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-latest-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-latest-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-beta-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-experimental-beta-0304": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-experimental-beta-0304-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-experimental-beta-0304-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-experimental-beta-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-experimental-beta-non-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-experimental-beta-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-non-reasoning-gv2": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-non-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-reasoning-gv2": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.20-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.3-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000},
"grok-4.5": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"grok-4.5-latest": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"grok-4.6": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"grok-4.7": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"grok-build-0.1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000},
"grok-build-latest": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"grok-code-fast": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000},
"grok-code-fast-1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000},
"grok-code-fast-1-0825": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000},
"kimi-k2.5": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 3000000},
"kimi-k2.6": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000},
"kimi-k2.7-code": {ContextTokens: 229376, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000},
"kimi-k3": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000},
"llama-3.3-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 850000, OutputCostPerMTokMicroUSD: 1200000},
"llama3.1-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 600000},
"llama3.1-8b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 100000},
"moonshot-v1-128k": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000},
"moonshot-v1-128k-vision-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000},
"moonshot-v1-32k": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000},
"moonshot-v1-32k-vision-preview": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000},
"moonshot-v1-8k": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000},
"moonshot-v1-8k-vision-preview": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000},
"moonshot-v1-auto": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000},
"o1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000},
"o3": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000},
"o3-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000},
"o4-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000},
"qwen-3-32b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 800000},
"qwen-3.8-27b": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 990000, OutputCostPerMTokMicroUSD: 1490000},
"qwen-coder": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1500000},
"qwen-max": {ContextTokens: 30720, InputCostPerMTokMicroUSD: 1600000, OutputCostPerMTokMicroUSD: 6400000},
"qwen-plus": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000},
"qwen-turbo": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000},
"qwen-turbo-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000},
"qwen3-next-80b-a3b-instruct": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000},
"qwen3-next-80b-a3b-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000},
"qwen3-vl-235b-a22b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000},
"qwen3-vl-235b-a22b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 4000000},
"qwen3-vl-32b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 640000},
"qwen3-vl-32b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 2870000},
"qwen3.7-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 7500000},
"qwen3.8-flash": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 470000},
"qwen3.8-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000},
"qwen3.8-omni-flash": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 470000},
"qwq-plus": {ContextTokens: 98304, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 2400000},
"text-embedding-3-large": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 130000, OutputCostPerMTokMicroUSD: 0},
"text-embedding-3-small": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 20000, OutputCostPerMTokMicroUSD: 0},
"text-embedding-ada-002": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0},
"text-embedding-ada-002-v2": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0},
}
registry.go
+9 -2
View File
@@ -664,6 +664,9 @@ type modelRegistry struct {
// cheap nil-map lookup (always false/zero) that adds no branching
// cost to the v0.1 no-aliases case.
aliases map[string]string
// requirePrefix is Config.RequireModelPrefix: resolve and listFor treat
// a bare id as unaddressable, leaving "provider/model" and aliases.
requirePrefix bool
// modelMeta is the validated Config.ModelMeta config-override map
// (feature v0.23), built once by newModelRegistry via
// validateModelMeta and never mutated afterward. nil (not merely
@@ -882,6 +885,7 @@ func newModelRegistry(adapters map[string]providerAdapter, cfg *Config, log func
return nil, err
}
m.aliases = aliases
m.requirePrefix = cfg.RequireModelPrefix
modelMeta, err := validateModelMeta(cfg.ModelMeta)
if err != nil {
@@ -1603,6 +1607,9 @@ func (m *modelRegistry) resolve(id string, grp *group) (providerAdapter, string,
if providerName, rest, ok := m.splitConfiguredProvider(id); ok {
return m.resolveAgainst(providerName, rest, id, "", grp, errModelUnknown)
}
if m.requirePrefix {
return nil, "", "", errModelUnknown
}
providerName, ok := m.bareWinner(id, grp)
if !ok {
return nil, "", "", errModelUnknown
@@ -1981,10 +1988,10 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any {
// that branch (harmless — matchesGlob is cheap), but still
// REQUIRED for the single-grant fast path, which never consults
// id at all.
if gwOK && grp.allowsProviderModel(groupWinner, id) && !shadowed && !ambiguousBareID {
if gwOK && grp.allowsProviderModel(groupWinner, id) && !shadowed && !ambiguousBareID && !m.requirePrefix {
out = append(out, modelObject(id, groupWinner, m.resolveMetaFor(groupWinner, id)))
}
if len(visible) < 2 && !shadowed && !ambiguousBareID {
if len(visible) < 2 && !shadowed && !ambiguousBareID && !m.requirePrefix {
continue // only one provider serving id is visible to grp: no disambiguating prefix to add
}
for _, p := range visible {
registry_requireprefix_test.go
+115
View File
@@ -0,0 +1,115 @@
package traefikllmgateway
import (
"errors"
"sort"
"testing"
)
func newRequirePrefixRegistry(t *testing.T, require bool) (*modelRegistry, map[string]providerAdapter) {
t.Helper()
adapters := map[string]providerAdapter{
"alpha": newFakeAdapter("alpha"),
"beta": newFakeAdapter("beta"),
}
cfg := &Config{
RequireModelPrefix: require,
ModelAliases: map[string]string{"fast": "alpha/solo", "shared-alias": "shared"},
Providers: map[string]*ProviderConfig{
"alpha": {Models: []string{"shared", "solo"}},
"beta": {Models: []string{"shared", "other"}},
},
}
reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {})
if err != nil {
t.Fatalf("newModelRegistry: %v", err)
}
return reg, adapters
}
func TestModelRegistry_RequireModelPrefix_Resolve(t *testing.T) {
t.Parallel()
reg, adapters := newRequirePrefixRegistry(t, true)
grp := allowAllGroup()
for _, id := range []string{"solo", "shared", "other"} {
if _, _, _, err := reg.resolve(id, grp); !errors.Is(err, errModelUnknown) {
t.Errorf("resolve(%q) err = %v, want errModelUnknown", id, err)
}
}
cases := []struct {
id string
provider string
canon string
}{
{"alpha/solo", "alpha", "alpha/solo"},
{"beta/shared", "beta", "beta/shared"},
{"fast", "alpha", "alpha/solo"},
{"shared-alias", "alpha", "alpha/shared"},
}
for _, tc := range cases {
adapter, _, canon, err := reg.resolve(tc.id, grp)
if err != nil {
t.Errorf("resolve(%q): %v", tc.id, err)
continue
}
if adapter != adapters[tc.provider] || canon != tc.canon {
t.Errorf("resolve(%q) = %v %q, want %s %q", tc.id, adapter, canon, tc.provider, tc.canon)
}
}
}
func TestModelRegistry_RequireModelPrefix_ListFor_NoBareIDs(t *testing.T) {
t.Parallel()
reg, _ := newRequirePrefixRegistry(t, true)
var got []string
for _, entry := range reg.listFor(allowAllGroup()) {
got = append(got, entry["id"].(string)) //nolint:forcetypeassert // modelObject always sets id to a string
}
sort.Strings(got)
want := []string{"alpha/shared", "alpha/solo", "beta/other", "beta/shared", "fast", "shared-alias"}
if len(got) != len(want) {
t.Fatalf("listFor ids = %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("listFor ids = %v, want %v", got, want)
}
}
}
func TestModelRegistry_RequireModelPrefix_ListFor_RespectsProviderAllowList(t *testing.T) {
t.Parallel()
reg, _ := newRequirePrefixRegistry(t, true)
grp := &group{name: "g", providers: []string{"beta"}}
for _, entry := range reg.listFor(grp) {
id := entry["id"].(string) //nolint:forcetypeassert // modelObject always sets id to a string
if entry["owned_by"] != "beta" {
t.Errorf("entry %q owned_by %v, want only beta-owned entries", id, entry["owned_by"])
}
}
if _, _, _, err := reg.resolve("alpha/solo", grp); !errors.Is(err, errModelDenied) {
t.Errorf("resolve(alpha/solo) err = %v, want errModelDenied", err)
}
}
func TestModelRegistry_RequireModelPrefix_Off_KeepsBareIDs(t *testing.T) {
t.Parallel()
reg, _ := newRequirePrefixRegistry(t, false)
if _, _, canon, err := reg.resolve("solo", allowAllGroup()); err != nil || canon != "alpha/solo" {
t.Errorf("resolve(solo) = %q, %v, want alpha/solo, nil", canon, err)
}
var sawBare bool
for _, entry := range reg.listFor(allowAllGroup()) {
if entry["id"] == "solo" {
sawBare = true
}
}
if !sawBare {
t.Error("listFor lost bare id \"solo\" with requireModelPrefix off")
}
}
sse.go
+13 -5
View File
@@ -164,7 +164,7 @@ func readSSE(r io.Reader, fn func(sseEvent) error) error {
// suite, the SSE streaming path); tools/yaegi-check never exercises
// a live streaming response.
event = ""
dataLines = nil
dataLines = dataLines[:0]
hasData = false
dataBytes = 0
if !fire {
@@ -315,12 +315,20 @@ func splitSSELines(b []byte) [][]byte {
// "\r" of any kind ever reaches the wire from here, regardless of what
// readSSE upstream of this call did or did not already split it on.
func (s *sseWriter) writeData(b []byte) error {
lines := splitSSELines(b)
buf := make([]byte, 0, len(b)+len(lines)*len(sseDataPrefix)+1)
for _, line := range lines {
var buf []byte
if !bytes.ContainsAny(b, "\r\n") {
buf = make([]byte, 0, len(sseDataPrefix)+len(b)+2)
buf = append(buf, sseDataPrefix...)
buf = append(buf, line...)
buf = append(buf, b...)
buf = append(buf, '\n')
} else {
lines := splitSSELines(b)
buf = make([]byte, 0, len(b)+len(lines)*len(sseDataPrefix)+1)
for _, line := range lines {
buf = append(buf, sseDataPrefix...)
buf = append(buf, line...)
buf = append(buf, '\n')
}
}
buf = append(buf, '\n')
tools/pricing-sync/main.go
+72 -4
View File
@@ -57,6 +57,14 @@
// a defensive fallback, not just a hardcoded "sample_spec" skip, so
// a future upstream schema-example entry with some other key name
// degrades the same way.
// 4. Freshness: an entry is dropped when LiteLLM's own deprecation_date
// has already passed, or when a full calendar date embedded in its
// bare id ("claude-opus-4-1-20250805", "gpt-4o-2024-11-20") is
// older than the -max-age-months flag (default 12; 0 turns the whole
// rule off). LiteLLM carries no release date, so an id with no such
// date and no deprecation_date is always kept: an undated alias is
// never dropped on a guess. An id in alwaysKeep, or in the -keep
// flag's comma-separated list, is exempt.
//
// # Bare-id collision resolution
//
@@ -95,6 +103,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"go/format"
"io"
@@ -102,6 +111,7 @@ import (
"net/http"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
@@ -129,6 +139,51 @@ const microUSDPerUSDPerToken = 1e12
// this specific, known case.
const sampleSpecKey = "sample_spec"
// defaultMaxAgeMonths is the -max-age-months flag's default.
const defaultMaxAgeMonths = 12
// alwaysKeep lists bare ids the freshness rule never drops, because they
// are still served after their age or deprecation date says otherwise.
var alwaysKeep = map[string]bool{
"claude-sonnet-4-5-20250929": true,
}
// pruneOptions is the freshness rule's configuration. maxAgeMonths <= 0
// turns the rule off.
type pruneOptions struct {
now time.Time
keep map[string]bool
maxAgeMonths int
}
// idDatePattern finds a full calendar date in a model id, as YYYYMMDD or
// YYYY-MM-DD, delimited by the id's ends or a "-" or "_".
var idDatePattern = regexp.MustCompile(`(?:^|[-_])(20\d{2})-?(0[1-9]|1[0-2])-?(0[1-9]|[12]\d|3[01])(?:$|[-_])`)
// isStale reports whether an entry fails the freshness rule: its
// deprecation_date (YYYY-MM-DD) is before opts.now, or a date embedded in
// bareID is more than opts.maxAgeMonths before it. Ids in opts.keep and
// alwaysKeep never fail.
func isStale(bareID, deprecationDate string, opts pruneOptions) bool {
if opts.maxAgeMonths <= 0 || opts.keep[bareID] || alwaysKeep[bareID] {
return false
}
if deprecationDate != "" {
if d, err := time.Parse("2006-01-02", deprecationDate); err == nil && d.Before(opts.now) {
return true
}
}
m := idDatePattern.FindStringSubmatch(bareID)
if m == nil {
return false
}
d, err := time.Parse("20060102", m[1]+m[2]+m[3])
if err != nil {
return false
}
return d.Before(opts.now.AddDate(0, -opts.maxAgeMonths, 0))
}
// allowedProviders is the pruning rule's provider allowlist — see the
// package doc comment for the full rationale and the qwen/alibaba ->
// dashscope mapping.
@@ -218,6 +273,7 @@ func applyNamingBridges(table map[string]builtinEntry, winners map[string]candid
// max_input_tokens/max_tokens as a JSON float (2000000.0) — decoding
// those into *int would fail outright and drop the entry.
type litellmEntry struct {
DeprecationDate string `json:"deprecation_date"`
InputCostPerToken *float64 `json:"input_cost_per_token"`
OutputCostPerToken *float64 `json:"output_cost_per_token"`
MaxInputTokens *float64 `json:"max_input_tokens"`
@@ -254,9 +310,18 @@ func main() {
}
func run() error {
maxAge := flag.Int("max-age-months", defaultMaxAgeMonths, "drop entries older than this many months; 0 disables the freshness rule")
keepList := flag.String("keep", "", "comma-separated bare ids the freshness rule never drops")
flag.Parse()
repoRoot := "."
if len(os.Args) > 1 {
repoRoot = os.Args[1]
if flag.NArg() > 0 {
repoRoot = flag.Arg(0)
}
keep := map[string]bool{}
for _, id := range strings.Split(*keepList, ",") {
if id = strings.TrimSpace(id); id != "" {
keep[id] = true
}
}
outPath := filepath.Join(repoRoot, "pricing_data_gen.go")
@@ -268,7 +333,7 @@ func run() error {
return fmt.Errorf("fetch %s: %w", litellmSourceURL, err)
}
table, totalUpstream, err := parseAndPrune(data)
table, totalUpstream, err := parseAndPrune(data, pruneOptions{now: time.Now(), maxAgeMonths: *maxAge, keep: keep})
if err != nil {
return fmt.Errorf("parse and prune: %w", err)
}
@@ -319,7 +384,7 @@ func fetchLiteLLMPrices(ctx context.Context, url string) ([]byte, error) {
// chooseWinner's deterministic bare-id collision resolution. It returns
// the pruned table and the total number of top-level keys data carried
// (including ones this function skips), for the summary line run prints.
func parseAndPrune(data []byte) (map[string]builtinEntry, int, error) {
func parseAndPrune(data []byte, opts pruneOptions) (map[string]builtinEntry, int, error) {
var raw map[string]json.RawMessage
if err := json.Unmarshal(data, &raw); err != nil {
return nil, 0, fmt.Errorf("decode top-level object: %w", err)
@@ -346,6 +411,9 @@ func parseAndPrune(data []byte) (map[string]builtinEntry, int, error) {
}
bare, isBareKey := bareModelID(key, e.LiteLLMProvider)
if isStale(bare, e.DeprecationDate, opts) {
continue
}
byBare[bare] = append(byBare[bare], candidate{
fullKey: key,
provider: e.LiteLLMProvider,
tools/pricing-sync/main_test.go
+80 -1
View File
@@ -6,6 +6,12 @@ import (
"reflect"
"strings"
"testing"
"time"
)
var (
fixtureNow = time.Date(2026, time.August, 1, 0, 0, 0, 0, time.UTC)
fixtureOpts = pruneOptions{now: fixtureNow, maxAgeMonths: defaultMaxAgeMonths}
)
// fixtureJSON is a small, hand-built stand-in for LiteLLM's real
@@ -116,7 +122,7 @@ const fixtureUpstreamKeyCount = 17
// out, covering every pruning-rule branch fixtureJSON's own doc comment
// lists.
func TestParseAndPrune_Golden(t *testing.T) {
got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON))
got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON), fixtureOpts)
if err != nil {
t.Fatalf("parseAndPrune: %v", err)
}
@@ -264,3 +270,76 @@ func TestContentDigest(t *testing.T) {
t.Errorf("len(contentDigest(...)) = %d, want %d", len(a), contentDigestLen)
}
}
func TestIsStale(t *testing.T) {
now := time.Date(2026, time.September, 29, 0, 0, 0, 0, time.UTC)
tests := []struct {
name, id, deprecated string
want bool
}{
{"undated alias kept", "claude-opus-4-1", "", false},
{"recent compact date kept", "claude-sonnet-5-20260401", "", false},
{"old compact date dropped", "claude-opus-4-1-20250805", "", true},
{"old dashed date dropped", "gpt-4o-2024-11-20", "", true},
{"just inside cutoff kept", "model-2025-10-01", "", false},
{"just outside cutoff dropped", "model-2025-09-28", "", true},
{"date mid-id dropped", "gpt-4o-2024-11-20-preview", "", true},
{"short MMDD suffix ignored", "grok-4.20-experimental-beta-0304", "", false},
{"invalid month ignored", "model-20251301", "", false},
{"past deprecation dropped", "gpt-4-turbo", "2025-06-01", true},
{"future deprecation kept", "gpt-4-turbo", "2027-01-01", false},
{"bad deprecation format ignored", "gpt-4-turbo", "soon", false},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
opts := pruneOptions{now: now, maxAgeMonths: defaultMaxAgeMonths}
if got := isStale(tc.id, tc.deprecated, opts); got != tc.want {
t.Errorf("isStale(%q, %q) = %v, want %v", tc.id, tc.deprecated, got, tc.want)
}
})
}
}
func TestParseAndPrune_DropsStaleEntries(t *testing.T) {
const data = `{
"gpt-fresh": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6},
"gpt-old-2024-05-13": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6},
"gpt-dead": {"litellm_provider": "openai", "mode": "chat", "deprecation_date": "2026-01-01", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}
}`
got, _, err := parseAndPrune([]byte(data), fixtureOpts)
if err != nil {
t.Fatalf("parseAndPrune: %v", err)
}
if len(got) != 1 {
t.Fatalf("kept %d entries %v, want only gpt-fresh", len(got), got)
}
if _, ok := got["gpt-fresh"]; !ok {
t.Errorf("gpt-fresh missing from %v", got)
}
}
func TestIsStale_Options(t *testing.T) {
now := time.Date(2026, time.September, 29, 0, 0, 0, 0, time.UTC)
const old = "gpt-4o-2024-08-06"
tests := []struct {
name string
opts pruneOptions
id string
want bool
}{
{"default drops old", pruneOptions{now: now, maxAgeMonths: 12}, old, true},
{"zero disables rule", pruneOptions{now: now, maxAgeMonths: 0}, old, false},
{"negative disables rule", pruneOptions{now: now, maxAgeMonths: -1}, old, false},
{"longer window keeps", pruneOptions{now: now, maxAgeMonths: 36}, old, false},
{"shorter window drops more", pruneOptions{now: now, maxAgeMonths: 3}, "model-2026-03-01", true},
{"keep flag exempts", pruneOptions{now: now, maxAgeMonths: 12, keep: map[string]bool{old: true}}, old, false},
{"alwaysKeep exempts", pruneOptions{now: now, maxAgeMonths: 12}, "claude-sonnet-4-5-20250929", false},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := isStale(tc.id, "", tc.opts); got != tc.want {
t.Errorf("isStale(%q) = %v, want %v", tc.id, got, tc.want)
}
})
}
}
translate_anthropic.go
+102 -2
View File
@@ -3,7 +3,9 @@ package traefikllmgateway
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"unicode/utf8"
)
// anthropicAPIVersion is the "anthropic-version" header value every request
@@ -695,6 +697,7 @@ type anthropicStreamState struct {
toolOrdinals map[int]int
id string
model string
content contentChunker
u usage
created int64
nextToolOrdinal int
@@ -705,7 +708,9 @@ type anthropicStreamState struct {
// created the response timestamp every emitted chunk carries — the
// adapter fixes both once, before the first upstream event arrives.
func newAnthropicStreamState(model string, created int64) *anthropicStreamState {
return &anthropicStreamState{model: model, created: created, toolOrdinals: map[int]int{}}
st := &anthropicStreamState{model: model, created: created, toolOrdinals: map[int]int{}}
st.content.set(chatCompletionIDPrefix, model, created)
return st
}
// usage returns the prompt/completion token counts captured so far, from
@@ -741,6 +746,100 @@ func (st *anthropicStreamState) chunk(delta map[string]any, finishReason *string
return b
}
// contentChunkHead frames a text-delta chunk with the same alphabetical
// key order json.Marshal gives a map, so the bytes are identical to the
// chunk method's output for map[string]any{"content": text}.
const contentChunkHead = `{"choices":[{"delta":{"content":`
// contentChunker builds text-delta chunks without allocating the envelope
// maps a state's chunk method needs for arbitrary deltas.
type contentChunker struct {
// tail is the per-stream suffix after the delta text, rebuilt by set.
tail []byte
}
// set fixes the envelope fields every later chunk carries; call it again
// whenever the stream id changes.
func (c *contentChunker) set(id, model string, created int64) {
idb, ierr := json.Marshal(id)
mb, merr := json.Marshal(model)
if ierr != nil || merr != nil {
panic("llmgateway: stream chunk failed to marshal id or model")
}
t := append([]byte(nil), `},"finish_reason":null,"index":0}],"created":`...)
t = strconv.AppendInt(t, created, 10)
t = append(t, `,"id":`...)
t = append(t, idb...)
t = append(t, `,"model":`...)
t = append(t, mb...)
t = append(t, `,"object":"chat.completion.chunk"}`...)
c.tail = t
}
func (c *contentChunker) build(text string) []byte {
out := make([]byte, 0, len(contentChunkHead)+len(text)+len(text)/8+2+len(c.tail))
out = append(out, contentChunkHead...)
out = appendJSONString(out, text)
return append(out, c.tail...)
}
const jsonHex = "0123456789abcdef"
// appendJSONString appends s as a quoted JSON string using
// encoding/json's default escaping (HTML characters, U+2028/U+2029 and
// invalid UTF-8 included), so its output equals json.Marshal(s).
func appendJSONString(dst []byte, s string) []byte {
dst = append(dst, '"')
start := 0
for i := 0; i < len(s); {
b := s[i]
if b < utf8.RuneSelf {
if b >= 0x20 && b != '"' && b != '\\' && b != '<' && b != '>' && b != '&' {
i++
continue
}
dst = append(dst, s[start:i]...)
switch b {
case '"', '\\':
dst = append(dst, '\\', b)
case '\b':
dst = append(dst, '\\', 'b')
case '\f':
dst = append(dst, '\\', 'f')
case '\n':
dst = append(dst, '\\', 'n')
case '\r':
dst = append(dst, '\\', 'r')
case '\t':
dst = append(dst, '\\', 't')
default:
dst = append(dst, '\\', 'u', '0', '0', jsonHex[b>>4], jsonHex[b&0xF])
}
i++
start = i
continue
}
r, size := utf8.DecodeRuneInString(s[i:])
if r == utf8.RuneError && size == 1 {
dst = append(dst, s[start:i]...)
dst = append(dst, "\xef\xbf\xbd"...)
i += size
start = i
continue
}
if r == '\u2028' || r == '\u2029' {
dst = append(dst, s[start:i]...)
dst = append(dst, '\\', 'u', '2', '0', '2', jsonHex[r&0xF])
i += size
start = i
continue
}
i += size
}
dst = append(dst, s[start:]...)
return append(dst, '"')
}
// usageChunk builds the final usage-only "chat.completion.chunk" payload —
// empty "choices" plus a populated "usage" — mirroring the shape
// provider_openai.go's forwardStream relays from a native OpenAI upstream
@@ -854,6 +953,7 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) {
return nil, fmt.Errorf("%w: decode anthropic message_start event: %w", errUpstream, err)
}
st.id = p.Message.ID
st.content.set(chatCompletionIDPrefix+st.id, st.model, st.created)
st.u.prompt = p.Message.Usage.totalInputTokens()
return [][]byte{st.chunk(map[string]any{"role": "assistant"}, nil)}, nil
@@ -886,7 +986,7 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) {
}
switch p.Delta.Type {
case "text_delta":
return [][]byte{st.chunk(map[string]any{"content": p.Delta.Text}, nil)}, nil
return [][]byte{st.content.build(p.Delta.Text)}, nil
case "input_json_delta":
ordinal, ok := st.toolOrdinals[p.Index]
if !ok {
translate_anthropic_test.go
+62
View File
@@ -1,6 +1,7 @@
package traefikllmgateway
import (
"bytes"
"context"
"encoding/json"
"errors"
@@ -12,6 +13,7 @@ import (
"sort"
"strings"
"testing"
"unicode/utf8"
"github.com/stretchr/testify/assert"
)
@@ -1127,3 +1129,63 @@ func TestOpenAIRequestFromAnthropic_NoTools_OmitsToolsAndToolChoiceFields(t *tes
t.Errorf(`out["tool_choice"] = %v, want the key absent when no tool_choice was requested`, out["tool_choice"])
}
}
// sameJSON reports whether got and want encode the same value. Invalid UTF-8
// input is escaped as \ufffd by encoding/json before Go 1.26 and written as
// the raw replacement rune after, so those inputs compare semantically.
func sameJSON(t *testing.T, input string, got, want []byte) bool {
t.Helper()
if bytes.Equal(got, want) {
return true
}
if utf8.ValidString(input) {
return false
}
var g, w any
if err := json.Unmarshal(got, &g); err != nil {
return false
}
if err := json.Unmarshal(want, &w); err != nil {
return false
}
return reflect.DeepEqual(g, w)
}
func TestContentChunkerMatchesChunk(t *testing.T) {
texts := []string{"", "Hello", "a \"quoted\" <b>&amp;</b>", "line\nbreak\ttab", "unicode é 日本 \u2028\u2029", "bad\xffutf8"}
for _, id := range []string{"", "msg_01", "we\"ird"} {
ast := newAnthropicStreamState("gw-model", 1734000000)
ast.id = id
ast.content.set(chatCompletionIDPrefix+id, ast.model, ast.created)
gst := newGeminiStreamState("gw-model", 1734000000)
gst.id = id
gst.content.set(chatCompletionIDPrefix+id, gst.model, gst.created)
for _, text := range texts {
gotA := ast.content.build(text)
if want := ast.chunk(map[string]any{"content": text}, nil); !sameJSON(t, text, gotA, want) {
t.Errorf("anthropic id=%q text=%q:\n got %s\nwant %s", id, text, gotA, want)
}
gotG := gst.content.build(text)
if want := gst.chunk(map[string]any{"content": text}, nil); !sameJSON(t, text, gotG, want) {
t.Errorf("gemini id=%q text=%q:\n got %s\nwant %s", id, text, gotG, want)
}
}
}
}
func TestAppendJSONStringMatchesMarshal(t *testing.T) {
var cases []string
for r := rune(0); r < 0x250; r++ {
cases = append(cases, string(r), "a"+string(r)+"b")
}
cases = append(cases, "\u2028", "\u2029", "x\u2028y\u2029z", "bad\xffutf8", "\xc3", "\xe2\x80", "\U0001F600 emoji", "", "<script>alert('&')</script>")
for _, s := range cases {
want, err := json.Marshal(s)
if err != nil {
t.Fatalf("Marshal(%q): %v", s, err)
}
if got := appendJSONString(nil, s); !sameJSON(t, s, got, want) {
t.Errorf("appendJSONString(%q) = %s, want %s", s, got, want)
}
}
}
translate_gemini.go
+6 -2
View File
@@ -655,6 +655,7 @@ func (m *geminiUsageMetadata) safeCompletion() int64 {
type geminiStreamState struct {
id string
model string
content contentChunker
u usage
created int64
// nextToolOrdinal is the per-stream contiguous tool-call counter lesson
@@ -681,7 +682,9 @@ type geminiStreamState struct {
// shares one stable, non-empty id instead of drifting between an empty
// string and whatever a later chunk happens to report.
func newGeminiStreamState(model string, created int64) *geminiStreamState {
return &geminiStreamState{model: model, created: created, id: geminiResponseIDOrFallback("", created)}
st := &geminiStreamState{model: model, created: created, id: geminiResponseIDOrFallback("", created)}
st.content.set(chatCompletionIDPrefix+st.id, model, created)
return st
}
// usage returns the prompt/completion token counts captured so far, from
@@ -783,6 +786,7 @@ func (st *geminiStreamState) translate(ev sseEvent) ([][]byte, error) {
isFirstChunk := !st.startedRole
if isFirstChunk && resp.ResponseID != "" {
st.id = resp.ResponseID
st.content.set(chatCompletionIDPrefix+st.id, st.model, st.created)
}
if resp.UsageMetadata != nil {
@@ -832,7 +836,7 @@ func (st *geminiStreamState) translate(ev sseEvent) ([][]byte, error) {
st.nextToolOrdinal++
chunks = append(chunks, st.chunk(delta, nil))
case p.Text != "":
chunks = append(chunks, st.chunk(map[string]any{"content": p.Text}, nil))
chunks = append(chunks, st.content.build(p.Text))
}
}