diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 572bc7c..008db70 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,7 +18,7 @@ jobs: pr-checks: uses: lukaszraczylo/shared-actions/.github/workflows/go-pr.yaml@main with: - go-version: "1.25.x" + go-version: "1.26.x" coverage-threshold: 75 secrets: inherit @@ -40,7 +40,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v6 with: - go-version: "1.25.x" + go-version: "1.26.x" cache: true - name: yaegi-check diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5f844a..93d4043 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,5 +24,5 @@ jobs: release: uses: lukaszraczylo/shared-actions/.github/workflows/go-release.yaml@main with: - go-version: "1.25.x" + go-version: "1.26.x" secrets: inherit diff --git a/Makefile b/Makefile index 156ee62..c91e6b8 100644 --- a/Makefile +++ b/Makefile @@ -40,8 +40,12 @@ yaegi-check: # pruning rule and naming-bridge mechanism. Not run by CI (network # fetch, and the generated table is committed); an operator re-runs this # by hand when LiteLLM's own pricing drifts. +# PRICING_MAX_AGE_MONTHS drops entries older than N months (0 = keep all); +# PRICING_KEEP is a comma-separated list of bare ids exempt from that rule. +PRICING_MAX_AGE_MONTHS ?= 12 +PRICING_KEEP ?= pricing-sync: - cd tools/pricing-sync && GOWORK=off go run . $(CURDIR) + cd tools/pricing-sync && GOWORK=off go run . -max-age-months=$(PRICING_MAX_AGE_MONTHS) -keep=$(PRICING_KEEP) $(CURDIR) gofmt -w pricing_data_gen.go INTEGRATION_COMPOSE := integration/docker-compose.yml diff --git a/README.md b/README.md index b34b432..c4408f3 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,7 @@ config accepts them as YAML, which decodes to the same JSON shape. | `admin` | `*AdminConfig` | `nil` (disabled) | Read-only admin dashboard — see [Admin](#admin). `nil` or `enabled: false` means the `/admin*` routes are not registered at all. | | `metrics` | `*MetricsConfig` | `nil` (disabled) | Prometheus text-exposition endpoint — see [Metrics](#metrics). `nil` or `enabled: false` means the metrics route is not registered at all. | | `modelAliases` | `map[string]string` | `{}` | Operator-defined alias id → target model id — see [Model aliases](#model-aliases). Omitted or empty means no aliases, byte-identical to a gateway built before this field existed. | +| `requireModelPrefix` | `bool` | `false` | When `true`, a model resolves only as `provider/model` or as a `modelAliases` id. A bare id returns 404 `unknown model` and `GET /v1/models` lists no bare ids. `false` keeps the sorted-first bare-id collision winner. | | `modelMeta` | `map[string]*ModelMetaConfig` | `{}` | Per-model context-window and cost overrides, keyed by exact `provider/model` or a bare model id — see [Model metadata](#model-metadata). Omitted or empty means no overrides; every model's metadata still resolves through discovery and the built-in table. | | `passthroughUnknown` | `bool` | `false` | `false`: a request matching none of the plugin's routes gets a 404 JSON envelope. `true`: it falls through to the router's own backing service. **Does not cover the three media routes** — see the upgrade note below. | | `maxInFlightBodyRequests` | `int` | `0` (self-tuned) | Caps how many unified/media requests may concurrently be inside their own request-body read+decode step — see [Body-admission limit](#body-admission-limit). `0` self-tunes from the host's own `GOMAXPROCS`; a positive value pins an exact cap instead, clamped to a maximum of 10,000. | diff --git a/bench_test.go b/bench_test.go new file mode 100644 index 0000000..ec2ccf6 --- /dev/null +++ b/bench_test.go @@ -0,0 +1,226 @@ +package traefikllmgateway + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +const benchStreamDeltas = 500 + +func benchOpenAIRequest() map[string]any { + return map[string]any{ + "model": "bench-model", + "max_tokens": float64(1024), + "temperature": 0.7, + "messages": []any{ + map[string]any{"role": "system", "content": "You are a helpful assistant."}, + map[string]any{"role": "user", "content": "What is the weather in Paris and Berlin?"}, + map[string]any{"role": "assistant", "content": "", "tool_calls": []any{ + map[string]any{"id": "call_1", "type": "function", "function": map[string]any{"name": "get_weather", "arguments": `{"city":"Paris"}`}}, + map[string]any{"id": "call_2", "type": "function", "function": map[string]any{"name": "get_weather", "arguments": `{"city":"Berlin"}`}}, + }}, + map[string]any{"role": "tool", "tool_call_id": "call_1", "content": "18C, cloudy"}, + map[string]any{"role": "tool", "tool_call_id": "call_2", "content": "15C, rain"}, + map[string]any{"role": "user", "content": "Summarise both in one sentence."}, + }, + "tools": []any{ + map[string]any{"type": "function", "function": map[string]any{ + "name": "get_weather", + "description": "Get the weather for a city", + "parameters": map[string]any{ + "type": "object", + "properties": map[string]any{"city": map[string]any{"type": "string"}}, + "required": []any{"city"}, + }, + }}, + }, + "tool_choice": "auto", + } +} + +func benchAnthropicStream() []byte { + var sb strings.Builder + sb.WriteString("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01B\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"claude-opus-5\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n") + sb.WriteString("event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n") + for i := 0; i < benchStreamDeltas; i++ { + fmt.Fprintf(&sb, "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" token%d\"}}\n\n", i) + } + sb.WriteString("event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n") + sb.WriteString("event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"output_tokens\":500}}\n\n") + sb.WriteString("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n") + return []byte(sb.String()) +} + +func benchGeminiStream() []byte { + var sb strings.Builder + for i := 0; i < benchStreamDeltas; i++ { + fmt.Fprintf(&sb, "data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\" token%d\"}]}}],\"responseId\":\"resp_B\"}\n\n", i) + } + sb.WriteString("data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[]},\"finishReason\":\"STOP\"}],\"usageMetadata\":{\"promptTokenCount\":5,\"candidatesTokenCount\":500}}\n\n") + return []byte(sb.String()) +} + +func BenchmarkAnthropicRequestFromOpenAI(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := anthropicRequestFromOpenAI(benchOpenAIRequest()); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkGeminiRequestFromOpenAI(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := geminiRequestFromOpenAI(benchOpenAIRequest()); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkOpenAIResponseFromAnthropic(b *testing.B) { + body := []byte(`{"id":"msg_01XYZ","type":"message","role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."},{"type":"tool_use","id":"tu_1","name":"get_weather","input":{"city":"Paris"}}],"model":"claude-opus-5","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":8}}`) + b.ReportAllocs() + b.SetBytes(int64(len(body))) + for i := 0; i < b.N; i++ { + if _, _, err := openAIResponseFromAnthropic(body, "gw-model", 1734000000); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkOpenAIResponseFromGemini(b *testing.B) { + body := []byte(`{"candidates":[{"content":{"role":"model","parts":[{"text":"The capital of France is Paris."},{"functionCall":{"name":"get_weather","args":{"city":"Paris"}}}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":12,"candidatesTokenCount":8},"responseId":"resp_1"}`) + b.ReportAllocs() + b.SetBytes(int64(len(body))) + for i := 0; i < b.N; i++ { + if _, _, err := openAIResponseFromGemini(body, "gw-model", 1734000000); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkReadSSE(b *testing.B) { + stream := benchAnthropicStream() + b.ReportAllocs() + b.SetBytes(int64(len(stream))) + for i := 0; i < b.N; i++ { + if err := readSSE(bytes.NewReader(stream), func(sseEvent) error { return nil }); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkAnthropicStreamTranslate(b *testing.B) { + stream := benchAnthropicStream() + b.ReportAllocs() + b.SetBytes(int64(len(stream))) + for i := 0; i < b.N; i++ { + st := newAnthropicStreamState("gw-model", 1734000000) + err := readSSE(bytes.NewReader(stream), func(ev sseEvent) error { + _, terr := st.translate(ev) + return terr + }) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkGeminiStreamTranslate(b *testing.B) { + stream := benchGeminiStream() + b.ReportAllocs() + b.SetBytes(int64(len(stream))) + for i := 0; i < b.N; i++ { + st := newGeminiStreamState("gw-model", 1734000000) + err := readSSE(bytes.NewReader(stream), func(ev sseEvent) error { + _, terr := st.translate(ev) + return terr + }) + if err != nil { + b.Fatal(err) + } + } +} + +func newBenchGateway(b *testing.B, provider *ProviderConfig) http.Handler { + b.Helper() + devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0) + if err != nil { + b.Fatalf("open %s: %v", os.DevNull, err) + } + origStderr := os.Stderr + os.Stderr = devNull + b.Cleanup(func() { + os.Stderr = origStderr + _ = devNull.Close() + }) + cfg := CreateConfig() + cfg.Providers = map[string]*ProviderConfig{"bench": provider} + cfg.Groups = map[string]*GroupConfig{"default": {}} + cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "alice", Group: "default", APIKey: "sk-alice", Limits: &LimitsConfig{}}}} + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}) + h, err := New(context.Background(), next, cfg, "llmgw") + if err != nil { + b.Fatalf("New: %v", err) + } + return h +} + +func benchChatRequest(payload []byte) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(payload)) + req.Header.Set("Authorization", "Bearer sk-alice") + req.Header.Set("Content-Type", "application/json") + return req +} + +func BenchmarkUnifiedChatOpenAI(b *testing.B) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"x","object":"chat.completion","model":"bench-model","choices":[{"index":0,"message":{"role":"assistant","content":"hello"},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5}}`)) + })) + b.Cleanup(upstream.Close) + h := newBenchGateway(b, &ProviderConfig{Type: "openai", BaseURL: upstream.URL, APIKey: "k", Models: []string{"bench-model"}}) + payload := []byte(`{"model":"bench-model","messages":[{"role":"user","content":"hi"}]}`) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, benchChatRequest(payload)) + if rec.Code != http.StatusOK { + b.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + } +} + +func BenchmarkUnifiedChatAnthropicStream(b *testing.B) { + stream := benchAnthropicStream() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write(stream) + })) + b.Cleanup(upstream.Close) + h := newBenchGateway(b, &ProviderConfig{Type: "anthropic", BaseURL: upstream.URL, APIKey: "k", Models: []string{"bench-model"}}) + payload := []byte(`{"model":"bench-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`) + + b.ReportAllocs() + b.SetBytes(int64(len(stream))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, benchChatRequest(payload)) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "[DONE]") { + b.Fatalf("status = %d, body tail = %.200s", rec.Code, rec.Body.String()) + } + } +} diff --git a/llmgateway.go b/llmgateway.go index 53de021..8901039 100644 --- a/llmgateway.go +++ b/llmgateway.go @@ -55,6 +55,11 @@ type Config struct { // (validateModelAliases, registry.go). ModelAliases map[string]string `json:"modelAliases,omitempty"` Providers map[string]*ProviderConfig `json:"providers,omitempty"` + // RequireModelPrefix, when true, makes every model addressable only as + // "provider/model" or as a ModelAliases id: a bare discovered id is not + // resolved (404 unknown model) and is not listed by GET /v1/models. It + // removes the sorted-first bare-id collision winner. Defaults to false. + RequireModelPrefix bool `json:"requireModelPrefix,omitempty"` // ModelMeta declares per-model metadata overrides (feature v0.23): // context window size and per-token cost, keyed by an exact // "provider/model" id or a bare model id (applying wherever that bare diff --git a/pricing_data_gen.go b/pricing_data_gen.go index f2b4f3f..7101eb9 100644 --- a/pricing_data_gen.go +++ b/pricing_data_gen.go @@ -4,247 +4,183 @@ package traefikllmgateway // model_prices_and_context_window.json — DO NOT EDIT. // // Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json -// Source digest: sha256:0e8d23239c27 -// 236 entries kept (pruned from 3111 upstream entries — see +// Source digest: sha256:9f7b337871ed +// 172 entries kept (pruned from 4422 upstream entries — see // tools/pricing-sync/main.go's own doc comment for the exact pruning // rule and bare-id collision resolution). // // Regenerate with: make pricing-sync var builtinModelMetaTable = map[string]builtinModelMeta{ - "MiniMax-M2": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "MiniMax-M2.1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "MiniMax-M2.1-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M2.1-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M2.5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "MiniMax-M2.5-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M2.5-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "chat-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "chatgpt-4o-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-3-7-sonnet-20250219": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-3-haiku-20240307": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 1250000}, - "claude-3-opus-20240229": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-4-opus-20250514": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-4-sonnet-20250514": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-fable-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-haiku-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, - "claude-haiku-4-5-20251001": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, - "claude-mythos-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-mythos-preview": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-opus-4-1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-opus-4-1-20250805": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-opus-4-20250514": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-opus-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-5-20251101": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-6-20260205": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-7": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-7-20260416": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-8": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-sonnet-4-20250514": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-4-5-20250929": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "daybreak-blue-latest": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "daybreak-red-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, - "deepseek-chat": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, - "deepseek-coder": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 140000, OutputCostPerMTokMicroUSD: 280000}, - "deepseek-r1": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 550000, OutputCostPerMTokMicroUSD: 2190000}, - "deepseek-reasoner": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, - "deepseek-v3": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 270000, OutputCostPerMTokMicroUSD: 1100000}, - "deepseek-v3.2": {ContextTokens: 163840, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 400000}, - "deepseek-v4-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 440000, OutputCostPerMTokMicroUSD: 1320000}, - "deepseek-v4-flash-0731": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 400000}, - "deepseek-v4-pro": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1320000, OutputCostPerMTokMicroUSD: 3960000}, - "ft:gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-3.5-turbo-0613": {ContextTokens: 4096, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "ft:gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 12000000}, - "ft:gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 3200000}, - "ft:gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 800000}, - "ft:gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000}, - "ft:gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000}, - "ft:gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "ft:o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 16000000}, - "glm-5.1": {ContextTokens: 202745, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, - "glm-5.2": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, - "gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, - "gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, - "gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "gpt-3.5-turbo-16k": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 4000000}, - "gpt-4": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "gpt-4-0125-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-0314": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "gpt-4-1106-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-turbo": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-turbo-2024-04-09": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-turbo-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4.1": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "gpt-4.1-mini": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, - "gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, - "gpt-4.1-nano": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-4o": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-2024-05-13": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, - "gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-audio-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-audio-preview-2025-06-03": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-audio-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-search-preview-2025-03-11": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-search-preview-2025-03-11": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-chat": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, - "gpt-5-mini-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, - "gpt-5-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-5-nano-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-5-search-api": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-search-api-2025-10-14": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.1": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.1-2025-11-13": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.1-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.2": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.2-2025-12-11": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.2-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.3-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.4": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, - "gpt-5.4-2026-03-05": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, - "gpt-5.4-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, - "gpt-5.4-mini-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, - "gpt-5.4-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, - "gpt-5.4-nano-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, - "gpt-5.5": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.5-2026-04-23": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.6": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.6-cyber": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, - "gpt-5.6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1200000}, - "gpt-5.6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.6-terra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 12000000}, - "gpt-audio": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio-1.5": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio-2025-08-28": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, - "gpt-audio-mini-2025-10-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, - "gpt-audio-mini-2025-12-15": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, - "gpt-oss-120b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 350000, OutputCostPerMTokMicroUSD: 750000}, - "grok-2": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-1212": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-vision": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-vision-1212": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-vision-latest": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-3": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-3-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-3-fast-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "grok-3-fast-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "grok-3-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-3-mini": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 500000}, - "grok-3-mini-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 500000}, - "grok-3-mini-fast": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 4000000}, - "grok-3-mini-fast-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 4000000}, - "grok-3-mini-fast-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 4000000}, - "grok-3-mini-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-4-0709": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-4-1-fast": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-1-fast-non-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-1-fast-non-reasoning-latest": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-1-fast-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-1-fast-reasoning-latest": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-fast-non-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-fast-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-latest": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-4.20-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-multi-agent-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-multi-agent-beta-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.3-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.5": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, - "grok-4.5-latest": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, - "grok-4.6": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, - "grok-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-build-0.1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "grok-code-fast": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "grok-code-fast-1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "grok-code-fast-1-0825": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "grok-vision-beta": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, - "kimi-k2-0711-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000}, - "kimi-k2-0905-preview": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000}, - "kimi-k2-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000}, - "kimi-k2-thinking-turbo": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 1150000, OutputCostPerMTokMicroUSD: 8000000}, - "kimi-k2-turbo-preview": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 1150000, OutputCostPerMTokMicroUSD: 8000000}, - "kimi-k2.5": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 3000000}, - "kimi-k2.6": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000}, - "kimi-k2.7-code": {ContextTokens: 229376, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000}, - "kimi-k3": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "kimi-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "kimi-latest-128k": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "kimi-latest-32k": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, - "kimi-latest-8k": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, - "kimi-thinking-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000}, - "llama-3.3-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 850000, OutputCostPerMTokMicroUSD: 1200000}, - "llama3.1-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 600000}, - "llama3.1-8b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 100000}, - "moonshot-v1-128k": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "moonshot-v1-128k-0430": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "moonshot-v1-128k-vision-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "moonshot-v1-32k": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, - "moonshot-v1-32k-0430": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, - "moonshot-v1-32k-vision-preview": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, - "moonshot-v1-8k": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, - "moonshot-v1-8k-0430": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, - "moonshot-v1-8k-vision-preview": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, - "moonshot-v1-auto": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "o1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000}, - "o1-2024-12-17": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000}, - "o3": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "o3-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "o3-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "o3-mini-2025-01-31": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "o4-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "qwen-3-32b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 800000}, - "qwen-coder": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1500000}, - "qwen-max": {ContextTokens: 30720, InputCostPerMTokMicroUSD: 1600000, OutputCostPerMTokMicroUSD: 6400000}, - "qwen-plus": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-plus-2025-01-25": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-plus-2025-04-28": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-plus-2025-07-14": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-turbo": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen-turbo-2024-11-01": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen-turbo-2025-04-28": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen-turbo-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen3-next-80b-a3b-instruct": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen3-next-80b-a3b-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen3-vl-235b-a22b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, - "qwen3-vl-235b-a22b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 4000000}, - "qwen3-vl-32b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 640000}, - "qwen3-vl-32b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 2870000}, - "qwen3.7-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 7500000}, - "qwen3.8-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, - "qwq-plus": {ContextTokens: 98304, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 2400000}, - "text-embedding-3-large": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 130000, OutputCostPerMTokMicroUSD: 0}, - "text-embedding-3-small": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 20000, OutputCostPerMTokMicroUSD: 0}, - "text-embedding-ada-002": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0}, - "text-embedding-ada-002-v2": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0}, - "zai-glm-4.6": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2250000, OutputCostPerMTokMicroUSD: 2750000}, - "zai-glm-4.7": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2250000, OutputCostPerMTokMicroUSD: 2750000}, + "MiniMax-M2": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "MiniMax-M2.1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "MiniMax-M2.1-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M2.1-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M2.5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "MiniMax-M2.5-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M2.5-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "chat-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, + "claude-fable-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-fable-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-haiku-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, + "claude-haiku-4-5-20251001": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, + "claude-mythos-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-mythos-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-opus-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-5-20251101": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-6-20260205": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-7": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-7-20260416": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-8": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "claude-sonnet-4-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "claude-sonnet-4-5-20250929": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "claude-sonnet-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "claude-sonnet-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, + "claude-sonnet-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, + "daybreak-blue-latest": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "daybreak-red-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, + "deepseek-chat": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, + "deepseek-coder": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 140000, OutputCostPerMTokMicroUSD: 280000}, + "deepseek-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "deepseek-r1": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 550000, OutputCostPerMTokMicroUSD: 2190000}, + "deepseek-reasoner": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, + "deepseek-v3": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 270000, OutputCostPerMTokMicroUSD: 1100000}, + "deepseek-v3.2": {ContextTokens: 163840, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 400000}, + "deepseek-v4-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "deepseek-v4-flash-0731": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 400000}, + "deepseek-v4-flash-vision-exp": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "deepseek-v4-pro": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1320000, OutputCostPerMTokMicroUSD: 3960000}, + "ft:gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-3.5-turbo-0613": {ContextTokens: 4096, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, + "gemma-4-31b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 990000, OutputCostPerMTokMicroUSD: 1490000}, + "glm-5.1": {ContextTokens: 202745, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, + "glm-5.2": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, + "gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, + "gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, + "gpt-3.5-turbo-16k": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 4000000}, + "gpt-4": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, + "gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, + "gpt-4-1106-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-4-turbo": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-4.1": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, + "gpt-4.1-mini": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, + "gpt-4.1-nano": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, + "gpt-4o": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-4o-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, + "gpt-4o-mini-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, + "gpt-4o-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, + "gpt-5-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, + "gpt-5-search-api": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5-search-api-2025-10-14": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.1": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.1-2025-11-13": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.2": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, + "gpt-5.2-2025-12-11": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, + "gpt-5.4": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, + "gpt-5.4-2026-03-05": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, + "gpt-5.4-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, + "gpt-5.4-mini-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, + "gpt-5.4-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, + "gpt-5.4-nano-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, + "gpt-5.5": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-5.5-2026-04-23": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-5.5-cyber": {ContextTokens: 0, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, + "gpt-5.6": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "gpt-5.6-cyber": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, + "gpt-5.6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1200000}, + "gpt-5.6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "gpt-5.6-terra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 12000000}, + "gpt-6-astra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "gpt-6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 500000}, + "gpt-6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio-1.5": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, + "gpt-audio-mini-2025-12-15": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, + "gpt-oss-120b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 350000, OutputCostPerMTokMicroUSD: 750000}, + "gpt-rosalind-research": {ContextTokens: 0, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "grok-4.20": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-latest-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-latest-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-0304": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-0304-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-0304-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-non-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-non-reasoning-gv2": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-non-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-reasoning-gv2": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.3-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.5": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-4.5-latest": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-4.6": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-4.7": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-build-0.1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, + "grok-build-latest": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-code-fast": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, + "grok-code-fast-1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, + "grok-code-fast-1-0825": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, + "kimi-k2.5": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 3000000}, + "kimi-k2.6": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000}, + "kimi-k2.7-code": {ContextTokens: 229376, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000}, + "kimi-k3": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "llama-3.3-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 850000, OutputCostPerMTokMicroUSD: 1200000}, + "llama3.1-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 600000}, + "llama3.1-8b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 100000}, + "moonshot-v1-128k": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, + "moonshot-v1-128k-vision-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, + "moonshot-v1-32k": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, + "moonshot-v1-32k-vision-preview": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, + "moonshot-v1-8k": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, + "moonshot-v1-8k-vision-preview": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, + "moonshot-v1-auto": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, + "o1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000}, + "o3": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, + "o3-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, + "o4-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, + "qwen-3-32b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 800000}, + "qwen-3.8-27b": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 990000, OutputCostPerMTokMicroUSD: 1490000}, + "qwen-coder": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1500000}, + "qwen-max": {ContextTokens: 30720, InputCostPerMTokMicroUSD: 1600000, OutputCostPerMTokMicroUSD: 6400000}, + "qwen-plus": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, + "qwen-turbo": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, + "qwen-turbo-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, + "qwen3-next-80b-a3b-instruct": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, + "qwen3-next-80b-a3b-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, + "qwen3-vl-235b-a22b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, + "qwen3-vl-235b-a22b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 4000000}, + "qwen3-vl-32b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 640000}, + "qwen3-vl-32b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 2870000}, + "qwen3.7-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 7500000}, + "qwen3.8-flash": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 470000}, + "qwen3.8-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "qwen3.8-omni-flash": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 470000}, + "qwq-plus": {ContextTokens: 98304, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 2400000}, + "text-embedding-3-large": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 130000, OutputCostPerMTokMicroUSD: 0}, + "text-embedding-3-small": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 20000, OutputCostPerMTokMicroUSD: 0}, + "text-embedding-ada-002": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0}, + "text-embedding-ada-002-v2": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0}, } diff --git a/registry.go b/registry.go index faa5f38..71eeb47 100644 --- a/registry.go +++ b/registry.go @@ -664,6 +664,9 @@ type modelRegistry struct { // cheap nil-map lookup (always false/zero) that adds no branching // cost to the v0.1 no-aliases case. aliases map[string]string + // requirePrefix is Config.RequireModelPrefix: resolve and listFor treat + // a bare id as unaddressable, leaving "provider/model" and aliases. + requirePrefix bool // modelMeta is the validated Config.ModelMeta config-override map // (feature v0.23), built once by newModelRegistry via // validateModelMeta and never mutated afterward. nil (not merely @@ -882,6 +885,7 @@ func newModelRegistry(adapters map[string]providerAdapter, cfg *Config, log func return nil, err } m.aliases = aliases + m.requirePrefix = cfg.RequireModelPrefix modelMeta, err := validateModelMeta(cfg.ModelMeta) if err != nil { @@ -1603,6 +1607,9 @@ func (m *modelRegistry) resolve(id string, grp *group) (providerAdapter, string, if providerName, rest, ok := m.splitConfiguredProvider(id); ok { return m.resolveAgainst(providerName, rest, id, "", grp, errModelUnknown) } + if m.requirePrefix { + return nil, "", "", errModelUnknown + } providerName, ok := m.bareWinner(id, grp) if !ok { return nil, "", "", errModelUnknown @@ -1981,10 +1988,10 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any { // that branch (harmless — matchesGlob is cheap), but still // REQUIRED for the single-grant fast path, which never consults // id at all. - if gwOK && grp.allowsProviderModel(groupWinner, id) && !shadowed && !ambiguousBareID { + if gwOK && grp.allowsProviderModel(groupWinner, id) && !shadowed && !ambiguousBareID && !m.requirePrefix { out = append(out, modelObject(id, groupWinner, m.resolveMetaFor(groupWinner, id))) } - if len(visible) < 2 && !shadowed && !ambiguousBareID { + if len(visible) < 2 && !shadowed && !ambiguousBareID && !m.requirePrefix { continue // only one provider serving id is visible to grp: no disambiguating prefix to add } for _, p := range visible { diff --git a/registry_requireprefix_test.go b/registry_requireprefix_test.go new file mode 100644 index 0000000..721415a --- /dev/null +++ b/registry_requireprefix_test.go @@ -0,0 +1,115 @@ +package traefikllmgateway + +import ( + "errors" + "sort" + "testing" +) + +func newRequirePrefixRegistry(t *testing.T, require bool) (*modelRegistry, map[string]providerAdapter) { + t.Helper() + adapters := map[string]providerAdapter{ + "alpha": newFakeAdapter("alpha"), + "beta": newFakeAdapter("beta"), + } + cfg := &Config{ + RequireModelPrefix: require, + ModelAliases: map[string]string{"fast": "alpha/solo", "shared-alias": "shared"}, + Providers: map[string]*ProviderConfig{ + "alpha": {Models: []string{"shared", "solo"}}, + "beta": {Models: []string{"shared", "other"}}, + }, + } + reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) + if err != nil { + t.Fatalf("newModelRegistry: %v", err) + } + return reg, adapters +} + +func TestModelRegistry_RequireModelPrefix_Resolve(t *testing.T) { + t.Parallel() + reg, adapters := newRequirePrefixRegistry(t, true) + grp := allowAllGroup() + + for _, id := range []string{"solo", "shared", "other"} { + if _, _, _, err := reg.resolve(id, grp); !errors.Is(err, errModelUnknown) { + t.Errorf("resolve(%q) err = %v, want errModelUnknown", id, err) + } + } + + cases := []struct { + id string + provider string + canon string + }{ + {"alpha/solo", "alpha", "alpha/solo"}, + {"beta/shared", "beta", "beta/shared"}, + {"fast", "alpha", "alpha/solo"}, + {"shared-alias", "alpha", "alpha/shared"}, + } + for _, tc := range cases { + adapter, _, canon, err := reg.resolve(tc.id, grp) + if err != nil { + t.Errorf("resolve(%q): %v", tc.id, err) + continue + } + if adapter != adapters[tc.provider] || canon != tc.canon { + t.Errorf("resolve(%q) = %v %q, want %s %q", tc.id, adapter, canon, tc.provider, tc.canon) + } + } +} + +func TestModelRegistry_RequireModelPrefix_ListFor_NoBareIDs(t *testing.T) { + t.Parallel() + reg, _ := newRequirePrefixRegistry(t, true) + + var got []string + for _, entry := range reg.listFor(allowAllGroup()) { + got = append(got, entry["id"].(string)) //nolint:forcetypeassert // modelObject always sets id to a string + } + sort.Strings(got) + want := []string{"alpha/shared", "alpha/solo", "beta/other", "beta/shared", "fast", "shared-alias"} + if len(got) != len(want) { + t.Fatalf("listFor ids = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("listFor ids = %v, want %v", got, want) + } + } +} + +func TestModelRegistry_RequireModelPrefix_ListFor_RespectsProviderAllowList(t *testing.T) { + t.Parallel() + reg, _ := newRequirePrefixRegistry(t, true) + grp := &group{name: "g", providers: []string{"beta"}} + + for _, entry := range reg.listFor(grp) { + id := entry["id"].(string) //nolint:forcetypeassert // modelObject always sets id to a string + if entry["owned_by"] != "beta" { + t.Errorf("entry %q owned_by %v, want only beta-owned entries", id, entry["owned_by"]) + } + } + if _, _, _, err := reg.resolve("alpha/solo", grp); !errors.Is(err, errModelDenied) { + t.Errorf("resolve(alpha/solo) err = %v, want errModelDenied", err) + } +} + +func TestModelRegistry_RequireModelPrefix_Off_KeepsBareIDs(t *testing.T) { + t.Parallel() + reg, _ := newRequirePrefixRegistry(t, false) + + if _, _, canon, err := reg.resolve("solo", allowAllGroup()); err != nil || canon != "alpha/solo" { + t.Errorf("resolve(solo) = %q, %v, want alpha/solo, nil", canon, err) + } + var sawBare bool + for _, entry := range reg.listFor(allowAllGroup()) { + if entry["id"] == "solo" { + sawBare = true + } + } + if !sawBare { + t.Error("listFor lost bare id \"solo\" with requireModelPrefix off") + } +} diff --git a/sse.go b/sse.go index 6ef961d..424b935 100644 --- a/sse.go +++ b/sse.go @@ -164,7 +164,7 @@ func readSSE(r io.Reader, fn func(sseEvent) error) error { // suite, the SSE streaming path); tools/yaegi-check never exercises // a live streaming response. event = "" - dataLines = nil + dataLines = dataLines[:0] hasData = false dataBytes = 0 if !fire { @@ -315,12 +315,20 @@ func splitSSELines(b []byte) [][]byte { // "\r" of any kind ever reaches the wire from here, regardless of what // readSSE upstream of this call did or did not already split it on. func (s *sseWriter) writeData(b []byte) error { - lines := splitSSELines(b) - buf := make([]byte, 0, len(b)+len(lines)*len(sseDataPrefix)+1) - for _, line := range lines { + var buf []byte + if !bytes.ContainsAny(b, "\r\n") { + buf = make([]byte, 0, len(sseDataPrefix)+len(b)+2) buf = append(buf, sseDataPrefix...) - buf = append(buf, line...) + buf = append(buf, b...) buf = append(buf, '\n') + } else { + lines := splitSSELines(b) + buf = make([]byte, 0, len(b)+len(lines)*len(sseDataPrefix)+1) + for _, line := range lines { + buf = append(buf, sseDataPrefix...) + buf = append(buf, line...) + buf = append(buf, '\n') + } } buf = append(buf, '\n') diff --git a/tools/pricing-sync/main.go b/tools/pricing-sync/main.go index 1a33128..8b4c56e 100644 --- a/tools/pricing-sync/main.go +++ b/tools/pricing-sync/main.go @@ -57,6 +57,14 @@ // a defensive fallback, not just a hardcoded "sample_spec" skip, so // a future upstream schema-example entry with some other key name // degrades the same way. +// 4. Freshness: an entry is dropped when LiteLLM's own deprecation_date +// has already passed, or when a full calendar date embedded in its +// bare id ("claude-opus-4-1-20250805", "gpt-4o-2024-11-20") is +// older than the -max-age-months flag (default 12; 0 turns the whole +// rule off). LiteLLM carries no release date, so an id with no such +// date and no deprecation_date is always kept: an undated alias is +// never dropped on a guess. An id in alwaysKeep, or in the -keep +// flag's comma-separated list, is exempt. // // # Bare-id collision resolution // @@ -95,6 +103,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "flag" "fmt" "go/format" "io" @@ -102,6 +111,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "sort" "strings" "time" @@ -129,6 +139,51 @@ const microUSDPerUSDPerToken = 1e12 // this specific, known case. const sampleSpecKey = "sample_spec" +// defaultMaxAgeMonths is the -max-age-months flag's default. +const defaultMaxAgeMonths = 12 + +// alwaysKeep lists bare ids the freshness rule never drops, because they +// are still served after their age or deprecation date says otherwise. +var alwaysKeep = map[string]bool{ + "claude-sonnet-4-5-20250929": true, +} + +// pruneOptions is the freshness rule's configuration. maxAgeMonths <= 0 +// turns the rule off. +type pruneOptions struct { + now time.Time + keep map[string]bool + maxAgeMonths int +} + +// idDatePattern finds a full calendar date in a model id, as YYYYMMDD or +// YYYY-MM-DD, delimited by the id's ends or a "-" or "_". +var idDatePattern = regexp.MustCompile(`(?:^|[-_])(20\d{2})-?(0[1-9]|1[0-2])-?(0[1-9]|[12]\d|3[01])(?:$|[-_])`) + +// isStale reports whether an entry fails the freshness rule: its +// deprecation_date (YYYY-MM-DD) is before opts.now, or a date embedded in +// bareID is more than opts.maxAgeMonths before it. Ids in opts.keep and +// alwaysKeep never fail. +func isStale(bareID, deprecationDate string, opts pruneOptions) bool { + if opts.maxAgeMonths <= 0 || opts.keep[bareID] || alwaysKeep[bareID] { + return false + } + if deprecationDate != "" { + if d, err := time.Parse("2006-01-02", deprecationDate); err == nil && d.Before(opts.now) { + return true + } + } + m := idDatePattern.FindStringSubmatch(bareID) + if m == nil { + return false + } + d, err := time.Parse("20060102", m[1]+m[2]+m[3]) + if err != nil { + return false + } + return d.Before(opts.now.AddDate(0, -opts.maxAgeMonths, 0)) +} + // allowedProviders is the pruning rule's provider allowlist — see the // package doc comment for the full rationale and the qwen/alibaba -> // dashscope mapping. @@ -218,6 +273,7 @@ func applyNamingBridges(table map[string]builtinEntry, winners map[string]candid // max_input_tokens/max_tokens as a JSON float (2000000.0) — decoding // those into *int would fail outright and drop the entry. type litellmEntry struct { + DeprecationDate string `json:"deprecation_date"` InputCostPerToken *float64 `json:"input_cost_per_token"` OutputCostPerToken *float64 `json:"output_cost_per_token"` MaxInputTokens *float64 `json:"max_input_tokens"` @@ -254,9 +310,18 @@ func main() { } func run() error { + maxAge := flag.Int("max-age-months", defaultMaxAgeMonths, "drop entries older than this many months; 0 disables the freshness rule") + keepList := flag.String("keep", "", "comma-separated bare ids the freshness rule never drops") + flag.Parse() repoRoot := "." - if len(os.Args) > 1 { - repoRoot = os.Args[1] + if flag.NArg() > 0 { + repoRoot = flag.Arg(0) + } + keep := map[string]bool{} + for _, id := range strings.Split(*keepList, ",") { + if id = strings.TrimSpace(id); id != "" { + keep[id] = true + } } outPath := filepath.Join(repoRoot, "pricing_data_gen.go") @@ -268,7 +333,7 @@ func run() error { return fmt.Errorf("fetch %s: %w", litellmSourceURL, err) } - table, totalUpstream, err := parseAndPrune(data) + table, totalUpstream, err := parseAndPrune(data, pruneOptions{now: time.Now(), maxAgeMonths: *maxAge, keep: keep}) if err != nil { return fmt.Errorf("parse and prune: %w", err) } @@ -319,7 +384,7 @@ func fetchLiteLLMPrices(ctx context.Context, url string) ([]byte, error) { // chooseWinner's deterministic bare-id collision resolution. It returns // the pruned table and the total number of top-level keys data carried // (including ones this function skips), for the summary line run prints. -func parseAndPrune(data []byte) (map[string]builtinEntry, int, error) { +func parseAndPrune(data []byte, opts pruneOptions) (map[string]builtinEntry, int, error) { var raw map[string]json.RawMessage if err := json.Unmarshal(data, &raw); err != nil { return nil, 0, fmt.Errorf("decode top-level object: %w", err) @@ -346,6 +411,9 @@ func parseAndPrune(data []byte) (map[string]builtinEntry, int, error) { } bare, isBareKey := bareModelID(key, e.LiteLLMProvider) + if isStale(bare, e.DeprecationDate, opts) { + continue + } byBare[bare] = append(byBare[bare], candidate{ fullKey: key, provider: e.LiteLLMProvider, diff --git a/tools/pricing-sync/main_test.go b/tools/pricing-sync/main_test.go index adad3e9..9337f6e 100644 --- a/tools/pricing-sync/main_test.go +++ b/tools/pricing-sync/main_test.go @@ -6,6 +6,12 @@ import ( "reflect" "strings" "testing" + "time" +) + +var ( + fixtureNow = time.Date(2026, time.August, 1, 0, 0, 0, 0, time.UTC) + fixtureOpts = pruneOptions{now: fixtureNow, maxAgeMonths: defaultMaxAgeMonths} ) // fixtureJSON is a small, hand-built stand-in for LiteLLM's real @@ -116,7 +122,7 @@ const fixtureUpstreamKeyCount = 17 // out, covering every pruning-rule branch fixtureJSON's own doc comment // lists. func TestParseAndPrune_Golden(t *testing.T) { - got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON)) + got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON), fixtureOpts) if err != nil { t.Fatalf("parseAndPrune: %v", err) } @@ -264,3 +270,76 @@ func TestContentDigest(t *testing.T) { t.Errorf("len(contentDigest(...)) = %d, want %d", len(a), contentDigestLen) } } + +func TestIsStale(t *testing.T) { + now := time.Date(2026, time.September, 29, 0, 0, 0, 0, time.UTC) + tests := []struct { + name, id, deprecated string + want bool + }{ + {"undated alias kept", "claude-opus-4-1", "", false}, + {"recent compact date kept", "claude-sonnet-5-20260401", "", false}, + {"old compact date dropped", "claude-opus-4-1-20250805", "", true}, + {"old dashed date dropped", "gpt-4o-2024-11-20", "", true}, + {"just inside cutoff kept", "model-2025-10-01", "", false}, + {"just outside cutoff dropped", "model-2025-09-28", "", true}, + {"date mid-id dropped", "gpt-4o-2024-11-20-preview", "", true}, + {"short MMDD suffix ignored", "grok-4.20-experimental-beta-0304", "", false}, + {"invalid month ignored", "model-20251301", "", false}, + {"past deprecation dropped", "gpt-4-turbo", "2025-06-01", true}, + {"future deprecation kept", "gpt-4-turbo", "2027-01-01", false}, + {"bad deprecation format ignored", "gpt-4-turbo", "soon", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + opts := pruneOptions{now: now, maxAgeMonths: defaultMaxAgeMonths} + if got := isStale(tc.id, tc.deprecated, opts); got != tc.want { + t.Errorf("isStale(%q, %q) = %v, want %v", tc.id, tc.deprecated, got, tc.want) + } + }) + } +} + +func TestParseAndPrune_DropsStaleEntries(t *testing.T) { + const data = `{ + "gpt-fresh": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + "gpt-old-2024-05-13": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + "gpt-dead": {"litellm_provider": "openai", "mode": "chat", "deprecation_date": "2026-01-01", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} +}` + got, _, err := parseAndPrune([]byte(data), fixtureOpts) + if err != nil { + t.Fatalf("parseAndPrune: %v", err) + } + if len(got) != 1 { + t.Fatalf("kept %d entries %v, want only gpt-fresh", len(got), got) + } + if _, ok := got["gpt-fresh"]; !ok { + t.Errorf("gpt-fresh missing from %v", got) + } +} + +func TestIsStale_Options(t *testing.T) { + now := time.Date(2026, time.September, 29, 0, 0, 0, 0, time.UTC) + const old = "gpt-4o-2024-08-06" + tests := []struct { + name string + opts pruneOptions + id string + want bool + }{ + {"default drops old", pruneOptions{now: now, maxAgeMonths: 12}, old, true}, + {"zero disables rule", pruneOptions{now: now, maxAgeMonths: 0}, old, false}, + {"negative disables rule", pruneOptions{now: now, maxAgeMonths: -1}, old, false}, + {"longer window keeps", pruneOptions{now: now, maxAgeMonths: 36}, old, false}, + {"shorter window drops more", pruneOptions{now: now, maxAgeMonths: 3}, "model-2026-03-01", true}, + {"keep flag exempts", pruneOptions{now: now, maxAgeMonths: 12, keep: map[string]bool{old: true}}, old, false}, + {"alwaysKeep exempts", pruneOptions{now: now, maxAgeMonths: 12}, "claude-sonnet-4-5-20250929", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := isStale(tc.id, "", tc.opts); got != tc.want { + t.Errorf("isStale(%q) = %v, want %v", tc.id, got, tc.want) + } + }) + } +} diff --git a/translate_anthropic.go b/translate_anthropic.go index 4f70183..c1be72a 100644 --- a/translate_anthropic.go +++ b/translate_anthropic.go @@ -3,7 +3,9 @@ package traefikllmgateway import ( "encoding/json" "fmt" + "strconv" "strings" + "unicode/utf8" ) // anthropicAPIVersion is the "anthropic-version" header value every request @@ -695,6 +697,7 @@ type anthropicStreamState struct { toolOrdinals map[int]int id string model string + content contentChunker u usage created int64 nextToolOrdinal int @@ -705,7 +708,9 @@ type anthropicStreamState struct { // created the response timestamp every emitted chunk carries — the // adapter fixes both once, before the first upstream event arrives. func newAnthropicStreamState(model string, created int64) *anthropicStreamState { - return &anthropicStreamState{model: model, created: created, toolOrdinals: map[int]int{}} + st := &anthropicStreamState{model: model, created: created, toolOrdinals: map[int]int{}} + st.content.set(chatCompletionIDPrefix, model, created) + return st } // usage returns the prompt/completion token counts captured so far, from @@ -741,6 +746,100 @@ func (st *anthropicStreamState) chunk(delta map[string]any, finishReason *string return b } +// contentChunkHead frames a text-delta chunk with the same alphabetical +// key order json.Marshal gives a map, so the bytes are identical to the +// chunk method's output for map[string]any{"content": text}. +const contentChunkHead = `{"choices":[{"delta":{"content":` + +// contentChunker builds text-delta chunks without allocating the envelope +// maps a state's chunk method needs for arbitrary deltas. +type contentChunker struct { + // tail is the per-stream suffix after the delta text, rebuilt by set. + tail []byte +} + +// set fixes the envelope fields every later chunk carries; call it again +// whenever the stream id changes. +func (c *contentChunker) set(id, model string, created int64) { + idb, ierr := json.Marshal(id) + mb, merr := json.Marshal(model) + if ierr != nil || merr != nil { + panic("llmgateway: stream chunk failed to marshal id or model") + } + t := append([]byte(nil), `},"finish_reason":null,"index":0}],"created":`...) + t = strconv.AppendInt(t, created, 10) + t = append(t, `,"id":`...) + t = append(t, idb...) + t = append(t, `,"model":`...) + t = append(t, mb...) + t = append(t, `,"object":"chat.completion.chunk"}`...) + c.tail = t +} + +func (c *contentChunker) build(text string) []byte { + out := make([]byte, 0, len(contentChunkHead)+len(text)+len(text)/8+2+len(c.tail)) + out = append(out, contentChunkHead...) + out = appendJSONString(out, text) + return append(out, c.tail...) +} + +const jsonHex = "0123456789abcdef" + +// appendJSONString appends s as a quoted JSON string using +// encoding/json's default escaping (HTML characters, U+2028/U+2029 and +// invalid UTF-8 included), so its output equals json.Marshal(s). +func appendJSONString(dst []byte, s string) []byte { + dst = append(dst, '"') + start := 0 + for i := 0; i < len(s); { + b := s[i] + if b < utf8.RuneSelf { + if b >= 0x20 && b != '"' && b != '\\' && b != '<' && b != '>' && b != '&' { + i++ + continue + } + dst = append(dst, s[start:i]...) + switch b { + case '"', '\\': + dst = append(dst, '\\', b) + case '\b': + dst = append(dst, '\\', 'b') + case '\f': + dst = append(dst, '\\', 'f') + case '\n': + dst = append(dst, '\\', 'n') + case '\r': + dst = append(dst, '\\', 'r') + case '\t': + dst = append(dst, '\\', 't') + default: + dst = append(dst, '\\', 'u', '0', '0', jsonHex[b>>4], jsonHex[b&0xF]) + } + i++ + start = i + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + dst = append(dst, s[start:i]...) + dst = append(dst, "\xef\xbf\xbd"...) + i += size + start = i + continue + } + if r == '\u2028' || r == '\u2029' { + dst = append(dst, s[start:i]...) + dst = append(dst, '\\', 'u', '2', '0', '2', jsonHex[r&0xF]) + i += size + start = i + continue + } + i += size + } + dst = append(dst, s[start:]...) + return append(dst, '"') +} + // usageChunk builds the final usage-only "chat.completion.chunk" payload — // empty "choices" plus a populated "usage" — mirroring the shape // provider_openai.go's forwardStream relays from a native OpenAI upstream @@ -854,6 +953,7 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) { return nil, fmt.Errorf("%w: decode anthropic message_start event: %w", errUpstream, err) } st.id = p.Message.ID + st.content.set(chatCompletionIDPrefix+st.id, st.model, st.created) st.u.prompt = p.Message.Usage.totalInputTokens() return [][]byte{st.chunk(map[string]any{"role": "assistant"}, nil)}, nil @@ -886,7 +986,7 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) { } switch p.Delta.Type { case "text_delta": - return [][]byte{st.chunk(map[string]any{"content": p.Delta.Text}, nil)}, nil + return [][]byte{st.content.build(p.Delta.Text)}, nil case "input_json_delta": ordinal, ok := st.toolOrdinals[p.Index] if !ok { diff --git a/translate_anthropic_test.go b/translate_anthropic_test.go index d5e8e53..0286434 100644 --- a/translate_anthropic_test.go +++ b/translate_anthropic_test.go @@ -1,6 +1,7 @@ package traefikllmgateway import ( + "bytes" "context" "encoding/json" "errors" @@ -12,6 +13,7 @@ import ( "sort" "strings" "testing" + "unicode/utf8" "github.com/stretchr/testify/assert" ) @@ -1127,3 +1129,63 @@ func TestOpenAIRequestFromAnthropic_NoTools_OmitsToolsAndToolChoiceFields(t *tes t.Errorf(`out["tool_choice"] = %v, want the key absent when no tool_choice was requested`, out["tool_choice"]) } } + +// sameJSON reports whether got and want encode the same value. Invalid UTF-8 +// input is escaped as \ufffd by encoding/json before Go 1.26 and written as +// the raw replacement rune after, so those inputs compare semantically. +func sameJSON(t *testing.T, input string, got, want []byte) bool { + t.Helper() + if bytes.Equal(got, want) { + return true + } + if utf8.ValidString(input) { + return false + } + var g, w any + if err := json.Unmarshal(got, &g); err != nil { + return false + } + if err := json.Unmarshal(want, &w); err != nil { + return false + } + return reflect.DeepEqual(g, w) +} + +func TestContentChunkerMatchesChunk(t *testing.T) { + texts := []string{"", "Hello", "a \"quoted\" &", "line\nbreak\ttab", "unicode é 日本 \u2028\u2029", "bad\xffutf8"} + for _, id := range []string{"", "msg_01", "we\"ird"} { + ast := newAnthropicStreamState("gw-model", 1734000000) + ast.id = id + ast.content.set(chatCompletionIDPrefix+id, ast.model, ast.created) + gst := newGeminiStreamState("gw-model", 1734000000) + gst.id = id + gst.content.set(chatCompletionIDPrefix+id, gst.model, gst.created) + for _, text := range texts { + gotA := ast.content.build(text) + if want := ast.chunk(map[string]any{"content": text}, nil); !sameJSON(t, text, gotA, want) { + t.Errorf("anthropic id=%q text=%q:\n got %s\nwant %s", id, text, gotA, want) + } + gotG := gst.content.build(text) + if want := gst.chunk(map[string]any{"content": text}, nil); !sameJSON(t, text, gotG, want) { + t.Errorf("gemini id=%q text=%q:\n got %s\nwant %s", id, text, gotG, want) + } + } + } +} + +func TestAppendJSONStringMatchesMarshal(t *testing.T) { + var cases []string + for r := rune(0); r < 0x250; r++ { + cases = append(cases, string(r), "a"+string(r)+"b") + } + cases = append(cases, "\u2028", "\u2029", "x\u2028y\u2029z", "bad\xffutf8", "\xc3", "\xe2\x80", "\U0001F600 emoji", "", "") + for _, s := range cases { + want, err := json.Marshal(s) + if err != nil { + t.Fatalf("Marshal(%q): %v", s, err) + } + if got := appendJSONString(nil, s); !sameJSON(t, s, got, want) { + t.Errorf("appendJSONString(%q) = %s, want %s", s, got, want) + } + } +} diff --git a/translate_gemini.go b/translate_gemini.go index 48279ad..dedb038 100644 --- a/translate_gemini.go +++ b/translate_gemini.go @@ -655,6 +655,7 @@ func (m *geminiUsageMetadata) safeCompletion() int64 { type geminiStreamState struct { id string model string + content contentChunker u usage created int64 // nextToolOrdinal is the per-stream contiguous tool-call counter lesson @@ -681,7 +682,9 @@ type geminiStreamState struct { // shares one stable, non-empty id instead of drifting between an empty // string and whatever a later chunk happens to report. func newGeminiStreamState(model string, created int64) *geminiStreamState { - return &geminiStreamState{model: model, created: created, id: geminiResponseIDOrFallback("", created)} + st := &geminiStreamState{model: model, created: created, id: geminiResponseIDOrFallback("", created)} + st.content.set(chatCompletionIDPrefix+st.id, model, created) + return st } // usage returns the prompt/completion token counts captured so far, from @@ -783,6 +786,7 @@ func (st *geminiStreamState) translate(ev sseEvent) ([][]byte, error) { isFirstChunk := !st.startedRole if isFirstChunk && resp.ResponseID != "" { st.id = resp.ResponseID + st.content.set(chatCompletionIDPrefix+st.id, st.model, st.created) } if resp.UsageMetadata != nil { @@ -832,7 +836,7 @@ func (st *geminiStreamState) translate(ev sseEvent) ([][]byte, error) { st.nextToolOrdinal++ chunks = append(chunks, st.chunk(delta, nil)) case p.Text != "": - chunks = append(chunks, st.chunk(map[string]any{"content": p.Text}, nil)) + chunks = append(chunks, st.content.build(p.Text)) } }