From f5482317d7e4cf487e200a9b6baa1672f09ed91f Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 12:07:50 +0100 Subject: [PATCH 01/15] test(bench): add benchmarks for translators, SSE and unified route --- bench_test.go | 226 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 bench_test.go diff --git a/bench_test.go b/bench_test.go new file mode 100644 index 0000000..ec2ccf6 --- /dev/null +++ b/bench_test.go @@ -0,0 +1,226 @@ +package traefikllmgateway + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +const benchStreamDeltas = 500 + +func benchOpenAIRequest() map[string]any { + return map[string]any{ + "model": "bench-model", + "max_tokens": float64(1024), + "temperature": 0.7, + "messages": []any{ + map[string]any{"role": "system", "content": "You are a helpful assistant."}, + map[string]any{"role": "user", "content": "What is the weather in Paris and Berlin?"}, + map[string]any{"role": "assistant", "content": "", "tool_calls": []any{ + map[string]any{"id": "call_1", "type": "function", "function": map[string]any{"name": "get_weather", "arguments": `{"city":"Paris"}`}}, + map[string]any{"id": "call_2", "type": "function", "function": map[string]any{"name": "get_weather", "arguments": `{"city":"Berlin"}`}}, + }}, + map[string]any{"role": "tool", "tool_call_id": "call_1", "content": "18C, cloudy"}, + map[string]any{"role": "tool", "tool_call_id": "call_2", "content": "15C, rain"}, + map[string]any{"role": "user", "content": "Summarise both in one sentence."}, + }, + "tools": []any{ + map[string]any{"type": "function", "function": map[string]any{ + "name": "get_weather", + "description": "Get the weather for a city", + "parameters": map[string]any{ + "type": "object", + "properties": map[string]any{"city": map[string]any{"type": "string"}}, + "required": []any{"city"}, + }, + }}, + }, + "tool_choice": "auto", + } +} + +func benchAnthropicStream() []byte { + var sb strings.Builder + sb.WriteString("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01B\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"claude-opus-5\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n") + sb.WriteString("event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n") + for i := 0; i < benchStreamDeltas; i++ { + fmt.Fprintf(&sb, "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" token%d\"}}\n\n", i) + } + sb.WriteString("event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n") + sb.WriteString("event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"output_tokens\":500}}\n\n") + sb.WriteString("event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n") + return []byte(sb.String()) +} + +func benchGeminiStream() []byte { + var sb strings.Builder + for i := 0; i < benchStreamDeltas; i++ { + fmt.Fprintf(&sb, "data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\" token%d\"}]}}],\"responseId\":\"resp_B\"}\n\n", i) + } + sb.WriteString("data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[]},\"finishReason\":\"STOP\"}],\"usageMetadata\":{\"promptTokenCount\":5,\"candidatesTokenCount\":500}}\n\n") + return []byte(sb.String()) +} + +func BenchmarkAnthropicRequestFromOpenAI(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := anthropicRequestFromOpenAI(benchOpenAIRequest()); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkGeminiRequestFromOpenAI(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := geminiRequestFromOpenAI(benchOpenAIRequest()); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkOpenAIResponseFromAnthropic(b *testing.B) { + body := []byte(`{"id":"msg_01XYZ","type":"message","role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."},{"type":"tool_use","id":"tu_1","name":"get_weather","input":{"city":"Paris"}}],"model":"claude-opus-5","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":8}}`) + b.ReportAllocs() + b.SetBytes(int64(len(body))) + for i := 0; i < b.N; i++ { + if _, _, err := openAIResponseFromAnthropic(body, "gw-model", 1734000000); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkOpenAIResponseFromGemini(b *testing.B) { + body := []byte(`{"candidates":[{"content":{"role":"model","parts":[{"text":"The capital of France is Paris."},{"functionCall":{"name":"get_weather","args":{"city":"Paris"}}}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":12,"candidatesTokenCount":8},"responseId":"resp_1"}`) + b.ReportAllocs() + b.SetBytes(int64(len(body))) + for i := 0; i < b.N; i++ { + if _, _, err := openAIResponseFromGemini(body, "gw-model", 1734000000); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkReadSSE(b *testing.B) { + stream := benchAnthropicStream() + b.ReportAllocs() + b.SetBytes(int64(len(stream))) + for i := 0; i < b.N; i++ { + if err := readSSE(bytes.NewReader(stream), func(sseEvent) error { return nil }); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkAnthropicStreamTranslate(b *testing.B) { + stream := benchAnthropicStream() + b.ReportAllocs() + b.SetBytes(int64(len(stream))) + for i := 0; i < b.N; i++ { + st := newAnthropicStreamState("gw-model", 1734000000) + err := readSSE(bytes.NewReader(stream), func(ev sseEvent) error { + _, terr := st.translate(ev) + return terr + }) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkGeminiStreamTranslate(b *testing.B) { + stream := benchGeminiStream() + b.ReportAllocs() + b.SetBytes(int64(len(stream))) + for i := 0; i < b.N; i++ { + st := newGeminiStreamState("gw-model", 1734000000) + err := readSSE(bytes.NewReader(stream), func(ev sseEvent) error { + _, terr := st.translate(ev) + return terr + }) + if err != nil { + b.Fatal(err) + } + } +} + +func newBenchGateway(b *testing.B, provider *ProviderConfig) http.Handler { + b.Helper() + devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0) + if err != nil { + b.Fatalf("open %s: %v", os.DevNull, err) + } + origStderr := os.Stderr + os.Stderr = devNull + b.Cleanup(func() { + os.Stderr = origStderr + _ = devNull.Close() + }) + cfg := CreateConfig() + cfg.Providers = map[string]*ProviderConfig{"bench": provider} + cfg.Groups = map[string]*GroupConfig{"default": {}} + cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "alice", Group: "default", APIKey: "sk-alice", Limits: &LimitsConfig{}}}} + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}) + h, err := New(context.Background(), next, cfg, "llmgw") + if err != nil { + b.Fatalf("New: %v", err) + } + return h +} + +func benchChatRequest(payload []byte) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(payload)) + req.Header.Set("Authorization", "Bearer sk-alice") + req.Header.Set("Content-Type", "application/json") + return req +} + +func BenchmarkUnifiedChatOpenAI(b *testing.B) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"x","object":"chat.completion","model":"bench-model","choices":[{"index":0,"message":{"role":"assistant","content":"hello"},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5}}`)) + })) + b.Cleanup(upstream.Close) + h := newBenchGateway(b, &ProviderConfig{Type: "openai", BaseURL: upstream.URL, APIKey: "k", Models: []string{"bench-model"}}) + payload := []byte(`{"model":"bench-model","messages":[{"role":"user","content":"hi"}]}`) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, benchChatRequest(payload)) + if rec.Code != http.StatusOK { + b.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + } +} + +func BenchmarkUnifiedChatAnthropicStream(b *testing.B) { + stream := benchAnthropicStream() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write(stream) + })) + b.Cleanup(upstream.Close) + h := newBenchGateway(b, &ProviderConfig{Type: "anthropic", BaseURL: upstream.URL, APIKey: "k", Models: []string{"bench-model"}}) + payload := []byte(`{"model":"bench-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`) + + b.ReportAllocs() + b.SetBytes(int64(len(stream))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, benchChatRequest(payload)) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "[DONE]") { + b.Fatalf("status = %d, body tail = %.200s", rec.Code, rec.Body.String()) + } + } +} From f94fe1dad581c0ad5451e4b227ba24bcdb4bb70b Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 12:16:20 +0100 Subject: [PATCH 02/15] perf(anthropic): build text-delta stream chunks without envelope maps --- translate_anthropic.go | 42 +++++++++++++++++++++++++++++++++++-- translate_anthropic_test.go | 16 ++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/translate_anthropic.go b/translate_anthropic.go index 4f70183..313fd4b 100644 --- a/translate_anthropic.go +++ b/translate_anthropic.go @@ -3,6 +3,7 @@ package traefikllmgateway import ( "encoding/json" "fmt" + "strconv" "strings" ) @@ -692,7 +693,10 @@ type anthropicStreamState struct { // index also counts any preceding non-tool_use blocks (a tool_use // block right after a text block is content-block index 1, but it is // still the first — index 0 — tool call). - toolOrdinals map[int]int + toolOrdinals map[int]int + // contentTail caches the per-stream suffix contentChunk appends after + // the delta text; nil until first use and reset when id changes. + contentTail []byte id string model string u usage @@ -741,6 +745,39 @@ func (st *anthropicStreamState) chunk(delta map[string]any, finishReason *string return b } +// contentChunkHead frames a text-delta chunk with +// the same alphabetical key order json.Marshal gives chunk's map, so the +// bytes are identical to chunk(map[string]any{"content": text}, nil). +const contentChunkHead = `{"choices":[{"delta":{"content":` + +// contentChunk builds a text-delta chunk without allocating the envelope +// maps chunk needs for arbitrary deltas. +func (st *anthropicStreamState) contentChunk(text string) []byte { + tb, err := json.Marshal(text) + if err != nil { + panic(fmt.Sprintf("llmgateway: anthropic stream chunk failed to marshal: %v", err)) + } + if st.contentTail == nil { + idb, ierr := json.Marshal(chatCompletionIDPrefix + st.id) + mb, merr := json.Marshal(st.model) + if ierr != nil || merr != nil { + panic("llmgateway: anthropic stream chunk failed to marshal id or model") + } + t := append([]byte(nil), `},"finish_reason":null,"index":0}],"created":`...) + t = strconv.AppendInt(t, st.created, 10) + t = append(t, `,"id":`...) + t = append(t, idb...) + t = append(t, `,"model":`...) + t = append(t, mb...) + t = append(t, `,"object":"chat.completion.chunk"}`...) + st.contentTail = t + } + out := make([]byte, 0, len(contentChunkHead)+len(tb)+len(st.contentTail)) + out = append(out, contentChunkHead...) + out = append(out, tb...) + return append(out, st.contentTail...) +} + // usageChunk builds the final usage-only "chat.completion.chunk" payload — // empty "choices" plus a populated "usage" — mirroring the shape // provider_openai.go's forwardStream relays from a native OpenAI upstream @@ -854,6 +891,7 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) { return nil, fmt.Errorf("%w: decode anthropic message_start event: %w", errUpstream, err) } st.id = p.Message.ID + st.contentTail = nil st.u.prompt = p.Message.Usage.totalInputTokens() return [][]byte{st.chunk(map[string]any{"role": "assistant"}, nil)}, nil @@ -886,7 +924,7 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) { } switch p.Delta.Type { case "text_delta": - return [][]byte{st.chunk(map[string]any{"content": p.Delta.Text}, nil)}, nil + return [][]byte{st.contentChunk(p.Delta.Text)}, nil case "input_json_delta": ordinal, ok := st.toolOrdinals[p.Index] if !ok { diff --git a/translate_anthropic_test.go b/translate_anthropic_test.go index d5e8e53..13280c4 100644 --- a/translate_anthropic_test.go +++ b/translate_anthropic_test.go @@ -1,6 +1,7 @@ package traefikllmgateway import ( + "bytes" "context" "encoding/json" "errors" @@ -1127,3 +1128,18 @@ func TestOpenAIRequestFromAnthropic_NoTools_OmitsToolsAndToolChoiceFields(t *tes t.Errorf(`out["tool_choice"] = %v, want the key absent when no tool_choice was requested`, out["tool_choice"]) } } + +func TestAnthropicContentChunkMatchesChunk(t *testing.T) { + texts := []string{"", "Hello", "a \"quoted\" &", "line\nbreak\ttab", "unicode é 日本 \u2028\u2029", "bad\xffutf8"} + for _, id := range []string{"", "msg_01", "we\"ird"} { + st := newAnthropicStreamState("gw-model", 1734000000) + st.id = id + for _, text := range texts { + got := st.contentChunk(text) + want := st.chunk(map[string]any{"content": text}, nil) + if !bytes.Equal(got, want) { + t.Errorf("id=%q text=%q:\n got %s\nwant %s", id, text, got, want) + } + } + } +} From baabd5d3bbda2c4b793f093cbfc3a3fddb90e59f Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 12:21:50 +0100 Subject: [PATCH 03/15] perf(gemini): share text-delta chunk builder with the anthropic stream --- translate_anthropic.go | 70 +++++++++++++++++++++---------------- translate_anthropic_test.go | 21 +++++++---- translate_gemini.go | 8 +++-- 3 files changed, 59 insertions(+), 40 deletions(-) diff --git a/translate_anthropic.go b/translate_anthropic.go index 313fd4b..4935b9d 100644 --- a/translate_anthropic.go +++ b/translate_anthropic.go @@ -693,10 +693,8 @@ type anthropicStreamState struct { // index also counts any preceding non-tool_use blocks (a tool_use // block right after a text block is content-block index 1, but it is // still the first — index 0 — tool call). - toolOrdinals map[int]int - // contentTail caches the per-stream suffix contentChunk appends after - // the delta text; nil until first use and reset when id changes. - contentTail []byte + toolOrdinals map[int]int + content contentChunker id string model string u usage @@ -709,7 +707,9 @@ type anthropicStreamState struct { // created the response timestamp every emitted chunk carries — the // adapter fixes both once, before the first upstream event arrives. func newAnthropicStreamState(model string, created int64) *anthropicStreamState { - return &anthropicStreamState{model: model, created: created, toolOrdinals: map[int]int{}} + st := &anthropicStreamState{model: model, created: created, toolOrdinals: map[int]int{}} + st.content.set(chatCompletionIDPrefix, model, created) + return st } // usage returns the prompt/completion token counts captured so far, from @@ -745,37 +745,45 @@ func (st *anthropicStreamState) chunk(delta map[string]any, finishReason *string return b } -// contentChunkHead frames a text-delta chunk with -// the same alphabetical key order json.Marshal gives chunk's map, so the -// bytes are identical to chunk(map[string]any{"content": text}, nil). +// contentChunkHead frames a text-delta chunk with the same alphabetical +// key order json.Marshal gives a map, so the bytes are identical to the +// chunk method's output for map[string]any{"content": text}. const contentChunkHead = `{"choices":[{"delta":{"content":` -// contentChunk builds a text-delta chunk without allocating the envelope -// maps chunk needs for arbitrary deltas. -func (st *anthropicStreamState) contentChunk(text string) []byte { +// contentChunker builds text-delta chunks without allocating the envelope +// maps a state's chunk method needs for arbitrary deltas. +type contentChunker struct { + // tail is the per-stream suffix after the delta text, rebuilt by set. + tail []byte +} + +// set fixes the envelope fields every later chunk carries; call it again +// whenever the stream id changes. +func (c *contentChunker) set(id, model string, created int64) { + idb, ierr := json.Marshal(id) + mb, merr := json.Marshal(model) + if ierr != nil || merr != nil { + panic("llmgateway: stream chunk failed to marshal id or model") + } + t := append([]byte(nil), `},"finish_reason":null,"index":0}],"created":`...) + t = strconv.AppendInt(t, created, 10) + t = append(t, `,"id":`...) + t = append(t, idb...) + t = append(t, `,"model":`...) + t = append(t, mb...) + t = append(t, `,"object":"chat.completion.chunk"}`...) + c.tail = t +} + +func (c *contentChunker) build(text string) []byte { tb, err := json.Marshal(text) if err != nil { - panic(fmt.Sprintf("llmgateway: anthropic stream chunk failed to marshal: %v", err)) + panic(fmt.Sprintf("llmgateway: stream chunk failed to marshal text: %v", err)) } - if st.contentTail == nil { - idb, ierr := json.Marshal(chatCompletionIDPrefix + st.id) - mb, merr := json.Marshal(st.model) - if ierr != nil || merr != nil { - panic("llmgateway: anthropic stream chunk failed to marshal id or model") - } - t := append([]byte(nil), `},"finish_reason":null,"index":0}],"created":`...) - t = strconv.AppendInt(t, st.created, 10) - t = append(t, `,"id":`...) - t = append(t, idb...) - t = append(t, `,"model":`...) - t = append(t, mb...) - t = append(t, `,"object":"chat.completion.chunk"}`...) - st.contentTail = t - } - out := make([]byte, 0, len(contentChunkHead)+len(tb)+len(st.contentTail)) + out := make([]byte, 0, len(contentChunkHead)+len(tb)+len(c.tail)) out = append(out, contentChunkHead...) out = append(out, tb...) - return append(out, st.contentTail...) + return append(out, c.tail...) } // usageChunk builds the final usage-only "chat.completion.chunk" payload — @@ -891,7 +899,7 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) { return nil, fmt.Errorf("%w: decode anthropic message_start event: %w", errUpstream, err) } st.id = p.Message.ID - st.contentTail = nil + st.content.set(chatCompletionIDPrefix+st.id, st.model, st.created) st.u.prompt = p.Message.Usage.totalInputTokens() return [][]byte{st.chunk(map[string]any{"role": "assistant"}, nil)}, nil @@ -924,7 +932,7 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) { } switch p.Delta.Type { case "text_delta": - return [][]byte{st.contentChunk(p.Delta.Text)}, nil + return [][]byte{st.content.build(p.Delta.Text)}, nil case "input_json_delta": ordinal, ok := st.toolOrdinals[p.Index] if !ok { diff --git a/translate_anthropic_test.go b/translate_anthropic_test.go index 13280c4..d630812 100644 --- a/translate_anthropic_test.go +++ b/translate_anthropic_test.go @@ -1129,16 +1129,23 @@ func TestOpenAIRequestFromAnthropic_NoTools_OmitsToolsAndToolChoiceFields(t *tes } } -func TestAnthropicContentChunkMatchesChunk(t *testing.T) { +func TestContentChunkerMatchesChunk(t *testing.T) { texts := []string{"", "Hello", "a \"quoted\" &", "line\nbreak\ttab", "unicode é 日本 \u2028\u2029", "bad\xffutf8"} for _, id := range []string{"", "msg_01", "we\"ird"} { - st := newAnthropicStreamState("gw-model", 1734000000) - st.id = id + ast := newAnthropicStreamState("gw-model", 1734000000) + ast.id = id + ast.content.set(chatCompletionIDPrefix+id, ast.model, ast.created) + gst := newGeminiStreamState("gw-model", 1734000000) + gst.id = id + gst.content.set(chatCompletionIDPrefix+id, gst.model, gst.created) for _, text := range texts { - got := st.contentChunk(text) - want := st.chunk(map[string]any{"content": text}, nil) - if !bytes.Equal(got, want) { - t.Errorf("id=%q text=%q:\n got %s\nwant %s", id, text, got, want) + gotA := ast.content.build(text) + if want := ast.chunk(map[string]any{"content": text}, nil); !bytes.Equal(gotA, want) { + t.Errorf("anthropic id=%q text=%q:\n got %s\nwant %s", id, text, gotA, want) + } + gotG := gst.content.build(text) + if want := gst.chunk(map[string]any{"content": text}, nil); !bytes.Equal(gotG, want) { + t.Errorf("gemini id=%q text=%q:\n got %s\nwant %s", id, text, gotG, want) } } } diff --git a/translate_gemini.go b/translate_gemini.go index 48279ad..4bbe95e 100644 --- a/translate_gemini.go +++ b/translate_gemini.go @@ -661,6 +661,7 @@ type geminiStreamState struct { // (a) requires. nextToolOrdinal int startedRole bool + content contentChunker // sawFunctionCall latches true the first time any chunk in this stream // carries a functionCall part, and stays true for the rest of the // stream. Gemini's finishReason is "STOP" whether the turn ended in @@ -681,7 +682,9 @@ type geminiStreamState struct { // shares one stable, non-empty id instead of drifting between an empty // string and whatever a later chunk happens to report. func newGeminiStreamState(model string, created int64) *geminiStreamState { - return &geminiStreamState{model: model, created: created, id: geminiResponseIDOrFallback("", created)} + st := &geminiStreamState{model: model, created: created, id: geminiResponseIDOrFallback("", created)} + st.content.set(chatCompletionIDPrefix+st.id, model, created) + return st } // usage returns the prompt/completion token counts captured so far, from @@ -783,6 +786,7 @@ func (st *geminiStreamState) translate(ev sseEvent) ([][]byte, error) { isFirstChunk := !st.startedRole if isFirstChunk && resp.ResponseID != "" { st.id = resp.ResponseID + st.content.set(chatCompletionIDPrefix+st.id, st.model, st.created) } if resp.UsageMetadata != nil { @@ -832,7 +836,7 @@ func (st *geminiStreamState) translate(ev sseEvent) ([][]byte, error) { st.nextToolOrdinal++ chunks = append(chunks, st.chunk(delta, nil)) case p.Text != "": - chunks = append(chunks, st.chunk(map[string]any{"content": p.Text}, nil)) + chunks = append(chunks, st.content.build(p.Text)) } } From 2cd686a669716f4d469df9cd3718c0e9a1a046b1 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 12:26:15 +0100 Subject: [PATCH 04/15] perf(stream): append text deltas as JSON without an intermediate Marshal --- translate_anthropic.go | 66 +++++++++++++++++++++++++++++++++---- translate_anthropic_test.go | 17 ++++++++++ 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/translate_anthropic.go b/translate_anthropic.go index 4935b9d..672ca73 100644 --- a/translate_anthropic.go +++ b/translate_anthropic.go @@ -5,6 +5,7 @@ import ( "fmt" "strconv" "strings" + "unicode/utf8" ) // anthropicAPIVersion is the "anthropic-version" header value every request @@ -776,16 +777,69 @@ func (c *contentChunker) set(id, model string, created int64) { } func (c *contentChunker) build(text string) []byte { - tb, err := json.Marshal(text) - if err != nil { - panic(fmt.Sprintf("llmgateway: stream chunk failed to marshal text: %v", err)) - } - out := make([]byte, 0, len(contentChunkHead)+len(tb)+len(c.tail)) + out := make([]byte, 0, len(contentChunkHead)+len(text)+len(text)/8+2+len(c.tail)) out = append(out, contentChunkHead...) - out = append(out, tb...) + out = appendJSONString(out, text) return append(out, c.tail...) } +const jsonHex = "0123456789abcdef" + +// appendJSONString appends s as a quoted JSON string using +// encoding/json's default escaping (HTML characters, U+2028/U+2029 and +// invalid UTF-8 included), so its output equals json.Marshal(s). +func appendJSONString(dst []byte, s string) []byte { + dst = append(dst, '"') + start := 0 + for i := 0; i < len(s); { + b := s[i] + if b < utf8.RuneSelf { + if b >= 0x20 && b != '"' && b != '\\' && b != '<' && b != '>' && b != '&' { + i++ + continue + } + dst = append(dst, s[start:i]...) + switch b { + case '"', '\\': + dst = append(dst, '\\', b) + case '\b': + dst = append(dst, '\\', 'b') + case '\f': + dst = append(dst, '\\', 'f') + case '\n': + dst = append(dst, '\\', 'n') + case '\r': + dst = append(dst, '\\', 'r') + case '\t': + dst = append(dst, '\\', 't') + default: + dst = append(dst, '\\', 'u', '0', '0', jsonHex[b>>4], jsonHex[b&0xF]) + } + i++ + start = i + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + dst = append(dst, s[start:i]...) + dst = append(dst, "\xef\xbf\xbd"...) + i += size + start = i + continue + } + if r == '\u2028' || r == '\u2029' { + dst = append(dst, s[start:i]...) + dst = append(dst, '\\', 'u', '2', '0', '2', jsonHex[r&0xF]) + i += size + start = i + continue + } + i += size + } + dst = append(dst, s[start:]...) + return append(dst, '"') +} + // usageChunk builds the final usage-only "chat.completion.chunk" payload — // empty "choices" plus a populated "usage" — mirroring the shape // provider_openai.go's forwardStream relays from a native OpenAI upstream diff --git a/translate_anthropic_test.go b/translate_anthropic_test.go index d630812..757b112 100644 --- a/translate_anthropic_test.go +++ b/translate_anthropic_test.go @@ -1150,3 +1150,20 @@ func TestContentChunkerMatchesChunk(t *testing.T) { } } } + +func TestAppendJSONStringMatchesMarshal(t *testing.T) { + var cases []string + for r := rune(0); r < 0x250; r++ { + cases = append(cases, string(r), "a"+string(r)+"b") + } + cases = append(cases, "\u2028", "\u2029", "x\u2028y\u2029z", "bad\xffutf8", "\xc3", "\xe2\x80", "\U0001F600 emoji", "", "") + for _, s := range cases { + want, err := json.Marshal(s) + if err != nil { + t.Fatalf("Marshal(%q): %v", s, err) + } + if got := appendJSONString(nil, s); !bytes.Equal(got, want) { + t.Errorf("appendJSONString(%q) = %s, want %s", s, got, want) + } + } +} From c25c6e38e1bd834b868c01503e6e556bcd036d47 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 12:31:04 +0100 Subject: [PATCH 05/15] perf(sse): skip line splitting for single-line data events --- sse.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sse.go b/sse.go index 6ef961d..38b70a5 100644 --- a/sse.go +++ b/sse.go @@ -315,12 +315,20 @@ func splitSSELines(b []byte) [][]byte { // "\r" of any kind ever reaches the wire from here, regardless of what // readSSE upstream of this call did or did not already split it on. func (s *sseWriter) writeData(b []byte) error { - lines := splitSSELines(b) - buf := make([]byte, 0, len(b)+len(lines)*len(sseDataPrefix)+1) - for _, line := range lines { + var buf []byte + if !bytes.ContainsAny(b, "\r\n") { + buf = make([]byte, 0, len(sseDataPrefix)+len(b)+2) buf = append(buf, sseDataPrefix...) - buf = append(buf, line...) + buf = append(buf, b...) buf = append(buf, '\n') + } else { + lines := splitSSELines(b) + buf = make([]byte, 0, len(b)+len(lines)*len(sseDataPrefix)+1) + for _, line := range lines { + buf = append(buf, sseDataPrefix...) + buf = append(buf, line...) + buf = append(buf, '\n') + } } buf = append(buf, '\n') From e12cec8bf9244482d238845e592f87b4a28698cf Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 12:31:07 +0100 Subject: [PATCH 06/15] style(stream): reorder stream state fields for fieldalignment --- translate_anthropic.go | 2 +- translate_gemini.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/translate_anthropic.go b/translate_anthropic.go index 672ca73..c1be72a 100644 --- a/translate_anthropic.go +++ b/translate_anthropic.go @@ -695,9 +695,9 @@ type anthropicStreamState struct { // block right after a text block is content-block index 1, but it is // still the first — index 0 — tool call). toolOrdinals map[int]int - content contentChunker id string model string + content contentChunker u usage created int64 nextToolOrdinal int diff --git a/translate_gemini.go b/translate_gemini.go index 4bbe95e..dedb038 100644 --- a/translate_gemini.go +++ b/translate_gemini.go @@ -655,13 +655,13 @@ func (m *geminiUsageMetadata) safeCompletion() int64 { type geminiStreamState struct { id string model string + content contentChunker u usage created int64 // nextToolOrdinal is the per-stream contiguous tool-call counter lesson // (a) requires. nextToolOrdinal int startedRole bool - content contentChunker // sawFunctionCall latches true the first time any chunk in this stream // carries a functionCall part, and stays true for the rest of the // stream. Gemini's finishReason is "STOP" whether the turn ended in From b8772d3d73b01434329754cda6387a9c502639e8 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 12:35:05 +0100 Subject: [PATCH 07/15] perf(sse): reuse the data-line buffer across events in readSSE --- sse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sse.go b/sse.go index 38b70a5..424b935 100644 --- a/sse.go +++ b/sse.go @@ -164,7 +164,7 @@ func readSSE(r io.Reader, fn func(sseEvent) error) error { // suite, the SSE streaming path); tools/yaegi-check never exercises // a live streaming response. event = "" - dataLines = nil + dataLines = dataLines[:0] hasData = false dataBytes = 0 if !fire { From 0a9522dfa71e62b90aa75b293b4fbbec5c59f3ed Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 13:23:06 +0100 Subject: [PATCH 08/15] chore(pricing): resync the builtin table from LiteLLM Adds 39 models, including Claude 5.5, GPT-6, Grok 4.20 variants and Qwen 3.8. Drops 65 models that LiteLLM no longer lists, for example Grok 2 and 3, old Kimi and Claude 3 ids. Updates 6 prices. --- pricing_data_gen.go | 451 +++++++++++++++++++++----------------------- 1 file changed, 213 insertions(+), 238 deletions(-) diff --git a/pricing_data_gen.go b/pricing_data_gen.go index f2b4f3f..b7913da 100644 --- a/pricing_data_gen.go +++ b/pricing_data_gen.go @@ -4,247 +4,222 @@ package traefikllmgateway // model_prices_and_context_window.json — DO NOT EDIT. // // Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json -// Source digest: sha256:0e8d23239c27 -// 236 entries kept (pruned from 3111 upstream entries — see +// Source digest: sha256:9f7b337871ed +// 211 entries kept (pruned from 4422 upstream entries — see // tools/pricing-sync/main.go's own doc comment for the exact pruning // rule and bare-id collision resolution). // // Regenerate with: make pricing-sync var builtinModelMetaTable = map[string]builtinModelMeta{ - "MiniMax-M2": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "MiniMax-M2.1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "MiniMax-M2.1-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M2.1-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M2.5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "MiniMax-M2.5-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M2.5-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "chat-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "chatgpt-4o-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-3-7-sonnet-20250219": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-3-haiku-20240307": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 1250000}, - "claude-3-opus-20240229": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-4-opus-20250514": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-4-sonnet-20250514": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-fable-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-haiku-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, - "claude-haiku-4-5-20251001": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, - "claude-mythos-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-mythos-preview": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-opus-4-1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-opus-4-1-20250805": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-opus-4-20250514": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 75000000}, - "claude-opus-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-5-20251101": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-6-20260205": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-7": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-7-20260416": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-8": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-sonnet-4-20250514": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-4-5-20250929": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "daybreak-blue-latest": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "daybreak-red-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, - "deepseek-chat": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, - "deepseek-coder": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 140000, OutputCostPerMTokMicroUSD: 280000}, - "deepseek-r1": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 550000, OutputCostPerMTokMicroUSD: 2190000}, - "deepseek-reasoner": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, - "deepseek-v3": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 270000, OutputCostPerMTokMicroUSD: 1100000}, - "deepseek-v3.2": {ContextTokens: 163840, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 400000}, - "deepseek-v4-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 440000, OutputCostPerMTokMicroUSD: 1320000}, - "deepseek-v4-flash-0731": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 400000}, - "deepseek-v4-pro": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1320000, OutputCostPerMTokMicroUSD: 3960000}, - "ft:gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-3.5-turbo-0613": {ContextTokens: 4096, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "ft:gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 12000000}, - "ft:gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 3200000}, - "ft:gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 800000}, - "ft:gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000}, - "ft:gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000}, - "ft:gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "ft:o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 16000000}, - "glm-5.1": {ContextTokens: 202745, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, - "glm-5.2": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, - "gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, - "gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, - "gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "gpt-3.5-turbo-16k": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 4000000}, - "gpt-4": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "gpt-4-0125-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-0314": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "gpt-4-1106-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-turbo": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-turbo-2024-04-09": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-turbo-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4.1": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "gpt-4.1-mini": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, - "gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, - "gpt-4.1-nano": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-4o": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-2024-05-13": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, - "gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-audio-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-audio-preview-2025-06-03": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-audio-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-search-preview-2025-03-11": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-search-preview-2025-03-11": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-chat": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, - "gpt-5-mini-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, - "gpt-5-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-5-nano-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-5-search-api": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-search-api-2025-10-14": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.1": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.1-2025-11-13": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.1-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.2": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.2-2025-12-11": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.2-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.3-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.4": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, - "gpt-5.4-2026-03-05": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, - "gpt-5.4-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, - "gpt-5.4-mini-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, - "gpt-5.4-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, - "gpt-5.4-nano-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, - "gpt-5.5": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.5-2026-04-23": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.6": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.6-cyber": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, - "gpt-5.6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1200000}, - "gpt-5.6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.6-terra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 12000000}, - "gpt-audio": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio-1.5": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio-2025-08-28": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, - "gpt-audio-mini-2025-10-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, - "gpt-audio-mini-2025-12-15": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, - "gpt-oss-120b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 350000, OutputCostPerMTokMicroUSD: 750000}, - "grok-2": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-1212": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-vision": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-vision-1212": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-2-vision-latest": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "grok-3": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-3-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-3-fast-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "grok-3-fast-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "grok-3-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-3-mini": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 500000}, - "grok-3-mini-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 500000}, - "grok-3-mini-fast": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 4000000}, - "grok-3-mini-fast-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 4000000}, - "grok-3-mini-fast-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 4000000}, - "grok-3-mini-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-4-0709": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-4-1-fast": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-1-fast-non-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-1-fast-non-reasoning-latest": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-1-fast-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-1-fast-reasoning-latest": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-fast-non-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-fast-reasoning": {ContextTokens: 2000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 500000}, - "grok-4-latest": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-4.20-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-multi-agent-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-multi-agent-beta-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.3-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.5": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, - "grok-4.5-latest": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, - "grok-4.6": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, - "grok-beta": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, - "grok-build-0.1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "grok-code-fast": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "grok-code-fast-1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "grok-code-fast-1-0825": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "grok-vision-beta": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, - "kimi-k2-0711-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000}, - "kimi-k2-0905-preview": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000}, - "kimi-k2-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000}, - "kimi-k2-thinking-turbo": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 1150000, OutputCostPerMTokMicroUSD: 8000000}, - "kimi-k2-turbo-preview": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 1150000, OutputCostPerMTokMicroUSD: 8000000}, - "kimi-k2.5": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 3000000}, - "kimi-k2.6": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000}, - "kimi-k2.7-code": {ContextTokens: 229376, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000}, - "kimi-k3": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "kimi-latest": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "kimi-latest-128k": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "kimi-latest-32k": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, - "kimi-latest-8k": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, - "kimi-thinking-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2500000}, - "llama-3.3-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 850000, OutputCostPerMTokMicroUSD: 1200000}, - "llama3.1-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 600000}, - "llama3.1-8b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 100000}, - "moonshot-v1-128k": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "moonshot-v1-128k-0430": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "moonshot-v1-128k-vision-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "moonshot-v1-32k": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, - "moonshot-v1-32k-0430": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, - "moonshot-v1-32k-vision-preview": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, - "moonshot-v1-8k": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, - "moonshot-v1-8k-0430": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, - "moonshot-v1-8k-vision-preview": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, - "moonshot-v1-auto": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, - "o1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000}, - "o1-2024-12-17": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000}, - "o3": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "o3-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "o3-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "o3-mini-2025-01-31": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "o4-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "qwen-3-32b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 800000}, - "qwen-coder": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1500000}, - "qwen-max": {ContextTokens: 30720, InputCostPerMTokMicroUSD: 1600000, OutputCostPerMTokMicroUSD: 6400000}, - "qwen-plus": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-plus-2025-01-25": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-plus-2025-04-28": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-plus-2025-07-14": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-turbo": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen-turbo-2024-11-01": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen-turbo-2025-04-28": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen-turbo-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen3-next-80b-a3b-instruct": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen3-next-80b-a3b-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen3-vl-235b-a22b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, - "qwen3-vl-235b-a22b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 4000000}, - "qwen3-vl-32b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 640000}, - "qwen3-vl-32b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 2870000}, - "qwen3.7-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 7500000}, - "qwen3.8-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, - "qwq-plus": {ContextTokens: 98304, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 2400000}, - "text-embedding-3-large": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 130000, OutputCostPerMTokMicroUSD: 0}, - "text-embedding-3-small": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 20000, OutputCostPerMTokMicroUSD: 0}, - "text-embedding-ada-002": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0}, - "text-embedding-ada-002-v2": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0}, - "zai-glm-4.6": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2250000, OutputCostPerMTokMicroUSD: 2750000}, - "zai-glm-4.7": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2250000, OutputCostPerMTokMicroUSD: 2750000}, + "MiniMax-M2": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "MiniMax-M2.1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "MiniMax-M2.1-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M2.1-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M2.5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "MiniMax-M2.5-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M2.5-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "chat-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, + "claude-fable-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-fable-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-haiku-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, + "claude-haiku-4-5-20251001": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, + "claude-mythos-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-mythos-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-mythos-preview": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-opus-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-5-20251101": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-6-20260205": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-7": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-7-20260416": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-8": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "claude-sonnet-4-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "claude-sonnet-4-5-20250929": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "claude-sonnet-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "claude-sonnet-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, + "claude-sonnet-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, + "computer-use-preview": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 12000000}, + "daybreak-blue-latest": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "daybreak-red-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, + "deepseek-chat": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, + "deepseek-coder": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 140000, OutputCostPerMTokMicroUSD: 280000}, + "deepseek-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "deepseek-r1": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 550000, OutputCostPerMTokMicroUSD: 2190000}, + "deepseek-reasoner": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, + "deepseek-v3": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 270000, OutputCostPerMTokMicroUSD: 1100000}, + "deepseek-v3.2": {ContextTokens: 163840, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 400000}, + "deepseek-v4-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "deepseek-v4-flash-0731": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 400000}, + "deepseek-v4-flash-vision-exp": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "deepseek-v4-pro": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1320000, OutputCostPerMTokMicroUSD: 3960000}, + "ft:gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-3.5-turbo-0613": {ContextTokens: 4096, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, + "ft:gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 12000000}, + "ft:gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 3200000}, + "ft:gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 800000}, + "ft:gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000}, + "ft:gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000}, + "ft:gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "ft:o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 16000000}, + "gemma-4-31b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 990000, OutputCostPerMTokMicroUSD: 1490000}, + "glm-5.1": {ContextTokens: 202745, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, + "glm-5.2": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, + "gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, + "gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, + "gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, + "gpt-3.5-turbo-16k": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 4000000}, + "gpt-4": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, + "gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, + "gpt-4-1106-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-4-turbo": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-4-turbo-2024-04-09": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-4.1": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, + "gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, + "gpt-4.1-mini": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, + "gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, + "gpt-4.1-nano": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, + "gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, + "gpt-4o": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-4o-2024-05-13": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, + "gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-4o-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-4o-audio-preview-2025-06-03": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-4o-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, + "gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, + "gpt-4o-mini-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, + "gpt-4o-mini-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, + "gpt-4o-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5-chat": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, + "gpt-5-mini-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, + "gpt-5-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, + "gpt-5-nano-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, + "gpt-5-search-api": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5-search-api-2025-10-14": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.1": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.1-2025-11-13": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.1-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.2": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, + "gpt-5.2-2025-12-11": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, + "gpt-5.2-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, + "gpt-5.3-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, + "gpt-5.4": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, + "gpt-5.4-2026-03-05": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, + "gpt-5.4-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, + "gpt-5.4-mini-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, + "gpt-5.4-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, + "gpt-5.4-nano-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, + "gpt-5.5": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-5.5-2026-04-23": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-5.5-cyber": {ContextTokens: 0, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, + "gpt-5.6": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "gpt-5.6-cyber": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, + "gpt-5.6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1200000}, + "gpt-5.6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "gpt-5.6-terra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 12000000}, + "gpt-6-astra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "gpt-6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 500000}, + "gpt-6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio-1.5": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio-2025-08-28": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, + "gpt-audio-mini-2025-12-15": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, + "gpt-oss-120b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 350000, OutputCostPerMTokMicroUSD: 750000}, + "gpt-rosalind-research": {ContextTokens: 0, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "grok-4.20": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-latest-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-latest-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-0304": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-0304-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-0304-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-non-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-non-reasoning-gv2": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-non-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-reasoning-gv2": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-reasoning-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.3-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.5": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-4.5-latest": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-4.6": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-4.7": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-build-0.1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, + "grok-build-latest": {ContextTokens: 500000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "grok-code-fast": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, + "grok-code-fast-1": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, + "grok-code-fast-1-0825": {ContextTokens: 256000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, + "kimi-k2.5": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 3000000}, + "kimi-k2.6": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000}, + "kimi-k2.7-code": {ContextTokens: 229376, InputCostPerMTokMicroUSD: 950000, OutputCostPerMTokMicroUSD: 4000000}, + "kimi-k3": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "llama-3.3-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 850000, OutputCostPerMTokMicroUSD: 1200000}, + "llama3.1-70b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 600000}, + "llama3.1-8b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 100000}, + "moonshot-v1-128k": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, + "moonshot-v1-128k-vision-preview": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, + "moonshot-v1-32k": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, + "moonshot-v1-32k-vision-preview": {ContextTokens: 32768, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 3000000}, + "moonshot-v1-8k": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, + "moonshot-v1-8k-vision-preview": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, + "moonshot-v1-auto": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, + "o1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000}, + "o1-2024-12-17": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000}, + "o3": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, + "o3-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, + "o3-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, + "o3-mini-2025-01-31": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, + "o4-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, + "o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, + "qwen-3-32b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 800000}, + "qwen-3.8-27b": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 990000, OutputCostPerMTokMicroUSD: 1490000}, + "qwen-coder": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1500000}, + "qwen-max": {ContextTokens: 30720, InputCostPerMTokMicroUSD: 1600000, OutputCostPerMTokMicroUSD: 6400000}, + "qwen-plus": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, + "qwen-plus-2025-01-25": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, + "qwen-plus-2025-04-28": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, + "qwen-plus-2025-07-14": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, + "qwen-turbo": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, + "qwen-turbo-2024-11-01": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, + "qwen-turbo-2025-04-28": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, + "qwen-turbo-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, + "qwen3-next-80b-a3b-instruct": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, + "qwen3-next-80b-a3b-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, + "qwen3-vl-235b-a22b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, + "qwen3-vl-235b-a22b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 4000000}, + "qwen3-vl-32b-instruct": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 640000}, + "qwen3-vl-32b-thinking": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 160000, OutputCostPerMTokMicroUSD: 2870000}, + "qwen3.7-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 7500000}, + "qwen3.8-flash": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 470000}, + "qwen3.8-max": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 6000000}, + "qwen3.8-omni-flash": {ContextTokens: 991808, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 470000}, + "qwq-plus": {ContextTokens: 98304, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 2400000}, + "text-embedding-3-large": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 130000, OutputCostPerMTokMicroUSD: 0}, + "text-embedding-3-small": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 20000, OutputCostPerMTokMicroUSD: 0}, + "text-embedding-ada-002": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0}, + "text-embedding-ada-002-v2": {ContextTokens: 8191, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 0}, } From 4681596281df46820294beed6283f41c0a323317 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 13:49:13 +0100 Subject: [PATCH 09/15] feat(pricing-sync): drop entries older than 12 months An entry is dropped when its deprecation date has passed, or when a full date in its id is over 12 months old. Ids with no date signal stay. --- tools/pricing-sync/main.go | 43 +++++++++++++++++++++++++-- tools/pricing-sync/main_test.go | 51 ++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/tools/pricing-sync/main.go b/tools/pricing-sync/main.go index 1a33128..8cdc63f 100644 --- a/tools/pricing-sync/main.go +++ b/tools/pricing-sync/main.go @@ -57,6 +57,12 @@ // a defensive fallback, not just a hardcoded "sample_spec" skip, so // a future upstream schema-example entry with some other key name // degrades the same way. +// 4. Freshness: an entry is dropped when LiteLLM's own deprecation_date +// has already passed, or when a full calendar date embedded in its +// bare id ("claude-opus-4-1-20250805", "gpt-4o-2024-11-20") is +// older than maxModelAgeMonths. LiteLLM carries no release date, so +// an id with no such date and no deprecation_date is always kept: +// an undated alias is never dropped on a guess. // // # Bare-id collision resolution // @@ -102,6 +108,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "sort" "strings" "time" @@ -129,6 +136,34 @@ const microUSDPerUSDPerToken = 1e12 // this specific, known case. const sampleSpecKey = "sample_spec" +// maxModelAgeMonths is the freshness rule's age cutoff for a date embedded +// in a bare model id. +const maxModelAgeMonths = 12 + +// idDatePattern finds a full calendar date in a model id, as YYYYMMDD or +// YYYY-MM-DD, delimited by the id's ends or a "-" or "_". +var idDatePattern = regexp.MustCompile(`(?:^|[-_])(20\d{2})-?(0[1-9]|1[0-2])-?(0[1-9]|[12]\d|3[01])(?:$|[-_])`) + +// isStale reports whether an entry fails the freshness rule: its +// deprecation_date (YYYY-MM-DD) is before now, or a date embedded in +// bareID is more than maxModelAgeMonths before now. +func isStale(bareID, deprecationDate string, now time.Time) bool { + if deprecationDate != "" { + if d, err := time.Parse("2006-01-02", deprecationDate); err == nil && d.Before(now) { + return true + } + } + m := idDatePattern.FindStringSubmatch(bareID) + if m == nil { + return false + } + d, err := time.Parse("20060102", m[1]+m[2]+m[3]) + if err != nil { + return false + } + return d.Before(now.AddDate(0, -maxModelAgeMonths, 0)) +} + // allowedProviders is the pruning rule's provider allowlist — see the // package doc comment for the full rationale and the qwen/alibaba -> // dashscope mapping. @@ -218,6 +253,7 @@ func applyNamingBridges(table map[string]builtinEntry, winners map[string]candid // max_input_tokens/max_tokens as a JSON float (2000000.0) — decoding // those into *int would fail outright and drop the entry. type litellmEntry struct { + DeprecationDate string `json:"deprecation_date"` InputCostPerToken *float64 `json:"input_cost_per_token"` OutputCostPerToken *float64 `json:"output_cost_per_token"` MaxInputTokens *float64 `json:"max_input_tokens"` @@ -268,7 +304,7 @@ func run() error { return fmt.Errorf("fetch %s: %w", litellmSourceURL, err) } - table, totalUpstream, err := parseAndPrune(data) + table, totalUpstream, err := parseAndPrune(data, time.Now()) if err != nil { return fmt.Errorf("parse and prune: %w", err) } @@ -319,7 +355,7 @@ func fetchLiteLLMPrices(ctx context.Context, url string) ([]byte, error) { // chooseWinner's deterministic bare-id collision resolution. It returns // the pruned table and the total number of top-level keys data carried // (including ones this function skips), for the summary line run prints. -func parseAndPrune(data []byte) (map[string]builtinEntry, int, error) { +func parseAndPrune(data []byte, now time.Time) (map[string]builtinEntry, int, error) { var raw map[string]json.RawMessage if err := json.Unmarshal(data, &raw); err != nil { return nil, 0, fmt.Errorf("decode top-level object: %w", err) @@ -346,6 +382,9 @@ func parseAndPrune(data []byte) (map[string]builtinEntry, int, error) { } bare, isBareKey := bareModelID(key, e.LiteLLMProvider) + if isStale(bare, e.DeprecationDate, now) { + continue + } byBare[bare] = append(byBare[bare], candidate{ fullKey: key, provider: e.LiteLLMProvider, diff --git a/tools/pricing-sync/main_test.go b/tools/pricing-sync/main_test.go index adad3e9..5886f8a 100644 --- a/tools/pricing-sync/main_test.go +++ b/tools/pricing-sync/main_test.go @@ -6,8 +6,11 @@ import ( "reflect" "strings" "testing" + "time" ) +var fixtureNow = time.Date(2026, time.August, 1, 0, 0, 0, 0, time.UTC) + // fixtureJSON is a small, hand-built stand-in for LiteLLM's real // model_prices_and_context_window.json, covering every branch of the // pruning rule documented on this package (main.go's own doc comment): @@ -116,7 +119,7 @@ const fixtureUpstreamKeyCount = 17 // out, covering every pruning-rule branch fixtureJSON's own doc comment // lists. func TestParseAndPrune_Golden(t *testing.T) { - got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON)) + got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON), fixtureNow) if err != nil { t.Fatalf("parseAndPrune: %v", err) } @@ -264,3 +267,49 @@ func TestContentDigest(t *testing.T) { t.Errorf("len(contentDigest(...)) = %d, want %d", len(a), contentDigestLen) } } + +func TestIsStale(t *testing.T) { + now := time.Date(2026, time.September, 29, 0, 0, 0, 0, time.UTC) + tests := []struct { + name, id, deprecated string + want bool + }{ + {"undated alias kept", "claude-opus-4-1", "", false}, + {"recent compact date kept", "claude-sonnet-5-20260401", "", false}, + {"old compact date dropped", "claude-opus-4-1-20250805", "", true}, + {"old dashed date dropped", "gpt-4o-2024-11-20", "", true}, + {"just inside cutoff kept", "model-2025-10-01", "", false}, + {"just outside cutoff dropped", "model-2025-09-28", "", true}, + {"date mid-id dropped", "gpt-4o-2024-11-20-preview", "", true}, + {"short MMDD suffix ignored", "grok-4.20-experimental-beta-0304", "", false}, + {"invalid month ignored", "model-20251301", "", false}, + {"past deprecation dropped", "gpt-4-turbo", "2025-06-01", true}, + {"future deprecation kept", "gpt-4-turbo", "2027-01-01", false}, + {"bad deprecation format ignored", "gpt-4-turbo", "soon", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := isStale(tc.id, tc.deprecated, now); got != tc.want { + t.Errorf("isStale(%q, %q) = %v, want %v", tc.id, tc.deprecated, got, tc.want) + } + }) + } +} + +func TestParseAndPrune_DropsStaleEntries(t *testing.T) { + const data = `{ + "gpt-fresh": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + "gpt-old-2024-05-13": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + "gpt-dead": {"litellm_provider": "openai", "mode": "chat", "deprecation_date": "2026-01-01", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} +}` + got, _, err := parseAndPrune([]byte(data), fixtureNow) + if err != nil { + t.Fatalf("parseAndPrune: %v", err) + } + if len(got) != 1 { + t.Fatalf("kept %d entries %v, want only gpt-fresh", len(got), got) + } + if _, ok := got["gpt-fresh"]; !ok { + t.Errorf("gpt-fresh missing from %v", got) + } +} From 4edf9be804e06f2231182871cd6fe32a70bd37d5 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 13:49:15 +0100 Subject: [PATCH 10/15] chore(pricing): regenerate the builtin table with the age filter Keeps 171 entries, down from 211. Drops mostly dated snapshot ids. --- pricing_data_gen.go | 260 +++++++++++++++++++------------------------- 1 file changed, 110 insertions(+), 150 deletions(-) diff --git a/pricing_data_gen.go b/pricing_data_gen.go index b7913da..7937fa1 100644 --- a/pricing_data_gen.go +++ b/pricing_data_gen.go @@ -5,152 +5,121 @@ package traefikllmgateway // // Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json // Source digest: sha256:9f7b337871ed -// 211 entries kept (pruned from 4422 upstream entries — see +// 171 entries kept (pruned from 4422 upstream entries — see // tools/pricing-sync/main.go's own doc comment for the exact pruning // rule and bare-id collision resolution). // // Regenerate with: make pricing-sync var builtinModelMetaTable = map[string]builtinModelMeta{ - "MiniMax-M2": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "MiniMax-M2.1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "MiniMax-M2.1-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M2.1-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M2.5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "MiniMax-M2.5-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M2.5-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, - "MiniMax-M3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "chat-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "claude-fable-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-fable-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-haiku-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, - "claude-haiku-4-5-20251001": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, - "claude-mythos-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-mythos-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-mythos-preview": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "claude-opus-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-5-20251101": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-6-20260205": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-7": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-7-20260416": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-4-8": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "claude-opus-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, - "claude-sonnet-4-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-4-5-20250929": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, - "claude-sonnet-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "claude-sonnet-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "computer-use-preview": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 12000000}, - "daybreak-blue-latest": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, - "daybreak-red-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, - "deepseek-chat": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, - "deepseek-coder": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 140000, OutputCostPerMTokMicroUSD: 280000}, - "deepseek-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "deepseek-r1": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 550000, OutputCostPerMTokMicroUSD: 2190000}, - "deepseek-reasoner": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, - "deepseek-v3": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 270000, OutputCostPerMTokMicroUSD: 1100000}, - "deepseek-v3.2": {ContextTokens: 163840, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 400000}, - "deepseek-v4-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "deepseek-v4-flash-0731": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 400000}, - "deepseek-v4-flash-vision-exp": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "deepseek-v4-pro": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1320000, OutputCostPerMTokMicroUSD: 3960000}, - "ft:gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-3.5-turbo-0613": {ContextTokens: 4096, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, - "ft:gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "ft:gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 12000000}, - "ft:gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 800000, OutputCostPerMTokMicroUSD: 3200000}, - "ft:gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 800000}, - "ft:gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000}, - "ft:gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 3750000, OutputCostPerMTokMicroUSD: 15000000}, - "ft:gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, - "ft:o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 16000000}, - "gemma-4-31b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 990000, OutputCostPerMTokMicroUSD: 1490000}, - "glm-5.1": {ContextTokens: 202745, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, - "glm-5.2": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, - "gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, - "gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, - "gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 2000000}, - "gpt-3.5-turbo-16k": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 4000000}, - "gpt-4": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, - "gpt-4-1106-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-turbo": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4-turbo-2024-04-09": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-4.1": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "gpt-4.1-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "gpt-4.1-mini": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, - "gpt-4.1-mini-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, - "gpt-4.1-nano": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-4.1-nano-2025-04-14": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-4o": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-2024-05-13": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 15000000}, - "gpt-4o-2024-08-06": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-2024-11-20": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-audio-preview-2025-06-03": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-4o-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-2024-07-18": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-audio-preview-2024-12-17": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-mini-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, - "gpt-4o-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-chat": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, - "gpt-5-mini-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, - "gpt-5-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-5-nano-2025-08-07": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, - "gpt-5-search-api": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5-search-api-2025-10-14": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.1": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.1-2025-11-13": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.1-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-5.2": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.2-2025-12-11": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.2-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.3-chat-latest": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, - "gpt-5.4": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, - "gpt-5.4-2026-03-05": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, - "gpt-5.4-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, - "gpt-5.4-mini-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, - "gpt-5.4-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, - "gpt-5.4-nano-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, - "gpt-5.5": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.5-2026-04-23": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, - "gpt-5.5-cyber": {ContextTokens: 0, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, - "gpt-5.6": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, - "gpt-5.6-cyber": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, - "gpt-5.6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1200000}, - "gpt-5.6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, - "gpt-5.6-terra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 12000000}, - "gpt-6-astra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, - "gpt-6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 500000}, - "gpt-6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio-1.5": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio-2025-08-28": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, - "gpt-audio-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, - "gpt-audio-mini-2025-12-15": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, - "gpt-oss-120b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 350000, OutputCostPerMTokMicroUSD: 750000}, - "gpt-rosalind-research": {ContextTokens: 0, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, - "grok-4.20": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-latest-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-latest-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-beta-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, - "grok-4.20-experimental-beta-0304": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "MiniMax-M2": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "MiniMax-M2.1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "MiniMax-M2.1-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M2.1-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M2.5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "MiniMax-M2.5-highspeed": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M2.5-lightning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 2400000}, + "MiniMax-M3": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "chat-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, + "claude-fable-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-fable-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-haiku-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, + "claude-haiku-4-5-20251001": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1000000, OutputCostPerMTokMicroUSD: 5000000}, + "claude-mythos-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-mythos-5-1": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "claude-opus-4-5": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-5-20251101": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-6-20260205": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-7": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-7-20260416": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-4-8": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "claude-opus-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "claude-sonnet-4-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "claude-sonnet-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "claude-sonnet-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, + "claude-sonnet-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, + "daybreak-blue-latest": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "daybreak-red-latest": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, + "deepseek-chat": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, + "deepseek-coder": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 140000, OutputCostPerMTokMicroUSD: 280000}, + "deepseek-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "deepseek-r1": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 550000, OutputCostPerMTokMicroUSD: 2190000}, + "deepseek-reasoner": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 420000}, + "deepseek-v3": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 270000, OutputCostPerMTokMicroUSD: 1100000}, + "deepseek-v3.2": {ContextTokens: 163840, InputCostPerMTokMicroUSD: 280000, OutputCostPerMTokMicroUSD: 400000}, + "deepseek-v4-flash": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "deepseek-v4-flash-0731": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 400000}, + "deepseek-v4-flash-vision-exp": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1200000}, + "deepseek-v4-pro": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1320000, OutputCostPerMTokMicroUSD: 3960000}, + "ft:gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-3.5-turbo-0613": {ContextTokens: 4096, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-3.5-turbo-1106": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 6000000}, + "ft:gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, + "gemma-4-31b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 990000, OutputCostPerMTokMicroUSD: 1490000}, + "glm-5.1": {ContextTokens: 202745, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, + "glm-5.2": {ContextTokens: 1048576, InputCostPerMTokMicroUSD: 1400000, OutputCostPerMTokMicroUSD: 4400000}, + "gpt-3.5-turbo": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, + "gpt-3.5-turbo-0125": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 500000, OutputCostPerMTokMicroUSD: 1500000}, + "gpt-3.5-turbo-16k": {ContextTokens: 16385, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 4000000}, + "gpt-4": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, + "gpt-4-0613": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 30000000, OutputCostPerMTokMicroUSD: 60000000}, + "gpt-4-1106-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-4-turbo": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-4.1": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, + "gpt-4.1-mini": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1600000}, + "gpt-4.1-nano": {ContextTokens: 1047576, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 400000}, + "gpt-4o": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-4o-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, + "gpt-4o-mini-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 600000}, + "gpt-4o-search-preview": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 250000, OutputCostPerMTokMicroUSD: 2000000}, + "gpt-5-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 400000}, + "gpt-5-search-api": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5-search-api-2025-10-14": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.1": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.1-2025-11-13": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-5.2": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, + "gpt-5.2-2025-12-11": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 1750000, OutputCostPerMTokMicroUSD: 14000000}, + "gpt-5.4": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, + "gpt-5.4-2026-03-05": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 15000000}, + "gpt-5.4-mini": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, + "gpt-5.4-mini-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 750000, OutputCostPerMTokMicroUSD: 4500000}, + "gpt-5.4-nano": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, + "gpt-5.4-nano-2026-03-17": {ContextTokens: 272000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1250000}, + "gpt-5.5": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-5.5-2026-04-23": {ContextTokens: 1050000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 30000000}, + "gpt-5.5-cyber": {ContextTokens: 0, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, + "gpt-5.6": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "gpt-5.6-cyber": {ContextTokens: 400000, InputCostPerMTokMicroUSD: 12500000, OutputCostPerMTokMicroUSD: 75000000}, + "gpt-5.6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 1200000}, + "gpt-5.6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, + "gpt-5.6-terra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 12000000}, + "gpt-6-astra": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 10000000, OutputCostPerMTokMicroUSD: 50000000}, + "gpt-6-luna": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 100000, OutputCostPerMTokMicroUSD: 500000}, + "gpt-6-sol": {ContextTokens: 922000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio-1.5": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 2500000, OutputCostPerMTokMicroUSD: 10000000}, + "gpt-audio-mini": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, + "gpt-audio-mini-2025-12-15": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 600000, OutputCostPerMTokMicroUSD: 2400000}, + "gpt-oss-120b": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 350000, OutputCostPerMTokMicroUSD: 750000}, + "gpt-rosalind-research": {ContextTokens: 0, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, + "grok-4.20": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-0309": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-0309-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-0309-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-latest-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-latest-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-beta-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, + "grok-4.20-experimental-beta-0304": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, "grok-4.20-experimental-beta-0304-non-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, "grok-4.20-experimental-beta-0304-reasoning": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, "grok-4.20-experimental-beta-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 1250000, OutputCostPerMTokMicroUSD: 2500000}, @@ -188,24 +157,15 @@ var builtinModelMetaTable = map[string]builtinModelMeta{ "moonshot-v1-8k-vision-preview": {ContextTokens: 8192, InputCostPerMTokMicroUSD: 200000, OutputCostPerMTokMicroUSD: 2000000}, "moonshot-v1-auto": {ContextTokens: 131072, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 5000000}, "o1": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000}, - "o1-2024-12-17": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 15000000, OutputCostPerMTokMicroUSD: 60000000}, "o3": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, - "o3-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 8000000}, "o3-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "o3-mini-2025-01-31": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, "o4-mini": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, - "o4-mini-2025-04-16": {ContextTokens: 200000, InputCostPerMTokMicroUSD: 1100000, OutputCostPerMTokMicroUSD: 4400000}, "qwen-3-32b": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 800000}, "qwen-3.8-27b": {ContextTokens: 65536, InputCostPerMTokMicroUSD: 990000, OutputCostPerMTokMicroUSD: 1490000}, "qwen-coder": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 300000, OutputCostPerMTokMicroUSD: 1500000}, "qwen-max": {ContextTokens: 30720, InputCostPerMTokMicroUSD: 1600000, OutputCostPerMTokMicroUSD: 6400000}, "qwen-plus": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-plus-2025-01-25": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-plus-2025-04-28": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, - "qwen-plus-2025-07-14": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 400000, OutputCostPerMTokMicroUSD: 1200000}, "qwen-turbo": {ContextTokens: 129024, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen-turbo-2024-11-01": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, - "qwen-turbo-2025-04-28": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, "qwen-turbo-latest": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 50000, OutputCostPerMTokMicroUSD: 200000}, "qwen3-next-80b-a3b-instruct": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, "qwen3-next-80b-a3b-thinking": {ContextTokens: 262144, InputCostPerMTokMicroUSD: 150000, OutputCostPerMTokMicroUSD: 1200000}, From d4322e914c1b08ee96a3cfa37de498e516ccb6a8 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 13:58:20 +0100 Subject: [PATCH 11/15] feat(pricing-sync): make the age cutoff and keep list configurable Add the -max-age-months and -keep flags, with PRICING_MAX_AGE_MONTHS and PRICING_KEEP make variables. A built-in list keeps ids that are still served. --- Makefile | 6 +++- tools/pricing-sync/main.go | 61 ++++++++++++++++++++++++--------- tools/pricing-sync/main_test.go | 38 +++++++++++++++++--- 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 156ee62..c91e6b8 100644 --- a/Makefile +++ b/Makefile @@ -40,8 +40,12 @@ yaegi-check: # pruning rule and naming-bridge mechanism. Not run by CI (network # fetch, and the generated table is committed); an operator re-runs this # by hand when LiteLLM's own pricing drifts. +# PRICING_MAX_AGE_MONTHS drops entries older than N months (0 = keep all); +# PRICING_KEEP is a comma-separated list of bare ids exempt from that rule. +PRICING_MAX_AGE_MONTHS ?= 12 +PRICING_KEEP ?= pricing-sync: - cd tools/pricing-sync && GOWORK=off go run . $(CURDIR) + cd tools/pricing-sync && GOWORK=off go run . -max-age-months=$(PRICING_MAX_AGE_MONTHS) -keep=$(PRICING_KEEP) $(CURDIR) gofmt -w pricing_data_gen.go INTEGRATION_COMPOSE := integration/docker-compose.yml diff --git a/tools/pricing-sync/main.go b/tools/pricing-sync/main.go index 8cdc63f..8b4c56e 100644 --- a/tools/pricing-sync/main.go +++ b/tools/pricing-sync/main.go @@ -60,9 +60,11 @@ // 4. Freshness: an entry is dropped when LiteLLM's own deprecation_date // has already passed, or when a full calendar date embedded in its // bare id ("claude-opus-4-1-20250805", "gpt-4o-2024-11-20") is -// older than maxModelAgeMonths. LiteLLM carries no release date, so -// an id with no such date and no deprecation_date is always kept: -// an undated alias is never dropped on a guess. +// older than the -max-age-months flag (default 12; 0 turns the whole +// rule off). LiteLLM carries no release date, so an id with no such +// date and no deprecation_date is always kept: an undated alias is +// never dropped on a guess. An id in alwaysKeep, or in the -keep +// flag's comma-separated list, is exempt. // // # Bare-id collision resolution // @@ -101,6 +103,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "flag" "fmt" "go/format" "io" @@ -136,20 +139,37 @@ const microUSDPerUSDPerToken = 1e12 // this specific, known case. const sampleSpecKey = "sample_spec" -// maxModelAgeMonths is the freshness rule's age cutoff for a date embedded -// in a bare model id. -const maxModelAgeMonths = 12 +// defaultMaxAgeMonths is the -max-age-months flag's default. +const defaultMaxAgeMonths = 12 + +// alwaysKeep lists bare ids the freshness rule never drops, because they +// are still served after their age or deprecation date says otherwise. +var alwaysKeep = map[string]bool{ + "claude-sonnet-4-5-20250929": true, +} + +// pruneOptions is the freshness rule's configuration. maxAgeMonths <= 0 +// turns the rule off. +type pruneOptions struct { + now time.Time + keep map[string]bool + maxAgeMonths int +} // idDatePattern finds a full calendar date in a model id, as YYYYMMDD or // YYYY-MM-DD, delimited by the id's ends or a "-" or "_". var idDatePattern = regexp.MustCompile(`(?:^|[-_])(20\d{2})-?(0[1-9]|1[0-2])-?(0[1-9]|[12]\d|3[01])(?:$|[-_])`) // isStale reports whether an entry fails the freshness rule: its -// deprecation_date (YYYY-MM-DD) is before now, or a date embedded in -// bareID is more than maxModelAgeMonths before now. -func isStale(bareID, deprecationDate string, now time.Time) bool { +// deprecation_date (YYYY-MM-DD) is before opts.now, or a date embedded in +// bareID is more than opts.maxAgeMonths before it. Ids in opts.keep and +// alwaysKeep never fail. +func isStale(bareID, deprecationDate string, opts pruneOptions) bool { + if opts.maxAgeMonths <= 0 || opts.keep[bareID] || alwaysKeep[bareID] { + return false + } if deprecationDate != "" { - if d, err := time.Parse("2006-01-02", deprecationDate); err == nil && d.Before(now) { + if d, err := time.Parse("2006-01-02", deprecationDate); err == nil && d.Before(opts.now) { return true } } @@ -161,7 +181,7 @@ func isStale(bareID, deprecationDate string, now time.Time) bool { if err != nil { return false } - return d.Before(now.AddDate(0, -maxModelAgeMonths, 0)) + return d.Before(opts.now.AddDate(0, -opts.maxAgeMonths, 0)) } // allowedProviders is the pruning rule's provider allowlist — see the @@ -290,9 +310,18 @@ func main() { } func run() error { + maxAge := flag.Int("max-age-months", defaultMaxAgeMonths, "drop entries older than this many months; 0 disables the freshness rule") + keepList := flag.String("keep", "", "comma-separated bare ids the freshness rule never drops") + flag.Parse() repoRoot := "." - if len(os.Args) > 1 { - repoRoot = os.Args[1] + if flag.NArg() > 0 { + repoRoot = flag.Arg(0) + } + keep := map[string]bool{} + for _, id := range strings.Split(*keepList, ",") { + if id = strings.TrimSpace(id); id != "" { + keep[id] = true + } } outPath := filepath.Join(repoRoot, "pricing_data_gen.go") @@ -304,7 +333,7 @@ func run() error { return fmt.Errorf("fetch %s: %w", litellmSourceURL, err) } - table, totalUpstream, err := parseAndPrune(data, time.Now()) + table, totalUpstream, err := parseAndPrune(data, pruneOptions{now: time.Now(), maxAgeMonths: *maxAge, keep: keep}) if err != nil { return fmt.Errorf("parse and prune: %w", err) } @@ -355,7 +384,7 @@ func fetchLiteLLMPrices(ctx context.Context, url string) ([]byte, error) { // chooseWinner's deterministic bare-id collision resolution. It returns // the pruned table and the total number of top-level keys data carried // (including ones this function skips), for the summary line run prints. -func parseAndPrune(data []byte, now time.Time) (map[string]builtinEntry, int, error) { +func parseAndPrune(data []byte, opts pruneOptions) (map[string]builtinEntry, int, error) { var raw map[string]json.RawMessage if err := json.Unmarshal(data, &raw); err != nil { return nil, 0, fmt.Errorf("decode top-level object: %w", err) @@ -382,7 +411,7 @@ func parseAndPrune(data []byte, now time.Time) (map[string]builtinEntry, int, er } bare, isBareKey := bareModelID(key, e.LiteLLMProvider) - if isStale(bare, e.DeprecationDate, now) { + if isStale(bare, e.DeprecationDate, opts) { continue } byBare[bare] = append(byBare[bare], candidate{ diff --git a/tools/pricing-sync/main_test.go b/tools/pricing-sync/main_test.go index 5886f8a..9337f6e 100644 --- a/tools/pricing-sync/main_test.go +++ b/tools/pricing-sync/main_test.go @@ -9,7 +9,10 @@ import ( "time" ) -var fixtureNow = time.Date(2026, time.August, 1, 0, 0, 0, 0, time.UTC) +var ( + fixtureNow = time.Date(2026, time.August, 1, 0, 0, 0, 0, time.UTC) + fixtureOpts = pruneOptions{now: fixtureNow, maxAgeMonths: defaultMaxAgeMonths} +) // fixtureJSON is a small, hand-built stand-in for LiteLLM's real // model_prices_and_context_window.json, covering every branch of the @@ -119,7 +122,7 @@ const fixtureUpstreamKeyCount = 17 // out, covering every pruning-rule branch fixtureJSON's own doc comment // lists. func TestParseAndPrune_Golden(t *testing.T) { - got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON), fixtureNow) + got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON), fixtureOpts) if err != nil { t.Fatalf("parseAndPrune: %v", err) } @@ -289,7 +292,8 @@ func TestIsStale(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - if got := isStale(tc.id, tc.deprecated, now); got != tc.want { + opts := pruneOptions{now: now, maxAgeMonths: defaultMaxAgeMonths} + if got := isStale(tc.id, tc.deprecated, opts); got != tc.want { t.Errorf("isStale(%q, %q) = %v, want %v", tc.id, tc.deprecated, got, tc.want) } }) @@ -302,7 +306,7 @@ func TestParseAndPrune_DropsStaleEntries(t *testing.T) { "gpt-old-2024-05-13": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, "gpt-dead": {"litellm_provider": "openai", "mode": "chat", "deprecation_date": "2026-01-01", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} }` - got, _, err := parseAndPrune([]byte(data), fixtureNow) + got, _, err := parseAndPrune([]byte(data), fixtureOpts) if err != nil { t.Fatalf("parseAndPrune: %v", err) } @@ -313,3 +317,29 @@ func TestParseAndPrune_DropsStaleEntries(t *testing.T) { t.Errorf("gpt-fresh missing from %v", got) } } + +func TestIsStale_Options(t *testing.T) { + now := time.Date(2026, time.September, 29, 0, 0, 0, 0, time.UTC) + const old = "gpt-4o-2024-08-06" + tests := []struct { + name string + opts pruneOptions + id string + want bool + }{ + {"default drops old", pruneOptions{now: now, maxAgeMonths: 12}, old, true}, + {"zero disables rule", pruneOptions{now: now, maxAgeMonths: 0}, old, false}, + {"negative disables rule", pruneOptions{now: now, maxAgeMonths: -1}, old, false}, + {"longer window keeps", pruneOptions{now: now, maxAgeMonths: 36}, old, false}, + {"shorter window drops more", pruneOptions{now: now, maxAgeMonths: 3}, "model-2026-03-01", true}, + {"keep flag exempts", pruneOptions{now: now, maxAgeMonths: 12, keep: map[string]bool{old: true}}, old, false}, + {"alwaysKeep exempts", pruneOptions{now: now, maxAgeMonths: 12}, "claude-sonnet-4-5-20250929", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := isStale(tc.id, "", tc.opts); got != tc.want { + t.Errorf("isStale(%q) = %v, want %v", tc.id, got, tc.want) + } + }) + } +} From 6be3f090320efa4059994f47d00198bfe79e9acf Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Tue, 29 Sep 2026 13:58:22 +0100 Subject: [PATCH 12/15] chore(pricing): keep claude-sonnet-4-5-20250929 in the builtin table --- pricing_data_gen.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pricing_data_gen.go b/pricing_data_gen.go index 7937fa1..7101eb9 100644 --- a/pricing_data_gen.go +++ b/pricing_data_gen.go @@ -5,7 +5,7 @@ package traefikllmgateway // // Source: https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json // Source digest: sha256:9f7b337871ed -// 171 entries kept (pruned from 4422 upstream entries — see +// 172 entries kept (pruned from 4422 upstream entries — see // tools/pricing-sync/main.go's own doc comment for the exact pruning // rule and bare-id collision resolution). // @@ -36,6 +36,7 @@ var builtinModelMetaTable = map[string]builtinModelMeta{ "claude-opus-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 5000000, OutputCostPerMTokMicroUSD: 25000000}, "claude-opus-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 4000000, OutputCostPerMTokMicroUSD: 20000000}, "claude-sonnet-4-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, + "claude-sonnet-4-5-20250929": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, "claude-sonnet-4-6": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 3000000, OutputCostPerMTokMicroUSD: 15000000}, "claude-sonnet-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, "claude-sonnet-5-5": {ContextTokens: 1000000, InputCostPerMTokMicroUSD: 2000000, OutputCostPerMTokMicroUSD: 10000000}, From 622686091732645cdf18d10c7367a288c09e7d01 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Thu, 15 Oct 2026 21:48:18 +0100 Subject: [PATCH 13/15] feat(registry): add requireModelPrefix to disable bare model ids When the flag is true, a model resolves only as provider/model or as a modelAliases id. A bare id returns 404 unknown model. GET /v1/models lists every model in provider/model form and lists no bare ids. The flag defaults to false, so existing deployments keep the sorted-first bare-id collision winner. --- README.md | 1 + llmgateway.go | 5 ++ registry.go | 11 +++- registry_requireprefix_test.go | 115 +++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 registry_requireprefix_test.go diff --git a/README.md b/README.md index b34b432..c4408f3 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,7 @@ config accepts them as YAML, which decodes to the same JSON shape. | `admin` | `*AdminConfig` | `nil` (disabled) | Read-only admin dashboard — see [Admin](#admin). `nil` or `enabled: false` means the `/admin*` routes are not registered at all. | | `metrics` | `*MetricsConfig` | `nil` (disabled) | Prometheus text-exposition endpoint — see [Metrics](#metrics). `nil` or `enabled: false` means the metrics route is not registered at all. | | `modelAliases` | `map[string]string` | `{}` | Operator-defined alias id → target model id — see [Model aliases](#model-aliases). Omitted or empty means no aliases, byte-identical to a gateway built before this field existed. | +| `requireModelPrefix` | `bool` | `false` | When `true`, a model resolves only as `provider/model` or as a `modelAliases` id. A bare id returns 404 `unknown model` and `GET /v1/models` lists no bare ids. `false` keeps the sorted-first bare-id collision winner. | | `modelMeta` | `map[string]*ModelMetaConfig` | `{}` | Per-model context-window and cost overrides, keyed by exact `provider/model` or a bare model id — see [Model metadata](#model-metadata). Omitted or empty means no overrides; every model's metadata still resolves through discovery and the built-in table. | | `passthroughUnknown` | `bool` | `false` | `false`: a request matching none of the plugin's routes gets a 404 JSON envelope. `true`: it falls through to the router's own backing service. **Does not cover the three media routes** — see the upgrade note below. | | `maxInFlightBodyRequests` | `int` | `0` (self-tuned) | Caps how many unified/media requests may concurrently be inside their own request-body read+decode step — see [Body-admission limit](#body-admission-limit). `0` self-tunes from the host's own `GOMAXPROCS`; a positive value pins an exact cap instead, clamped to a maximum of 10,000. | diff --git a/llmgateway.go b/llmgateway.go index 53de021..8901039 100644 --- a/llmgateway.go +++ b/llmgateway.go @@ -55,6 +55,11 @@ type Config struct { // (validateModelAliases, registry.go). ModelAliases map[string]string `json:"modelAliases,omitempty"` Providers map[string]*ProviderConfig `json:"providers,omitempty"` + // RequireModelPrefix, when true, makes every model addressable only as + // "provider/model" or as a ModelAliases id: a bare discovered id is not + // resolved (404 unknown model) and is not listed by GET /v1/models. It + // removes the sorted-first bare-id collision winner. Defaults to false. + RequireModelPrefix bool `json:"requireModelPrefix,omitempty"` // ModelMeta declares per-model metadata overrides (feature v0.23): // context window size and per-token cost, keyed by an exact // "provider/model" id or a bare model id (applying wherever that bare diff --git a/registry.go b/registry.go index faa5f38..71eeb47 100644 --- a/registry.go +++ b/registry.go @@ -664,6 +664,9 @@ type modelRegistry struct { // cheap nil-map lookup (always false/zero) that adds no branching // cost to the v0.1 no-aliases case. aliases map[string]string + // requirePrefix is Config.RequireModelPrefix: resolve and listFor treat + // a bare id as unaddressable, leaving "provider/model" and aliases. + requirePrefix bool // modelMeta is the validated Config.ModelMeta config-override map // (feature v0.23), built once by newModelRegistry via // validateModelMeta and never mutated afterward. nil (not merely @@ -882,6 +885,7 @@ func newModelRegistry(adapters map[string]providerAdapter, cfg *Config, log func return nil, err } m.aliases = aliases + m.requirePrefix = cfg.RequireModelPrefix modelMeta, err := validateModelMeta(cfg.ModelMeta) if err != nil { @@ -1603,6 +1607,9 @@ func (m *modelRegistry) resolve(id string, grp *group) (providerAdapter, string, if providerName, rest, ok := m.splitConfiguredProvider(id); ok { return m.resolveAgainst(providerName, rest, id, "", grp, errModelUnknown) } + if m.requirePrefix { + return nil, "", "", errModelUnknown + } providerName, ok := m.bareWinner(id, grp) if !ok { return nil, "", "", errModelUnknown @@ -1981,10 +1988,10 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any { // that branch (harmless — matchesGlob is cheap), but still // REQUIRED for the single-grant fast path, which never consults // id at all. - if gwOK && grp.allowsProviderModel(groupWinner, id) && !shadowed && !ambiguousBareID { + if gwOK && grp.allowsProviderModel(groupWinner, id) && !shadowed && !ambiguousBareID && !m.requirePrefix { out = append(out, modelObject(id, groupWinner, m.resolveMetaFor(groupWinner, id))) } - if len(visible) < 2 && !shadowed && !ambiguousBareID { + if len(visible) < 2 && !shadowed && !ambiguousBareID && !m.requirePrefix { continue // only one provider serving id is visible to grp: no disambiguating prefix to add } for _, p := range visible { diff --git a/registry_requireprefix_test.go b/registry_requireprefix_test.go new file mode 100644 index 0000000..721415a --- /dev/null +++ b/registry_requireprefix_test.go @@ -0,0 +1,115 @@ +package traefikllmgateway + +import ( + "errors" + "sort" + "testing" +) + +func newRequirePrefixRegistry(t *testing.T, require bool) (*modelRegistry, map[string]providerAdapter) { + t.Helper() + adapters := map[string]providerAdapter{ + "alpha": newFakeAdapter("alpha"), + "beta": newFakeAdapter("beta"), + } + cfg := &Config{ + RequireModelPrefix: require, + ModelAliases: map[string]string{"fast": "alpha/solo", "shared-alias": "shared"}, + Providers: map[string]*ProviderConfig{ + "alpha": {Models: []string{"shared", "solo"}}, + "beta": {Models: []string{"shared", "other"}}, + }, + } + reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) + if err != nil { + t.Fatalf("newModelRegistry: %v", err) + } + return reg, adapters +} + +func TestModelRegistry_RequireModelPrefix_Resolve(t *testing.T) { + t.Parallel() + reg, adapters := newRequirePrefixRegistry(t, true) + grp := allowAllGroup() + + for _, id := range []string{"solo", "shared", "other"} { + if _, _, _, err := reg.resolve(id, grp); !errors.Is(err, errModelUnknown) { + t.Errorf("resolve(%q) err = %v, want errModelUnknown", id, err) + } + } + + cases := []struct { + id string + provider string + canon string + }{ + {"alpha/solo", "alpha", "alpha/solo"}, + {"beta/shared", "beta", "beta/shared"}, + {"fast", "alpha", "alpha/solo"}, + {"shared-alias", "alpha", "alpha/shared"}, + } + for _, tc := range cases { + adapter, _, canon, err := reg.resolve(tc.id, grp) + if err != nil { + t.Errorf("resolve(%q): %v", tc.id, err) + continue + } + if adapter != adapters[tc.provider] || canon != tc.canon { + t.Errorf("resolve(%q) = %v %q, want %s %q", tc.id, adapter, canon, tc.provider, tc.canon) + } + } +} + +func TestModelRegistry_RequireModelPrefix_ListFor_NoBareIDs(t *testing.T) { + t.Parallel() + reg, _ := newRequirePrefixRegistry(t, true) + + var got []string + for _, entry := range reg.listFor(allowAllGroup()) { + got = append(got, entry["id"].(string)) //nolint:forcetypeassert // modelObject always sets id to a string + } + sort.Strings(got) + want := []string{"alpha/shared", "alpha/solo", "beta/other", "beta/shared", "fast", "shared-alias"} + if len(got) != len(want) { + t.Fatalf("listFor ids = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("listFor ids = %v, want %v", got, want) + } + } +} + +func TestModelRegistry_RequireModelPrefix_ListFor_RespectsProviderAllowList(t *testing.T) { + t.Parallel() + reg, _ := newRequirePrefixRegistry(t, true) + grp := &group{name: "g", providers: []string{"beta"}} + + for _, entry := range reg.listFor(grp) { + id := entry["id"].(string) //nolint:forcetypeassert // modelObject always sets id to a string + if entry["owned_by"] != "beta" { + t.Errorf("entry %q owned_by %v, want only beta-owned entries", id, entry["owned_by"]) + } + } + if _, _, _, err := reg.resolve("alpha/solo", grp); !errors.Is(err, errModelDenied) { + t.Errorf("resolve(alpha/solo) err = %v, want errModelDenied", err) + } +} + +func TestModelRegistry_RequireModelPrefix_Off_KeepsBareIDs(t *testing.T) { + t.Parallel() + reg, _ := newRequirePrefixRegistry(t, false) + + if _, _, canon, err := reg.resolve("solo", allowAllGroup()); err != nil || canon != "alpha/solo" { + t.Errorf("resolve(solo) = %q, %v, want alpha/solo, nil", canon, err) + } + var sawBare bool + for _, entry := range reg.listFor(allowAllGroup()) { + if entry["id"] == "solo" { + sawBare = true + } + } + if !sawBare { + t.Error("listFor lost bare id \"solo\" with requireModelPrefix off") + } +} From 06769904f449bc52e233533566665f9954503e6b Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Thu, 15 Oct 2026 21:59:15 +0100 Subject: [PATCH 14/15] ci: build with Go 1.26 staticcheck and govulncheck at latest require Go 1.26.0 or newer. The runner used Go 1.25, so both tools failed to install. --- .github/workflows/pr.yaml | 4 ++-- .github/workflows/release.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 572bc7c..008db70 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,7 +18,7 @@ jobs: pr-checks: uses: lukaszraczylo/shared-actions/.github/workflows/go-pr.yaml@main with: - go-version: "1.25.x" + go-version: "1.26.x" coverage-threshold: 75 secrets: inherit @@ -40,7 +40,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v6 with: - go-version: "1.25.x" + go-version: "1.26.x" cache: true - name: yaegi-check diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5f844a..93d4043 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,5 +24,5 @@ jobs: release: uses: lukaszraczylo/shared-actions/.github/workflows/go-release.yaml@main with: - go-version: "1.25.x" + go-version: "1.26.x" secrets: inherit From 77d5d1f9d0578c29cce18fa44b71a92f4b7c80b0 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Thu, 15 Oct 2026 21:59:16 +0100 Subject: [PATCH 15/15] test(translate): compare invalid UTF-8 JSON by value Go 1.25 json.Marshal escapes invalid UTF-8 as \ufffd. Go 1.26 writes the raw replacement rune. The streaming encoder writes the raw rune. The tests now compare these inputs as decoded values and keep byte comparison for valid input. --- translate_anthropic_test.go | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/translate_anthropic_test.go b/translate_anthropic_test.go index 757b112..0286434 100644 --- a/translate_anthropic_test.go +++ b/translate_anthropic_test.go @@ -13,6 +13,7 @@ import ( "sort" "strings" "testing" + "unicode/utf8" "github.com/stretchr/testify/assert" ) @@ -1129,6 +1130,27 @@ func TestOpenAIRequestFromAnthropic_NoTools_OmitsToolsAndToolChoiceFields(t *tes } } +// sameJSON reports whether got and want encode the same value. Invalid UTF-8 +// input is escaped as \ufffd by encoding/json before Go 1.26 and written as +// the raw replacement rune after, so those inputs compare semantically. +func sameJSON(t *testing.T, input string, got, want []byte) bool { + t.Helper() + if bytes.Equal(got, want) { + return true + } + if utf8.ValidString(input) { + return false + } + var g, w any + if err := json.Unmarshal(got, &g); err != nil { + return false + } + if err := json.Unmarshal(want, &w); err != nil { + return false + } + return reflect.DeepEqual(g, w) +} + func TestContentChunkerMatchesChunk(t *testing.T) { texts := []string{"", "Hello", "a \"quoted\" &", "line\nbreak\ttab", "unicode é 日本 \u2028\u2029", "bad\xffutf8"} for _, id := range []string{"", "msg_01", "we\"ird"} { @@ -1140,11 +1162,11 @@ func TestContentChunkerMatchesChunk(t *testing.T) { gst.content.set(chatCompletionIDPrefix+id, gst.model, gst.created) for _, text := range texts { gotA := ast.content.build(text) - if want := ast.chunk(map[string]any{"content": text}, nil); !bytes.Equal(gotA, want) { + if want := ast.chunk(map[string]any{"content": text}, nil); !sameJSON(t, text, gotA, want) { t.Errorf("anthropic id=%q text=%q:\n got %s\nwant %s", id, text, gotA, want) } gotG := gst.content.build(text) - if want := gst.chunk(map[string]any{"content": text}, nil); !bytes.Equal(gotG, want) { + if want := gst.chunk(map[string]any{"content": text}, nil); !sameJSON(t, text, gotG, want) { t.Errorf("gemini id=%q text=%q:\n got %s\nwant %s", id, text, gotG, want) } } @@ -1162,7 +1184,7 @@ func TestAppendJSONStringMatchesMarshal(t *testing.T) { if err != nil { t.Fatalf("Marshal(%q): %v", s, err) } - if got := appendJSONString(nil, s); !bytes.Equal(got, want) { + if got := appendJSONString(nil, s); !sameJSON(t, s, got, want) { t.Errorf("appendJSONString(%q) = %s, want %s", s, got, want) } }