diff --git a/tools/pricing-sync/main.go b/tools/pricing-sync/main.go index 1a33128..8cdc63f 100644 --- a/tools/pricing-sync/main.go +++ b/tools/pricing-sync/main.go @@ -57,6 +57,12 @@ // a defensive fallback, not just a hardcoded "sample_spec" skip, so // a future upstream schema-example entry with some other key name // degrades the same way. +// 4. Freshness: an entry is dropped when LiteLLM's own deprecation_date +// has already passed, or when a full calendar date embedded in its +// bare id ("claude-opus-4-1-20250805", "gpt-4o-2024-11-20") is +// older than maxModelAgeMonths. LiteLLM carries no release date, so +// an id with no such date and no deprecation_date is always kept: +// an undated alias is never dropped on a guess. // // # Bare-id collision resolution // @@ -102,6 +108,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "sort" "strings" "time" @@ -129,6 +136,34 @@ const microUSDPerUSDPerToken = 1e12 // this specific, known case. const sampleSpecKey = "sample_spec" +// maxModelAgeMonths is the freshness rule's age cutoff for a date embedded +// in a bare model id. +const maxModelAgeMonths = 12 + +// idDatePattern finds a full calendar date in a model id, as YYYYMMDD or +// YYYY-MM-DD, delimited by the id's ends or a "-" or "_". +var idDatePattern = regexp.MustCompile(`(?:^|[-_])(20\d{2})-?(0[1-9]|1[0-2])-?(0[1-9]|[12]\d|3[01])(?:$|[-_])`) + +// isStale reports whether an entry fails the freshness rule: its +// deprecation_date (YYYY-MM-DD) is before now, or a date embedded in +// bareID is more than maxModelAgeMonths before now. +func isStale(bareID, deprecationDate string, now time.Time) bool { + if deprecationDate != "" { + if d, err := time.Parse("2006-01-02", deprecationDate); err == nil && d.Before(now) { + return true + } + } + m := idDatePattern.FindStringSubmatch(bareID) + if m == nil { + return false + } + d, err := time.Parse("20060102", m[1]+m[2]+m[3]) + if err != nil { + return false + } + return d.Before(now.AddDate(0, -maxModelAgeMonths, 0)) +} + // allowedProviders is the pruning rule's provider allowlist — see the // package doc comment for the full rationale and the qwen/alibaba -> // dashscope mapping. @@ -218,6 +253,7 @@ func applyNamingBridges(table map[string]builtinEntry, winners map[string]candid // max_input_tokens/max_tokens as a JSON float (2000000.0) — decoding // those into *int would fail outright and drop the entry. type litellmEntry struct { + DeprecationDate string `json:"deprecation_date"` InputCostPerToken *float64 `json:"input_cost_per_token"` OutputCostPerToken *float64 `json:"output_cost_per_token"` MaxInputTokens *float64 `json:"max_input_tokens"` @@ -268,7 +304,7 @@ func run() error { return fmt.Errorf("fetch %s: %w", litellmSourceURL, err) } - table, totalUpstream, err := parseAndPrune(data) + table, totalUpstream, err := parseAndPrune(data, time.Now()) if err != nil { return fmt.Errorf("parse and prune: %w", err) } @@ -319,7 +355,7 @@ func fetchLiteLLMPrices(ctx context.Context, url string) ([]byte, error) { // chooseWinner's deterministic bare-id collision resolution. It returns // the pruned table and the total number of top-level keys data carried // (including ones this function skips), for the summary line run prints. -func parseAndPrune(data []byte) (map[string]builtinEntry, int, error) { +func parseAndPrune(data []byte, now time.Time) (map[string]builtinEntry, int, error) { var raw map[string]json.RawMessage if err := json.Unmarshal(data, &raw); err != nil { return nil, 0, fmt.Errorf("decode top-level object: %w", err) @@ -346,6 +382,9 @@ func parseAndPrune(data []byte) (map[string]builtinEntry, int, error) { } bare, isBareKey := bareModelID(key, e.LiteLLMProvider) + if isStale(bare, e.DeprecationDate, now) { + continue + } byBare[bare] = append(byBare[bare], candidate{ fullKey: key, provider: e.LiteLLMProvider, diff --git a/tools/pricing-sync/main_test.go b/tools/pricing-sync/main_test.go index adad3e9..5886f8a 100644 --- a/tools/pricing-sync/main_test.go +++ b/tools/pricing-sync/main_test.go @@ -6,8 +6,11 @@ import ( "reflect" "strings" "testing" + "time" ) +var fixtureNow = time.Date(2026, time.August, 1, 0, 0, 0, 0, time.UTC) + // fixtureJSON is a small, hand-built stand-in for LiteLLM's real // model_prices_and_context_window.json, covering every branch of the // pruning rule documented on this package (main.go's own doc comment): @@ -116,7 +119,7 @@ const fixtureUpstreamKeyCount = 17 // out, covering every pruning-rule branch fixtureJSON's own doc comment // lists. func TestParseAndPrune_Golden(t *testing.T) { - got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON)) + got, totalUpstream, err := parseAndPrune([]byte(fixtureJSON), fixtureNow) if err != nil { t.Fatalf("parseAndPrune: %v", err) } @@ -264,3 +267,49 @@ func TestContentDigest(t *testing.T) { t.Errorf("len(contentDigest(...)) = %d, want %d", len(a), contentDigestLen) } } + +func TestIsStale(t *testing.T) { + now := time.Date(2026, time.September, 29, 0, 0, 0, 0, time.UTC) + tests := []struct { + name, id, deprecated string + want bool + }{ + {"undated alias kept", "claude-opus-4-1", "", false}, + {"recent compact date kept", "claude-sonnet-5-20260401", "", false}, + {"old compact date dropped", "claude-opus-4-1-20250805", "", true}, + {"old dashed date dropped", "gpt-4o-2024-11-20", "", true}, + {"just inside cutoff kept", "model-2025-10-01", "", false}, + {"just outside cutoff dropped", "model-2025-09-28", "", true}, + {"date mid-id dropped", "gpt-4o-2024-11-20-preview", "", true}, + {"short MMDD suffix ignored", "grok-4.20-experimental-beta-0304", "", false}, + {"invalid month ignored", "model-20251301", "", false}, + {"past deprecation dropped", "gpt-4-turbo", "2025-06-01", true}, + {"future deprecation kept", "gpt-4-turbo", "2027-01-01", false}, + {"bad deprecation format ignored", "gpt-4-turbo", "soon", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := isStale(tc.id, tc.deprecated, now); got != tc.want { + t.Errorf("isStale(%q, %q) = %v, want %v", tc.id, tc.deprecated, got, tc.want) + } + }) + } +} + +func TestParseAndPrune_DropsStaleEntries(t *testing.T) { + const data = `{ + "gpt-fresh": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + "gpt-old-2024-05-13": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + "gpt-dead": {"litellm_provider": "openai", "mode": "chat", "deprecation_date": "2026-01-01", "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} +}` + got, _, err := parseAndPrune([]byte(data), fixtureNow) + if err != nil { + t.Fatalf("parseAndPrune: %v", err) + } + if len(got) != 1 { + t.Fatalf("kept %d entries %v, want only gpt-fresh", len(got), got) + } + if _, ok := got["gpt-fresh"]; !ok { + t.Errorf("gpt-fresh missing from %v", got) + } +}