package traefikllmgateway import ( "fmt" "math" "sort" "sync" ) // tokensPerMillion is the denominator for the providers' per-1M-token // pricing unit. const tokensPerMillion = 1_000_000 // builtinPricing is the built-in per-model price table, in USD per 1M // tokens (InputPerM, OutputPerM). Source: each provider's own published // list prices, as recorded 2026-08. Approximate: providers change prices // without notice, and this table is not refreshed automatically — set the // middleware config's "pricing" overrides for billing-grade accuracy. var builtinPricing = map[string]ModelPricing{ "gpt-5": {InputPerM: 1.25, OutputPerM: 10}, "gpt-5-mini": {InputPerM: 0.25, OutputPerM: 2}, "gpt-4o": {InputPerM: 2.5, OutputPerM: 10}, "o3": {InputPerM: 2, OutputPerM: 8}, "claude-opus-4-5": {InputPerM: 5, OutputPerM: 25}, "claude-sonnet-4-5": {InputPerM: 3, OutputPerM: 15}, "claude-haiku-4-5": {InputPerM: 1, OutputPerM: 5}, "gemini-2.5-pro": {InputPerM: 1.25, OutputPerM: 10}, "gemini-2.5-flash": {InputPerM: 0.3, OutputPerM: 2.5}, "grok-4": {InputPerM: 3, OutputPerM: 15}, } // unknownModelWarnCap bounds warnedModels. The model id in // costMicrosKnownFor's calls comes from client-supplied request bodies (a // group with an empty Models allowlist accepts any model string), so // without a cap an attacker or a buggy client cycling through model names // could grow warnedModels without bound. Once the cap is reached, // warnUnknownModelFor stops recording new names and fires its warn func // once more with a summary message instead of a per-model one. const unknownModelWarnCap = 128 // warnCapMessage is the single summary warning fired once the dedup set // in warnedModels reaches unknownModelWarnCap. const warnCapMessage = "pricing: unknown-model warning cap reached; further unknown models unwarned" // warnedModels tracks which unknown model ids have already triggered // a warn func, so a gateway processing many requests for the same // unpriced model logs the warning once per process lifetime rather than // once per request. warnCapNotified tracks whether the one-time summary // warning has already fired once the cap is reached. Package-level and // mutex-guarded by design: costMicrosKnownFor is called from the request path // with no natural place to thread a longer-lived cache through, and the // dedup set must survive across requests and across limiter instances, // not reset per call. var ( warnedModelsMu sync.Mutex warnedModels = map[string]bool{} warnCapNotified bool ) // costMicrosKnownFor computes the micro-USD cost of u's token usage for // model, and whether a price was actually FOUND. An unknown model returns // (0, false) — a zero that means "cannot be priced", which is NOT the // same value a genuinely free model produces, even though both are 0. // Looks up model's price via lookupPricing (overrides, then // builtinPricing, then builtinModelMetaTable); an unknown model triggers // one warn call per model id, up to unknownModelWarnCap distinct names. // // Security audit run-1, finding F-1 (high): an earlier single-int64- // returning costMicros conflated "unpriced" with "genuinely zero cost", // and limiter.account's own `if costMicros != 0` guard then skipped every // cost-counter write for an unpriced model. checkAndCount reads exactly // those never-written counters, so `used < limit` stayed permanently true // and a configured costPerDayUSD/costPerMonthUSD could never fire. Any // caller gating a SPEND control must handle ok == false. // // warn is the caller's own per-instance sink (a *Gateway's g.pricingWarn, // logger.go), never a package global: Traefik constructs one Gateway per // middleware instance, and a shared global made the most recently // constructed instance log every other instance's warnings (finding F11 / // review-auth F8, 2026-09 review). The unknown-model dedup (warnedModels) // stays process-global, so a model already warned about under one // instance is not warned about again under another. func costMicrosKnownFor(model string, u usage, overrides map[string]*ModelPricing, warn func(string)) (int64, bool) { price, ok := lookupPricing(model, overrides) if !ok { warnUnknownModelFor(model, warn) return 0, false } return computeCostMicros(price, u), true } // computeCostMicros is costMicrosKnownFor's arithmetic: price's two per-1M-token rates, each scaled to // micro-USD and rounded to the nearest micro-USD (a bare truncating cast // can undershoot a price like 8.2 by one micro-USD due to float64 // representation), multiplied by their own token count, SUMMED, and only // THEN divided back down and rounded to the nearest micro-USD (findings // F9/F10, 2026-09 review). // // F9: the previous version divided input and output separately // (tokens*priceMicrosPerM/tokensPerMillion, truncating each), which // silently billed 0 for any single request under ~50 tokens at a // realistic per-token price — up to ~2 micro-USD of truncation error per // request (one per direction), enough that account (limits.go) skipped // the cost-counter write entirely (`if costMicros != 0`) for genuinely // billable-but-cheap traffic. Rounding the COMBINED total once instead // halves the worst-case error and stops a nonzero combined cost from // truncating away to a written 0. // // F10: the previous unchecked int64 multiply overflowed for a token count // above roughly 1.2e11 at a $75/M price — entirely upstream-controlled, // since the token counts come from the provider's own reported usage // (limits.go's account, security audit run-1) — silently wrapping to a // negative or garbage value; a negative result was then clamped to 0 by // account, permanently masking the true cost. saturatingMulInt64/ // saturatingAddInt64 (below) clamp each multiply and the final sum to // [math.MinInt64, math.MaxInt64] instead of wrapping, so an absurd // upstream-reported token count bills the maximum representable cost // rather than a wrapped-around near-zero or negative one. func computeCostMicros(price ModelPricing, u usage) int64 { inputMicrosPerM := int64(math.Round(price.InputPerM * tokensPerMillion)) outputMicrosPerM := int64(math.Round(price.OutputPerM * tokensPerMillion)) inputCost := saturatingMulInt64(u.prompt, inputMicrosPerM) outputCost := saturatingMulInt64(u.completion, outputMicrosPerM) total := saturatingAddInt64(inputCost, outputCost) return roundedDivInt64(total, tokensPerMillion) } // roundedDivInt64 divides total by denom, rounded to the nearest integer // (half away from zero) rather than truncated toward zero — see // computeCostMicros' own F9 discussion. denom is always tokensPerMillion // (a positive compile-time constant) at this function's one call site. // // Deliberately computed via quotient/remainder rather than the more // obvious "(total + denom/2) / denom": total can legitimately arrive at // math.MaxInt64 (computeCostMicros' own saturatingAddInt64 clamps there // on overflow, finding F10), and adding denom/2 to it before dividing // would itself silently overflow right back — exactly the wrap-around // bug this whole function exists to avoid one step earlier. Working from // the remainder instead never adds to total at all. func roundedDivInt64(total, denom int64) int64 { q := total / denom r := total % denom if r < 0 { r = -r } if r*2 >= denom { if total >= 0 { q++ } else { q-- } } return q } // saturatingMulInt64 returns a*b, clamped to math.MinInt64/math.MaxInt64 // instead of wrapping on overflow (finding F10, 2026-09 review) — a // simple post-multiply check (divide back and compare), not math/bits: // this plugin runs interpreted under Yaegi (COMMON.md's own "stdlib only, // avoid reflection-heavy code" guidance), and a division-based overflow // check needs nothing beyond the four basic int64 operators. func saturatingMulInt64(a, b int64) int64 { if a == 0 || b == 0 { return 0 } p := a * b if p/b != a { if (a > 0) == (b > 0) { return math.MaxInt64 } return math.MinInt64 } return p } // saturatingAddInt64 returns a+b, clamped to math.MinInt64/math.MaxInt64 // instead of wrapping on overflow (finding F10, 2026-09 review) — the // standard "the sum moved the wrong way" overflow test for two's- // complement addition. func saturatingAddInt64(a, b int64) int64 { sum := a + b if b > 0 && sum < a { return math.MaxInt64 } if b < 0 && sum > a { return math.MinInt64 } return sum } // priceSourceOverride, priceSourceBuiltin, priceSourceLitellm, // priceSourceFree and priceSourceUnpriced enumerate every value // billingPriceSource (below) and GET /admin/api/catalog's own // adminCatalogModel.PriceSource field (admin_catalog.go) can report — // lookupPricingSource's own three lookup layers, plus modelMetaFree's // explicit "free", plus "no price found at all". A model whose id // merely carries the freeTierSuffix (":free", modelmeta.go) resolves to // priceSourceUnpriced here — that DISPLAY-only free-tier naming // convention never counts as a real billing source on its own; GET // /admin/api/catalog surfaces it through its own separate DisplayFree // field instead (DECISIONS Q3 — see billingPriceSource's own doc // comment below). const ( priceSourceOverride = "override" priceSourceBuiltin = "builtin" priceSourceLitellm = "litellm" priceSourceFree = "free" priceSourceUnpriced = "unpriced" ) // lookupPricingSource resolves model's price exactly like lookupPricing // (below, now a thin wrapper around this), additionally reporting WHICH // layer matched: priceSourceOverride, priceSourceBuiltin, or // priceSourceLitellm, in that same precedence order — an operator // override always wins, builtinPricing (curated, more likely to be // current) wins over the generated table next, and the generated table // is only ever the last resort (F2, 2026-09 review — see lookupPricing's // own doc comment, unchanged below, for the full per-layer reasoning). // ok is false, with source == "", when no layer prices model at all. // This function's own resolution is byte-for-byte identical to // lookupPricing's prior direct implementation; it only ADDS the source // label GET /admin/api/catalog needs (billingPriceSource, below). func lookupPricingSource(model string, overrides map[string]*ModelPricing) (price ModelPricing, source string, ok bool) { if p, ok := overrides[model]; ok && p != nil { return *p, priceSourceOverride, true } if p, ok := builtinPricing[model]; ok { return p, priceSourceBuiltin, true } if m, ok := builtinModelMetaTable[model]; ok && (m.InputCostPerMTokMicroUSD > 0 || m.OutputCostPerMTokMicroUSD > 0) { return ModelPricing{ InputPerM: microUSDPerMTokToUSD(m.InputCostPerMTokMicroUSD), OutputPerM: microUSDPerMTokToUSD(m.OutputCostPerMTokMicroUSD), }, priceSourceLitellm, true } return ModelPricing{}, "", false } // lookupPricing resolves model's price: overrides first (a nil entry does // not count as a match, so a caller can't accidentally price a model at // zero by leaving a map entry nil), then builtinPricing, then // builtinModelMetaTable (pricing_data_gen.go's 236-entry LiteLLM-synced // table — review finding F2, 2026-09 review): builtinPricing alone only // ever carried 10 flagship models, so an operator serving anything else // the generated table already knows a cost for (gpt-4.1, o4-mini, // deepseek-chat, and 200+ others) billed 0 for it even though GET // /v1/models — which reads the identical table via resolveModelMeta, // modelmeta.go — advertised a real price for the exact same model, // letting AllowUnpricedWithCostBudget's cost guard refuse the request as // "unpriced" in the same breath the catalog called it priced. Converted // from the table's integer micro-USD-per-million-tokens unit to // ModelPricing's own float64-USD-per-million-tokens unit via // microUSDPerMTokToUSD (modelmeta.go) — the identical conversion GET // /v1/models' own pricing view already applies to this same table. A // zero-valued cost field in the table means "no data" (builtinModelMeta's // own doc comment), not "free" — at least one side must be positive // before an entry is trusted, the same rule resolveModelMeta's own cost // switch already applies to it. Precedence is unchanged by this addition: // an operator override always wins, builtinPricing (curated, more likely // to be current) wins over the generated table next, and the generated // table is only ever the last resort. // // A thin wrapper around lookupPricingSource (above, GET /admin/api/catalog // feature) since the extension here needed both the price and which layer // produced it — this call site only ever wanted the price. func lookupPricing(model string, overrides map[string]*ModelPricing) (ModelPricing, bool) { price, _, ok := lookupPricingSource(model, overrides) return price, ok } // billingPriceSource reports the SAME price/source billing itself would // charge for a request serving canonical ("provider/model") / bare (the // upstream model id alone) — mirroring unifiedCostMicros' own resolution // order EXACTLY (routes_unified.go): modelMetaFree first (an operator // modelMeta:{free:true} entry always wins, priceSourceFree, zero price), // then lookupPricingSource against canonical, then against bare, then // priceSourceUnpriced (ModelPricing{}) when neither carries a price at // any layer. GET /admin/api/catalog's own "billing truth" PriceSource // field (admin_catalog.go) calls this directly so the catalog view and // actual billing can never disagree about which price a model resolves // to — the same guarantee modelMetaFree's own doc comment already // promises between unifiedCostMicros and priceKnown. // // A ":free"-suffixed model id (freeTierSuffix, modelmeta.go) is NOT // specially handled here: unifiedCostMicros itself never consults that // suffix either (it only affects resolveModelMeta's DISPLAY-path cost // resolution, GET /v1/models), so such an id resolves through the normal // override/builtin/litellm/unpriced layers exactly like any other id — // usually priceSourceUnpriced, since the suffix rarely matches a real // priced entry. A caller wanting to also surface that naming convention // (DECISIONS Q3: "':free' suffix models: report priceSource 'unpriced' + // displayFree; billing unchanged") checks strings.HasSuffix(bare, // freeTierSuffix) itself, alongside this result (admin_catalog.go) — that // flag is display-only and must never change what this function reports // as the billing source. func billingPriceSource(canonical, bare string, overrides map[string]*ModelPricing, meta map[string]*ModelMetaConfig) (source string, price ModelPricing) { if modelMetaFree(canonical, bare, meta) { return priceSourceFree, ModelPricing{} } if p, src, ok := lookupPricingSource(canonical, overrides); ok { return src, p } if p, src, ok := lookupPricingSource(bare, overrides); ok { return src, p } return priceSourceUnpriced, ModelPricing{} } // validatePricing validates every entry in raw (Config.Pricing overrides) // and returns it unchanged — a constructor error rather than a silently // unenforceable cost budget (finding F4, 2026-09 review): costMicrosKnownFor // had no validation of its own, so a negative, NaN, or Inf price still // passed priceKnown/modelPriceKnown's own "is this model priced" check // (modelmeta.go) yet produced a meaningless or negative cost — permanently // defeating any costPerDayUSD/costPerMonthUSD budget configured for that // model (limits.go's account already clamps a negative cost to 0 and logs // it on every request, but by then the budget has already never fired, // silently, since construction). Mirrors LimitsConfig.validate's own // "non-nil, finite, >= 0" rule (limits.go) applied to InputPerM/OutputPerM // instead of a count/cost limit. raw is walked in sorted key order, // matching validateModelMeta's own convention (modelmeta.go), so a config // with more than one invalid entry always reports the same one first, // deterministically across repeated runs. func validatePricing(raw map[string]*ModelPricing) (map[string]*ModelPricing, error) { if len(raw) == 0 { return raw, nil } keys := make([]string, 0, len(raw)) for k := range raw { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { p := raw[k] if p == nil { return nil, fmt.Errorf("llmgateway: pricing: entry %q must not be nil", k) } fields := []struct { name string v float64 }{ {"inputPerM", p.InputPerM}, {"outputPerM", p.OutputPerM}, } for _, f := range fields { if math.IsNaN(f.v) || math.IsInf(f.v, 0) { return nil, fmt.Errorf("llmgateway: pricing: entry %q: %s must be a finite number, got %v", k, f.name, f.v) } if f.v < 0 { return nil, fmt.Errorf("llmgateway: pricing: entry %q: %s must not be negative, got %v", k, f.name, f.v) } } } return raw, nil } // warnUnknownModelFor calls warn for model, once per process lifetime, via // the process-global dedup set (warnedModels) — deliberately not scoped // per-caller, so a model already warned about under one Gateway // instance's warn func is not warned about again under a different // instance's (finding F11 / review-auth F8's own "keep the process-global // dedup cap if needed" allowance; costMicrosKnownFor passes its caller's // own warn func). Once // warnedModels reaches unknownModelWarnCap, it stops recording new model // ids and instead fires a single one-time summary warning via // warnCapMessage — after that, further unknown models are silently // unwarned rather than growing the set forever. func warnUnknownModelFor(model string, warn func(string)) { warnedModelsMu.Lock() defer warnedModelsMu.Unlock() if warnedModels[model] { return } if len(warnedModels) >= unknownModelWarnCap { if !warnCapNotified { warnCapNotified = true warn(warnCapMessage) } return } warnedModels[model] = true warn(model) }