diff --git a/llmgateway.go b/llmgateway.go index 3425a5c..e284d5b 100644 --- a/llmgateway.go +++ b/llmgateway.go @@ -36,6 +36,20 @@ type Config struct { // model id's resolution changes. Validated at construction // (validateModelAliases, registry.go). ModelAliases map[string]string `json:"modelAliases,omitempty"` + // ModelMeta declares per-model metadata overrides (feature v0.23): + // context window size and per-token cost, keyed by an exact + // "provider/model" id or a bare model id (applying wherever that bare + // id resolves — modelmeta.go's lookupModelMetaConfig checks the exact + // form first). This ALWAYS wins over discovery-captured context or + // the built-in LiteLLM-synced table (resolveModelMeta, modelmeta.go) + // — distinct from Pricing above, which drives request cost + // ACCOUNTING (costMicros); ModelMeta drives metadata EXPOSURE + // (GET /v1/models' context_window/pricing extension fields, the + // admin dashboard) and never affects billing. nil/omitted preserves + // prior behavior exactly: every model's metadata falls through to + // discovery/builtin/absent, same as before this feature existed. + // Validated at construction (validateModelMeta, modelmeta.go). + ModelMeta map[string]*ModelMetaConfig `json:"modelMeta,omitempty"` // Retry is a struct value, not a pointer, because its own Enabled // field is the on/off signal (unlike Redis/Users, where the block's // mere presence is the signal) — so its tag omits "omitempty": @@ -51,12 +65,23 @@ type Config struct { // ProviderConfig describes one upstream LLM provider. type ProviderConfig struct { - Type string `json:"type"` - BaseURL string `json:"baseUrl,omitempty"` - APIKey string `json:"apiKey"` - DiscoveryInterval string `json:"discoveryInterval,omitempty"` - Models []string `json:"models,omitempty"` - Discovery bool `json:"discovery,omitempty"` + Type string `json:"type"` + BaseURL string `json:"baseUrl,omitempty"` + APIKey string `json:"apiKey"` + DiscoveryInterval string `json:"discoveryInterval,omitempty"` + // MetadataPath is an optional second discovery endpoint (feature + // v0.23), fetched alongside the provider's normal listModels call + // when set: a path such as "/api/v0/models" (LM Studio's own, + // non-OpenAI-compatible models endpoint) that reports per-model + // context length. Only openai-type adapters read this field + // (provider_openai.go's openaiAdapter.fetchModelMetadata) — an + // anthropic/gemini provider configuring it has no effect. Empty (the + // default) captures no metadata, preserving prior behavior exactly. + // A failed or absent fetch is always non-fatal to discovery itself: + // no metadata is captured, nothing else changes. + MetadataPath string `json:"metadataPath,omitempty"` + Models []string `json:"models,omitempty"` + Discovery bool `json:"discovery,omitempty"` } // GroupConfig describes a group's access and limits. Empty/omitted @@ -183,6 +208,25 @@ type ModelPricing struct { OutputPerM float64 `json:"outputPerM"` } +// ModelMetaConfig overrides one model's context window and per-token +// cost (feature v0.23, Config.ModelMeta) — layered ahead of discovery- +// captured context and the built-in LiteLLM-synced table +// (resolveModelMeta, modelmeta.go). ContextTokens left at 0 falls +// through to the next layer rather than overriding with an explicit +// unknown; the same holds for InputCostPerMTokMicroUSD/ +// OutputCostPerMTokMicroUSD when Free is false. Free, when true, +// explicitly zeroes both cost fields regardless of whatever they are +// set to — a KNOWN zero cost, distinct from the "not overridden here" +// fallback a zero cost field means when Free is false. Setting Free +// true together with a non-zero cost field is rejected at construction +// (validateModelMeta) as a contradictory config. +type ModelMetaConfig struct { + ContextTokens int `json:"contextTokens,omitempty"` + InputCostPerMTokMicroUSD int64 `json:"inputCostPerMTokMicroUsd,omitempty"` + OutputCostPerMTokMicroUSD int64 `json:"outputCostPerMTokMicroUsd,omitempty"` + Free bool `json:"free,omitempty"` +} + // TargetConfig is a proxied upstream target (MCP server). type TargetConfig struct { URL string `json:"url"` diff --git a/modelmeta.go b/modelmeta.go new file mode 100644 index 0000000..a7d5fcc --- /dev/null +++ b/modelmeta.go @@ -0,0 +1,176 @@ +package traefikllmgateway + +import ( + "fmt" + "sort" +) + +// microUSDPerUSD scales a micro-USD integer (ModelMetaConfig/ +// builtinModelMeta's storage unit — integer math avoids the float-drift +// pricing.go's costMicros already documents for the cost-ACCOUNTING path) +// to plain USD for exposure in a JSON API. Kept as its own named constant +// rather than reusing pricing.go's tokensPerMillion: the two are +// numerically identical (1_000_000) but represent different physical +// quantities — one a token count, this one a currency scale — and must +// not be conflated just because they share a value. +const microUSDPerUSD = 1_000_000 + +// resolvedModelMeta is resolveModelMeta's result: a model's context +// window and per-token cost, each resolved independently through its own +// layering (feature v0.23). ContextKnown/CostKnown are authoritative — a +// caller must check them, never infer "unknown" from a zero numeric +// field, since a free model's cost fields are legitimately zero AND +// known (ModelMetaConfig.Free). +type resolvedModelMeta struct { + ContextTokens int + InputCostPerMTokMicroUSD int64 + OutputCostPerMTokMicroUSD int64 + ContextKnown bool + CostKnown bool +} + +// resolveModelMeta layers per-model metadata resolution (feature v0.23): +// +// - Context: an operator's modelMeta config override +// (lookupModelMetaConfig: an exact "provider/model" key, then a bare +// "model" key) wins when it sets ContextTokens > 0; otherwise +// discoveredContext (only when discoveredKnown — a provider's own +// discovery-captured context, registry.go's providerState. +// discoveredContext, populated only for a provider whose adapter +// implements modelMetadataFetcher and returned a value for this +// model); otherwise the built-in table (builtinModelMetaTable, +// pricing_data_gen.go), keyed by bare model id only — an operator's +// own provider name never matches LiteLLM's canonical provider +// names, so an exact "provider/model" form makes no sense for that +// table the way it does for cfg; otherwise unknown. +// - Cost: the config override wins when it sets Free (an explicit, +// known zero) or either cost field > 0; otherwise the built-in +// table. Discovery never reports cost, so there is no discovery +// layer for it. +// +// Absent at every applicable layer is a genuinely unknown value and is +// never invented — ContextKnown/CostKnown both stay false in that case. +func resolveModelMeta(provider, model string, cfg map[string]*ModelMetaConfig, discoveredContext int, discoveredKnown bool) resolvedModelMeta { + var out resolvedModelMeta + override := lookupModelMetaConfig(provider, model, cfg) + builtin, builtinOK := builtinModelMetaTable[model] + + switch { + case override != nil && override.ContextTokens > 0: + out.ContextTokens, out.ContextKnown = override.ContextTokens, true + case discoveredKnown && discoveredContext > 0: + out.ContextTokens, out.ContextKnown = discoveredContext, true + case builtinOK && builtin.ContextTokens > 0: + out.ContextTokens, out.ContextKnown = builtin.ContextTokens, true + } + + switch { + case override != nil && override.Free: + out.CostKnown = true // both cost fields stay 0 — explicit, known zero + case override != nil && (override.InputCostPerMTokMicroUSD > 0 || override.OutputCostPerMTokMicroUSD > 0): + out.InputCostPerMTokMicroUSD = override.InputCostPerMTokMicroUSD + out.OutputCostPerMTokMicroUSD = override.OutputCostPerMTokMicroUSD + out.CostKnown = true + case builtinOK && (builtin.InputCostPerMTokMicroUSD > 0 || builtin.OutputCostPerMTokMicroUSD > 0): + out.InputCostPerMTokMicroUSD = builtin.InputCostPerMTokMicroUSD + out.OutputCostPerMTokMicroUSD = builtin.OutputCostPerMTokMicroUSD + out.CostKnown = true + } + return out +} + +// lookupModelMetaConfig resolves cfg's config-override layer for +// (provider, model): an exact "provider/model" key wins over a bare +// "model" key, matching this feature's own spec precedence (a key names +// either an exact provider/model pair or a bare id that applies wherever +// that id resolves). Returns nil (no override at this layer) when cfg is +// nil or neither key is present, or when the matched entry is itself a +// nil pointer (a malformed-but-decoded config value, treated as absent +// rather than dereferenced). +func lookupModelMetaConfig(provider, model string, cfg map[string]*ModelMetaConfig) *ModelMetaConfig { + if cfg == nil { + return nil + } + if v, ok := cfg[provider+"/"+model]; ok && v != nil { + return v + } + if v, ok := cfg[model]; ok && v != nil { + return v + } + return nil +} + +// resolveAliasModelMeta applies modelMeta's alias-inheritance rule +// (feature v0.23): an alias inherits its target's fully resolved +// metadata (resolveModelMeta against the alias's resolved provider/ +// bare-model target) UNLESS cfg carries a modelMeta entry keyed by the +// alias id itself — that entry's own fields win over the inherited ones, +// independently per field (an alias-level entry that sets only context +// still inherits the target's cost, and vice versa), the same +// override-wins-per-field shape resolveModelMeta itself applies between +// its own layers. +func resolveAliasModelMeta(alias, targetProvider, targetModel string, cfg map[string]*ModelMetaConfig, discoveredContext int, discoveredKnown bool) resolvedModelMeta { + out := resolveModelMeta(targetProvider, targetModel, cfg, discoveredContext, discoveredKnown) + if cfg == nil { + return out + } + override, ok := cfg[alias] + if !ok || override == nil { + return out + } + if override.ContextTokens > 0 { + out.ContextTokens, out.ContextKnown = override.ContextTokens, true + } + switch { + case override.Free: + out.InputCostPerMTokMicroUSD, out.OutputCostPerMTokMicroUSD, out.CostKnown = 0, 0, true + case override.InputCostPerMTokMicroUSD > 0 || override.OutputCostPerMTokMicroUSD > 0: + out.InputCostPerMTokMicroUSD = override.InputCostPerMTokMicroUSD + out.OutputCostPerMTokMicroUSD = override.OutputCostPerMTokMicroUSD + out.CostKnown = true + } + return out +} + +// microUSDPerMTokToUSD converts a micro-USD-per-million-tokens integer +// (ModelMetaConfig/builtinModelMeta's storage unit) to plain +// USD-per-million-tokens for exposure in a JSON API (GET /v1/models' +// "pricing" extension field: "floats in USD per MTok"). +func microUSDPerMTokToUSD(micro int64) float64 { + return float64(micro) / microUSDPerUSD +} + +// validateModelMeta validates cfg.ModelMeta (feature v0.23) and returns +// it unchanged (nil when raw is empty — the zero-behavior-change +// default, matching validateModelAliases' own convention, registry.go). +// raw is iterated in sorted key order, not Go's randomized map order, so +// a config with more than one invalid entry always reports the same one +// first, deterministically, across repeated runs. +func validateModelMeta(raw map[string]*ModelMetaConfig) (map[string]*ModelMetaConfig, error) { + if len(raw) == 0 { + return nil, nil + } + + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + v := raw[k] + if v == nil { + return nil, fmt.Errorf("llmgateway: modelMeta: entry %q must not be nil", k) + } + if v.ContextTokens < 0 { + return nil, fmt.Errorf("llmgateway: modelMeta: entry %q: contextTokens must not be negative", k) + } + if v.InputCostPerMTokMicroUSD < 0 || v.OutputCostPerMTokMicroUSD < 0 { + return nil, fmt.Errorf("llmgateway: modelMeta: entry %q: cost fields must not be negative", k) + } + if v.Free && (v.InputCostPerMTokMicroUSD != 0 || v.OutputCostPerMTokMicroUSD != 0) { + return nil, fmt.Errorf("llmgateway: modelMeta: entry %q: free=true is contradictory with a non-zero cost field", k) + } + } + return raw, nil +} diff --git a/modelmeta_test.go b/modelmeta_test.go new file mode 100644 index 0000000..5114b25 --- /dev/null +++ b/modelmeta_test.go @@ -0,0 +1,327 @@ +package traefikllmgateway + +import ( + "strings" + "testing" +) + +// TestResolveModelMeta_Layering is the feature's core table-driven test: +// config override > discovered > builtin > absent, checked independently +// for context and for cost (discovery never reports cost, so cost only +// has three layers). +func TestResolveModelMeta_Layering(t *testing.T) { + const provider = "openai" + const model = "test-model" + + cases := []struct { + cfg map[string]*ModelMetaConfig + name string + discoveredCtx int + wantCtx int + wantInput int64 + wantOutput int64 + discoveredKnown bool + wantCtxKnown bool + wantCostKnown bool + }{ + { + name: "everything absent", + wantCtxKnown: false, + }, + { + name: "discovered context only, no cost anywhere", + discoveredCtx: 4096, + discoveredKnown: true, + wantCtx: 4096, + wantCtxKnown: true, + }, + { + name: "config override wins over discovered", + cfg: map[string]*ModelMetaConfig{ + model: {ContextTokens: 8192}, + }, + discoveredCtx: 4096, + discoveredKnown: true, + wantCtx: 8192, + wantCtxKnown: true, + }, + { + name: "config exact provider/model key wins over bare key", + cfg: map[string]*ModelMetaConfig{ + model: {ContextTokens: 1000}, + provider + "/" + model: {ContextTokens: 2000}, + }, + wantCtx: 2000, + wantCtxKnown: true, + }, + { + name: "config override sets cost, discovery ignored for cost", + cfg: map[string]*ModelMetaConfig{ + model: {InputCostPerMTokMicroUSD: 100, OutputCostPerMTokMicroUSD: 200}, + }, + wantInput: 100, + wantOutput: 200, + wantCostKnown: true, + }, + { + name: "free config override zeroes cost explicitly, known", + cfg: map[string]*ModelMetaConfig{ + model: {Free: true}, + }, + wantInput: 0, + wantOutput: 0, + wantCostKnown: true, + }, + { + name: "builtin table used when nothing else known", + wantCtx: 0, // "test-model" is not in builtinModelMetaTable + wantCostKnown: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := resolveModelMeta(provider, model, c.cfg, c.discoveredCtx, c.discoveredKnown) + if got.ContextKnown != c.wantCtxKnown || got.ContextTokens != c.wantCtx { + t.Errorf("context = (%d, known=%v), want (%d, known=%v)", got.ContextTokens, got.ContextKnown, c.wantCtx, c.wantCtxKnown) + } + if got.CostKnown != c.wantCostKnown || got.InputCostPerMTokMicroUSD != c.wantInput || got.OutputCostPerMTokMicroUSD != c.wantOutput { + t.Errorf("cost = (%d, %d, known=%v), want (%d, %d, known=%v)", + got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown, + c.wantInput, c.wantOutput, c.wantCostKnown) + } + }) + } +} + +// TestResolveModelMeta_BuiltinTable exercises the real, generated +// builtinModelMetaTable directly (pricing_data_gen.go) for both context +// and cost, and confirms an operator's config override still wins over +// it. +func TestResolveModelMeta_BuiltinTable(t *testing.T) { + const bareID = "deepseek-chat" // a stable, real LiteLLM entry (deepseek's own bare key) + builtin, ok := builtinModelMetaTable[bareID] + if !ok || builtin.ContextTokens <= 0 || builtin.InputCostPerMTokMicroUSD <= 0 { + t.Fatalf("builtinModelMetaTable[%q] = %+v, ok=%v — test fixture assumption broke, pick another stable id", bareID, builtin, ok) + } + + got := resolveModelMeta("acme", bareID, nil, 0, false) + if !got.ContextKnown || got.ContextTokens != builtin.ContextTokens { + t.Errorf("context = (%d, known=%v), want (%d, known=true)", got.ContextTokens, got.ContextKnown, builtin.ContextTokens) + } + if !got.CostKnown || got.InputCostPerMTokMicroUSD != builtin.InputCostPerMTokMicroUSD || got.OutputCostPerMTokMicroUSD != builtin.OutputCostPerMTokMicroUSD { + t.Errorf("cost = (%d, %d, known=%v), want (%d, %d, known=true)", + got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown, + builtin.InputCostPerMTokMicroUSD, builtin.OutputCostPerMTokMicroUSD) + } + + overridden := resolveModelMeta("acme", bareID, map[string]*ModelMetaConfig{bareID: {ContextTokens: 1}}, 0, false) + if overridden.ContextTokens != 1 { + t.Errorf("config override over builtin: context = %d, want 1", overridden.ContextTokens) + } + // Cost is untouched by the context-only override — still falls + // through to the builtin table (independent per-field layering). + if overridden.InputCostPerMTokMicroUSD != builtin.InputCostPerMTokMicroUSD { + t.Errorf("cost should still fall through to builtin when override sets only context, got %d, want %d", overridden.InputCostPerMTokMicroUSD, builtin.InputCostPerMTokMicroUSD) + } +} + +// TestLookupModelMetaConfig covers lookupModelMetaConfig's own +// precedence and nil-safety directly. +func TestLookupModelMetaConfig(t *testing.T) { + t.Run("nil cfg", func(t *testing.T) { + if got := lookupModelMetaConfig("p", "m", nil); got != nil { + t.Errorf("got %+v, want nil", got) + } + }) + t.Run("nil entry treated as absent", func(t *testing.T) { + cfg := map[string]*ModelMetaConfig{"m": nil} + if got := lookupModelMetaConfig("p", "m", cfg); got != nil { + t.Errorf("got %+v, want nil (nil map value must not be dereferenced)", got) + } + }) + t.Run("bare key match", func(t *testing.T) { + want := &ModelMetaConfig{ContextTokens: 5} + cfg := map[string]*ModelMetaConfig{"m": want} + if got := lookupModelMetaConfig("p", "m", cfg); got != want { + t.Errorf("got %+v, want %+v", got, want) + } + }) + t.Run("exact provider/model beats bare", func(t *testing.T) { + bare := &ModelMetaConfig{ContextTokens: 1} + exact := &ModelMetaConfig{ContextTokens: 2} + cfg := map[string]*ModelMetaConfig{"m": bare, "p/m": exact} + if got := lookupModelMetaConfig("p", "m", cfg); got != exact { + t.Errorf("got %+v, want the exact-key entry %+v", got, exact) + } + }) +} + +// TestResolveAliasModelMeta_Inheritance asserts an alias inherits its +// target's fully resolved metadata, and that an alias-level modelMeta +// entry overrides the inherited values independently per field. +func TestResolveAliasModelMeta_Inheritance(t *testing.T) { + const alias = "aliased/coding" + const targetProvider = "anthropic" + const targetModel = "claude-x" + + t.Run("no alias entry: inherits target fully", func(t *testing.T) { + cfg := map[string]*ModelMetaConfig{ + targetProvider + "/" + targetModel: {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3_000_000, OutputCostPerMTokMicroUSD: 15_000_000}, + } + got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false) + if got.ContextTokens != 200000 || !got.ContextKnown { + t.Errorf("context = (%d, %v), want (200000, true)", got.ContextTokens, got.ContextKnown) + } + if got.InputCostPerMTokMicroUSD != 3_000_000 || got.OutputCostPerMTokMicroUSD != 15_000_000 || !got.CostKnown { + t.Errorf("cost = (%d, %d, %v), want (3000000, 15000000, true)", got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown) + } + }) + + t.Run("alias entry overrides context only, cost still inherited", func(t *testing.T) { + cfg := map[string]*ModelMetaConfig{ + targetProvider + "/" + targetModel: {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3_000_000, OutputCostPerMTokMicroUSD: 15_000_000}, + alias: {ContextTokens: 999}, + } + got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false) + if got.ContextTokens != 999 { + t.Errorf("context = %d, want 999 (alias override)", got.ContextTokens) + } + if got.InputCostPerMTokMicroUSD != 3_000_000 || !got.CostKnown { + t.Errorf("cost should still be inherited from target: got (%d, known=%v)", got.InputCostPerMTokMicroUSD, got.CostKnown) + } + }) + + t.Run("alias entry sets free, overrides inherited cost", func(t *testing.T) { + cfg := map[string]*ModelMetaConfig{ + targetProvider + "/" + targetModel: {InputCostPerMTokMicroUSD: 3_000_000, OutputCostPerMTokMicroUSD: 15_000_000}, + alias: {Free: true}, + } + got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false) + if !got.CostKnown || got.InputCostPerMTokMicroUSD != 0 || got.OutputCostPerMTokMicroUSD != 0 { + t.Errorf("cost = (%d, %d, known=%v), want (0, 0, true)", got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown) + } + }) + + t.Run("nil cfg never panics", func(t *testing.T) { + got := resolveAliasModelMeta(alias, targetProvider, targetModel, nil, 0, false) + if got.ContextKnown || got.CostKnown { + t.Errorf("got %+v, want everything unknown", got) + } + }) + + t.Run("nil alias entry value treated as absent, still inherits", func(t *testing.T) { + cfg := map[string]*ModelMetaConfig{ + targetProvider + "/" + targetModel: {ContextTokens: 111}, + alias: nil, + } + got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false) + if got.ContextTokens != 111 || !got.ContextKnown { + t.Errorf("got %+v, want inherited ContextTokens=111 (nil alias entry must not panic or override)", got) + } + }) + + t.Run("alias entry sets non-free cost directly, overrides inherited cost", func(t *testing.T) { + cfg := map[string]*ModelMetaConfig{ + targetProvider + "/" + targetModel: {InputCostPerMTokMicroUSD: 1, OutputCostPerMTokMicroUSD: 2}, + alias: {InputCostPerMTokMicroUSD: 100, OutputCostPerMTokMicroUSD: 200}, + } + got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false) + if !got.CostKnown || got.InputCostPerMTokMicroUSD != 100 || got.OutputCostPerMTokMicroUSD != 200 { + t.Errorf("got cost=(%d, %d, known=%v), want (100, 200, true) — alias's own non-free cost override", got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown) + } + }) +} + +// TestMicroUSDPerMTokToUSD pins the currency conversion. +func TestMicroUSDPerMTokToUSD(t *testing.T) { + cases := []struct { + micro int64 + want float64 + }{ + {0, 0}, + {1_250_000, 1.25}, + {10_000_000, 10}, + } + for _, c := range cases { + if got := microUSDPerMTokToUSD(c.micro); got != c.want { + t.Errorf("microUSDPerMTokToUSD(%d) = %v, want %v", c.micro, got, c.want) + } + } +} + +// TestValidateModelMeta covers construction-time validation: the +// zero-behavior-change nil/empty case, every rejection rule, and a +// deterministic first-error-wins ordering across multiple bad entries. +func TestValidateModelMeta(t *testing.T) { + t.Run("nil/empty preserves prior behavior", func(t *testing.T) { + got, err := validateModelMeta(nil) + if err != nil || got != nil { + t.Errorf("validateModelMeta(nil) = (%v, %v), want (nil, nil)", got, err) + } + got, err = validateModelMeta(map[string]*ModelMetaConfig{}) + if err != nil || got != nil { + t.Errorf("validateModelMeta({}) = (%v, %v), want (nil, nil)", got, err) + } + }) + + t.Run("valid entries pass through unchanged", func(t *testing.T) { + raw := map[string]*ModelMetaConfig{ + "m1": {ContextTokens: 100, InputCostPerMTokMicroUSD: 1, OutputCostPerMTokMicroUSD: 2}, + "m2": {Free: true}, + } + got, err := validateModelMeta(raw) + if err != nil { + t.Fatalf("validateModelMeta: %v", err) + } + if len(got) != 2 { + t.Errorf("got %d entries, want 2", len(got)) + } + }) + + t.Run("nil entry rejected", func(t *testing.T) { + _, err := validateModelMeta(map[string]*ModelMetaConfig{"m": nil}) + if err == nil { + t.Fatal("want error for nil entry") + } + }) + + t.Run("negative context rejected", func(t *testing.T) { + _, err := validateModelMeta(map[string]*ModelMetaConfig{"m": {ContextTokens: -1}}) + if err == nil { + t.Fatal("want error for negative contextTokens") + } + }) + + t.Run("negative cost rejected", func(t *testing.T) { + _, err := validateModelMeta(map[string]*ModelMetaConfig{"m": {InputCostPerMTokMicroUSD: -1}}) + if err == nil { + t.Fatal("want error for negative input cost") + } + _, err = validateModelMeta(map[string]*ModelMetaConfig{"m": {OutputCostPerMTokMicroUSD: -1}}) + if err == nil { + t.Fatal("want error for negative output cost") + } + }) + + t.Run("free with non-zero cost rejected as contradictory", func(t *testing.T) { + _, err := validateModelMeta(map[string]*ModelMetaConfig{"m": {Free: true, InputCostPerMTokMicroUSD: 1}}) + if err == nil { + t.Fatal("want error for free:true with a non-zero cost field") + } + }) + + t.Run("multiple invalid entries report the first in sorted key order", func(t *testing.T) { + raw := map[string]*ModelMetaConfig{ + "zzz": {ContextTokens: -1}, + "aaa": {ContextTokens: -1}, + } + _, err := validateModelMeta(raw) + if err == nil { + t.Fatal("want error") + } + if got := err.Error(); !strings.Contains(got, `"aaa"`) { + t.Errorf("error = %q, want it to name the sorted-first key %q", got, "aaa") + } + }) +} diff --git a/provider_openai.go b/provider_openai.go index a2d18c7..ffd6beb 100644 --- a/provider_openai.go +++ b/provider_openai.go @@ -23,6 +23,12 @@ type openaiAdapter struct { adapterName string baseURL string apiKey string + // metadataPath is ProviderConfig.MetadataPath (feature v0.23), set by + // buildAdapters (providers.go) after construction — empty (the + // default) makes fetchModelMetadata a no-op, so every existing + // newOpenAIAdapter call site (every test, and any caller that never + // sets this) keeps behaving exactly as before this feature existed. + metadataPath string } // newOpenAIAdapter returns an openaiAdapter for provider name, with base as @@ -401,3 +407,72 @@ func (a *openaiAdapter) listModels(ctx context.Context) ([]string, error) { } return ids, nil } + +// lmStudioModelsPayload is the wire shape of LM Studio's own, native +// models endpoint (ProviderConfig.MetadataPath — a real deployment sets +// it to "/api/v0/models"), verified live against a running LM Studio +// instance (2026-08): {"data":[{"id":...,"max_context_length":262144, +// "loaded_context_length":4096}, ...]}. Distinct from modelsPayload +// above, which is OpenAI's own /v1/models shape (LM Studio also serves +// that, unchanged, but it carries no per-model context metadata) — this +// is LM Studio's separate, additional endpoint this feature (v0.23) +// opts into reading. +type lmStudioModelsPayload struct { + Data []struct { + ID string `json:"id"` + MaxContextLength int `json:"max_context_length"` + LoadedContextLength int `json:"loaded_context_length"` + } `json:"data"` +} + +// fetchModelMetadata implements modelMetadataFetcher (feature v0.23, +// registry.go): when a.metadataPath is configured, fetches it and +// returns each reported model id's discovery-captured context length. +// It prefers loaded_context_length when the upstream reports one greater +// than zero — the context actually usable right now, which a runtime can +// configure smaller than the model's own maximum under VRAM-constrained +// settings — and falls back to max_context_length otherwise. a. +// metadataPath left empty (the default; every provider except an +// operator's opt-in) is a fast no-op: (nil, nil), never an error, so +// registry.go's captureModelMetadata can invoke this unconditionally on +// every openai-type adapter without a config check of its own. A +// non-2xx response or a decode failure returns an error — the caller +// treats it as non-fatal, per this feature's own "absent/failed = no +// metadata captured" ruling. +func (a *openaiAdapter) fetchModelMetadata(ctx context.Context) (map[string]int, error) { + if a.metadataPath == "" { + return nil, nil + } + + resp, err := upstreamJSON(ctx, a.client, http.MethodGet, a.baseURL+a.metadataPath, a.requestHeaders(false), nil, a.retry) + if err != nil { + return nil, err + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, newProviderHTTPError(resp) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes)) + if err != nil { + return nil, fmt.Errorf("%w: read metadata response body: %w", errUpstream, err) + } + + var payload lmStudioModelsPayload + if err := json.Unmarshal(body, &payload); err != nil { + return nil, fmt.Errorf("%w: decode metadata response: %w", errUpstream, err) + } + + out := make(map[string]int, len(payload.Data)) + for _, d := range payload.Data { + ctxLen := d.MaxContextLength + if d.LoadedContextLength > 0 { + ctxLen = d.LoadedContextLength + } + if ctxLen > 0 { + out[d.ID] = ctxLen + } + } + return out, nil +} diff --git a/provider_openai_test.go b/provider_openai_test.go index ed0b257..93a2978 100644 --- a/provider_openai_test.go +++ b/provider_openai_test.go @@ -528,6 +528,106 @@ func TestOpenAIAdapter_ListModels(t *testing.T) { } } +// TestOpenAIAdapter_FetchModelMetadata covers fetchModelMetadata (feature +// v0.23) against an httptest fixture shaped like LM Studio's real, live +// "/api/v0/models" response (task brief: llm/vlm models report +// max_context_length 262144, qwen2.5-0.5b 32768, embeddings 512-8192) — +// including the loaded_context_length-preferred-over-max_context_length +// rule, an entry with no loaded_context_length at all, and a +// metadataPath left unset (the default: a fast no-op, no HTTP call at +// all). +func TestOpenAIAdapter_FetchModelMetadata(t *testing.T) { + t.Run("metadataPath unset is a fast no-op", func(t *testing.T) { + called := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + a := newOpenAIAdapter("p1", srv.URL, "sk-test") + got, err := a.fetchModelMetadata(context.Background()) + if err != nil || got != nil { + t.Errorf("fetchModelMetadata = (%v, %v), want (nil, nil)", got, err) + } + if called { + t.Error("unset metadataPath must never issue an HTTP request") + } + }) + + t.Run("LM Studio shaped fixture: prefers loaded over max, falls back when absent", func(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[ + {"id":"qwen/qwen3-vl-30b","max_context_length":262144,"loaded_context_length":262144}, + {"id":"qwen2.5-0.5b","max_context_length":32768,"loaded_context_length":4096}, + {"id":"text-embedding-nomic-embed-text-v1.5","max_context_length":2048,"loaded_context_length":512}, + {"id":"no-loaded-field","max_context_length":8192}, + {"id":"zero-everything","max_context_length":0,"loaded_context_length":0} + ]}`)) + })) + defer srv.Close() + + a := newOpenAIAdapter("lmstudio", srv.URL, "") + a.metadataPath = "/api/v0/models" + got, err := a.fetchModelMetadata(context.Background()) + if err != nil { + t.Fatalf("fetchModelMetadata: %v", err) + } + if gotPath != "/api/v0/models" { + t.Errorf("request path = %q, want /api/v0/models", gotPath) + } + + want := map[string]int{ + "qwen/qwen3-vl-30b": 262144, + "qwen2.5-0.5b": 4096, // loaded_context_length preferred over max_context_length + "text-embedding-nomic-embed-text-v1.5": 512, + "no-loaded-field": 8192, // falls back to max_context_length + // "zero-everything" omitted entirely: both fields are 0. + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %#v, want %#v", got, want) + } + }) + + t.Run("non-2xx response is an error, not a captured empty map", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + a := newOpenAIAdapter("p1", srv.URL, "") + a.metadataPath = "/api/v0/models" + if _, err := a.fetchModelMetadata(context.Background()); err == nil { + t.Error("want an error for a non-2xx metadata response") + } + }) + + t.Run("malformed JSON body is a decode error", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{not json`)) + })) + defer srv.Close() + + a := newOpenAIAdapter("p1", srv.URL, "") + a.metadataPath = "/api/v0/models" + if _, err := a.fetchModelMetadata(context.Background()); err == nil { + t.Error("want a decode error for a malformed metadata response body") + } + }) + + t.Run("unreachable upstream is an error", func(t *testing.T) { + a := newOpenAIAdapter("p1", "http://127.0.0.1:1", "") // reserved, never listening + a.metadataPath = "/api/v0/models" + if _, err := a.fetchModelMetadata(context.Background()); err == nil { + t.Error("want an error when the metadata endpoint is unreachable") + } + }) +} + // TestOpenAIAdapter_Keyless proves a resolved-empty API key sends no // Authorization header at all, not one with an empty token — the // operator's own keyless gateway is a real deployment target. @@ -591,6 +691,23 @@ func TestBuildAdapters(t *testing.T) { } }) + t.Run("good: metadataPath is wired onto the openai adapter (feature v0.23)", func(t *testing.T) { + cfg := &Config{Providers: map[string]*ProviderConfig{ + "lmstudio": {Type: "openai", MetadataPath: "/api/v0/models"}, + }} + adapters, err := buildAdapters(cfg) + if err != nil { + t.Fatalf("buildAdapters: %v", err) + } + a, ok := adapters["lmstudio"].(*openaiAdapter) + if !ok { + t.Fatalf("adapters[%q] is not *openaiAdapter", "lmstudio") + } + if a.metadataPath != "/api/v0/models" { + t.Errorf("metadataPath = %q, want %q", a.metadataPath, "/api/v0/models") + } + }) + t.Run("good: explicit base URL trailing slash trimmed", func(t *testing.T) { cfg := &Config{Providers: map[string]*ProviderConfig{ "p1": {Type: "openai", BaseURL: "https://custom.example.com/", APIKey: "sk-test"}, diff --git a/providers.go b/providers.go index af7ee87..fbce066 100644 --- a/providers.go +++ b/providers.go @@ -417,6 +417,7 @@ func buildAdapters(cfg *Config) (map[string]providerAdapter, error) { case providerTypeOpenAI: a := newOpenAIAdapter(name, base, apiKey) a.retry = policy + a.metadataPath = pc.MetadataPath adapters[name] = a case providerTypeAnthropic: a, err := newAnthropicAdapter(name, base, apiKey) diff --git a/registry.go b/registry.go index 55f1c22..10acab3 100644 --- a/registry.go +++ b/registry.go @@ -46,6 +46,17 @@ type providerState struct { lastRefresh time.Time explicit map[string]bool discovered map[string]bool + // discoveredContext holds discovery-captured per-model context + // lengths (feature v0.23, ProviderConfig.MetadataPath — currently + // only openai-type adapters ever populate this, via + // modelMetadataFetcher), keyed by upstream model id. nil for a + // provider with no metadataPath configured, or whose adapter does + // not implement modelMetadataFetcher at all — contextFor treats a + // nil map the same as an absent entry: (0, false). Stale-while- + // error, the same convention discovered itself already uses: a + // failed capture leaves the previous map in place rather than + // clearing it (captureModelMetadata, below). + discoveredContext map[string]int // lastErr is the most recent finishRefresh call's error message, or // "" when that call succeeded (or discovery is disabled and // finishRefresh was never called at all). Read by snapshot for the @@ -99,6 +110,26 @@ func (st *providerState) knownIDs() []string { return st.knownIDsLocked() } +// contextFor returns id's discovery-captured context length and whether +// one is known (feature v0.23) — mirroring hasModel's own locked-read +// shape. +func (st *providerState) contextFor(id string) (int, bool) { + st.mu.Lock() + defer st.mu.Unlock() + n, ok := st.discoveredContext[id] + return n, ok +} + +// setDiscoveredContext replaces st's discoveredContext wholesale on a +// successful metadata capture (feature v0.23, captureModelMetadata below) +// — the same replace-on-success, stale-while-error shape finishRefresh +// already applies to discovered. +func (st *providerState) setDiscoveredContext(m map[string]int) { + st.mu.Lock() + defer st.mu.Unlock() + st.discoveredContext = m +} + // tryBeginRefresh reports whether now is far enough past lastRefresh (or // this is the first refresh) to start a new one, and if so marks the // provider inFlight so a concurrent caller cannot start a second one. The @@ -192,7 +223,16 @@ type modelRegistry struct { // configured no aliases at all, so every alias-aware code path is a // cheap nil-map lookup (always false/zero) that adds no branching // cost to the v0.1 no-aliases case. - aliases map[string]string + aliases map[string]string + // modelMeta is the validated Config.ModelMeta config-override map + // (feature v0.23), built once by newModelRegistry via + // validateModelMeta and never mutated afterward. nil (not merely + // empty) when the operator configured no metadata overrides at all, + // so resolveModelMeta's config-override layer is a cheap nil-map + // lookup (always a miss) that adds no branching cost to the + // no-overrides case — the same nil-vs-empty convention aliases above + // already establishes. + modelMeta map[string]*ModelMetaConfig nowFn func() time.Time warned map[string]bool providerNames []string @@ -260,6 +300,12 @@ func newModelRegistry(adapters map[string]providerAdapter, cfg *Config, log func } m.aliases = aliases + modelMeta, err := validateModelMeta(cfg.ModelMeta) + if err != nil { + return nil, err + } + m.modelMeta = modelMeta + return m, nil } @@ -355,14 +401,69 @@ func (m *modelRegistry) warmFill(ctx context.Context) { } fctx, cancel := context.WithTimeout(ctx, warmFillTimeout) ids, err := m.adapters[name].listModels(fctx) - cancel() st.finishRefresh(m.now(), ids, err) if err != nil { m.log("%s", fmt.Sprintf("model registry: initial discovery for provider %q failed: %v", name, err)) } + m.captureModelMetadata(fctx, name, m.adapters[name], st) + cancel() } } +// modelMetadataFetcher is an OPTIONAL providerAdapter capability (feature +// v0.23): an adapter that can also fetch per-model discovery metadata +// (currently: context window) beyond the plain model-id list listModels +// returns. Only openai-type adapters configured with ProviderConfig. +// MetadataPath implement it (provider_openai.go's openaiAdapter) — +// anthropic/gemini adapters do not, so captureModelMetadata checks for +// this via a comma-ok type assertion rather than every providerAdapter +// implementation growing a no-op method for it. +// +// Safe under Yaegi despite the documented Trap-4 restriction +// (matchesSentinel's own doc comment, limits.go): that trap is a comma-ok +// assertion of a COMPILED concrete value (a stdlib type, e.g. +// *fmt.wrapErrors or *url.Error) against an INTERPRETER-declared +// interface, which silently returns ok=false. Every adapter's dynamic +// type behind the providerAdapter interface here (*openaiAdapter, +// *anthropicAdapter, *geminiAdapter) is itself plugin-declared and +// interpreted right alongside modelMetadataFetcher — an +// interpreted-concrete-value-against-interpreted-interface assertion, +// the opposite, unaffected direction. +type modelMetadataFetcher interface { + // fetchModelMetadata returns discovery-captured per-model context + // lengths, keyed by upstream model id. A nil map with a nil error + // means the adapter has nothing to report (e.g. metadataPath left + // unconfigured) — a fast, cheap no-op the caller can invoke + // unconditionally on every adapter that implements this interface. + fetchModelMetadata(ctx context.Context) (map[string]int, error) +} + +// captureModelMetadata is warmFill/refreshProvider's shared, best-effort +// discovery-metadata step (feature v0.23): when adapter also implements +// modelMetadataFetcher (see that interface's own doc comment for why +// this assertion is yaegi-safe), it fetches per-model context lengths +// and records them on st. A fetch error, or an adapter that does not +// implement the interface at all, is intentionally non-fatal beyond one +// log line — per this feature's own "absent/failed = no metadata +// captured" ruling, it must never fail discovery itself, which +// listModels (warmFill/refreshProvider's own caller) already accounts +// for independently. +func (m *modelRegistry) captureModelMetadata(ctx context.Context, name string, adapter providerAdapter, st *providerState) { + mf, ok := adapter.(modelMetadataFetcher) + if !ok { + return + } + meta, err := mf.fetchModelMetadata(ctx) + if err != nil { + m.log("%s", fmt.Sprintf("model registry: metadata capture for provider %q failed: %v", name, err)) + return + } + if meta == nil { + return + } + st.setDiscoveredContext(meta) +} + // maybeRefresh is called at every ServeHTTP entry. For each discovery- // enabled provider whose last refresh is older than its interval (and that // is not already mid-refresh), it spawns exactly one background goroutine @@ -424,6 +525,7 @@ func (m *modelRegistry) refreshProvider(name string, st *providerState, adapter fctx, cancel := context.WithTimeout(context.Background(), backgroundRefreshTimeout) defer cancel() ids, err = adapter.listModels(fctx) + m.captureModelMetadata(fctx, name, adapter, st) } // splitConfiguredProvider reports whether id has "prefix/rest" form where @@ -697,7 +799,7 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any { for id, provs := range owners { winner := provs[0] if grp.allowsModel(id) && grp.allowsProvider(winner) { - out = append(out, modelObject(id, winner)) + out = append(out, modelObject(id, winner, m.resolveMetaFor(winner, id))) listed[id] = true } if len(provs) < 2 { @@ -710,7 +812,7 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any { // prefixed form and its bare suffix, same as resolveAgainst does // for the equivalent client request. if (grp.allowsModel(pid) || grp.allowsModel(id)) && grp.allowsProvider(p) { - out = append(out, modelObject(pid, p)) + out = append(out, modelObject(pid, p, m.resolveMetaFor(p, id))) listed[pid] = true } } @@ -746,8 +848,8 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any { if err != nil { continue } - providerName, _, _ := strings.Cut(canonical, "/") - out = append(out, modelObject(alias, providerName)) + providerName, bareTarget, _ := strings.Cut(canonical, "/") + out = append(out, modelObject(alias, providerName, m.resolveMetaForAlias(alias, providerName, bareTarget))) } sort.Slice(out, func(i, j int) bool { @@ -756,9 +858,73 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any { return out } -// modelObject builds one OpenAI-compatible model list entry. -func modelObject(id, ownedBy string) map[string]any { - return map[string]any{"id": id, "object": "model", "owned_by": ownedBy} +// modelObject builds one OpenAI-compatible model list entry, extended +// with two OpenAI-compat-safe extension fields (feature v0.23): +// "context_window" (int) and "pricing" ({"input_per_mtok_usd", +// "output_per_mtok_usd"} floats, USD per million tokens) — both omitted +// entirely when meta reports them unknown, never emitted as a +// misleading zero. "pricing" IS emitted with zeros for an explicitly +// free model (meta.CostKnown true, both cost fields 0) — that is a +// known, meaningful zero, not an absent one. +func modelObject(id, ownedBy string, meta resolvedModelMeta) map[string]any { + obj := map[string]any{"id": id, "object": "model", "owned_by": ownedBy} + if meta.ContextKnown { + obj["context_window"] = meta.ContextTokens + } + if meta.CostKnown { + obj["pricing"] = map[string]any{ + "input_per_mtok_usd": microUSDPerMTokToUSD(meta.InputCostPerMTokMicroUSD), + "output_per_mtok_usd": microUSDPerMTokToUSD(meta.OutputCostPerMTokMicroUSD), + } + } + return obj +} + +// resolveMetaFor resolves (provider, bareModel)'s metadata (feature +// v0.23), supplying this registry's config-override map and that +// provider's own discovery-captured context for bareModel. +func (m *modelRegistry) resolveMetaFor(provider, bareModel string) resolvedModelMeta { + var ctxVal int + var ctxKnown bool + if st, ok := m.states[provider]; ok { + ctxVal, ctxKnown = st.contextFor(bareModel) + } + return resolveModelMeta(provider, bareModel, m.modelMeta, ctxVal, ctxKnown) +} + +// resolveMetaForAliasName resolves alias's metadata for the admin +// dashboard's unrestricted view (feature v0.23, hover-detail +// refinement): unlike listFor's request-authorized alias handling, this +// ignores group authorization entirely (an internal, unrestricted +// &group{} — GroupConfig's own documented "empty means all" semantics, +// llmgateway.go), matching buildAdminOverview's existing behavior of +// showing every configured provider/alias unconditionally, with no +// group filtering anywhere else in its response either. Returns an +// entirely-unknown resolvedModelMeta when alias is not a configured +// alias at all, or its target does not resolve to any known model yet. +func (m *modelRegistry) resolveMetaForAliasName(alias string) resolvedModelMeta { + target, ok := m.aliases[alias] + if !ok { + return resolvedModelMeta{} + } + _, _, canonical, err := m.resolveAliasTarget(alias, target, &group{}) + if err != nil { + return resolvedModelMeta{} + } + providerName, bareTarget, _ := strings.Cut(canonical, "/") + return m.resolveMetaForAlias(alias, providerName, bareTarget) +} + +// resolveMetaForAlias resolves alias's metadata (feature v0.23) via +// resolveAliasModelMeta: target-inherited unless the alias itself has +// its own modelMeta entry — see that function's own doc comment. +func (m *modelRegistry) resolveMetaForAlias(alias, targetProvider, targetModel string) resolvedModelMeta { + var ctxVal int + var ctxKnown bool + if st, ok := m.states[targetProvider]; ok { + ctxVal, ctxKnown = st.contextFor(targetModel) + } + return resolveAliasModelMeta(alias, targetProvider, targetModel, m.modelMeta, ctxVal, ctxKnown) } // providerSnapshot is one provider's read-only view for the admin diff --git a/registry_modelmeta_test.go b/registry_modelmeta_test.go new file mode 100644 index 0000000..49aa248 --- /dev/null +++ b/registry_modelmeta_test.go @@ -0,0 +1,290 @@ +package traefikllmgateway + +import ( + "context" + "testing" + "time" +) + +// fakeMetaAdapter wraps fakeAdapter, additionally implementing +// modelMetadataFetcher (feature v0.23) — a test double for an +// openai-type adapter configured with metadataPath, exercising +// warmFill/refreshProvider's captureModelMetadata wiring without a real +// HTTP fetch. Every other fakeAdapter-consuming test is unaffected: only +// tests that explicitly construct a fakeMetaAdapter get this extra +// capability. +type fakeMetaAdapter struct { + *fakeAdapter + fetchModelMetadataFn func(context.Context) (map[string]int, error) +} + +func newFakeMetaAdapter(name string) *fakeMetaAdapter { + return &fakeMetaAdapter{fakeAdapter: newFakeAdapter(name)} +} + +func (f *fakeMetaAdapter) fetchModelMetadata(ctx context.Context) (map[string]int, error) { + if f.fetchModelMetadataFn == nil { + return nil, nil + } + return f.fetchModelMetadataFn(ctx) +} + +// TestModelRegistry_WarmFill_CapturesModelMetadata proves warmFill's +// captureModelMetadata wiring: a discovery-enabled provider whose +// adapter implements modelMetadataFetcher has its returned per-model +// context lengths recorded on providerState, reachable through +// resolveMetaFor. +func TestModelRegistry_WarmFill_CapturesModelMetadata(t *testing.T) { + t.Parallel() + fa := newFakeMetaAdapter("lmstudio") + fa.listModelsFn = func(context.Context) ([]string, error) { + return []string{"qwen2.5-0.5b"}, nil + } + fa.fetchModelMetadataFn = func(context.Context) (map[string]int, error) { + return map[string]int{"qwen2.5-0.5b": 32768}, nil + } + adapters := map[string]providerAdapter{"lmstudio": fa} + cfg := &Config{Providers: map[string]*ProviderConfig{"lmstudio": {Discovery: true, MetadataPath: "/api/v0/models"}}} + reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) + if err != nil { + t.Fatalf("newModelRegistry: %v", err) + } + + reg.warmFill(context.Background()) + + ctxVal, ok := reg.states["lmstudio"].contextFor("qwen2.5-0.5b") + if !ok || ctxVal != 32768 { + t.Errorf("contextFor(qwen2.5-0.5b) = (%d, %v), want (32768, true)", ctxVal, ok) + } + + meta := reg.resolveMetaFor("lmstudio", "qwen2.5-0.5b") + if !meta.ContextKnown || meta.ContextTokens != 32768 { + t.Errorf("resolveMetaFor = %+v, want ContextKnown=true ContextTokens=32768", meta) + } +} + +// TestModelRegistry_WarmFill_MetadataFetchErrorNonFatal proves a failed +// metadata capture is non-fatal to discovery itself (feature v0.23's own +// "absent/failed = no metadata captured" ruling): listModels still +// succeeds and populates the provider's known model set, one log line +// records the metadata failure, and contextFor simply reports unknown. +func TestModelRegistry_WarmFill_MetadataFetchErrorNonFatal(t *testing.T) { + t.Parallel() + fa := newFakeMetaAdapter("lmstudio") + fa.listModelsFn = func(context.Context) ([]string, error) { + return []string{"m1"}, nil + } + fa.fetchModelMetadataFn = func(context.Context) (map[string]int, error) { + return nil, errUpstream + } + adapters := map[string]providerAdapter{"lmstudio": fa} + cfg := &Config{Providers: map[string]*ProviderConfig{"lmstudio": {Discovery: true, MetadataPath: "/api/v0/models"}}} + log := &recordingLog{} + reg, err := newModelRegistry(adapters, cfg, log.fn) + if err != nil { + t.Fatalf("newModelRegistry: %v", err) + } + + reg.warmFill(context.Background()) + + if !reg.states["lmstudio"].hasModel("m1") { + t.Error("discovery's own model set must still populate despite a metadata capture failure") + } + if _, ok := reg.states["lmstudio"].contextFor("m1"); ok { + t.Error("contextFor must report unknown after a failed metadata capture") + } + if log.count() == 0 { + t.Error("want at least one log line recording the metadata capture failure") + } +} + +// TestModelRegistry_MaybeRefresh_CapturesModelMetadata proves the SAME +// wiring runs on the background refresh path (refreshProvider), not just +// the synchronous warmFill — captureModelMetadata is shared between both. +func TestModelRegistry_MaybeRefresh_CapturesModelMetadata(t *testing.T) { + t.Parallel() + fa := newFakeMetaAdapter("lmstudio") + fa.listModelsFn = func(context.Context) ([]string, error) { + return []string{"m1"}, nil + } + fa.fetchModelMetadataFn = func(context.Context) (map[string]int, error) { + return map[string]int{"m1": 8192}, nil + } + adapters := map[string]providerAdapter{"lmstudio": fa} + cfg := &Config{Providers: map[string]*ProviderConfig{"lmstudio": {Discovery: true, DiscoveryInterval: "1h", MetadataPath: "/api/v0/models"}}} + reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) + if err != nil { + t.Fatalf("newModelRegistry: %v", err) + } + + reg.maybeRefresh(context.Background()) + waitUntil(t, time.Second, func() bool { + _, ok := reg.states["lmstudio"].contextFor("m1") + return ok + }) + ctxVal, _ := reg.states["lmstudio"].contextFor("m1") + if ctxVal != 8192 { + t.Errorf("contextFor(m1) = %d, want 8192", ctxVal) + } +} + +// TestModelRegistry_ListFor_ExposesContextAndPricingFields proves +// listFor's modelObject output carries "context_window" and "pricing" +// when resolveMetaFor knows them (via a modelMeta config override here), +// and omits both entirely for a model nothing knows anything about. +func TestModelRegistry_ListFor_ExposesContextAndPricingFields(t *testing.T) { + t.Parallel() + adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")} + cfg := &Config{ + Providers: map[string]*ProviderConfig{"openai": {Models: []string{"gpt-test", "totally-unknown"}}}, + ModelMeta: map[string]*ModelMetaConfig{ + "gpt-test": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1_250_000, OutputCostPerMTokMicroUSD: 10_000_000}, + }, + } + reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) + if err != nil { + t.Fatalf("newModelRegistry: %v", err) + } + + got := reg.listFor(allowAllGroup()) + + var known, unknown map[string]any + for _, entry := range got { + switch entry["id"] { + case "gpt-test": + known = entry + case "totally-unknown": + unknown = entry + } + } + if known == nil || unknown == nil { + t.Fatalf("listFor missing expected entries: %v", got) + } + + if got := known["context_window"]; got != 128000 { + t.Errorf(`known["context_window"] = %v, want 128000`, got) + } + pricing, ok := known["pricing"].(map[string]any) + if !ok { + t.Fatalf(`known["pricing"] = %v, want a map`, known["pricing"]) + } + if pricing["input_per_mtok_usd"] != 1.25 || pricing["output_per_mtok_usd"] != 10.0 { + t.Errorf("pricing = %v, want {input_per_mtok_usd:1.25 output_per_mtok_usd:10}", pricing) + } + + if _, present := unknown["context_window"]; present { + t.Errorf(`unknown["context_window"] present (%v), want omitted entirely`, unknown["context_window"]) + } + if _, present := unknown["pricing"]; present { + t.Errorf(`unknown["pricing"] present (%v), want omitted entirely`, unknown["pricing"]) + } +} + +// TestModelRegistry_ListFor_Alias_InheritsTargetMetadata proves an +// alias's listFor entry carries its target's resolved metadata +// (modelObject via resolveMetaForAlias), and that an alias-level +// modelMeta entry overrides it. +func TestModelRegistry_ListFor_Alias_InheritsTargetMetadata(t *testing.T) { + t.Parallel() + adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")} + + t.Run("inherits target metadata with no alias override", func(t *testing.T) { + cfg := &Config{ + Providers: map[string]*ProviderConfig{"openai": {Models: []string{"gpt-test"}}}, + ModelAliases: map[string]string{"aliased/coding": "openai/gpt-test"}, + ModelMeta: map[string]*ModelMetaConfig{ + "openai/gpt-test": {ContextTokens: 128000, Free: true}, + }, + } + reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) + if err != nil { + t.Fatalf("newModelRegistry: %v", err) + } + got := reg.listFor(allowAllGroup()) + var alias map[string]any + for _, entry := range got { + if entry["id"] == "aliased/coding" { + alias = entry + } + } + if alias == nil { + t.Fatalf("listFor missing alias entry: %v", got) + } + if alias["context_window"] != 128000 { + t.Errorf(`alias["context_window"] = %v, want 128000 (inherited)`, alias["context_window"]) + } + pricing, ok := alias["pricing"].(map[string]any) + if !ok || pricing["input_per_mtok_usd"] != 0.0 { + t.Errorf(`alias["pricing"] = %v, want a free (zero) pricing object (inherited)`, alias["pricing"]) + } + }) + + t.Run("alias-level override wins over inherited context", func(t *testing.T) { + cfg := &Config{ + Providers: map[string]*ProviderConfig{"openai": {Models: []string{"gpt-test"}}}, + ModelAliases: map[string]string{"aliased/coding": "openai/gpt-test"}, + ModelMeta: map[string]*ModelMetaConfig{ + "openai/gpt-test": {ContextTokens: 128000}, + "aliased/coding": {ContextTokens: 999}, + }, + } + reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) + if err != nil { + t.Fatalf("newModelRegistry: %v", err) + } + got := reg.listFor(allowAllGroup()) + var alias map[string]any + for _, entry := range got { + if entry["id"] == "aliased/coding" { + alias = entry + } + } + if alias == nil { + t.Fatalf("listFor missing alias entry: %v", got) + } + if alias["context_window"] != 999 { + t.Errorf(`alias["context_window"] = %v, want 999 (alias override)`, alias["context_window"]) + } + }) +} + +// TestModelRegistry_ResolveMetaForAliasName covers the admin dashboard's +// unrestricted alias-metadata resolution directly: an unconfigured +// alias, one whose target has not resolved yet, and a real, resolvable +// one. +func TestModelRegistry_ResolveMetaForAliasName(t *testing.T) { + t.Parallel() + + t.Run("unconfigured alias reports everything unknown", func(t *testing.T) { + reg := newAliasTestRegistry(t, map[string]string{"aliased/coding": "openai/gpt-test"}, nil) + got := reg.resolveMetaForAliasName("not-an-alias") + if got.ContextKnown || got.CostKnown { + t.Errorf("got %+v, want everything unknown", got) + } + }) + + t.Run("unresolved target reports everything unknown", func(t *testing.T) { + reg := newAliasTestRegistry(t, map[string]string{"aliased/future": "openai/not-yet-discovered"}, nil) + got := reg.resolveMetaForAliasName("aliased/future") + if got.ContextKnown || got.CostKnown { + t.Errorf("got %+v, want everything unknown", got) + } + }) + + t.Run("resolvable alias ignores group authorization entirely", func(t *testing.T) { + adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")} + cfg := &Config{ + Providers: map[string]*ProviderConfig{"openai": {Models: []string{"gpt-test"}}}, + ModelAliases: map[string]string{"aliased/coding": "openai/gpt-test"}, + ModelMeta: map[string]*ModelMetaConfig{"openai/gpt-test": {ContextTokens: 55555}}, + } + reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) + if err != nil { + t.Fatalf("newModelRegistry: %v", err) + } + got := reg.resolveMetaForAliasName("aliased/coding") + if !got.ContextKnown || got.ContextTokens != 55555 { + t.Errorf("got %+v, want ContextKnown=true ContextTokens=55555", got) + } + }) +} diff --git a/routes_unified_modelmeta_test.go b/routes_unified_modelmeta_test.go new file mode 100644 index 0000000..da96d20 --- /dev/null +++ b/routes_unified_modelmeta_test.go @@ -0,0 +1,92 @@ +package traefikllmgateway + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +// TestHandleModels_ContextWindowAndPricing_FieldPresenceOmission is the +// HTTP-level counterpart to registry_modelmeta_test.go's listFor +// coverage: a real GET /v1/models request through the full Gateway, +// proving the OpenAI-compat-safe extension fields (feature v0.23: +// "context_window", "pricing") are present with the expected values for +// a model with known metadata, present-with-zeros for an explicitly free +// one, and entirely absent (not present as null or 0) for a model +// nothing knows anything about. +func TestHandleModels_ContextWindowAndPricing_FieldPresenceOmission(t *testing.T) { + t.Parallel() + cfg := CreateConfig() + cfg.Providers = map[string]*ProviderConfig{ + "openai": {Type: "openai", APIKey: "sk-test", Models: []string{"gpt-test", "free-model", "unknown-model"}}, + } + cfg.Groups = map[string]*GroupConfig{"default": {}} + cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "tester", Group: "default", APIKey: "sk-user"}}} + cfg.ModelMeta = map[string]*ModelMetaConfig{ + "gpt-test": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1_250_000, OutputCostPerMTokMicroUSD: 10_000_000}, + "free-model": {ContextTokens: 32768, Free: true}, + } + + h, err := New(context.Background(), http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), cfg, "llmgw") + if err != nil { + t.Fatalf("New: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + req.Header.Set("Authorization", "Bearer sk-user") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + + var body struct { + Data []map[string]any `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + + byID := make(map[string]map[string]any, len(body.Data)) + for _, entry := range body.Data { + id, _ := entry["id"].(string) + byID[id] = entry + } + + priced, ok := byID["gpt-test"] + if !ok { + t.Fatalf("response missing gpt-test: %v", body.Data) + } + if priced["context_window"] != 128000.0 { + t.Errorf(`gpt-test["context_window"] = %v, want 128000`, priced["context_window"]) + } + pricing, ok := priced["pricing"].(map[string]any) + if !ok || pricing["input_per_mtok_usd"] != 1.25 || pricing["output_per_mtok_usd"] != 10.0 { + t.Errorf(`gpt-test["pricing"] = %v, want {input_per_mtok_usd:1.25 output_per_mtok_usd:10}`, priced["pricing"]) + } + + free, ok := byID["free-model"] + if !ok { + t.Fatalf("response missing free-model: %v", body.Data) + } + freePricing, ok := free["pricing"].(map[string]any) + if !ok { + t.Fatalf(`free-model["pricing"] missing, want present with zeros (explicit free, not absent)`) + } + if freePricing["input_per_mtok_usd"] != 0.0 || freePricing["output_per_mtok_usd"] != 0.0 { + t.Errorf(`free-model["pricing"] = %v, want zeros`, freePricing) + } + + unknown, ok := byID["unknown-model"] + if !ok { + t.Fatalf("response missing unknown-model: %v", body.Data) + } + if _, present := unknown["context_window"]; present { + t.Errorf(`unknown-model["context_window"] present (%v), want entirely omitted`, unknown["context_window"]) + } + if _, present := unknown["pricing"]; present { + t.Errorf(`unknown-model["pricing"] present (%v), want entirely omitted`, unknown["pricing"]) + } +} diff --git a/tools/yaegi-check/main.go b/tools/yaegi-check/main.go index 1df550e..7c69757 100644 --- a/tools/yaegi-check/main.go +++ b/tools/yaegi-check/main.go @@ -88,6 +88,12 @@ const ( slowProviderModel = "gpt-test" slowUpstreamSleep = 600 * time.Millisecond slowRequestDeadline = 60 * time.Millisecond + // yaegiMetaContextTokens is the modelMeta config-override + // contextTokens value the attempt-accounting harness sets for + // testDataWantModel (feature v0.23) — a string, not an int, since it + // is spliced directly into attemptAccountingOverride's hand-built + // JSON text below. + yaegiMetaContextTokens = "128000" ) // excludedTopLevelDirs lists repo-root directories the GOPATH copy must @@ -221,6 +227,15 @@ func run() error { attemptAccountingOverride := `{"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `"]},"` + slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]}},` + + // modelMeta (feature v0.23): a config-override entry for + // testDataWantModel, so exerciseHandler's GET /v1/models + // assertion below proves resolveModelMeta's config-override + // layer, modelObject's context_window/pricing extension fields, + // and the whole registry.go/modelmeta.go resolution path all + // run correctly INTERPRETED, not merely compiled — cheap to add + // to this existing harness request, per this feature's own gate + // requirement. + `"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000}},` + `"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true}]}}` if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil { return fmt.Errorf("decode attempt-accounting harness override into the interpreted Config: %w", err) @@ -277,6 +292,19 @@ func exerciseHandler(handler http.Handler) error { if !strings.Contains(authedRec.Body.String(), testDataWantModel) { return fmt.Errorf("GET /v1/models body does not contain %q: %s", testDataWantModel, authedRec.Body.String()) } + // modelMeta (feature v0.23), interpreted: run() layers a modelMeta + // config override for testDataWantModel onto cfgVal before New() is + // ever called (attemptAccountingOverride above) — proving under the + // real Yaegi interpreter, not just `go test`, that resolveModelMeta's + // config-override layer and modelObject's context_window/pricing + // extension fields both run correctly through registry.go's + // listFor/resolveMetaFor. + if !strings.Contains(authedRec.Body.String(), `"context_window":128000`) { + return fmt.Errorf("GET /v1/models body does not contain the modelMeta-resolved context_window (feature v0.23 metadata harness): %s", authedRec.Body.String()) + } + if !strings.Contains(authedRec.Body.String(), `"input_per_mtok_usd":1.25`) || !strings.Contains(authedRec.Body.String(), `"output_per_mtok_usd":10`) { + return fmt.Errorf("GET /v1/models body does not contain the modelMeta-resolved pricing (feature v0.23 metadata harness): %s", authedRec.Body.String()) + } if err := exerciseAttemptAccounting(handler); err != nil { return err