mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-12 04:54:45 +00:00
feat(registry): model metadata capture and resolution
Config gets two new surfaces (llmgateway.go): Config.ModelMeta (map[string]*ModelMetaConfig, keyed by exact "provider/model" or a bare model id) for operator overrides, and ProviderConfig.MetadataPath for openai-type providers whose discovery should also capture per-model context (LM Studio's native /api/v0/models shape). modelmeta.go adds the layering itself: resolveModelMeta applies config override > discovered context > builtin table > absent for context, and config override/free > builtin > absent for cost (discovery never reports cost). resolveAliasModelMeta implements alias inheritance: an alias inherits its target's fully resolved metadata unless the alias itself has its own modelMeta entry, applied per field. validateModelMeta rejects negative values and a free:true/non-zero-cost contradiction at construction. registry.go wires this into the request path: providerState gains a discoveredContext map (stale-while-error, mirroring discovered's own convention) with contextFor/setDiscoveredContext; a new optional modelMetadataFetcher interface, type-asserted (interpreted-to-interpreted, documented as yaegi-safe against the Trap-4 restriction) rather than added to providerAdapter, lets warmFill/refreshProvider capture metadata alongside listModels without touching anthropic/gemini adapters at all. modelObject/listFor now attach resolved context_window/pricing to every GET /v1/models entry, including aliases (resolveMetaFor/ resolveMetaForAlias/resolveMetaForAliasName). provider_openai.go implements modelMetadataFetcher: fetchModelMetadata reads ProviderConfig.MetadataPath, preferring loaded_context_length over max_context_length when the LM Studio response reports both. providers.go wires MetadataPath onto the adapter in buildAdapters. tools/yaegi-check/main.go adds one cheap modelMeta config-override assertion to the existing attempt-accounting harness, proving resolveModelMeta's config-override layer and the context_window/pricing extension fields both run correctly under the real Yaegi interpreter.
This commit is contained in:
+50
-6
@@ -36,6 +36,20 @@ type Config struct {
|
||||
// model id's resolution changes. Validated at construction
|
||||
// (validateModelAliases, registry.go).
|
||||
ModelAliases map[string]string `json:"modelAliases,omitempty"`
|
||||
// ModelMeta declares per-model metadata overrides (feature v0.23):
|
||||
// context window size and per-token cost, keyed by an exact
|
||||
// "provider/model" id or a bare model id (applying wherever that bare
|
||||
// id resolves — modelmeta.go's lookupModelMetaConfig checks the exact
|
||||
// form first). This ALWAYS wins over discovery-captured context or
|
||||
// the built-in LiteLLM-synced table (resolveModelMeta, modelmeta.go)
|
||||
// — distinct from Pricing above, which drives request cost
|
||||
// ACCOUNTING (costMicros); ModelMeta drives metadata EXPOSURE
|
||||
// (GET /v1/models' context_window/pricing extension fields, the
|
||||
// admin dashboard) and never affects billing. nil/omitted preserves
|
||||
// prior behavior exactly: every model's metadata falls through to
|
||||
// discovery/builtin/absent, same as before this feature existed.
|
||||
// Validated at construction (validateModelMeta, modelmeta.go).
|
||||
ModelMeta map[string]*ModelMetaConfig `json:"modelMeta,omitempty"`
|
||||
// Retry is a struct value, not a pointer, because its own Enabled
|
||||
// field is the on/off signal (unlike Redis/Users, where the block's
|
||||
// mere presence is the signal) — so its tag omits "omitempty":
|
||||
@@ -51,12 +65,23 @@ type Config struct {
|
||||
|
||||
// ProviderConfig describes one upstream LLM provider.
|
||||
type ProviderConfig struct {
|
||||
Type string `json:"type"`
|
||||
BaseURL string `json:"baseUrl,omitempty"`
|
||||
APIKey string `json:"apiKey"`
|
||||
DiscoveryInterval string `json:"discoveryInterval,omitempty"`
|
||||
Models []string `json:"models,omitempty"`
|
||||
Discovery bool `json:"discovery,omitempty"`
|
||||
Type string `json:"type"`
|
||||
BaseURL string `json:"baseUrl,omitempty"`
|
||||
APIKey string `json:"apiKey"`
|
||||
DiscoveryInterval string `json:"discoveryInterval,omitempty"`
|
||||
// MetadataPath is an optional second discovery endpoint (feature
|
||||
// v0.23), fetched alongside the provider's normal listModels call
|
||||
// when set: a path such as "/api/v0/models" (LM Studio's own,
|
||||
// non-OpenAI-compatible models endpoint) that reports per-model
|
||||
// context length. Only openai-type adapters read this field
|
||||
// (provider_openai.go's openaiAdapter.fetchModelMetadata) — an
|
||||
// anthropic/gemini provider configuring it has no effect. Empty (the
|
||||
// default) captures no metadata, preserving prior behavior exactly.
|
||||
// A failed or absent fetch is always non-fatal to discovery itself:
|
||||
// no metadata is captured, nothing else changes.
|
||||
MetadataPath string `json:"metadataPath,omitempty"`
|
||||
Models []string `json:"models,omitempty"`
|
||||
Discovery bool `json:"discovery,omitempty"`
|
||||
}
|
||||
|
||||
// GroupConfig describes a group's access and limits. Empty/omitted
|
||||
@@ -183,6 +208,25 @@ type ModelPricing struct {
|
||||
OutputPerM float64 `json:"outputPerM"`
|
||||
}
|
||||
|
||||
// ModelMetaConfig overrides one model's context window and per-token
|
||||
// cost (feature v0.23, Config.ModelMeta) — layered ahead of discovery-
|
||||
// captured context and the built-in LiteLLM-synced table
|
||||
// (resolveModelMeta, modelmeta.go). ContextTokens left at 0 falls
|
||||
// through to the next layer rather than overriding with an explicit
|
||||
// unknown; the same holds for InputCostPerMTokMicroUSD/
|
||||
// OutputCostPerMTokMicroUSD when Free is false. Free, when true,
|
||||
// explicitly zeroes both cost fields regardless of whatever they are
|
||||
// set to — a KNOWN zero cost, distinct from the "not overridden here"
|
||||
// fallback a zero cost field means when Free is false. Setting Free
|
||||
// true together with a non-zero cost field is rejected at construction
|
||||
// (validateModelMeta) as a contradictory config.
|
||||
type ModelMetaConfig struct {
|
||||
ContextTokens int `json:"contextTokens,omitempty"`
|
||||
InputCostPerMTokMicroUSD int64 `json:"inputCostPerMTokMicroUsd,omitempty"`
|
||||
OutputCostPerMTokMicroUSD int64 `json:"outputCostPerMTokMicroUsd,omitempty"`
|
||||
Free bool `json:"free,omitempty"`
|
||||
}
|
||||
|
||||
// TargetConfig is a proxied upstream target (MCP server).
|
||||
type TargetConfig struct {
|
||||
URL string `json:"url"`
|
||||
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
package traefikllmgateway
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// microUSDPerUSD scales a micro-USD integer (ModelMetaConfig/
|
||||
// builtinModelMeta's storage unit — integer math avoids the float-drift
|
||||
// pricing.go's costMicros already documents for the cost-ACCOUNTING path)
|
||||
// to plain USD for exposure in a JSON API. Kept as its own named constant
|
||||
// rather than reusing pricing.go's tokensPerMillion: the two are
|
||||
// numerically identical (1_000_000) but represent different physical
|
||||
// quantities — one a token count, this one a currency scale — and must
|
||||
// not be conflated just because they share a value.
|
||||
const microUSDPerUSD = 1_000_000
|
||||
|
||||
// resolvedModelMeta is resolveModelMeta's result: a model's context
|
||||
// window and per-token cost, each resolved independently through its own
|
||||
// layering (feature v0.23). ContextKnown/CostKnown are authoritative — a
|
||||
// caller must check them, never infer "unknown" from a zero numeric
|
||||
// field, since a free model's cost fields are legitimately zero AND
|
||||
// known (ModelMetaConfig.Free).
|
||||
type resolvedModelMeta struct {
|
||||
ContextTokens int
|
||||
InputCostPerMTokMicroUSD int64
|
||||
OutputCostPerMTokMicroUSD int64
|
||||
ContextKnown bool
|
||||
CostKnown bool
|
||||
}
|
||||
|
||||
// resolveModelMeta layers per-model metadata resolution (feature v0.23):
|
||||
//
|
||||
// - Context: an operator's modelMeta config override
|
||||
// (lookupModelMetaConfig: an exact "provider/model" key, then a bare
|
||||
// "model" key) wins when it sets ContextTokens > 0; otherwise
|
||||
// discoveredContext (only when discoveredKnown — a provider's own
|
||||
// discovery-captured context, registry.go's providerState.
|
||||
// discoveredContext, populated only for a provider whose adapter
|
||||
// implements modelMetadataFetcher and returned a value for this
|
||||
// model); otherwise the built-in table (builtinModelMetaTable,
|
||||
// pricing_data_gen.go), keyed by bare model id only — an operator's
|
||||
// own provider name never matches LiteLLM's canonical provider
|
||||
// names, so an exact "provider/model" form makes no sense for that
|
||||
// table the way it does for cfg; otherwise unknown.
|
||||
// - Cost: the config override wins when it sets Free (an explicit,
|
||||
// known zero) or either cost field > 0; otherwise the built-in
|
||||
// table. Discovery never reports cost, so there is no discovery
|
||||
// layer for it.
|
||||
//
|
||||
// Absent at every applicable layer is a genuinely unknown value and is
|
||||
// never invented — ContextKnown/CostKnown both stay false in that case.
|
||||
func resolveModelMeta(provider, model string, cfg map[string]*ModelMetaConfig, discoveredContext int, discoveredKnown bool) resolvedModelMeta {
|
||||
var out resolvedModelMeta
|
||||
override := lookupModelMetaConfig(provider, model, cfg)
|
||||
builtin, builtinOK := builtinModelMetaTable[model]
|
||||
|
||||
switch {
|
||||
case override != nil && override.ContextTokens > 0:
|
||||
out.ContextTokens, out.ContextKnown = override.ContextTokens, true
|
||||
case discoveredKnown && discoveredContext > 0:
|
||||
out.ContextTokens, out.ContextKnown = discoveredContext, true
|
||||
case builtinOK && builtin.ContextTokens > 0:
|
||||
out.ContextTokens, out.ContextKnown = builtin.ContextTokens, true
|
||||
}
|
||||
|
||||
switch {
|
||||
case override != nil && override.Free:
|
||||
out.CostKnown = true // both cost fields stay 0 — explicit, known zero
|
||||
case override != nil && (override.InputCostPerMTokMicroUSD > 0 || override.OutputCostPerMTokMicroUSD > 0):
|
||||
out.InputCostPerMTokMicroUSD = override.InputCostPerMTokMicroUSD
|
||||
out.OutputCostPerMTokMicroUSD = override.OutputCostPerMTokMicroUSD
|
||||
out.CostKnown = true
|
||||
case builtinOK && (builtin.InputCostPerMTokMicroUSD > 0 || builtin.OutputCostPerMTokMicroUSD > 0):
|
||||
out.InputCostPerMTokMicroUSD = builtin.InputCostPerMTokMicroUSD
|
||||
out.OutputCostPerMTokMicroUSD = builtin.OutputCostPerMTokMicroUSD
|
||||
out.CostKnown = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// lookupModelMetaConfig resolves cfg's config-override layer for
|
||||
// (provider, model): an exact "provider/model" key wins over a bare
|
||||
// "model" key, matching this feature's own spec precedence (a key names
|
||||
// either an exact provider/model pair or a bare id that applies wherever
|
||||
// that id resolves). Returns nil (no override at this layer) when cfg is
|
||||
// nil or neither key is present, or when the matched entry is itself a
|
||||
// nil pointer (a malformed-but-decoded config value, treated as absent
|
||||
// rather than dereferenced).
|
||||
func lookupModelMetaConfig(provider, model string, cfg map[string]*ModelMetaConfig) *ModelMetaConfig {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
if v, ok := cfg[provider+"/"+model]; ok && v != nil {
|
||||
return v
|
||||
}
|
||||
if v, ok := cfg[model]; ok && v != nil {
|
||||
return v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveAliasModelMeta applies modelMeta's alias-inheritance rule
|
||||
// (feature v0.23): an alias inherits its target's fully resolved
|
||||
// metadata (resolveModelMeta against the alias's resolved provider/
|
||||
// bare-model target) UNLESS cfg carries a modelMeta entry keyed by the
|
||||
// alias id itself — that entry's own fields win over the inherited ones,
|
||||
// independently per field (an alias-level entry that sets only context
|
||||
// still inherits the target's cost, and vice versa), the same
|
||||
// override-wins-per-field shape resolveModelMeta itself applies between
|
||||
// its own layers.
|
||||
func resolveAliasModelMeta(alias, targetProvider, targetModel string, cfg map[string]*ModelMetaConfig, discoveredContext int, discoveredKnown bool) resolvedModelMeta {
|
||||
out := resolveModelMeta(targetProvider, targetModel, cfg, discoveredContext, discoveredKnown)
|
||||
if cfg == nil {
|
||||
return out
|
||||
}
|
||||
override, ok := cfg[alias]
|
||||
if !ok || override == nil {
|
||||
return out
|
||||
}
|
||||
if override.ContextTokens > 0 {
|
||||
out.ContextTokens, out.ContextKnown = override.ContextTokens, true
|
||||
}
|
||||
switch {
|
||||
case override.Free:
|
||||
out.InputCostPerMTokMicroUSD, out.OutputCostPerMTokMicroUSD, out.CostKnown = 0, 0, true
|
||||
case override.InputCostPerMTokMicroUSD > 0 || override.OutputCostPerMTokMicroUSD > 0:
|
||||
out.InputCostPerMTokMicroUSD = override.InputCostPerMTokMicroUSD
|
||||
out.OutputCostPerMTokMicroUSD = override.OutputCostPerMTokMicroUSD
|
||||
out.CostKnown = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// microUSDPerMTokToUSD converts a micro-USD-per-million-tokens integer
|
||||
// (ModelMetaConfig/builtinModelMeta's storage unit) to plain
|
||||
// USD-per-million-tokens for exposure in a JSON API (GET /v1/models'
|
||||
// "pricing" extension field: "floats in USD per MTok").
|
||||
func microUSDPerMTokToUSD(micro int64) float64 {
|
||||
return float64(micro) / microUSDPerUSD
|
||||
}
|
||||
|
||||
// validateModelMeta validates cfg.ModelMeta (feature v0.23) and returns
|
||||
// it unchanged (nil when raw is empty — the zero-behavior-change
|
||||
// default, matching validateModelAliases' own convention, registry.go).
|
||||
// raw is iterated in sorted key order, not Go's randomized map order, so
|
||||
// a config with more than one invalid entry always reports the same one
|
||||
// first, deterministically, across repeated runs.
|
||||
func validateModelMeta(raw map[string]*ModelMetaConfig) (map[string]*ModelMetaConfig, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(raw))
|
||||
for k := range raw {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
for _, k := range keys {
|
||||
v := raw[k]
|
||||
if v == nil {
|
||||
return nil, fmt.Errorf("llmgateway: modelMeta: entry %q must not be nil", k)
|
||||
}
|
||||
if v.ContextTokens < 0 {
|
||||
return nil, fmt.Errorf("llmgateway: modelMeta: entry %q: contextTokens must not be negative", k)
|
||||
}
|
||||
if v.InputCostPerMTokMicroUSD < 0 || v.OutputCostPerMTokMicroUSD < 0 {
|
||||
return nil, fmt.Errorf("llmgateway: modelMeta: entry %q: cost fields must not be negative", k)
|
||||
}
|
||||
if v.Free && (v.InputCostPerMTokMicroUSD != 0 || v.OutputCostPerMTokMicroUSD != 0) {
|
||||
return nil, fmt.Errorf("llmgateway: modelMeta: entry %q: free=true is contradictory with a non-zero cost field", k)
|
||||
}
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package traefikllmgateway
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestResolveModelMeta_Layering is the feature's core table-driven test:
|
||||
// config override > discovered > builtin > absent, checked independently
|
||||
// for context and for cost (discovery never reports cost, so cost only
|
||||
// has three layers).
|
||||
func TestResolveModelMeta_Layering(t *testing.T) {
|
||||
const provider = "openai"
|
||||
const model = "test-model"
|
||||
|
||||
cases := []struct {
|
||||
cfg map[string]*ModelMetaConfig
|
||||
name string
|
||||
discoveredCtx int
|
||||
wantCtx int
|
||||
wantInput int64
|
||||
wantOutput int64
|
||||
discoveredKnown bool
|
||||
wantCtxKnown bool
|
||||
wantCostKnown bool
|
||||
}{
|
||||
{
|
||||
name: "everything absent",
|
||||
wantCtxKnown: false,
|
||||
},
|
||||
{
|
||||
name: "discovered context only, no cost anywhere",
|
||||
discoveredCtx: 4096,
|
||||
discoveredKnown: true,
|
||||
wantCtx: 4096,
|
||||
wantCtxKnown: true,
|
||||
},
|
||||
{
|
||||
name: "config override wins over discovered",
|
||||
cfg: map[string]*ModelMetaConfig{
|
||||
model: {ContextTokens: 8192},
|
||||
},
|
||||
discoveredCtx: 4096,
|
||||
discoveredKnown: true,
|
||||
wantCtx: 8192,
|
||||
wantCtxKnown: true,
|
||||
},
|
||||
{
|
||||
name: "config exact provider/model key wins over bare key",
|
||||
cfg: map[string]*ModelMetaConfig{
|
||||
model: {ContextTokens: 1000},
|
||||
provider + "/" + model: {ContextTokens: 2000},
|
||||
},
|
||||
wantCtx: 2000,
|
||||
wantCtxKnown: true,
|
||||
},
|
||||
{
|
||||
name: "config override sets cost, discovery ignored for cost",
|
||||
cfg: map[string]*ModelMetaConfig{
|
||||
model: {InputCostPerMTokMicroUSD: 100, OutputCostPerMTokMicroUSD: 200},
|
||||
},
|
||||
wantInput: 100,
|
||||
wantOutput: 200,
|
||||
wantCostKnown: true,
|
||||
},
|
||||
{
|
||||
name: "free config override zeroes cost explicitly, known",
|
||||
cfg: map[string]*ModelMetaConfig{
|
||||
model: {Free: true},
|
||||
},
|
||||
wantInput: 0,
|
||||
wantOutput: 0,
|
||||
wantCostKnown: true,
|
||||
},
|
||||
{
|
||||
name: "builtin table used when nothing else known",
|
||||
wantCtx: 0, // "test-model" is not in builtinModelMetaTable
|
||||
wantCostKnown: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := resolveModelMeta(provider, model, c.cfg, c.discoveredCtx, c.discoveredKnown)
|
||||
if got.ContextKnown != c.wantCtxKnown || got.ContextTokens != c.wantCtx {
|
||||
t.Errorf("context = (%d, known=%v), want (%d, known=%v)", got.ContextTokens, got.ContextKnown, c.wantCtx, c.wantCtxKnown)
|
||||
}
|
||||
if got.CostKnown != c.wantCostKnown || got.InputCostPerMTokMicroUSD != c.wantInput || got.OutputCostPerMTokMicroUSD != c.wantOutput {
|
||||
t.Errorf("cost = (%d, %d, known=%v), want (%d, %d, known=%v)",
|
||||
got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown,
|
||||
c.wantInput, c.wantOutput, c.wantCostKnown)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveModelMeta_BuiltinTable exercises the real, generated
|
||||
// builtinModelMetaTable directly (pricing_data_gen.go) for both context
|
||||
// and cost, and confirms an operator's config override still wins over
|
||||
// it.
|
||||
func TestResolveModelMeta_BuiltinTable(t *testing.T) {
|
||||
const bareID = "deepseek-chat" // a stable, real LiteLLM entry (deepseek's own bare key)
|
||||
builtin, ok := builtinModelMetaTable[bareID]
|
||||
if !ok || builtin.ContextTokens <= 0 || builtin.InputCostPerMTokMicroUSD <= 0 {
|
||||
t.Fatalf("builtinModelMetaTable[%q] = %+v, ok=%v — test fixture assumption broke, pick another stable id", bareID, builtin, ok)
|
||||
}
|
||||
|
||||
got := resolveModelMeta("acme", bareID, nil, 0, false)
|
||||
if !got.ContextKnown || got.ContextTokens != builtin.ContextTokens {
|
||||
t.Errorf("context = (%d, known=%v), want (%d, known=true)", got.ContextTokens, got.ContextKnown, builtin.ContextTokens)
|
||||
}
|
||||
if !got.CostKnown || got.InputCostPerMTokMicroUSD != builtin.InputCostPerMTokMicroUSD || got.OutputCostPerMTokMicroUSD != builtin.OutputCostPerMTokMicroUSD {
|
||||
t.Errorf("cost = (%d, %d, known=%v), want (%d, %d, known=true)",
|
||||
got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown,
|
||||
builtin.InputCostPerMTokMicroUSD, builtin.OutputCostPerMTokMicroUSD)
|
||||
}
|
||||
|
||||
overridden := resolveModelMeta("acme", bareID, map[string]*ModelMetaConfig{bareID: {ContextTokens: 1}}, 0, false)
|
||||
if overridden.ContextTokens != 1 {
|
||||
t.Errorf("config override over builtin: context = %d, want 1", overridden.ContextTokens)
|
||||
}
|
||||
// Cost is untouched by the context-only override — still falls
|
||||
// through to the builtin table (independent per-field layering).
|
||||
if overridden.InputCostPerMTokMicroUSD != builtin.InputCostPerMTokMicroUSD {
|
||||
t.Errorf("cost should still fall through to builtin when override sets only context, got %d, want %d", overridden.InputCostPerMTokMicroUSD, builtin.InputCostPerMTokMicroUSD)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLookupModelMetaConfig covers lookupModelMetaConfig's own
|
||||
// precedence and nil-safety directly.
|
||||
func TestLookupModelMetaConfig(t *testing.T) {
|
||||
t.Run("nil cfg", func(t *testing.T) {
|
||||
if got := lookupModelMetaConfig("p", "m", nil); got != nil {
|
||||
t.Errorf("got %+v, want nil", got)
|
||||
}
|
||||
})
|
||||
t.Run("nil entry treated as absent", func(t *testing.T) {
|
||||
cfg := map[string]*ModelMetaConfig{"m": nil}
|
||||
if got := lookupModelMetaConfig("p", "m", cfg); got != nil {
|
||||
t.Errorf("got %+v, want nil (nil map value must not be dereferenced)", got)
|
||||
}
|
||||
})
|
||||
t.Run("bare key match", func(t *testing.T) {
|
||||
want := &ModelMetaConfig{ContextTokens: 5}
|
||||
cfg := map[string]*ModelMetaConfig{"m": want}
|
||||
if got := lookupModelMetaConfig("p", "m", cfg); got != want {
|
||||
t.Errorf("got %+v, want %+v", got, want)
|
||||
}
|
||||
})
|
||||
t.Run("exact provider/model beats bare", func(t *testing.T) {
|
||||
bare := &ModelMetaConfig{ContextTokens: 1}
|
||||
exact := &ModelMetaConfig{ContextTokens: 2}
|
||||
cfg := map[string]*ModelMetaConfig{"m": bare, "p/m": exact}
|
||||
if got := lookupModelMetaConfig("p", "m", cfg); got != exact {
|
||||
t.Errorf("got %+v, want the exact-key entry %+v", got, exact)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResolveAliasModelMeta_Inheritance asserts an alias inherits its
|
||||
// target's fully resolved metadata, and that an alias-level modelMeta
|
||||
// entry overrides the inherited values independently per field.
|
||||
func TestResolveAliasModelMeta_Inheritance(t *testing.T) {
|
||||
const alias = "aliased/coding"
|
||||
const targetProvider = "anthropic"
|
||||
const targetModel = "claude-x"
|
||||
|
||||
t.Run("no alias entry: inherits target fully", func(t *testing.T) {
|
||||
cfg := map[string]*ModelMetaConfig{
|
||||
targetProvider + "/" + targetModel: {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3_000_000, OutputCostPerMTokMicroUSD: 15_000_000},
|
||||
}
|
||||
got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false)
|
||||
if got.ContextTokens != 200000 || !got.ContextKnown {
|
||||
t.Errorf("context = (%d, %v), want (200000, true)", got.ContextTokens, got.ContextKnown)
|
||||
}
|
||||
if got.InputCostPerMTokMicroUSD != 3_000_000 || got.OutputCostPerMTokMicroUSD != 15_000_000 || !got.CostKnown {
|
||||
t.Errorf("cost = (%d, %d, %v), want (3000000, 15000000, true)", got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("alias entry overrides context only, cost still inherited", func(t *testing.T) {
|
||||
cfg := map[string]*ModelMetaConfig{
|
||||
targetProvider + "/" + targetModel: {ContextTokens: 200000, InputCostPerMTokMicroUSD: 3_000_000, OutputCostPerMTokMicroUSD: 15_000_000},
|
||||
alias: {ContextTokens: 999},
|
||||
}
|
||||
got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false)
|
||||
if got.ContextTokens != 999 {
|
||||
t.Errorf("context = %d, want 999 (alias override)", got.ContextTokens)
|
||||
}
|
||||
if got.InputCostPerMTokMicroUSD != 3_000_000 || !got.CostKnown {
|
||||
t.Errorf("cost should still be inherited from target: got (%d, known=%v)", got.InputCostPerMTokMicroUSD, got.CostKnown)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("alias entry sets free, overrides inherited cost", func(t *testing.T) {
|
||||
cfg := map[string]*ModelMetaConfig{
|
||||
targetProvider + "/" + targetModel: {InputCostPerMTokMicroUSD: 3_000_000, OutputCostPerMTokMicroUSD: 15_000_000},
|
||||
alias: {Free: true},
|
||||
}
|
||||
got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false)
|
||||
if !got.CostKnown || got.InputCostPerMTokMicroUSD != 0 || got.OutputCostPerMTokMicroUSD != 0 {
|
||||
t.Errorf("cost = (%d, %d, known=%v), want (0, 0, true)", got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil cfg never panics", func(t *testing.T) {
|
||||
got := resolveAliasModelMeta(alias, targetProvider, targetModel, nil, 0, false)
|
||||
if got.ContextKnown || got.CostKnown {
|
||||
t.Errorf("got %+v, want everything unknown", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil alias entry value treated as absent, still inherits", func(t *testing.T) {
|
||||
cfg := map[string]*ModelMetaConfig{
|
||||
targetProvider + "/" + targetModel: {ContextTokens: 111},
|
||||
alias: nil,
|
||||
}
|
||||
got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false)
|
||||
if got.ContextTokens != 111 || !got.ContextKnown {
|
||||
t.Errorf("got %+v, want inherited ContextTokens=111 (nil alias entry must not panic or override)", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("alias entry sets non-free cost directly, overrides inherited cost", func(t *testing.T) {
|
||||
cfg := map[string]*ModelMetaConfig{
|
||||
targetProvider + "/" + targetModel: {InputCostPerMTokMicroUSD: 1, OutputCostPerMTokMicroUSD: 2},
|
||||
alias: {InputCostPerMTokMicroUSD: 100, OutputCostPerMTokMicroUSD: 200},
|
||||
}
|
||||
got := resolveAliasModelMeta(alias, targetProvider, targetModel, cfg, 0, false)
|
||||
if !got.CostKnown || got.InputCostPerMTokMicroUSD != 100 || got.OutputCostPerMTokMicroUSD != 200 {
|
||||
t.Errorf("got cost=(%d, %d, known=%v), want (100, 200, true) — alias's own non-free cost override", got.InputCostPerMTokMicroUSD, got.OutputCostPerMTokMicroUSD, got.CostKnown)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestMicroUSDPerMTokToUSD pins the currency conversion.
|
||||
func TestMicroUSDPerMTokToUSD(t *testing.T) {
|
||||
cases := []struct {
|
||||
micro int64
|
||||
want float64
|
||||
}{
|
||||
{0, 0},
|
||||
{1_250_000, 1.25},
|
||||
{10_000_000, 10},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := microUSDPerMTokToUSD(c.micro); got != c.want {
|
||||
t.Errorf("microUSDPerMTokToUSD(%d) = %v, want %v", c.micro, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateModelMeta covers construction-time validation: the
|
||||
// zero-behavior-change nil/empty case, every rejection rule, and a
|
||||
// deterministic first-error-wins ordering across multiple bad entries.
|
||||
func TestValidateModelMeta(t *testing.T) {
|
||||
t.Run("nil/empty preserves prior behavior", func(t *testing.T) {
|
||||
got, err := validateModelMeta(nil)
|
||||
if err != nil || got != nil {
|
||||
t.Errorf("validateModelMeta(nil) = (%v, %v), want (nil, nil)", got, err)
|
||||
}
|
||||
got, err = validateModelMeta(map[string]*ModelMetaConfig{})
|
||||
if err != nil || got != nil {
|
||||
t.Errorf("validateModelMeta({}) = (%v, %v), want (nil, nil)", got, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid entries pass through unchanged", func(t *testing.T) {
|
||||
raw := map[string]*ModelMetaConfig{
|
||||
"m1": {ContextTokens: 100, InputCostPerMTokMicroUSD: 1, OutputCostPerMTokMicroUSD: 2},
|
||||
"m2": {Free: true},
|
||||
}
|
||||
got, err := validateModelMeta(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("validateModelMeta: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Errorf("got %d entries, want 2", len(got))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil entry rejected", func(t *testing.T) {
|
||||
_, err := validateModelMeta(map[string]*ModelMetaConfig{"m": nil})
|
||||
if err == nil {
|
||||
t.Fatal("want error for nil entry")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("negative context rejected", func(t *testing.T) {
|
||||
_, err := validateModelMeta(map[string]*ModelMetaConfig{"m": {ContextTokens: -1}})
|
||||
if err == nil {
|
||||
t.Fatal("want error for negative contextTokens")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("negative cost rejected", func(t *testing.T) {
|
||||
_, err := validateModelMeta(map[string]*ModelMetaConfig{"m": {InputCostPerMTokMicroUSD: -1}})
|
||||
if err == nil {
|
||||
t.Fatal("want error for negative input cost")
|
||||
}
|
||||
_, err = validateModelMeta(map[string]*ModelMetaConfig{"m": {OutputCostPerMTokMicroUSD: -1}})
|
||||
if err == nil {
|
||||
t.Fatal("want error for negative output cost")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("free with non-zero cost rejected as contradictory", func(t *testing.T) {
|
||||
_, err := validateModelMeta(map[string]*ModelMetaConfig{"m": {Free: true, InputCostPerMTokMicroUSD: 1}})
|
||||
if err == nil {
|
||||
t.Fatal("want error for free:true with a non-zero cost field")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("multiple invalid entries report the first in sorted key order", func(t *testing.T) {
|
||||
raw := map[string]*ModelMetaConfig{
|
||||
"zzz": {ContextTokens: -1},
|
||||
"aaa": {ContextTokens: -1},
|
||||
}
|
||||
_, err := validateModelMeta(raw)
|
||||
if err == nil {
|
||||
t.Fatal("want error")
|
||||
}
|
||||
if got := err.Error(); !strings.Contains(got, `"aaa"`) {
|
||||
t.Errorf("error = %q, want it to name the sorted-first key %q", got, "aaa")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -23,6 +23,12 @@ type openaiAdapter struct {
|
||||
adapterName string
|
||||
baseURL string
|
||||
apiKey string
|
||||
// metadataPath is ProviderConfig.MetadataPath (feature v0.23), set by
|
||||
// buildAdapters (providers.go) after construction — empty (the
|
||||
// default) makes fetchModelMetadata a no-op, so every existing
|
||||
// newOpenAIAdapter call site (every test, and any caller that never
|
||||
// sets this) keeps behaving exactly as before this feature existed.
|
||||
metadataPath string
|
||||
}
|
||||
|
||||
// newOpenAIAdapter returns an openaiAdapter for provider name, with base as
|
||||
@@ -401,3 +407,72 @@ func (a *openaiAdapter) listModels(ctx context.Context) ([]string, error) {
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// lmStudioModelsPayload is the wire shape of LM Studio's own, native
|
||||
// models endpoint (ProviderConfig.MetadataPath — a real deployment sets
|
||||
// it to "/api/v0/models"), verified live against a running LM Studio
|
||||
// instance (2026-08): {"data":[{"id":...,"max_context_length":262144,
|
||||
// "loaded_context_length":4096}, ...]}. Distinct from modelsPayload
|
||||
// above, which is OpenAI's own /v1/models shape (LM Studio also serves
|
||||
// that, unchanged, but it carries no per-model context metadata) — this
|
||||
// is LM Studio's separate, additional endpoint this feature (v0.23)
|
||||
// opts into reading.
|
||||
type lmStudioModelsPayload struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
MaxContextLength int `json:"max_context_length"`
|
||||
LoadedContextLength int `json:"loaded_context_length"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// fetchModelMetadata implements modelMetadataFetcher (feature v0.23,
|
||||
// registry.go): when a.metadataPath is configured, fetches it and
|
||||
// returns each reported model id's discovery-captured context length.
|
||||
// It prefers loaded_context_length when the upstream reports one greater
|
||||
// than zero — the context actually usable right now, which a runtime can
|
||||
// configure smaller than the model's own maximum under VRAM-constrained
|
||||
// settings — and falls back to max_context_length otherwise. a.
|
||||
// metadataPath left empty (the default; every provider except an
|
||||
// operator's opt-in) is a fast no-op: (nil, nil), never an error, so
|
||||
// registry.go's captureModelMetadata can invoke this unconditionally on
|
||||
// every openai-type adapter without a config check of its own. A
|
||||
// non-2xx response or a decode failure returns an error — the caller
|
||||
// treats it as non-fatal, per this feature's own "absent/failed = no
|
||||
// metadata captured" ruling.
|
||||
func (a *openaiAdapter) fetchModelMetadata(ctx context.Context) (map[string]int, error) {
|
||||
if a.metadataPath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
resp, err := upstreamJSON(ctx, a.client, http.MethodGet, a.baseURL+a.metadataPath, a.requestHeaders(false), nil, a.retry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, newProviderHTTPError(resp)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: read metadata response body: %w", errUpstream, err)
|
||||
}
|
||||
|
||||
var payload lmStudioModelsPayload
|
||||
if err := json.Unmarshal(body, &payload); err != nil {
|
||||
return nil, fmt.Errorf("%w: decode metadata response: %w", errUpstream, err)
|
||||
}
|
||||
|
||||
out := make(map[string]int, len(payload.Data))
|
||||
for _, d := range payload.Data {
|
||||
ctxLen := d.MaxContextLength
|
||||
if d.LoadedContextLength > 0 {
|
||||
ctxLen = d.LoadedContextLength
|
||||
}
|
||||
if ctxLen > 0 {
|
||||
out[d.ID] = ctxLen
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -528,6 +528,106 @@ func TestOpenAIAdapter_ListModels(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAIAdapter_FetchModelMetadata covers fetchModelMetadata (feature
|
||||
// v0.23) against an httptest fixture shaped like LM Studio's real, live
|
||||
// "/api/v0/models" response (task brief: llm/vlm models report
|
||||
// max_context_length 262144, qwen2.5-0.5b 32768, embeddings 512-8192) —
|
||||
// including the loaded_context_length-preferred-over-max_context_length
|
||||
// rule, an entry with no loaded_context_length at all, and a
|
||||
// metadataPath left unset (the default: a fast no-op, no HTTP call at
|
||||
// all).
|
||||
func TestOpenAIAdapter_FetchModelMetadata(t *testing.T) {
|
||||
t.Run("metadataPath unset is a fast no-op", func(t *testing.T) {
|
||||
called := false
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
called = true
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
a := newOpenAIAdapter("p1", srv.URL, "sk-test")
|
||||
got, err := a.fetchModelMetadata(context.Background())
|
||||
if err != nil || got != nil {
|
||||
t.Errorf("fetchModelMetadata = (%v, %v), want (nil, nil)", got, err)
|
||||
}
|
||||
if called {
|
||||
t.Error("unset metadataPath must never issue an HTTP request")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("LM Studio shaped fixture: prefers loaded over max, falls back when absent", func(t *testing.T) {
|
||||
var gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[
|
||||
{"id":"qwen/qwen3-vl-30b","max_context_length":262144,"loaded_context_length":262144},
|
||||
{"id":"qwen2.5-0.5b","max_context_length":32768,"loaded_context_length":4096},
|
||||
{"id":"text-embedding-nomic-embed-text-v1.5","max_context_length":2048,"loaded_context_length":512},
|
||||
{"id":"no-loaded-field","max_context_length":8192},
|
||||
{"id":"zero-everything","max_context_length":0,"loaded_context_length":0}
|
||||
]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
a := newOpenAIAdapter("lmstudio", srv.URL, "")
|
||||
a.metadataPath = "/api/v0/models"
|
||||
got, err := a.fetchModelMetadata(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("fetchModelMetadata: %v", err)
|
||||
}
|
||||
if gotPath != "/api/v0/models" {
|
||||
t.Errorf("request path = %q, want /api/v0/models", gotPath)
|
||||
}
|
||||
|
||||
want := map[string]int{
|
||||
"qwen/qwen3-vl-30b": 262144,
|
||||
"qwen2.5-0.5b": 4096, // loaded_context_length preferred over max_context_length
|
||||
"text-embedding-nomic-embed-text-v1.5": 512,
|
||||
"no-loaded-field": 8192, // falls back to max_context_length
|
||||
// "zero-everything" omitted entirely: both fields are 0.
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %#v, want %#v", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-2xx response is an error, not a captured empty map", func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
a := newOpenAIAdapter("p1", srv.URL, "")
|
||||
a.metadataPath = "/api/v0/models"
|
||||
if _, err := a.fetchModelMetadata(context.Background()); err == nil {
|
||||
t.Error("want an error for a non-2xx metadata response")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("malformed JSON body is a decode error", func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(`{not json`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
a := newOpenAIAdapter("p1", srv.URL, "")
|
||||
a.metadataPath = "/api/v0/models"
|
||||
if _, err := a.fetchModelMetadata(context.Background()); err == nil {
|
||||
t.Error("want a decode error for a malformed metadata response body")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unreachable upstream is an error", func(t *testing.T) {
|
||||
a := newOpenAIAdapter("p1", "http://127.0.0.1:1", "") // reserved, never listening
|
||||
a.metadataPath = "/api/v0/models"
|
||||
if _, err := a.fetchModelMetadata(context.Background()); err == nil {
|
||||
t.Error("want an error when the metadata endpoint is unreachable")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestOpenAIAdapter_Keyless proves a resolved-empty API key sends no
|
||||
// Authorization header at all, not one with an empty token — the
|
||||
// operator's own keyless gateway is a real deployment target.
|
||||
@@ -591,6 +691,23 @@ func TestBuildAdapters(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("good: metadataPath is wired onto the openai adapter (feature v0.23)", func(t *testing.T) {
|
||||
cfg := &Config{Providers: map[string]*ProviderConfig{
|
||||
"lmstudio": {Type: "openai", MetadataPath: "/api/v0/models"},
|
||||
}}
|
||||
adapters, err := buildAdapters(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("buildAdapters: %v", err)
|
||||
}
|
||||
a, ok := adapters["lmstudio"].(*openaiAdapter)
|
||||
if !ok {
|
||||
t.Fatalf("adapters[%q] is not *openaiAdapter", "lmstudio")
|
||||
}
|
||||
if a.metadataPath != "/api/v0/models" {
|
||||
t.Errorf("metadataPath = %q, want %q", a.metadataPath, "/api/v0/models")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("good: explicit base URL trailing slash trimmed", func(t *testing.T) {
|
||||
cfg := &Config{Providers: map[string]*ProviderConfig{
|
||||
"p1": {Type: "openai", BaseURL: "https://custom.example.com/", APIKey: "sk-test"},
|
||||
|
||||
@@ -417,6 +417,7 @@ func buildAdapters(cfg *Config) (map[string]providerAdapter, error) {
|
||||
case providerTypeOpenAI:
|
||||
a := newOpenAIAdapter(name, base, apiKey)
|
||||
a.retry = policy
|
||||
a.metadataPath = pc.MetadataPath
|
||||
adapters[name] = a
|
||||
case providerTypeAnthropic:
|
||||
a, err := newAnthropicAdapter(name, base, apiKey)
|
||||
|
||||
+175
-9
@@ -46,6 +46,17 @@ type providerState struct {
|
||||
lastRefresh time.Time
|
||||
explicit map[string]bool
|
||||
discovered map[string]bool
|
||||
// discoveredContext holds discovery-captured per-model context
|
||||
// lengths (feature v0.23, ProviderConfig.MetadataPath — currently
|
||||
// only openai-type adapters ever populate this, via
|
||||
// modelMetadataFetcher), keyed by upstream model id. nil for a
|
||||
// provider with no metadataPath configured, or whose adapter does
|
||||
// not implement modelMetadataFetcher at all — contextFor treats a
|
||||
// nil map the same as an absent entry: (0, false). Stale-while-
|
||||
// error, the same convention discovered itself already uses: a
|
||||
// failed capture leaves the previous map in place rather than
|
||||
// clearing it (captureModelMetadata, below).
|
||||
discoveredContext map[string]int
|
||||
// lastErr is the most recent finishRefresh call's error message, or
|
||||
// "" when that call succeeded (or discovery is disabled and
|
||||
// finishRefresh was never called at all). Read by snapshot for the
|
||||
@@ -99,6 +110,26 @@ func (st *providerState) knownIDs() []string {
|
||||
return st.knownIDsLocked()
|
||||
}
|
||||
|
||||
// contextFor returns id's discovery-captured context length and whether
|
||||
// one is known (feature v0.23) — mirroring hasModel's own locked-read
|
||||
// shape.
|
||||
func (st *providerState) contextFor(id string) (int, bool) {
|
||||
st.mu.Lock()
|
||||
defer st.mu.Unlock()
|
||||
n, ok := st.discoveredContext[id]
|
||||
return n, ok
|
||||
}
|
||||
|
||||
// setDiscoveredContext replaces st's discoveredContext wholesale on a
|
||||
// successful metadata capture (feature v0.23, captureModelMetadata below)
|
||||
// — the same replace-on-success, stale-while-error shape finishRefresh
|
||||
// already applies to discovered.
|
||||
func (st *providerState) setDiscoveredContext(m map[string]int) {
|
||||
st.mu.Lock()
|
||||
defer st.mu.Unlock()
|
||||
st.discoveredContext = m
|
||||
}
|
||||
|
||||
// tryBeginRefresh reports whether now is far enough past lastRefresh (or
|
||||
// this is the first refresh) to start a new one, and if so marks the
|
||||
// provider inFlight so a concurrent caller cannot start a second one. The
|
||||
@@ -192,7 +223,16 @@ type modelRegistry struct {
|
||||
// configured no aliases at all, so every alias-aware code path is a
|
||||
// cheap nil-map lookup (always false/zero) that adds no branching
|
||||
// cost to the v0.1 no-aliases case.
|
||||
aliases map[string]string
|
||||
aliases map[string]string
|
||||
// modelMeta is the validated Config.ModelMeta config-override map
|
||||
// (feature v0.23), built once by newModelRegistry via
|
||||
// validateModelMeta and never mutated afterward. nil (not merely
|
||||
// empty) when the operator configured no metadata overrides at all,
|
||||
// so resolveModelMeta's config-override layer is a cheap nil-map
|
||||
// lookup (always a miss) that adds no branching cost to the
|
||||
// no-overrides case — the same nil-vs-empty convention aliases above
|
||||
// already establishes.
|
||||
modelMeta map[string]*ModelMetaConfig
|
||||
nowFn func() time.Time
|
||||
warned map[string]bool
|
||||
providerNames []string
|
||||
@@ -260,6 +300,12 @@ func newModelRegistry(adapters map[string]providerAdapter, cfg *Config, log func
|
||||
}
|
||||
m.aliases = aliases
|
||||
|
||||
modelMeta, err := validateModelMeta(cfg.ModelMeta)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.modelMeta = modelMeta
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
@@ -355,14 +401,69 @@ func (m *modelRegistry) warmFill(ctx context.Context) {
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(ctx, warmFillTimeout)
|
||||
ids, err := m.adapters[name].listModels(fctx)
|
||||
cancel()
|
||||
st.finishRefresh(m.now(), ids, err)
|
||||
if err != nil {
|
||||
m.log("%s", fmt.Sprintf("model registry: initial discovery for provider %q failed: %v", name, err))
|
||||
}
|
||||
m.captureModelMetadata(fctx, name, m.adapters[name], st)
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// modelMetadataFetcher is an OPTIONAL providerAdapter capability (feature
|
||||
// v0.23): an adapter that can also fetch per-model discovery metadata
|
||||
// (currently: context window) beyond the plain model-id list listModels
|
||||
// returns. Only openai-type adapters configured with ProviderConfig.
|
||||
// MetadataPath implement it (provider_openai.go's openaiAdapter) —
|
||||
// anthropic/gemini adapters do not, so captureModelMetadata checks for
|
||||
// this via a comma-ok type assertion rather than every providerAdapter
|
||||
// implementation growing a no-op method for it.
|
||||
//
|
||||
// Safe under Yaegi despite the documented Trap-4 restriction
|
||||
// (matchesSentinel's own doc comment, limits.go): that trap is a comma-ok
|
||||
// assertion of a COMPILED concrete value (a stdlib type, e.g.
|
||||
// *fmt.wrapErrors or *url.Error) against an INTERPRETER-declared
|
||||
// interface, which silently returns ok=false. Every adapter's dynamic
|
||||
// type behind the providerAdapter interface here (*openaiAdapter,
|
||||
// *anthropicAdapter, *geminiAdapter) is itself plugin-declared and
|
||||
// interpreted right alongside modelMetadataFetcher — an
|
||||
// interpreted-concrete-value-against-interpreted-interface assertion,
|
||||
// the opposite, unaffected direction.
|
||||
type modelMetadataFetcher interface {
|
||||
// fetchModelMetadata returns discovery-captured per-model context
|
||||
// lengths, keyed by upstream model id. A nil map with a nil error
|
||||
// means the adapter has nothing to report (e.g. metadataPath left
|
||||
// unconfigured) — a fast, cheap no-op the caller can invoke
|
||||
// unconditionally on every adapter that implements this interface.
|
||||
fetchModelMetadata(ctx context.Context) (map[string]int, error)
|
||||
}
|
||||
|
||||
// captureModelMetadata is warmFill/refreshProvider's shared, best-effort
|
||||
// discovery-metadata step (feature v0.23): when adapter also implements
|
||||
// modelMetadataFetcher (see that interface's own doc comment for why
|
||||
// this assertion is yaegi-safe), it fetches per-model context lengths
|
||||
// and records them on st. A fetch error, or an adapter that does not
|
||||
// implement the interface at all, is intentionally non-fatal beyond one
|
||||
// log line — per this feature's own "absent/failed = no metadata
|
||||
// captured" ruling, it must never fail discovery itself, which
|
||||
// listModels (warmFill/refreshProvider's own caller) already accounts
|
||||
// for independently.
|
||||
func (m *modelRegistry) captureModelMetadata(ctx context.Context, name string, adapter providerAdapter, st *providerState) {
|
||||
mf, ok := adapter.(modelMetadataFetcher)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
meta, err := mf.fetchModelMetadata(ctx)
|
||||
if err != nil {
|
||||
m.log("%s", fmt.Sprintf("model registry: metadata capture for provider %q failed: %v", name, err))
|
||||
return
|
||||
}
|
||||
if meta == nil {
|
||||
return
|
||||
}
|
||||
st.setDiscoveredContext(meta)
|
||||
}
|
||||
|
||||
// maybeRefresh is called at every ServeHTTP entry. For each discovery-
|
||||
// enabled provider whose last refresh is older than its interval (and that
|
||||
// is not already mid-refresh), it spawns exactly one background goroutine
|
||||
@@ -424,6 +525,7 @@ func (m *modelRegistry) refreshProvider(name string, st *providerState, adapter
|
||||
fctx, cancel := context.WithTimeout(context.Background(), backgroundRefreshTimeout)
|
||||
defer cancel()
|
||||
ids, err = adapter.listModels(fctx)
|
||||
m.captureModelMetadata(fctx, name, adapter, st)
|
||||
}
|
||||
|
||||
// splitConfiguredProvider reports whether id has "prefix/rest" form where
|
||||
@@ -697,7 +799,7 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any {
|
||||
for id, provs := range owners {
|
||||
winner := provs[0]
|
||||
if grp.allowsModel(id) && grp.allowsProvider(winner) {
|
||||
out = append(out, modelObject(id, winner))
|
||||
out = append(out, modelObject(id, winner, m.resolveMetaFor(winner, id)))
|
||||
listed[id] = true
|
||||
}
|
||||
if len(provs) < 2 {
|
||||
@@ -710,7 +812,7 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any {
|
||||
// prefixed form and its bare suffix, same as resolveAgainst does
|
||||
// for the equivalent client request.
|
||||
if (grp.allowsModel(pid) || grp.allowsModel(id)) && grp.allowsProvider(p) {
|
||||
out = append(out, modelObject(pid, p))
|
||||
out = append(out, modelObject(pid, p, m.resolveMetaFor(p, id)))
|
||||
listed[pid] = true
|
||||
}
|
||||
}
|
||||
@@ -746,8 +848,8 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
providerName, _, _ := strings.Cut(canonical, "/")
|
||||
out = append(out, modelObject(alias, providerName))
|
||||
providerName, bareTarget, _ := strings.Cut(canonical, "/")
|
||||
out = append(out, modelObject(alias, providerName, m.resolveMetaForAlias(alias, providerName, bareTarget)))
|
||||
}
|
||||
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
@@ -756,9 +858,73 @@ func (m *modelRegistry) listFor(grp *group) []map[string]any {
|
||||
return out
|
||||
}
|
||||
|
||||
// modelObject builds one OpenAI-compatible model list entry.
|
||||
func modelObject(id, ownedBy string) map[string]any {
|
||||
return map[string]any{"id": id, "object": "model", "owned_by": ownedBy}
|
||||
// modelObject builds one OpenAI-compatible model list entry, extended
|
||||
// with two OpenAI-compat-safe extension fields (feature v0.23):
|
||||
// "context_window" (int) and "pricing" ({"input_per_mtok_usd",
|
||||
// "output_per_mtok_usd"} floats, USD per million tokens) — both omitted
|
||||
// entirely when meta reports them unknown, never emitted as a
|
||||
// misleading zero. "pricing" IS emitted with zeros for an explicitly
|
||||
// free model (meta.CostKnown true, both cost fields 0) — that is a
|
||||
// known, meaningful zero, not an absent one.
|
||||
func modelObject(id, ownedBy string, meta resolvedModelMeta) map[string]any {
|
||||
obj := map[string]any{"id": id, "object": "model", "owned_by": ownedBy}
|
||||
if meta.ContextKnown {
|
||||
obj["context_window"] = meta.ContextTokens
|
||||
}
|
||||
if meta.CostKnown {
|
||||
obj["pricing"] = map[string]any{
|
||||
"input_per_mtok_usd": microUSDPerMTokToUSD(meta.InputCostPerMTokMicroUSD),
|
||||
"output_per_mtok_usd": microUSDPerMTokToUSD(meta.OutputCostPerMTokMicroUSD),
|
||||
}
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
// resolveMetaFor resolves (provider, bareModel)'s metadata (feature
|
||||
// v0.23), supplying this registry's config-override map and that
|
||||
// provider's own discovery-captured context for bareModel.
|
||||
func (m *modelRegistry) resolveMetaFor(provider, bareModel string) resolvedModelMeta {
|
||||
var ctxVal int
|
||||
var ctxKnown bool
|
||||
if st, ok := m.states[provider]; ok {
|
||||
ctxVal, ctxKnown = st.contextFor(bareModel)
|
||||
}
|
||||
return resolveModelMeta(provider, bareModel, m.modelMeta, ctxVal, ctxKnown)
|
||||
}
|
||||
|
||||
// resolveMetaForAliasName resolves alias's metadata for the admin
|
||||
// dashboard's unrestricted view (feature v0.23, hover-detail
|
||||
// refinement): unlike listFor's request-authorized alias handling, this
|
||||
// ignores group authorization entirely (an internal, unrestricted
|
||||
// &group{} — GroupConfig's own documented "empty means all" semantics,
|
||||
// llmgateway.go), matching buildAdminOverview's existing behavior of
|
||||
// showing every configured provider/alias unconditionally, with no
|
||||
// group filtering anywhere else in its response either. Returns an
|
||||
// entirely-unknown resolvedModelMeta when alias is not a configured
|
||||
// alias at all, or its target does not resolve to any known model yet.
|
||||
func (m *modelRegistry) resolveMetaForAliasName(alias string) resolvedModelMeta {
|
||||
target, ok := m.aliases[alias]
|
||||
if !ok {
|
||||
return resolvedModelMeta{}
|
||||
}
|
||||
_, _, canonical, err := m.resolveAliasTarget(alias, target, &group{})
|
||||
if err != nil {
|
||||
return resolvedModelMeta{}
|
||||
}
|
||||
providerName, bareTarget, _ := strings.Cut(canonical, "/")
|
||||
return m.resolveMetaForAlias(alias, providerName, bareTarget)
|
||||
}
|
||||
|
||||
// resolveMetaForAlias resolves alias's metadata (feature v0.23) via
|
||||
// resolveAliasModelMeta: target-inherited unless the alias itself has
|
||||
// its own modelMeta entry — see that function's own doc comment.
|
||||
func (m *modelRegistry) resolveMetaForAlias(alias, targetProvider, targetModel string) resolvedModelMeta {
|
||||
var ctxVal int
|
||||
var ctxKnown bool
|
||||
if st, ok := m.states[targetProvider]; ok {
|
||||
ctxVal, ctxKnown = st.contextFor(targetModel)
|
||||
}
|
||||
return resolveAliasModelMeta(alias, targetProvider, targetModel, m.modelMeta, ctxVal, ctxKnown)
|
||||
}
|
||||
|
||||
// providerSnapshot is one provider's read-only view for the admin
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
package traefikllmgateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeMetaAdapter wraps fakeAdapter, additionally implementing
|
||||
// modelMetadataFetcher (feature v0.23) — a test double for an
|
||||
// openai-type adapter configured with metadataPath, exercising
|
||||
// warmFill/refreshProvider's captureModelMetadata wiring without a real
|
||||
// HTTP fetch. Every other fakeAdapter-consuming test is unaffected: only
|
||||
// tests that explicitly construct a fakeMetaAdapter get this extra
|
||||
// capability.
|
||||
type fakeMetaAdapter struct {
|
||||
*fakeAdapter
|
||||
fetchModelMetadataFn func(context.Context) (map[string]int, error)
|
||||
}
|
||||
|
||||
func newFakeMetaAdapter(name string) *fakeMetaAdapter {
|
||||
return &fakeMetaAdapter{fakeAdapter: newFakeAdapter(name)}
|
||||
}
|
||||
|
||||
func (f *fakeMetaAdapter) fetchModelMetadata(ctx context.Context) (map[string]int, error) {
|
||||
if f.fetchModelMetadataFn == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return f.fetchModelMetadataFn(ctx)
|
||||
}
|
||||
|
||||
// TestModelRegistry_WarmFill_CapturesModelMetadata proves warmFill's
|
||||
// captureModelMetadata wiring: a discovery-enabled provider whose
|
||||
// adapter implements modelMetadataFetcher has its returned per-model
|
||||
// context lengths recorded on providerState, reachable through
|
||||
// resolveMetaFor.
|
||||
func TestModelRegistry_WarmFill_CapturesModelMetadata(t *testing.T) {
|
||||
t.Parallel()
|
||||
fa := newFakeMetaAdapter("lmstudio")
|
||||
fa.listModelsFn = func(context.Context) ([]string, error) {
|
||||
return []string{"qwen2.5-0.5b"}, nil
|
||||
}
|
||||
fa.fetchModelMetadataFn = func(context.Context) (map[string]int, error) {
|
||||
return map[string]int{"qwen2.5-0.5b": 32768}, nil
|
||||
}
|
||||
adapters := map[string]providerAdapter{"lmstudio": fa}
|
||||
cfg := &Config{Providers: map[string]*ProviderConfig{"lmstudio": {Discovery: true, MetadataPath: "/api/v0/models"}}}
|
||||
reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatalf("newModelRegistry: %v", err)
|
||||
}
|
||||
|
||||
reg.warmFill(context.Background())
|
||||
|
||||
ctxVal, ok := reg.states["lmstudio"].contextFor("qwen2.5-0.5b")
|
||||
if !ok || ctxVal != 32768 {
|
||||
t.Errorf("contextFor(qwen2.5-0.5b) = (%d, %v), want (32768, true)", ctxVal, ok)
|
||||
}
|
||||
|
||||
meta := reg.resolveMetaFor("lmstudio", "qwen2.5-0.5b")
|
||||
if !meta.ContextKnown || meta.ContextTokens != 32768 {
|
||||
t.Errorf("resolveMetaFor = %+v, want ContextKnown=true ContextTokens=32768", meta)
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelRegistry_WarmFill_MetadataFetchErrorNonFatal proves a failed
|
||||
// metadata capture is non-fatal to discovery itself (feature v0.23's own
|
||||
// "absent/failed = no metadata captured" ruling): listModels still
|
||||
// succeeds and populates the provider's known model set, one log line
|
||||
// records the metadata failure, and contextFor simply reports unknown.
|
||||
func TestModelRegistry_WarmFill_MetadataFetchErrorNonFatal(t *testing.T) {
|
||||
t.Parallel()
|
||||
fa := newFakeMetaAdapter("lmstudio")
|
||||
fa.listModelsFn = func(context.Context) ([]string, error) {
|
||||
return []string{"m1"}, nil
|
||||
}
|
||||
fa.fetchModelMetadataFn = func(context.Context) (map[string]int, error) {
|
||||
return nil, errUpstream
|
||||
}
|
||||
adapters := map[string]providerAdapter{"lmstudio": fa}
|
||||
cfg := &Config{Providers: map[string]*ProviderConfig{"lmstudio": {Discovery: true, MetadataPath: "/api/v0/models"}}}
|
||||
log := &recordingLog{}
|
||||
reg, err := newModelRegistry(adapters, cfg, log.fn)
|
||||
if err != nil {
|
||||
t.Fatalf("newModelRegistry: %v", err)
|
||||
}
|
||||
|
||||
reg.warmFill(context.Background())
|
||||
|
||||
if !reg.states["lmstudio"].hasModel("m1") {
|
||||
t.Error("discovery's own model set must still populate despite a metadata capture failure")
|
||||
}
|
||||
if _, ok := reg.states["lmstudio"].contextFor("m1"); ok {
|
||||
t.Error("contextFor must report unknown after a failed metadata capture")
|
||||
}
|
||||
if log.count() == 0 {
|
||||
t.Error("want at least one log line recording the metadata capture failure")
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelRegistry_MaybeRefresh_CapturesModelMetadata proves the SAME
|
||||
// wiring runs on the background refresh path (refreshProvider), not just
|
||||
// the synchronous warmFill — captureModelMetadata is shared between both.
|
||||
func TestModelRegistry_MaybeRefresh_CapturesModelMetadata(t *testing.T) {
|
||||
t.Parallel()
|
||||
fa := newFakeMetaAdapter("lmstudio")
|
||||
fa.listModelsFn = func(context.Context) ([]string, error) {
|
||||
return []string{"m1"}, nil
|
||||
}
|
||||
fa.fetchModelMetadataFn = func(context.Context) (map[string]int, error) {
|
||||
return map[string]int{"m1": 8192}, nil
|
||||
}
|
||||
adapters := map[string]providerAdapter{"lmstudio": fa}
|
||||
cfg := &Config{Providers: map[string]*ProviderConfig{"lmstudio": {Discovery: true, DiscoveryInterval: "1h", MetadataPath: "/api/v0/models"}}}
|
||||
reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatalf("newModelRegistry: %v", err)
|
||||
}
|
||||
|
||||
reg.maybeRefresh(context.Background())
|
||||
waitUntil(t, time.Second, func() bool {
|
||||
_, ok := reg.states["lmstudio"].contextFor("m1")
|
||||
return ok
|
||||
})
|
||||
ctxVal, _ := reg.states["lmstudio"].contextFor("m1")
|
||||
if ctxVal != 8192 {
|
||||
t.Errorf("contextFor(m1) = %d, want 8192", ctxVal)
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelRegistry_ListFor_ExposesContextAndPricingFields proves
|
||||
// listFor's modelObject output carries "context_window" and "pricing"
|
||||
// when resolveMetaFor knows them (via a modelMeta config override here),
|
||||
// and omits both entirely for a model nothing knows anything about.
|
||||
func TestModelRegistry_ListFor_ExposesContextAndPricingFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")}
|
||||
cfg := &Config{
|
||||
Providers: map[string]*ProviderConfig{"openai": {Models: []string{"gpt-test", "totally-unknown"}}},
|
||||
ModelMeta: map[string]*ModelMetaConfig{
|
||||
"gpt-test": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1_250_000, OutputCostPerMTokMicroUSD: 10_000_000},
|
||||
},
|
||||
}
|
||||
reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatalf("newModelRegistry: %v", err)
|
||||
}
|
||||
|
||||
got := reg.listFor(allowAllGroup())
|
||||
|
||||
var known, unknown map[string]any
|
||||
for _, entry := range got {
|
||||
switch entry["id"] {
|
||||
case "gpt-test":
|
||||
known = entry
|
||||
case "totally-unknown":
|
||||
unknown = entry
|
||||
}
|
||||
}
|
||||
if known == nil || unknown == nil {
|
||||
t.Fatalf("listFor missing expected entries: %v", got)
|
||||
}
|
||||
|
||||
if got := known["context_window"]; got != 128000 {
|
||||
t.Errorf(`known["context_window"] = %v, want 128000`, got)
|
||||
}
|
||||
pricing, ok := known["pricing"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf(`known["pricing"] = %v, want a map`, known["pricing"])
|
||||
}
|
||||
if pricing["input_per_mtok_usd"] != 1.25 || pricing["output_per_mtok_usd"] != 10.0 {
|
||||
t.Errorf("pricing = %v, want {input_per_mtok_usd:1.25 output_per_mtok_usd:10}", pricing)
|
||||
}
|
||||
|
||||
if _, present := unknown["context_window"]; present {
|
||||
t.Errorf(`unknown["context_window"] present (%v), want omitted entirely`, unknown["context_window"])
|
||||
}
|
||||
if _, present := unknown["pricing"]; present {
|
||||
t.Errorf(`unknown["pricing"] present (%v), want omitted entirely`, unknown["pricing"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelRegistry_ListFor_Alias_InheritsTargetMetadata proves an
|
||||
// alias's listFor entry carries its target's resolved metadata
|
||||
// (modelObject via resolveMetaForAlias), and that an alias-level
|
||||
// modelMeta entry overrides it.
|
||||
func TestModelRegistry_ListFor_Alias_InheritsTargetMetadata(t *testing.T) {
|
||||
t.Parallel()
|
||||
adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")}
|
||||
|
||||
t.Run("inherits target metadata with no alias override", func(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Providers: map[string]*ProviderConfig{"openai": {Models: []string{"gpt-test"}}},
|
||||
ModelAliases: map[string]string{"aliased/coding": "openai/gpt-test"},
|
||||
ModelMeta: map[string]*ModelMetaConfig{
|
||||
"openai/gpt-test": {ContextTokens: 128000, Free: true},
|
||||
},
|
||||
}
|
||||
reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatalf("newModelRegistry: %v", err)
|
||||
}
|
||||
got := reg.listFor(allowAllGroup())
|
||||
var alias map[string]any
|
||||
for _, entry := range got {
|
||||
if entry["id"] == "aliased/coding" {
|
||||
alias = entry
|
||||
}
|
||||
}
|
||||
if alias == nil {
|
||||
t.Fatalf("listFor missing alias entry: %v", got)
|
||||
}
|
||||
if alias["context_window"] != 128000 {
|
||||
t.Errorf(`alias["context_window"] = %v, want 128000 (inherited)`, alias["context_window"])
|
||||
}
|
||||
pricing, ok := alias["pricing"].(map[string]any)
|
||||
if !ok || pricing["input_per_mtok_usd"] != 0.0 {
|
||||
t.Errorf(`alias["pricing"] = %v, want a free (zero) pricing object (inherited)`, alias["pricing"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("alias-level override wins over inherited context", func(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Providers: map[string]*ProviderConfig{"openai": {Models: []string{"gpt-test"}}},
|
||||
ModelAliases: map[string]string{"aliased/coding": "openai/gpt-test"},
|
||||
ModelMeta: map[string]*ModelMetaConfig{
|
||||
"openai/gpt-test": {ContextTokens: 128000},
|
||||
"aliased/coding": {ContextTokens: 999},
|
||||
},
|
||||
}
|
||||
reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatalf("newModelRegistry: %v", err)
|
||||
}
|
||||
got := reg.listFor(allowAllGroup())
|
||||
var alias map[string]any
|
||||
for _, entry := range got {
|
||||
if entry["id"] == "aliased/coding" {
|
||||
alias = entry
|
||||
}
|
||||
}
|
||||
if alias == nil {
|
||||
t.Fatalf("listFor missing alias entry: %v", got)
|
||||
}
|
||||
if alias["context_window"] != 999 {
|
||||
t.Errorf(`alias["context_window"] = %v, want 999 (alias override)`, alias["context_window"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestModelRegistry_ResolveMetaForAliasName covers the admin dashboard's
|
||||
// unrestricted alias-metadata resolution directly: an unconfigured
|
||||
// alias, one whose target has not resolved yet, and a real, resolvable
|
||||
// one.
|
||||
func TestModelRegistry_ResolveMetaForAliasName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("unconfigured alias reports everything unknown", func(t *testing.T) {
|
||||
reg := newAliasTestRegistry(t, map[string]string{"aliased/coding": "openai/gpt-test"}, nil)
|
||||
got := reg.resolveMetaForAliasName("not-an-alias")
|
||||
if got.ContextKnown || got.CostKnown {
|
||||
t.Errorf("got %+v, want everything unknown", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unresolved target reports everything unknown", func(t *testing.T) {
|
||||
reg := newAliasTestRegistry(t, map[string]string{"aliased/future": "openai/not-yet-discovered"}, nil)
|
||||
got := reg.resolveMetaForAliasName("aliased/future")
|
||||
if got.ContextKnown || got.CostKnown {
|
||||
t.Errorf("got %+v, want everything unknown", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("resolvable alias ignores group authorization entirely", func(t *testing.T) {
|
||||
adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")}
|
||||
cfg := &Config{
|
||||
Providers: map[string]*ProviderConfig{"openai": {Models: []string{"gpt-test"}}},
|
||||
ModelAliases: map[string]string{"aliased/coding": "openai/gpt-test"},
|
||||
ModelMeta: map[string]*ModelMetaConfig{"openai/gpt-test": {ContextTokens: 55555}},
|
||||
}
|
||||
reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatalf("newModelRegistry: %v", err)
|
||||
}
|
||||
got := reg.resolveMetaForAliasName("aliased/coding")
|
||||
if !got.ContextKnown || got.ContextTokens != 55555 {
|
||||
t.Errorf("got %+v, want ContextKnown=true ContextTokens=55555", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package traefikllmgateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestHandleModels_ContextWindowAndPricing_FieldPresenceOmission is the
|
||||
// HTTP-level counterpart to registry_modelmeta_test.go's listFor
|
||||
// coverage: a real GET /v1/models request through the full Gateway,
|
||||
// proving the OpenAI-compat-safe extension fields (feature v0.23:
|
||||
// "context_window", "pricing") are present with the expected values for
|
||||
// a model with known metadata, present-with-zeros for an explicitly free
|
||||
// one, and entirely absent (not present as null or 0) for a model
|
||||
// nothing knows anything about.
|
||||
func TestHandleModels_ContextWindowAndPricing_FieldPresenceOmission(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := CreateConfig()
|
||||
cfg.Providers = map[string]*ProviderConfig{
|
||||
"openai": {Type: "openai", APIKey: "sk-test", Models: []string{"gpt-test", "free-model", "unknown-model"}},
|
||||
}
|
||||
cfg.Groups = map[string]*GroupConfig{"default": {}}
|
||||
cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "tester", Group: "default", APIKey: "sk-user"}}}
|
||||
cfg.ModelMeta = map[string]*ModelMetaConfig{
|
||||
"gpt-test": {ContextTokens: 128000, InputCostPerMTokMicroUSD: 1_250_000, OutputCostPerMTokMicroUSD: 10_000_000},
|
||||
"free-model": {ContextTokens: 32768, Free: true},
|
||||
}
|
||||
|
||||
h, err := New(context.Background(), http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), cfg, "llmgw")
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
||||
req.Header.Set("Authorization", "Bearer sk-user")
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Data []map[string]any `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
|
||||
byID := make(map[string]map[string]any, len(body.Data))
|
||||
for _, entry := range body.Data {
|
||||
id, _ := entry["id"].(string)
|
||||
byID[id] = entry
|
||||
}
|
||||
|
||||
priced, ok := byID["gpt-test"]
|
||||
if !ok {
|
||||
t.Fatalf("response missing gpt-test: %v", body.Data)
|
||||
}
|
||||
if priced["context_window"] != 128000.0 {
|
||||
t.Errorf(`gpt-test["context_window"] = %v, want 128000`, priced["context_window"])
|
||||
}
|
||||
pricing, ok := priced["pricing"].(map[string]any)
|
||||
if !ok || pricing["input_per_mtok_usd"] != 1.25 || pricing["output_per_mtok_usd"] != 10.0 {
|
||||
t.Errorf(`gpt-test["pricing"] = %v, want {input_per_mtok_usd:1.25 output_per_mtok_usd:10}`, priced["pricing"])
|
||||
}
|
||||
|
||||
free, ok := byID["free-model"]
|
||||
if !ok {
|
||||
t.Fatalf("response missing free-model: %v", body.Data)
|
||||
}
|
||||
freePricing, ok := free["pricing"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf(`free-model["pricing"] missing, want present with zeros (explicit free, not absent)`)
|
||||
}
|
||||
if freePricing["input_per_mtok_usd"] != 0.0 || freePricing["output_per_mtok_usd"] != 0.0 {
|
||||
t.Errorf(`free-model["pricing"] = %v, want zeros`, freePricing)
|
||||
}
|
||||
|
||||
unknown, ok := byID["unknown-model"]
|
||||
if !ok {
|
||||
t.Fatalf("response missing unknown-model: %v", body.Data)
|
||||
}
|
||||
if _, present := unknown["context_window"]; present {
|
||||
t.Errorf(`unknown-model["context_window"] present (%v), want entirely omitted`, unknown["context_window"])
|
||||
}
|
||||
if _, present := unknown["pricing"]; present {
|
||||
t.Errorf(`unknown-model["pricing"] present (%v), want entirely omitted`, unknown["pricing"])
|
||||
}
|
||||
}
|
||||
@@ -88,6 +88,12 @@ const (
|
||||
slowProviderModel = "gpt-test"
|
||||
slowUpstreamSleep = 600 * time.Millisecond
|
||||
slowRequestDeadline = 60 * time.Millisecond
|
||||
// yaegiMetaContextTokens is the modelMeta config-override
|
||||
// contextTokens value the attempt-accounting harness sets for
|
||||
// testDataWantModel (feature v0.23) — a string, not an int, since it
|
||||
// is spliced directly into attemptAccountingOverride's hand-built
|
||||
// JSON text below.
|
||||
yaegiMetaContextTokens = "128000"
|
||||
)
|
||||
|
||||
// excludedTopLevelDirs lists repo-root directories the GOPATH copy must
|
||||
@@ -221,6 +227,15 @@ func run() error {
|
||||
|
||||
attemptAccountingOverride := `{"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `"]},"` +
|
||||
slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]}},` +
|
||||
// modelMeta (feature v0.23): a config-override entry for
|
||||
// testDataWantModel, so exerciseHandler's GET /v1/models
|
||||
// assertion below proves resolveModelMeta's config-override
|
||||
// layer, modelObject's context_window/pricing extension fields,
|
||||
// and the whole registry.go/modelmeta.go resolution path all
|
||||
// run correctly INTERPRETED, not merely compiled — cheap to add
|
||||
// to this existing harness request, per this feature's own gate
|
||||
// requirement.
|
||||
`"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000}},` +
|
||||
`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true}]}}`
|
||||
if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil {
|
||||
return fmt.Errorf("decode attempt-accounting harness override into the interpreted Config: %w", err)
|
||||
@@ -277,6 +292,19 @@ func exerciseHandler(handler http.Handler) error {
|
||||
if !strings.Contains(authedRec.Body.String(), testDataWantModel) {
|
||||
return fmt.Errorf("GET /v1/models body does not contain %q: %s", testDataWantModel, authedRec.Body.String())
|
||||
}
|
||||
// modelMeta (feature v0.23), interpreted: run() layers a modelMeta
|
||||
// config override for testDataWantModel onto cfgVal before New() is
|
||||
// ever called (attemptAccountingOverride above) — proving under the
|
||||
// real Yaegi interpreter, not just `go test`, that resolveModelMeta's
|
||||
// config-override layer and modelObject's context_window/pricing
|
||||
// extension fields both run correctly through registry.go's
|
||||
// listFor/resolveMetaFor.
|
||||
if !strings.Contains(authedRec.Body.String(), `"context_window":128000`) {
|
||||
return fmt.Errorf("GET /v1/models body does not contain the modelMeta-resolved context_window (feature v0.23 metadata harness): %s", authedRec.Body.String())
|
||||
}
|
||||
if !strings.Contains(authedRec.Body.String(), `"input_per_mtok_usd":1.25`) || !strings.Contains(authedRec.Body.String(), `"output_per_mtok_usd":10`) {
|
||||
return fmt.Errorf("GET /v1/models body does not contain the modelMeta-resolved pricing (feature v0.23 metadata harness): %s", authedRec.Body.String())
|
||||
}
|
||||
|
||||
if err := exerciseAttemptAccounting(handler); err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user