mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-16 05:37:28 +00:00
feat(registry): model metadata capture and resolution
Config gets two new surfaces (llmgateway.go): Config.ModelMeta (map[string]*ModelMetaConfig, keyed by exact "provider/model" or a bare model id) for operator overrides, and ProviderConfig.MetadataPath for openai-type providers whose discovery should also capture per-model context (LM Studio's native /api/v0/models shape). modelmeta.go adds the layering itself: resolveModelMeta applies config override > discovered context > builtin table > absent for context, and config override/free > builtin > absent for cost (discovery never reports cost). resolveAliasModelMeta implements alias inheritance: an alias inherits its target's fully resolved metadata unless the alias itself has its own modelMeta entry, applied per field. validateModelMeta rejects negative values and a free:true/non-zero-cost contradiction at construction. registry.go wires this into the request path: providerState gains a discoveredContext map (stale-while-error, mirroring discovered's own convention) with contextFor/setDiscoveredContext; a new optional modelMetadataFetcher interface, type-asserted (interpreted-to-interpreted, documented as yaegi-safe against the Trap-4 restriction) rather than added to providerAdapter, lets warmFill/refreshProvider capture metadata alongside listModels without touching anthropic/gemini adapters at all. modelObject/listFor now attach resolved context_window/pricing to every GET /v1/models entry, including aliases (resolveMetaFor/ resolveMetaForAlias/resolveMetaForAliasName). provider_openai.go implements modelMetadataFetcher: fetchModelMetadata reads ProviderConfig.MetadataPath, preferring loaded_context_length over max_context_length when the LM Studio response reports both. providers.go wires MetadataPath onto the adapter in buildAdapters. tools/yaegi-check/main.go adds one cheap modelMeta config-override assertion to the existing attempt-accounting harness, proving resolveModelMeta's config-override layer and the context_window/pricing extension fields both run correctly under the real Yaegi interpreter.
This commit is contained in:
@@ -88,6 +88,12 @@ const (
|
||||
slowProviderModel = "gpt-test"
|
||||
slowUpstreamSleep = 600 * time.Millisecond
|
||||
slowRequestDeadline = 60 * time.Millisecond
|
||||
// yaegiMetaContextTokens is the modelMeta config-override
|
||||
// contextTokens value the attempt-accounting harness sets for
|
||||
// testDataWantModel (feature v0.23) — a string, not an int, since it
|
||||
// is spliced directly into attemptAccountingOverride's hand-built
|
||||
// JSON text below.
|
||||
yaegiMetaContextTokens = "128000"
|
||||
)
|
||||
|
||||
// excludedTopLevelDirs lists repo-root directories the GOPATH copy must
|
||||
@@ -221,6 +227,15 @@ func run() error {
|
||||
|
||||
attemptAccountingOverride := `{"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `"]},"` +
|
||||
slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]}},` +
|
||||
// modelMeta (feature v0.23): a config-override entry for
|
||||
// testDataWantModel, so exerciseHandler's GET /v1/models
|
||||
// assertion below proves resolveModelMeta's config-override
|
||||
// layer, modelObject's context_window/pricing extension fields,
|
||||
// and the whole registry.go/modelmeta.go resolution path all
|
||||
// run correctly INTERPRETED, not merely compiled — cheap to add
|
||||
// to this existing harness request, per this feature's own gate
|
||||
// requirement.
|
||||
`"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000}},` +
|
||||
`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true}]}}`
|
||||
if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil {
|
||||
return fmt.Errorf("decode attempt-accounting harness override into the interpreted Config: %w", err)
|
||||
@@ -277,6 +292,19 @@ func exerciseHandler(handler http.Handler) error {
|
||||
if !strings.Contains(authedRec.Body.String(), testDataWantModel) {
|
||||
return fmt.Errorf("GET /v1/models body does not contain %q: %s", testDataWantModel, authedRec.Body.String())
|
||||
}
|
||||
// modelMeta (feature v0.23), interpreted: run() layers a modelMeta
|
||||
// config override for testDataWantModel onto cfgVal before New() is
|
||||
// ever called (attemptAccountingOverride above) — proving under the
|
||||
// real Yaegi interpreter, not just `go test`, that resolveModelMeta's
|
||||
// config-override layer and modelObject's context_window/pricing
|
||||
// extension fields both run correctly through registry.go's
|
||||
// listFor/resolveMetaFor.
|
||||
if !strings.Contains(authedRec.Body.String(), `"context_window":128000`) {
|
||||
return fmt.Errorf("GET /v1/models body does not contain the modelMeta-resolved context_window (feature v0.23 metadata harness): %s", authedRec.Body.String())
|
||||
}
|
||||
if !strings.Contains(authedRec.Body.String(), `"input_per_mtok_usd":1.25`) || !strings.Contains(authedRec.Body.String(), `"output_per_mtok_usd":10`) {
|
||||
return fmt.Errorf("GET /v1/models body does not contain the modelMeta-resolved pricing (feature v0.23 metadata harness): %s", authedRec.Body.String())
|
||||
}
|
||||
|
||||
if err := exerciseAttemptAccounting(handler); err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user