mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-19 05:47:48 +00:00
feat(registry): model metadata capture and resolution
Config gets two new surfaces (llmgateway.go): Config.ModelMeta (map[string]*ModelMetaConfig, keyed by exact "provider/model" or a bare model id) for operator overrides, and ProviderConfig.MetadataPath for openai-type providers whose discovery should also capture per-model context (LM Studio's native /api/v0/models shape). modelmeta.go adds the layering itself: resolveModelMeta applies config override > discovered context > builtin table > absent for context, and config override/free > builtin > absent for cost (discovery never reports cost). resolveAliasModelMeta implements alias inheritance: an alias inherits its target's fully resolved metadata unless the alias itself has its own modelMeta entry, applied per field. validateModelMeta rejects negative values and a free:true/non-zero-cost contradiction at construction. registry.go wires this into the request path: providerState gains a discoveredContext map (stale-while-error, mirroring discovered's own convention) with contextFor/setDiscoveredContext; a new optional modelMetadataFetcher interface, type-asserted (interpreted-to-interpreted, documented as yaegi-safe against the Trap-4 restriction) rather than added to providerAdapter, lets warmFill/refreshProvider capture metadata alongside listModels without touching anthropic/gemini adapters at all. modelObject/listFor now attach resolved context_window/pricing to every GET /v1/models entry, including aliases (resolveMetaFor/ resolveMetaForAlias/resolveMetaForAliasName). provider_openai.go implements modelMetadataFetcher: fetchModelMetadata reads ProviderConfig.MetadataPath, preferring loaded_context_length over max_context_length when the LM Studio response reports both. providers.go wires MetadataPath onto the adapter in buildAdapters. tools/yaegi-check/main.go adds one cheap modelMeta config-override assertion to the existing attempt-accounting harness, proving resolveModelMeta's config-override layer and the context_window/pricing extension fields both run correctly under the real Yaegi interpreter.
This commit is contained in:
@@ -23,6 +23,12 @@ type openaiAdapter struct {
|
||||
adapterName string
|
||||
baseURL string
|
||||
apiKey string
|
||||
// metadataPath is ProviderConfig.MetadataPath (feature v0.23), set by
|
||||
// buildAdapters (providers.go) after construction — empty (the
|
||||
// default) makes fetchModelMetadata a no-op, so every existing
|
||||
// newOpenAIAdapter call site (every test, and any caller that never
|
||||
// sets this) keeps behaving exactly as before this feature existed.
|
||||
metadataPath string
|
||||
}
|
||||
|
||||
// newOpenAIAdapter returns an openaiAdapter for provider name, with base as
|
||||
@@ -401,3 +407,72 @@ func (a *openaiAdapter) listModels(ctx context.Context) ([]string, error) {
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// lmStudioModelsPayload is the wire shape of LM Studio's own, native
|
||||
// models endpoint (ProviderConfig.MetadataPath — a real deployment sets
|
||||
// it to "/api/v0/models"), verified live against a running LM Studio
|
||||
// instance (2026-08): {"data":[{"id":...,"max_context_length":262144,
|
||||
// "loaded_context_length":4096}, ...]}. Distinct from modelsPayload
|
||||
// above, which is OpenAI's own /v1/models shape (LM Studio also serves
|
||||
// that, unchanged, but it carries no per-model context metadata) — this
|
||||
// is LM Studio's separate, additional endpoint this feature (v0.23)
|
||||
// opts into reading.
|
||||
type lmStudioModelsPayload struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
MaxContextLength int `json:"max_context_length"`
|
||||
LoadedContextLength int `json:"loaded_context_length"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// fetchModelMetadata implements modelMetadataFetcher (feature v0.23,
|
||||
// registry.go): when a.metadataPath is configured, fetches it and
|
||||
// returns each reported model id's discovery-captured context length.
|
||||
// It prefers loaded_context_length when the upstream reports one greater
|
||||
// than zero — the context actually usable right now, which a runtime can
|
||||
// configure smaller than the model's own maximum under VRAM-constrained
|
||||
// settings — and falls back to max_context_length otherwise. a.
|
||||
// metadataPath left empty (the default; every provider except an
|
||||
// operator's opt-in) is a fast no-op: (nil, nil), never an error, so
|
||||
// registry.go's captureModelMetadata can invoke this unconditionally on
|
||||
// every openai-type adapter without a config check of its own. A
|
||||
// non-2xx response or a decode failure returns an error — the caller
|
||||
// treats it as non-fatal, per this feature's own "absent/failed = no
|
||||
// metadata captured" ruling.
|
||||
func (a *openaiAdapter) fetchModelMetadata(ctx context.Context) (map[string]int, error) {
|
||||
if a.metadataPath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
resp, err := upstreamJSON(ctx, a.client, http.MethodGet, a.baseURL+a.metadataPath, a.requestHeaders(false), nil, a.retry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, newProviderHTTPError(resp)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: read metadata response body: %w", errUpstream, err)
|
||||
}
|
||||
|
||||
var payload lmStudioModelsPayload
|
||||
if err := json.Unmarshal(body, &payload); err != nil {
|
||||
return nil, fmt.Errorf("%w: decode metadata response: %w", errUpstream, err)
|
||||
}
|
||||
|
||||
out := make(map[string]int, len(payload.Data))
|
||||
for _, d := range payload.Data {
|
||||
ctxLen := d.MaxContextLength
|
||||
if d.LoadedContextLength > 0 {
|
||||
ctxLen = d.LoadedContextLength
|
||||
}
|
||||
if ctxLen > 0 {
|
||||
out[d.ID] = ctxLen
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user