mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-21 05:57:53 +00:00
perf(registry): cache the encoded /v1/models response per group
GET /v1/models cost 15.9ms, 21.3MB, 552,884 allocs per call at the
live 1,033-model catalog (registry.go's listFor plus routes_unified.go's
json.Encoder), uncached — and OpenAI SDK clients poll this endpoint per
session, so an unchanged catalog was being re-walked and re-marshaled
on every request.
Add modelRegistry.modelsJSON(grp), caching the fully encoded
{object:list,data:[...]} body per (*group, modelsGen). modelsGen
is an atomic counter bumped by a new modelRegistry.finishRefresh
wrapper around providerState.finishRefresh — the only two places that
ever mutate a provider's discovered set or discovery-captured metadata
(warmFill, refreshProvider) already funnel through providerState.
finishRefresh once per refresh cycle, so wrapping it is sufficient to
invalidate the cache on every possible change, including a
metadata-only update on an otherwise-failed refresh (captureModelMetadata
runs unconditionally alongside finishRefresh in both call sites).
Bumped unconditionally rather than only on success, since a failed
listModels fetch can still pair with a successful metadata capture.
Cached per *group pointer: authStore builds every group once, at
construction, and never rebuilds or replaces that map afterward — a
config reload constructs an entirely new Gateway (and modelRegistry)
with its own empty cache, so a *group key can never alias a different
group's catalog either within one registry's lifetime or across a
reload.
routes_unified.go's handleModels call site shrinks to a single write of
modelsJSON's result, replacing the previous inline listFor+json.Encode;
routes_unified.go is otherwise untouched (owned by a sibling change in
this same effort).
This commit is contained in:
+6
-5
@@ -39,7 +39,11 @@ const gatewayAliasKey = "__alias"
|
||||
|
||||
// handleModels implements GET /v1/models: an authenticated caller gets back
|
||||
// an OpenAI-compatible {"object":"list","data":[...]} envelope of the
|
||||
// models their group can see, via modelRegistry.listFor.
|
||||
// models their group can see, via modelRegistry.modelsJSON (perf finding
|
||||
// 3: modelsJSON caches the encoded body per (group, discovery generation)
|
||||
// so a request between discovery refreshes never re-walks listFor/
|
||||
// re-marshals — registry.go owns the cache; this call site just writes
|
||||
// its result).
|
||||
func (g *Gateway) handleModels(w http.ResponseWriter, r *http.Request) {
|
||||
u, grp, ok := g.auth.identify(r)
|
||||
g.logAuthEvent(ok, authEventUserName(u), r)
|
||||
@@ -49,10 +53,7 @@ func (g *Gateway) handleModels(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{ // headers already committed; nothing useful to do on encode failure
|
||||
"object": "list",
|
||||
"data": g.registry.listFor(grp),
|
||||
})
|
||||
_, _ = w.Write(g.registry.modelsJSON(grp)) // headers already committed; nothing useful to do on write failure
|
||||
}
|
||||
|
||||
// adapterCall abstracts providerAdapter.chatCompletion and .embeddings:
|
||||
|
||||
Reference in New Issue
Block a user