mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-19 05:47:48 +00:00
feat(admin): add fleet statistics and admin read endpoints
Counters (always on when admin is enabled): - last seen per user and group, written asynchronously after admission, throttled to one write per scope per minute per replica - hourly provider and model attempts, failures and timeouts - failovers per provider, rejections per hour, and unpriced (402) refusals - cache hits, misses and cost saved - request counts per target and caller Opt-in counters under admin.stats (both default off): - userModel: usage per user and model - latency: latency histograms per provider and model All counters ride the existing Redis batches. A test measures three round trips per request with defaults and with every statistic on. A cache hit takes two. A model request is no longer counted for failed failover candidates. New admin endpoints: - usage/series and usage/totals - performance, with percentiles from the histograms - catalog, with the price billing actually uses and its source - consumers, with key source, grants and model access - config, with secrets redacted The overview gains features and a pricing summary. Usage gains lastSeen. usage/models gains offset and more detail fields. Reads are chunked at 1600 keys per round trip and capped at 64000 keys per request. Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
This commit is contained in:
+38
-8
@@ -1196,6 +1196,9 @@ func (g *Gateway) handlePassthrough(w http.ResponseWriter, r *http.Request, u *u
|
||||
writeLimitViolation(w, violation)
|
||||
return
|
||||
}
|
||||
// P9 fix (admin dashboard redesign verify round): last-seen only for
|
||||
// an admitted request — recordLastSeen's own doc comment (limits.go).
|
||||
g.limiter.recordLastSeen(scopes, g.limiter.now())
|
||||
|
||||
// providerName is only ever passed here by ServeHTTP's dispatch, which
|
||||
// already confirmed it is a key of g.adapters before calling in.
|
||||
@@ -1247,14 +1250,23 @@ func (g *Gateway) handlePassthrough(w http.ResponseWriter, r *http.Request, u *u
|
||||
r = r.WithContext(withAttemptRecorder(r.Context(), func(resp *http.Response, attemptErr error) {
|
||||
g.limiter.recordProviderAttempt(providerName, "", resp, attemptErr)
|
||||
}))
|
||||
// feat: instrument upstream latency — gated on metricsEnabled, same
|
||||
// reasoning as runUnified's identical wiring (routes_unified.go). No
|
||||
// model here either, for the identical reason the attemptRecorder
|
||||
// above passes "": the upstream model lives in the response body,
|
||||
// read only after this attempt already resolved (comment above).
|
||||
if metricsEnabled(g.cfg) {
|
||||
// feat: instrument upstream latency — gated on metricsEnabled OR
|
||||
// admin.stats.latency (admin-redesign WP-A), same reasoning as
|
||||
// runUnified's identical wiring (routes_unified.go). No model here
|
||||
// either, for the identical reason the attemptRecorder above passes
|
||||
// "": the upstream model lives in the response body, read only after
|
||||
// this attempt already resolved (comment above) — so accountWith's own
|
||||
// opt-in latency block below only ever writes this request's PROVIDER
|
||||
// bucket, never a model one, for native passthrough.
|
||||
var latSample latencySample
|
||||
var hasLat bool
|
||||
if metricsEnabled(g.cfg) || g.limiter.statsLatency {
|
||||
r = r.WithContext(withLatencyRecorder(r.Context(), func(sample latencySample) {
|
||||
g.recordLatency(providerName, "", sample)
|
||||
latSample = sample
|
||||
hasLat = true
|
||||
if metricsEnabled(g.cfg) {
|
||||
g.recordLatency(providerName, "", sample)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -1335,7 +1347,25 @@ func (g *Gateway) handlePassthrough(w http.ResponseWriter, r *http.Request, u *u
|
||||
// withModelScope drops the model scope when respModel is empty (the
|
||||
// upstream reported no model id), so a passthrough reply the gateway
|
||||
// cannot attribute still accounts to user/group/total as before.
|
||||
g.limiter.account(withModelScope(scopes, canonical), respUsage, cost)
|
||||
//
|
||||
// noModelHistograms: true (P13 fix, admin dashboard redesign verify
|
||||
// round; plan §2(c): passthrough gets "provider histograms only") —
|
||||
// canonical here is providerName + "/" + an upstream-ECHOED model id,
|
||||
// not one of this gateway's own configured/catalogued models, so
|
||||
// keying the opt-in latency and umodel counter families off it would
|
||||
// grow both families' cardinality by whatever an upstream (or a
|
||||
// client spoofing its response shape) chooses to report, for entries
|
||||
// that can never appear in usage/models or usermodel totals anyway —
|
||||
// both rank off the configured catalog, never an arbitrary passthrough
|
||||
// id. The base model-scope req/tokin/tokout/cost counters below are
|
||||
// unaffected; only the two per-model-keyed opt-in families are
|
||||
// suppressed.
|
||||
extras := accountExtras{provider: providerName, noModelHistograms: true}
|
||||
if hasLat {
|
||||
extras.lat = latSample
|
||||
extras.hasLat = true
|
||||
}
|
||||
g.limiter.accountWith(withModelScope(scopes, canonical), respUsage, cost, extras)
|
||||
}
|
||||
|
||||
// proxyResult is what proxyUpstream reports back to its caller once it has
|
||||
|
||||
Reference in New Issue
Block a user