mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-20 05:56:56 +00:00
feat(admin): add fleet statistics and admin read endpoints
Counters (always on when admin is enabled): - last seen per user and group, written asynchronously after admission, throttled to one write per scope per minute per replica - hourly provider and model attempts, failures and timeouts - failovers per provider, rejections per hour, and unpriced (402) refusals - cache hits, misses and cost saved - request counts per target and caller Opt-in counters under admin.stats (both default off): - userModel: usage per user and model - latency: latency histograms per provider and model All counters ride the existing Redis batches. A test measures three round trips per request with defaults and with every statistic on. A cache hit takes two. A model request is no longer counted for failed failover candidates. New admin endpoints: - usage/series and usage/totals - performance, with percentiles from the histograms - catalog, with the price billing actually uses and its source - consumers, with key source, grants and model access - config, with secrets redacted The overview gains features and a pricing summary. Usage gains lastSeen. usage/models gains offset and more detail fields. Reads are chunked at 1600 keys per round trip and capped at 64000 keys per request. Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
This commit is contained in:
+50
-17
@@ -127,11 +127,18 @@ func (g *Gateway) handleImagesGenerations(w http.ResponseWriter, r *http.Request
|
||||
ctx := withAttemptRecorder(r.Context(), func(resp *http.Response, attemptErr error) {
|
||||
g.limiter.recordProviderAttempt(adapter.name(), upstreamModel, resp, attemptErr)
|
||||
})
|
||||
// feat: instrument upstream latency — gated on metricsEnabled, same
|
||||
// reasoning as runUnified's identical wiring (routes_unified.go).
|
||||
if metricsEnabled(g.cfg) {
|
||||
// feat: instrument upstream latency — gated on metricsEnabled OR
|
||||
// admin.stats.latency, same reasoning as runUnified's identical wiring
|
||||
// (routes_unified.go).
|
||||
var latSample latencySample
|
||||
var hasLat bool
|
||||
if metricsEnabled(g.cfg) || g.limiter.statsLatency {
|
||||
ctx = withLatencyRecorder(ctx, func(sample latencySample) {
|
||||
g.recordLatency(adapter.name(), upstreamModel, sample)
|
||||
latSample = sample
|
||||
hasLat = true
|
||||
if metricsEnabled(g.cfg) {
|
||||
g.recordLatency(adapter.name(), upstreamModel, sample)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := adapter.imagesGeneration(ctx, sw, req); err != nil {
|
||||
@@ -139,7 +146,7 @@ func (g *Gateway) handleImagesGenerations(w http.ResponseWriter, r *http.Request
|
||||
g.handleAdapterError(sw, err, adapter.name())
|
||||
return
|
||||
}
|
||||
g.recordMediaModelUsage(adapter.name(), upstreamModel)
|
||||
g.recordMediaModelUsage(adapter.name(), upstreamModel, latSample, hasLat)
|
||||
}
|
||||
|
||||
// handleAudioSpeech implements POST /v1/audio/speech (spec §3): an
|
||||
@@ -196,11 +203,18 @@ func (g *Gateway) handleAudioSpeech(w http.ResponseWriter, r *http.Request, u *u
|
||||
ctx := withAttemptRecorder(r.Context(), func(resp *http.Response, attemptErr error) {
|
||||
g.limiter.recordProviderAttempt(adapter.name(), upstreamModel, resp, attemptErr)
|
||||
})
|
||||
// feat: instrument upstream latency — gated on metricsEnabled, same
|
||||
// reasoning as runUnified's identical wiring (routes_unified.go).
|
||||
if metricsEnabled(g.cfg) {
|
||||
// feat: instrument upstream latency — gated on metricsEnabled OR
|
||||
// admin.stats.latency, same reasoning as runUnified's identical wiring
|
||||
// (routes_unified.go).
|
||||
var latSample latencySample
|
||||
var hasLat bool
|
||||
if metricsEnabled(g.cfg) || g.limiter.statsLatency {
|
||||
ctx = withLatencyRecorder(ctx, func(sample latencySample) {
|
||||
g.recordLatency(adapter.name(), upstreamModel, sample)
|
||||
latSample = sample
|
||||
hasLat = true
|
||||
if metricsEnabled(g.cfg) {
|
||||
g.recordLatency(adapter.name(), upstreamModel, sample)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := adapter.audioSpeech(ctx, sw, body, "application/json"); err != nil {
|
||||
@@ -208,7 +222,7 @@ func (g *Gateway) handleAudioSpeech(w http.ResponseWriter, r *http.Request, u *u
|
||||
g.handleAdapterError(sw, err, adapter.name())
|
||||
return
|
||||
}
|
||||
g.recordMediaModelUsage(adapter.name(), upstreamModel)
|
||||
g.recordMediaModelUsage(adapter.name(), upstreamModel, latSample, hasLat)
|
||||
}
|
||||
|
||||
// handleAudioTranscriptions implements POST /v1/audio/transcriptions
|
||||
@@ -304,11 +318,18 @@ func (g *Gateway) handleAudioTranscriptions(w http.ResponseWriter, r *http.Reque
|
||||
ctx := withAttemptRecorder(r.Context(), func(resp *http.Response, attemptErr error) {
|
||||
g.limiter.recordProviderAttempt(adapter.name(), upstreamModel, resp, attemptErr)
|
||||
})
|
||||
// feat: instrument upstream latency — gated on metricsEnabled, same
|
||||
// reasoning as runUnified's identical wiring (routes_unified.go).
|
||||
if metricsEnabled(g.cfg) {
|
||||
// feat: instrument upstream latency — gated on metricsEnabled OR
|
||||
// admin.stats.latency, same reasoning as runUnified's identical wiring
|
||||
// (routes_unified.go).
|
||||
var latSample latencySample
|
||||
var hasLat bool
|
||||
if metricsEnabled(g.cfg) || g.limiter.statsLatency {
|
||||
ctx = withLatencyRecorder(ctx, func(sample latencySample) {
|
||||
g.recordLatency(adapter.name(), upstreamModel, sample)
|
||||
latSample = sample
|
||||
hasLat = true
|
||||
if metricsEnabled(g.cfg) {
|
||||
g.recordLatency(adapter.name(), upstreamModel, sample)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := adapter.audioTranscription(ctx, sw, uploadBody, uploadContentType); err != nil {
|
||||
@@ -316,7 +337,7 @@ func (g *Gateway) handleAudioTranscriptions(w http.ResponseWriter, r *http.Reque
|
||||
g.handleAdapterError(sw, err, adapter.name())
|
||||
return
|
||||
}
|
||||
g.recordMediaModelUsage(adapter.name(), upstreamModel)
|
||||
g.recordMediaModelUsage(adapter.name(), upstreamModel, latSample, hasLat)
|
||||
}
|
||||
|
||||
// recordMediaModelUsage writes ONLY the kindModel scope's own request
|
||||
@@ -344,9 +365,21 @@ func (g *Gateway) handleAudioTranscriptions(w http.ResponseWriter, r *http.Reque
|
||||
// the same single-round-trip account already batches for every other
|
||||
// scope, called synchronously here, the same as every other account call
|
||||
// site (routes_unified.go, routes_passthrough.go).
|
||||
func (g *Gateway) recordMediaModelUsage(providerName, upstreamModel string) {
|
||||
//
|
||||
// lat/hasLat (admin-redesign WP-A) carry the same completed-upstream
|
||||
// latency sample runUnified/handlePassthrough thread into their own
|
||||
// accountWith call — recordMediaModelUsage passes them through via
|
||||
// accountExtras.provider/lat/hasLat, exactly like those two routes, so a
|
||||
// media route's own opt-in latency counters (admin.stats.latency) get
|
||||
// written from the identical call site, never a second one.
|
||||
func (g *Gateway) recordMediaModelUsage(providerName, upstreamModel string, lat latencySample, hasLat bool) {
|
||||
canonical := providerModelScopeID(providerName, upstreamModel)
|
||||
g.limiter.account(withModelScope(nil, canonical), usage{}, 0)
|
||||
extras := accountExtras{provider: providerName}
|
||||
if hasLat {
|
||||
extras.lat = lat
|
||||
extras.hasLat = true
|
||||
}
|
||||
g.limiter.accountWith(withModelScope(nil, canonical), usage{}, 0, extras)
|
||||
}
|
||||
|
||||
// readAdmittedCapped claims a body-admission slot (acquireBodyAdmission),
|
||||
|
||||
Reference in New Issue
Block a user