feat(admin): add fleet statistics and admin read endpoints

Counters (always on when admin is enabled):
- last seen per user and group, written asynchronously after
  admission, throttled to one write per scope per minute per replica
- hourly provider and model attempts, failures and timeouts
- failovers per provider, rejections per hour, and unpriced (402)
  refusals
- cache hits, misses and cost saved
- request counts per target and caller

Opt-in counters under admin.stats (both default off):
- userModel: usage per user and model
- latency: latency histograms per provider and model

All counters ride the existing Redis batches. A test measures three
round trips per request with defaults and with every statistic on. A
cache hit takes two. A model request is no longer counted for failed
failover candidates.

New admin endpoints:
- usage/series and usage/totals
- performance, with percentiles from the histograms
- catalog, with the price billing actually uses and its source
- consumers, with key source, grants and model access
- config, with secrets redacted

The overview gains features and a pricing summary. Usage gains
lastSeen. usage/models gains offset and more detail fields. Reads are
chunked at 1600 keys per round trip and capped at 64000 keys per
request.

Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
This commit is contained in:
2026-09-23 16:40:45 +01:00
parent 8ac20d8d5e
commit 65b2f121c4
40 changed files with 7723 additions and 506 deletions
+62
View File
@@ -1773,3 +1773,65 @@ func TestMetrics_TargetHealthy_UnaffectedByStoreOutage(t *testing.T) {
_, samples := parsePrometheusText(t, rec.Body.Bytes())
assertSample(t, samples, "llmgateway_target_healthy", map[string]string{"kind": "mcp", "target": "alpha"}, "1")
}
// --- admin-redesign WP-A: latencyBucketIndex ---
// TestLatencyBucketIndex_BoundaryValuesAndOverflow pins latencyBucketIndex
// (the helper accountWith's own opt-in per-bucket latency counters share
// with this file's in-process observeLatencyBucket, limits.go) against
// latencyBucketBounds' own exact boundary values: a duration equal to a
// bound belongs to THAT bound's own bucket (le semantics, matching
// Prometheus histogram convention), and anything past the last bound
// overflows to index len(latencyBucketBounds).
func TestLatencyBucketIndex_BoundaryValuesAndOverflow(t *testing.T) {
for i, bound := range latencyBucketBounds {
d := time.Duration(bound * float64(time.Second))
idx, overflow := latencyBucketIndex(d)
if overflow {
t.Errorf("bound[%d]=%v: overflow=true, want false", i, bound)
}
if idx != i {
t.Errorf("bound[%d]=%v: idx=%d, want %d", i, bound, idx, i)
}
}
idx, overflow := latencyBucketIndex(0)
if overflow || idx != 0 {
t.Errorf("d=0: idx=%d overflow=%v, want 0, false", idx, overflow)
}
last := latencyBucketBounds[len(latencyBucketBounds)-1]
past := time.Duration(last*float64(time.Second)) + time.Second
idx, overflow = latencyBucketIndex(past)
if !overflow || idx != len(latencyBucketBounds) {
t.Errorf("d past every bound: idx=%d overflow=%v, want %d, true", idx, overflow, len(latencyBucketBounds))
}
}
// TestObserveLatencyBucket_StillMatchesLatencyBucketIndexAfterRefactor
// proves observeLatencyBucket's own dense-bucket/overflow/sum/count
// bookkeeping stays byte-for-byte correct after being refactored to share
// latencyBucketIndex, for both an in-range and an overflowing sample.
func TestObserveLatencyBucket_StillMatchesLatencyBucketIndexAfterRefactor(t *testing.T) {
buckets := make([]int64, len(latencyBucketBounds))
var overflow int64
var sum float64
var count int64
observeLatencyBucket(buckets, &overflow, &sum, &count, 2*time.Second)
wantIdx, _ := latencyBucketIndex(2 * time.Second)
if buckets[wantIdx] != 1 {
t.Errorf("buckets[%d] = %d, want 1", wantIdx, buckets[wantIdx])
}
if count != 1 || sum != 2 {
t.Errorf("count=%d sum=%v, want 1, 2", count, sum)
}
observeLatencyBucket(buckets, &overflow, &sum, &count, 3600*time.Second)
if overflow != 1 {
t.Errorf("overflow = %d, want 1", overflow)
}
if count != 2 {
t.Errorf("count = %d, want 2", count)
}
}