feat(admin): add fleet statistics and admin read endpoints

Counters (always on when admin is enabled):
- last seen per user and group, written asynchronously after
  admission, throttled to one write per scope per minute per replica
- hourly provider and model attempts, failures and timeouts
- failovers per provider, rejections per hour, and unpriced (402)
  refusals
- cache hits, misses and cost saved
- request counts per target and caller

Opt-in counters under admin.stats (both default off):
- userModel: usage per user and model
- latency: latency histograms per provider and model

All counters ride the existing Redis batches. A test measures three
round trips per request with defaults and with every statistic on. A
cache hit takes two. A model request is no longer counted for failed
failover candidates.

New admin endpoints:
- usage/series and usage/totals
- performance, with percentiles from the histograms
- catalog, with the price billing actually uses and its source
- consumers, with key source, grants and model access
- config, with secrets redacted

The overview gains features and a pricing summary. Usage gains
lastSeen. usage/models gains offset and more detail fields. Reads are
chunked at 1600 keys per round trip and capped at 64000 keys per
request.

Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
This commit is contained in:
2026-09-23 16:40:45 +01:00
parent 8ac20d8d5e
commit 65b2f121c4
40 changed files with 7723 additions and 506 deletions
+72
View File
@@ -1249,3 +1249,75 @@ func TestHandleAudioTranscriptions_RecordsMediaModelUsage(t *testing.T) {
assert.True(t, ok)
assert.Equal(t, int64(1), modelReq, "kindModel req counter")
}
// --- admin-redesign WP-A step 5: media routes' own latency threading ---
// TestHandleImagesGenerations_Latency_WrittenWhenStatsLatencyOn proves
// recordMediaModelUsage threads the completed request's own latSample/
// hasLat through to accountWith — exactly one provider-level and one
// model-level latency duration bucket must have been incremented, when
// admin.stats.latency is on.
func TestHandleImagesGenerations_Latency_WrittenWhenStatsLatencyOn(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"created":1,"data":[{"url":"https://example.invalid/img.png"}]}`))
}))
defer srv.Close()
cfg := newMediaTestConfig(srv.URL, "img-test")
cfg.Admin = &AdminConfig{Enabled: true, Stats: &AdminStatsConfig{Latency: true}}
gw := newMediaTestGateway(t, cfg)
body := map[string]any{"model": "img-test", "prompt": "a cat"}
b, err := json.Marshal(body)
require.NoError(t, err)
req := httptest.NewRequest(http.MethodPost, imagesGenerationsPath, bytes.NewReader(b))
req.Header.Set("Authorization", "Bearer sk-alice")
rec := httptest.NewRecorder()
gw.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code, "body=%s", rec.Body.String())
now := time.Now()
var totalProviderLat, totalModelLat int64
for i := 0; i <= len(latencyBucketBounds); i++ {
if v, ok := gw.limiter.getCounter(kindProvider, "openai", latencyDurationMetric(i), windowDay, now); ok {
totalProviderLat += v
}
if v, ok := gw.limiter.getCounter(kindModel, "openai/img-test", latencyDurationMetric(i), windowDay, now); ok {
totalModelLat += v
}
}
assert.Equal(t, int64(1), totalProviderLat, "exactly one provider-level latency duration bucket must have been incremented")
assert.Equal(t, int64(1), totalModelLat, "exactly one model-level latency duration bucket must have been incremented")
}
// TestHandleImagesGenerations_Latency_OffByDefault proves no latency
// bucket is ever written when admin.stats.latency (and metrics) are both
// off — the default, additive-and-preserving behavior.
func TestHandleImagesGenerations_Latency_OffByDefault(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"created":1,"data":[{"url":"https://example.invalid/img.png"}]}`))
}))
defer srv.Close()
gw := newMediaTestGateway(t, newMediaTestConfig(srv.URL, "img-test"))
body := map[string]any{"model": "img-test", "prompt": "a cat"}
b, err := json.Marshal(body)
require.NoError(t, err)
req := httptest.NewRequest(http.MethodPost, imagesGenerationsPath, bytes.NewReader(b))
req.Header.Set("Authorization", "Bearer sk-alice")
rec := httptest.NewRecorder()
gw.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code, "body=%s", rec.Body.String())
now := time.Now()
for i := 0; i <= len(latencyBucketBounds); i++ {
v, ok := gw.limiter.getCounter(kindProvider, "openai", latencyDurationMetric(i), windowDay, now)
assert.True(t, ok)
assert.Equal(t, int64(0), v, "bucket %d must stay at 0 without admin.stats.latency", i)
}
}