feat(admin): add fleet statistics and admin read endpoints

Counters (always on when admin is enabled):
- last seen per user and group, written asynchronously after
  admission, throttled to one write per scope per minute per replica
- hourly provider and model attempts, failures and timeouts
- failovers per provider, rejections per hour, and unpriced (402)
  refusals
- cache hits, misses and cost saved
- request counts per target and caller

Opt-in counters under admin.stats (both default off):
- userModel: usage per user and model
- latency: latency histograms per provider and model

All counters ride the existing Redis batches. A test measures three
round trips per request with defaults and with every statistic on. A
cache hit takes two. A model request is no longer counted for failed
failover candidates.

New admin endpoints:
- usage/series and usage/totals
- performance, with percentiles from the histograms
- catalog, with the price billing actually uses and its source
- consumers, with key source, grants and model access
- config, with secrets redacted

The overview gains features and a pricing summary. Usage gains
lastSeen. usage/models gains offset and more detail fields. Reads are
chunked at 1600 keys per round trip and capped at 64000 keys per
request.

Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
This commit is contained in:
2026-09-23 16:40:45 +01:00
parent 8ac20d8d5e
commit 65b2f121c4
40 changed files with 7723 additions and 506 deletions
+15 -6
View File
@@ -467,6 +467,9 @@ func (g *Gateway) handleMCPFederated(w http.ResponseWriter, r *http.Request, u *
writeLimitViolation(w, violation)
return
}
// P9 fix (admin dashboard redesign verify round): last-seen only for
// an admitted request — recordLastSeen's own doc comment (limits.go).
g.limiter.recordLastSeen(scopes, g.limiter.now())
switch {
case req.Method == "initialize":
@@ -474,9 +477,9 @@ func (g *Gateway) handleMCPFederated(w http.ResponseWriter, r *http.Request, u *
case req.Method == "ping":
writeJSONRPCResult(w, format, req.ID, map[string]any{})
case req.Method == "tools/list":
g.mcpFederatedToolsList(w, format, r, req, allowedMCPServerNames(g.cfg, grp))
g.mcpFederatedToolsList(w, format, r, req, allowedMCPServerNames(g.cfg, grp), u.name)
case req.Method == "tools/call":
g.mcpFederatedToolsCall(w, format, r, req, grp)
g.mcpFederatedToolsCall(w, format, r, req, grp, u.name)
case strings.HasPrefix(req.Method, "notifications/"):
// A JSON-RPC notification carries no id and gets no response body
// by definition; MCP's Streamable HTTP transport answers a
@@ -1188,7 +1191,7 @@ const mcpFederatedFanoutConcurrency = 8
// reach, because every call for that name is routed elsewhere). Logs
// exactly which server's tool was dropped and which server tools/call
// actually routes this name to.
func (g *Gateway) mcpFederatedToolsList(w http.ResponseWriter, format mcpResponseFormat, r *http.Request, req jsonrpcRequest, names []string) {
func (g *Gateway) mcpFederatedToolsList(w http.ResponseWriter, format mcpResponseFormat, r *http.Request, req jsonrpcRequest, names []string, caller string) {
var (
mu sync.Mutex
wg sync.WaitGroup
@@ -1416,7 +1419,11 @@ func (g *Gateway) mcpFederatedToolsList(w http.ResponseWriter, format mcpRespons
for i, name := range dialed {
attempted[i] = limitScope{kind: targetKindMCP, id: name}
}
g.limiter.countTargetRequests(attempted)
// Target x caller (always-on-with-admin, DECISIONS): caller is
// handleMCPFederated's own u.name, threaded through so every
// dialed server's own tcaller counter attributes to the caller
// whose credential authorized this fan-out.
g.limiter.countTargetRequestsBy(caller, attempted)
}
if len(failed) > 0 {
@@ -1548,7 +1555,7 @@ func resolveFederatedTool(fullName string, allowedNames []string) (serverName, t
// single resolved server, once the call was actually attempted against
// it — the same "attempted, not necessarily succeeded" accounting
// mcpFederatedToolsList's own doc comment explains for its own fan-out.
func (g *Gateway) mcpFederatedToolsCall(w http.ResponseWriter, format mcpResponseFormat, r *http.Request, req jsonrpcRequest, grp *group) {
func (g *Gateway) mcpFederatedToolsCall(w http.ResponseWriter, format mcpResponseFormat, r *http.Request, req jsonrpcRequest, grp *group, caller string) {
var params mcpToolCallParams
if err := json.Unmarshal(req.Params, &params); err != nil {
writeJSONRPCErrorResponse(w, format, req.ID, jsonrpcInvalidParams, "invalid params")
@@ -1588,7 +1595,9 @@ func (g *Gateway) mcpFederatedToolsCall(w http.ResponseWriter, format mcpRespons
if err == nil || !errors.Is(err, context.Canceled) {
g.targetHealth.record(targetKindMCP, serverName, targetURL, err == nil, err, time.Since(probeStart), targetHealthSourceTraffic)
}
g.limiter.countTargetRequest(targetKindMCP, serverName)
// Target x caller (always-on-with-admin, DECISIONS): caller is
// handleMCPFederated's own u.name, threaded through.
g.limiter.countTargetRequestBy(caller, targetKindMCP, serverName)
if err != nil {
// P4 (review round 4): err here is commonly a *url.Error wrapping
// targetURL verbatim, including its query string (this gateway's