merge: Prometheus metrics endpoint with admin-key and CIDR auth

# Conflicts:
#	llmgateway.go
#	tools/yaegi-check/main.go
This commit is contained in:
2026-08-23 02:05:06 +01:00
10 changed files with 2674 additions and 26 deletions
+155 -2
View File
@@ -376,6 +376,22 @@ func run() error {
// config change must preserve prior behavior), so without this the
// exerciseFailover probe below would find failover-a's 500 returned
// to the client unchanged, never reaching failover-b at all.
// metricsProbeTrickyUserNameJSON: a user name containing a double
// quote, a backslash, and a newline — auth.go's buildEntry validates
// a name as non-empty only, never restricted to a URL-path-segment
// character set (see its own doc comment), so this is a real,
// legal config, not a contrived one. json.Marshal, not hand-built
// string concatenation, produces the correctly JSON-escaped literal
// to splice into attemptAccountingOverride below — this is JSON
// escaping for the CONFIG document, a separate concern from the
// Prometheus label escaping exerciseMetricsRoute actually verifies
// in the RENDERED /metrics body.
trickyNameJSON, err := json.Marshal(metricsProbeTrickyUserName)
if err != nil {
return fmt.Errorf("marshal metrics escaping probe user name: %w", err)
}
attemptAccountingOverride := `{"failover":{"enabled":true},"breaker":{"failureThreshold":2,"openDuration":"3s","maxOpenDuration":"6s"},` +
`"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `","` + builtinLookupModelID + `"]},` +
`"brk":{"type":"openai","baseUrl":"` + brkUpstream.URL + `","apiKey":"sk-up","discovery":true,"discoveryInterval":"1ms"},"` +
@@ -402,7 +418,22 @@ func run() error {
// one testDataWantModel already exercises.
`"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000}},` +
`"mcpServers":{"` + mcpProbeServerName + `":{"url":"` + mcpProbeUpstream.URL + `"}},` +
`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true}]}}`
// metrics (Prometheus text-exposition endpoint): enabled with
// modelLabel on, so exerciseMetricsRoute below exercises the
// opt-in per-(provider,model) breakdown under the interpreter
// too, not just the always-on families. allowedCIDRs names the
// exact block httptest.NewRequest's own default RemoteAddr
// (192.0.2.1:1234, verified empirically) falls inside, so the
// CIDR-bypass path is reachable with a plain httptest.NewRequest
// carrying no explicit RemoteAddr override.
`"metrics":{"enabled":true,"allowedCIDRs":["` + metricsProbeAllowedCIDR + `"],"modelLabel":true},` +
// The third inline user (metricsProbeTrickyUserName) exists
// purely so exerciseMetricsRoute can prove the escaping path
// under the interpreter — it makes no requests of its own, but
// still renders a real (zero-traffic) llmgateway_requests_total
// series, since writeUsageMetrics (metrics.go) lists every
// active user regardless of traffic.
`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true},{"name":` + string(trickyNameJSON) + `,"group":"default","apiKey":"sk-tricky"}]}}`
if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil {
return fmt.Errorf("decode attempt-accounting harness override into the interpreted Config: %w", err)
}
@@ -450,7 +481,17 @@ func run() error {
// exactly why this class of divergence needs its own interpreted
// probe: a compiled `go test` pass here would prove nothing about the
// interpreter.
return exerciseBreaker(handler, brkModelsHits, brkHealthy)
if err := exerciseBreaker(handler, brkModelsHits, brkHealthy); err != nil {
return err
}
// exerciseMetricsRoute runs LAST, after exerciseBreaker has already
// settled: GET /metrics runs registry.maybeRefresh like every other
// route (ServeHTTP's own unconditional call at entry), and
// exerciseBreaker's own hit-counting is timing-sensitive against the
// "brk" provider specifically — probing metrics first could perturb
// the very discovery-attempt counts that test samples. Ordering the
// metrics probe after avoids any such interaction.
return exerciseMetricsRoute(handler)
}
// builtinLookupModelID is a real, stable entry in the generated
@@ -476,6 +517,27 @@ const (
mcpProbeToolName = mcpProbeServerName + "_big"
)
// metricsProbeAllowedCIDR is the CIDR block attemptAccountingOverride's
// own metrics.allowedCIDRs names — see that literal's own comment for why
// this must contain httptest.NewRequest's default RemoteAddr.
const metricsProbeAllowedCIDR = "192.0.2.0/24"
// metricsProbeOutsideAddr is a source address deliberately OUTSIDE
// metricsProbeAllowedCIDR (TEST-NET-3, RFC 5737) — exerciseMetricsRoute's
// own proof that the allowlist is not simply matching everything.
const metricsProbeOutsideAddr = "203.0.113.9:5555"
// metricsProbeTrickyUserName is a configured user name containing a
// double quote, a backslash, and a newline — the exact character set
// escapeLabelValue (metrics.go) exists to handle, driven under the REAL
// interpreter (review fix, adversarial verification 2026-08-23): the
// compiled test suite already proves escaping correct
// (TestMetrics_LabelValuesWithSpecialChars_Escaped,
// TestEscapeLabelValue_InvalidUTF8_Injective), but this codebase's own
// history is that a compiled pass has repeatedly said nothing about the
// interpreted shape.
const metricsProbeTrickyUserName = "weird\"user\\name\nwith-newline"
// mcpProbeOversizeBytes is how much filler mcpProbeUpstream writes: over
// mcpBackendCallResponseMaxBytes (maxRequestBytes, 10MiB) so the response
// trips doBackendJSONRPC's cap. Kept as its own named value rather than
@@ -662,6 +724,97 @@ func exerciseBreaker(handler http.Handler, hits, healthy *int64) error {
return nil
}
// exerciseMetricsRoute drives GET /metrics against the interpreted
// handler and proves both its auth gate and its rendering run correctly
// under Yaegi — the metrics feature's own gate requirement: compiled
// tests have repeatedly passed on this codebase while the interpreted
// shape failed (matchesSentinel's own doc comment, limits.go, is the
// canonical example), so a metrics-specific probe is not optional here.
//
// Four requests, mirroring TestMetrics_GateMatrix (metrics_test.go)
// exactly so this harness and the compiled test suite assert the
// identical contract from two different angles:
// 1. unauthenticated, from an address outside metrics.allowedCIDRs -> 401
// 2. authenticated as "tester" (not admin) -> 403
// 3. authenticated as the admin user -> 200, body inspected
// 4. unauthenticated, from an address INSIDE metrics.allowedCIDRs -> 200
//
// Request 3's body is inspected for real, live-data content — not just a
// 200 status — because a Yaegi-specific bug in this codebase's history
// (matchesSentinel, limits.go) was a case where the interpreted code ran
// without panicking yet produced a WRONG answer; a bare status-code check
// would not have caught that class of failure here either. Specifically:
// the "openai" provider's name, and — since attemptAccountingOverride
// enables metrics.modelLabel — testDataWantModel's own id inside the
// opt-in provider-model breakdown, both of which can only appear via a
// live registry.snapshot() walk and a live providerModelScopeID join
// under the real interpreter, not a static string in this harness.
func exerciseMetricsRoute(handler http.Handler) error {
unauthReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
unauthReq.RemoteAddr = metricsProbeOutsideAddr
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, unauthReq)
if rec.Code != http.StatusUnauthorized {
return fmt.Errorf("GET /metrics (no key, outside allowedCIDRs): status = %d, want 401, body=%s", rec.Code, rec.Body.String())
}
nonAdminReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
nonAdminReq.RemoteAddr = metricsProbeOutsideAddr
nonAdminReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, nonAdminReq)
if rec.Code != http.StatusForbidden {
return fmt.Errorf("GET /metrics (non-admin key): status = %d, want 403, body=%s", rec.Code, rec.Body.String())
}
adminReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
adminReq.RemoteAddr = metricsProbeOutsideAddr
adminReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, adminReq)
if rec.Code != http.StatusOK {
return fmt.Errorf("GET /metrics (admin key): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
return fmt.Errorf("GET /metrics (admin key): Content-Type = %q, want a text/plain prefix", ct)
}
body := rec.Body.String()
// metricsProbeEscapedTrickyName is escapeLabelValue's OWN expected
// output for metricsProbeTrickyUserName — a raw Go string literal
// (backticks: every backslash below is a literal backslash
// character, not a Go escape), computed by hand from the format
// rule (\\ -> \\\\, " -> \\", newline -> \\n) rather than by calling
// the function itself, so this assertion cannot pass merely because
// the interpreted and this harness's own understanding of the rule
// happen to agree by construction.
const metricsProbeEscapedTrickyName = `weird\"user\\name\nwith-newline`
for _, want := range []string{
"# TYPE llmgateway_requests_total counter",
"# TYPE llmgateway_provider_healthy gauge",
`llmgateway_provider_attempts_total{provider="openai"}`,
"# TYPE llmgateway_provider_model_attempts_total counter",
`llmgateway_provider_model_attempts_total{provider="openai",model="` + testDataWantModel + `"}`,
`llmgateway_requests_total{scope_kind="user",scope_id="` + metricsProbeEscapedTrickyName + `"}`,
} {
if !strings.Contains(body, want) {
return fmt.Errorf("GET /metrics (admin key): body missing %q — interpreted rendering diverged from the compiled shape; body=%s", want, body)
}
}
if strings.Contains(body, metricsProbeTrickyUserName) {
return fmt.Errorf("GET /metrics (admin key): body contains the RAW, unescaped tricky user name — escapeLabelValue did not run interpreted; body=%s", body)
}
allowlistedReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) // default RemoteAddr, inside metricsProbeAllowedCIDR
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, allowlistedReq)
if rec.Code != http.StatusOK {
return fmt.Errorf("GET /metrics (no key, inside allowedCIDRs): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
fmt.Println("yaegi-check: GET /metrics auth gate and rendering both verified interpreted")
return nil
}
// exerciseHandler runs two real requests against the interpreted
// handler, drawn from the plugin's own .traefik.yml testData: an
// authenticated GET /v1/models, which must return the configured