traefik-llmgateway/metrics_test.go

package traefikllmgateway

import (
	"context"
	"net/http"
	"net/http/httptest"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"
)

// newMetricsTestConfig builds a Config with Metrics enabled at the default
// path, one provider, one group ("g"), a non-admin user ("alice") and an
// admin user ("admin1") — the minimal shape every gate/structure test
// below needs. Callers may mutate the returned Config (in particular
// cfg.Metrics) before calling New.
func newMetricsTestConfig() *Config {
	cfg := CreateConfig()
	cfg.Metrics = &MetricsConfig{Enabled: true}
	cfg.Providers = map[string]*ProviderConfig{
		"openai": {Type: "openai", BaseURL: "http://openai.invalid", APIKey: "sk-up", Models: []string{"gpt-test"}},
	}
	cfg.Groups = map[string]*GroupConfig{"g": {}}
	cfg.Users = &UsersConfig{Inline: []*UserConfig{
		{Name: "alice", Group: "g", APIKey: "sk-alice", Limits: &LimitsConfig{}},
		{Name: "admin1", Group: "g", APIKey: "sk-admin1", Admin: true},
	}}
	return cfg
}

// newMetricsGatewayHandle mirrors newAdminGatewayHandle (admin_test.go):
// builds the Gateway from cfg and returns both the http.Handler ServeHTTP
// uses and the concrete *Gateway, with recordProviderAttempt's store
// write forced synchronous so a test asserting on a resulting provider
// counter never races the production fire-and-forget spawn (see
// newAdminGatewayHandle's own doc comment for why).
func newMetricsGatewayHandle(t *testing.T, cfg *Config) (http.Handler, *Gateway) {
	t.Helper()
	next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTeapot) })
	h, err := New(context.Background(), next, cfg, "llmgw")
	if err != nil {
		t.Fatalf("New: %v", err)
	}
	gw, ok := h.(*Gateway)
	if !ok {
		t.Fatal("handler is not *Gateway")
	}
	gw.limiter.spawn = func(f func()) { f() }
	return h, gw
}

func metricsRequest(method, path, apiKey string) *http.Request {
	req := httptest.NewRequest(method, path, nil)
	if apiKey != "" {
		req.Header.Set("Authorization", "Bearer "+apiKey)
	}
	return req
}

// --- gate: disabled/custom-path/unauth/non-admin/admin/CIDR ---

func TestMetrics_Disabled_FallsThroughTo404(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	cfg.Metrics = nil
	h, _ := newMetricsGatewayHandle(t, cfg)

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusNotFound {
		t.Fatalf("status = %d, want 404 (metrics nil)", rec.Code)
	}
}

func TestMetrics_DisabledExplicitly_FallsThroughTo404(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	cfg.Metrics = &MetricsConfig{Enabled: false}
	h, _ := newMetricsGatewayHandle(t, cfg)

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusNotFound {
		t.Fatalf("status = %d, want 404 (metrics disabled, even with a valid admin key)", rec.Code)
	}
}

func TestMetrics_CustomPath(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	cfg.Metrics = &MetricsConfig{Enabled: true, Path: "/internal/metrics"}
	h, _ := newMetricsGatewayHandle(t, cfg)

	// The default path is no longer registered once Path is set.
	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusNotFound {
		t.Fatalf("default path status = %d, want 404 (Path overrides it)", rec.Code)
	}

	rec = httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, "/internal/metrics", "sk-admin1"))
	if rec.Code != http.StatusOK {
		t.Fatalf("custom path status = %d, want 200, body=%s", rec.Code, rec.Body.String())
	}
}

// TestMetrics_GateMatrix covers every auth outcome the brief calls out by
// name: unauthenticated is rejected, a non-admin key is rejected, an
// admin key is accepted, a source address inside AllowedCIDRs is accepted
// with no key at all, and a source address outside it is still rejected
// with no key.
func TestMetrics_GateMatrix(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	// httptest.NewRequest's default RemoteAddr is 192.0.2.1:1234 (verified
	// empirically, not assumed) — 192.0.2.0/24 is the allowlisted block
	// below, so a request built via metricsRequest with no explicit
	// RemoteAddr override lands inside it.
	cfg.Metrics.AllowedCIDRs = []string{"192.0.2.0/24"}
	h, _ := newMetricsGatewayHandle(t, cfg)

	t.Run("unauthenticated_non_allowlisted", func(t *testing.T) {
		req := metricsRequest(http.MethodGet, metricsPathDefault, "")
		req.RemoteAddr = "203.0.113.9:5555" // TEST-NET-3, outside 192.0.2.0/24
		rec := httptest.NewRecorder()
		h.ServeHTTP(rec, req)
		if rec.Code != http.StatusUnauthorized {
			t.Fatalf("status = %d, want 401", rec.Code)
		}
	})
	t.Run("authenticated_non_admin", func(t *testing.T) {
		req := metricsRequest(http.MethodGet, metricsPathDefault, "sk-alice")
		req.RemoteAddr = "203.0.113.9:5555" // outside the allowlist too — the key path must still gate on admin
		rec := httptest.NewRecorder()
		h.ServeHTTP(rec, req)
		if rec.Code != http.StatusForbidden {
			t.Fatalf("status = %d, want 403", rec.Code)
		}
	})
	t.Run("admin_key_from_non_allowlisted_address", func(t *testing.T) {
		req := metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1")
		req.RemoteAddr = "203.0.113.9:5555"
		rec := httptest.NewRecorder()
		h.ServeHTTP(rec, req)
		if rec.Code != http.StatusOK {
			t.Fatalf("status = %d, want 200 (admin bearer token always works), body=%s", rec.Code, rec.Body.String())
		}
	})
	t.Run("allowlisted_cidr_no_key", func(t *testing.T) {
		req := metricsRequest(http.MethodGet, metricsPathDefault, "") // default RemoteAddr, inside 192.0.2.0/24
		rec := httptest.NewRecorder()
		h.ServeHTTP(rec, req)
		if rec.Code != http.StatusOK {
			t.Fatalf("status = %d, want 200 (source address inside AllowedCIDRs), body=%s", rec.Code, rec.Body.String())
		}
	})
}

// TestMetrics_NoAllowlistConfigured_KeylessRequestRejected proves the
// CIDR bypass is opt-in: with no AllowedCIDRs at all, even the default
// httptest RemoteAddr (which the test above proves IS matched once
// configured) gets 401 without a key.
func TestMetrics_NoAllowlistConfigured_KeylessRequestRejected(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig() // no AllowedCIDRs
	h, _ := newMetricsGatewayHandle(t, cfg)

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, ""))
	if rec.Code != http.StatusUnauthorized {
		t.Fatalf("status = %d, want 401 (no allowlist configured)", rec.Code)
	}
}

func TestMetrics_InvalidCIDR_FailsConstruction(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	cfg.Metrics.AllowedCIDRs = []string{"not-a-cidr"}
	next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
	_, err := New(context.Background(), next, cfg, "llmgw")
	if err == nil {
		t.Fatal("New: want an error for a malformed allowedCIDRs entry, got nil")
	}
}

// --- structure: valid exposition format ---

// promSample is one parsed exposition-format sample line.
type promSample struct {
	name   string
	labels map[string]string
	value  string
}

// parsePrometheusText parses body as a Prometheus text-exposition
// document, failing the test on any structural violation. Label values
// are unescaped char-by-char, honoring \\, \", and \n exactly as the
// format (and escapeLabelValue, metrics.go) define them — a real,
// unescaped quote or embedded newline in a label value would desync this
// parser (an unterminated quote, or a value silently truncated where the
// injected control character landed), which is exactly the failure mode
// this test suite exists to catch.
func parsePrometheusText(t *testing.T, body []byte) (types map[string]string, samples []promSample) {
	t.Helper()
	types = map[string]string{}
	for _, line := range strings.Split(string(body), "\n") {
		if line == "" {
			continue
		}
		if rest, ok := strings.CutPrefix(line, "# TYPE "); ok {
			name, kind, ok := strings.Cut(rest, " ")
			if !ok {
				t.Fatalf("malformed TYPE line: %q", line)
			}
			types[name] = kind
			continue
		}
		if strings.HasPrefix(line, "#") {
			continue
		}
		samples = append(samples, parsePromSample(t, line))
	}
	return types, samples
}

func parsePromSample(t *testing.T, line string) promSample {
	t.Helper()
	i := 0
	for i < len(line) && line[i] != '{' && line[i] != ' ' {
		i++
	}
	name := line[:i]
	if name == "" {
		t.Fatalf("empty metric name in line: %q", line)
	}
	labels := map[string]string{}
	if i < len(line) && line[i] == '{' {
		i++
		for {
			if i >= len(line) {
				t.Fatalf("unterminated label block in line: %q", line)
			}
			if line[i] == '}' {
				i++
				break
			}
			lnStart := i
			for i < len(line) && line[i] != '=' {
				i++
			}
			if i >= len(line) {
				t.Fatalf("malformed label (no '=') in line: %q", line)
			}
			lname := line[lnStart:i]
			i++ // skip '='
			if i >= len(line) || line[i] != '"' {
				t.Fatalf("malformed label value (no opening quote) in line: %q", line)
			}
			i++ // skip opening quote
			var val strings.Builder
			for {
				if i >= len(line) {
					t.Fatalf("unterminated label value in line: %q", line)
				}
				c := line[i]
				if c == '\\' {
					i++
					if i >= len(line) {
						t.Fatalf("dangling escape in line: %q", line)
					}
					switch line[i] {
					case '\\':
						val.WriteByte('\\')
					case '"':
						val.WriteByte('"')
					case 'n':
						val.WriteByte('\n')
					default:
						t.Fatalf("unknown escape \\%c in line: %q", line[i], line)
					}
					i++
					continue
				}
				if c == '"' {
					i++
					break
				}
				val.WriteByte(c)
				i++
			}
			labels[lname] = val.String()
			if i < len(line) && line[i] == ',' {
				i++
				continue
			}
			if i < len(line) && line[i] == '}' {
				i++
				break
			}
			t.Fatalf("malformed label separator in line: %q", line)
		}
	}
	for i < len(line) && line[i] == ' ' {
		i++
	}
	value := line[i:]
	if value == "" {
		t.Fatalf("missing value in line: %q", line)
	}
	if _, err := strconv.ParseFloat(value, 64); err != nil {
		t.Fatalf("sample value %q does not parse as a number in line: %q", value, line)
	}
	return promSample{name: name, labels: labels, value: value}
}

// metricNameCharset is the character set a Prometheus metric name must
// match: [a-zA-Z_:][a-zA-Z0-9_:]*.
func validMetricName(name string) bool {
	if name == "" {
		return false
	}
	for i, r := range name {
		switch {
		case r == '_' || r == ':':
		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z':
		case r >= '0' && r <= '9' && i > 0:
		default:
			return false
		}
	}
	return true
}

// TestMetrics_OutputParsesAsValidExposition seeds real traffic (a
// successful request, a rejected one, and an opted-in model breakdown)
// so every metric family below actually has at least one sample line,
// then asserts structure rather than a golden blob: every metric name is
// well-formed, every family has a # TYPE line, every counter's name ends
// "_total", and no (name, label set) combination repeats.
func TestMetrics_OutputParsesAsValidExposition(t *testing.T) {
	t.Parallel()
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte(`{"id":"c1","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
	}))
	defer srv.Close()

	cfg := newMetricsTestConfig()
	cfg.Metrics.ModelLabel = true
	cfg.Providers["openai"].BaseURL = srv.URL
	cfg.Users.Inline[0].Limits = &LimitsConfig{RequestsPerMinute: 1}
	h, _ := newMetricsGatewayHandle(t, cfg)

	body := map[string]any{"model": "gpt-test", "messages": []any{map[string]any{"role": "user", "content": "hi"}}}
	// First request: admitted, and succeeds against the local httptest
	// server — seeds request/token/cost/provider counters via
	// checkAndCount+account+recordProviderAttempt.
	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, newUnifiedRequest(t, http.MethodPost, "/v1/chat/completions", "sk-alice", body))
	if rec.Code != http.StatusOK {
		t.Fatalf("first request status = %d, want 200, body=%s", rec.Code, rec.Body.String())
	}
	// Second request: alice's RequestsPerMinute:1 is now exhausted, so
	// this one is rejected — seeding llmgateway_rate_limit_rejections_total.
	rec = httptest.NewRecorder()
	h.ServeHTTP(rec, newUnifiedRequest(t, http.MethodPost, "/v1/chat/completions", "sk-alice", body))
	if rec.Code != http.StatusTooManyRequests {
		t.Fatalf("second request status = %d, want 429 (RequestsPerMinute:1 exhausted)", rec.Code)
	}

	rec = httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusOK {
		t.Fatalf("GET /metrics status = %d, want 200, body=%s", rec.Code, rec.Body.String())
	}
	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
		t.Errorf("Content-Type = %q, want text/plain prefix", ct)
	}

	types, samples := parsePrometheusText(t, rec.Body.Bytes())
	if len(samples) == 0 {
		t.Fatal("no samples parsed at all")
	}

	seen := map[string]bool{}
	for _, s := range samples {
		if !validMetricName(s.name) {
			t.Errorf("metric name %q does not match [a-zA-Z_:][a-zA-Z0-9_:]*", s.name)
		}
		kind, ok := types[s.name]
		if !ok {
			t.Errorf("sample %q has no # TYPE line", s.name)
		}
		if kind == "counter" && !strings.HasSuffix(s.name, "_total") {
			t.Errorf("counter %q does not end in _total", s.name)
		}

		labelKeys := make([]string, 0, len(s.labels))
		for k := range s.labels {
			labelKeys = append(labelKeys, k)
		}
		sort.Strings(labelKeys)
		var key strings.Builder
		key.WriteString(s.name)
		for _, k := range labelKeys {
			key.WriteByte('\x00')
			key.WriteString(k)
			key.WriteByte('=')
			key.WriteString(s.labels[k])
		}
		if seen[key.String()] {
			t.Errorf("duplicate series: %s%v", s.name, s.labels)
		}
		seen[key.String()] = true
	}

	wantFamilies := []string{
		"llmgateway_requests_total",
		"llmgateway_tokens_total",
		"llmgateway_cost_micro_usd_total",
		"llmgateway_budget_consumed_ratio",
		"llmgateway_provider_attempts_total",
		"llmgateway_provider_failures_total",
		"llmgateway_provider_healthy",
		"llmgateway_provider_model_attempts_total",
		"llmgateway_provider_model_failures_total",
		"llmgateway_rate_limit_rejections_total",
	}
	for _, name := range wantFamilies {
		if _, ok := types[name]; !ok {
			t.Errorf("expected family %q to have a # TYPE line, got none", name)
		}
	}
}

// --- label escaping: the correctness requirement, not a nicety ---

func TestEscapeLabelValue(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name string
		in   string
		want string
	}{
		{"plain", "gpt-test", "gpt-test"},
		{"quote", `say "hi"`, `say \"hi\"`},
		{"backslash", `C:\models\x`, `C:\\models\\x`},
		{"newline", "line1\nline2", `line1\nline2`},
		{"quote_then_backslash", `"\`, `\"\\`},
		{"backslash_then_quote", `\"`, `\\\"`},
		{"all_three", "a\"b\\c\nd", `a\"b\\c\nd`},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()
			if got := escapeLabelValue(tt.in); got != tt.want {
				t.Errorf("escapeLabelValue(%q) = %q, want %q", tt.in, got, tt.want)
			}
		})
	}
}

// TestMetrics_LabelValuesWithSpecialChars_Escaped is the end-to-end
// version of TestEscapeLabelValue: a real user name containing a double
// quote, a backslash, and a newline (auth.go's buildEntry doc comment
// confirms a user name is validated as non-empty only, never restricted
// to configNamePattern's URL-path-segment character set) flows all the
// way into the scope_id label of a real rendered document, and the
// document still parses.
func TestMetrics_LabelValuesWithSpecialChars_Escaped(t *testing.T) {
	t.Parallel()
	const trickyName = `weird"user\name` + "\nwith-newline"
	cfg := newMetricsTestConfig()
	cfg.Users.Inline = append(cfg.Users.Inline, &UserConfig{Name: trickyName, Group: "g", APIKey: "sk-tricky"})
	h, gw := newMetricsGatewayHandle(t, cfg)

	fixedNow := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC)
	gw.limiter.nowFn = func() time.Time { return fixedNow }
	gw.limiter.incrCounter("user", trickyName, metricReq, windowDay, fixedNow, 3, dayWindowTTL)

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
	}

	_, samples := parsePrometheusText(t, rec.Body.Bytes())
	var found *promSample
	for i, s := range samples {
		if s.name == "llmgateway_requests_total" && s.labels["scope_id"] == trickyName {
			found = &samples[i]
			break
		}
	}
	if found == nil {
		t.Fatalf("no llmgateway_requests_total sample with scope_id == %q found among %d samples", trickyName, len(samples))
	}
	if found.value != "3" {
		t.Errorf("value = %q, want 3", found.value)
	}
	// The raw bytes must never contain an unescaped copy of the tricky
	// name — only its escaped form — confirming this genuinely exercised
	// the escaping path rather than the name having been dropped/replaced.
	if strings.Contains(rec.Body.String(), trickyName) {
		t.Error("response body contains the raw, unescaped name — escaping did not run")
	}
}

// --- model label: opt-in ---

func TestMetrics_ModelLabel_AbsentByDefault(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	h, _ := newMetricsGatewayHandle(t, cfg)

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200", rec.Code)
	}
	body := rec.Body.String()
	if strings.Contains(body, "llmgateway_provider_model_attempts_total") {
		t.Error("llmgateway_provider_model_attempts_total present with ModelLabel unset (default off)")
	}
	if strings.Contains(body, "llmgateway_provider_model_failures_total") {
		t.Error("llmgateway_provider_model_failures_total present with ModelLabel unset (default off)")
	}
}

func TestMetrics_ModelLabel_PresentWhenEnabled(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	cfg.Metrics.ModelLabel = true
	h, _ := newMetricsGatewayHandle(t, cfg)

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200", rec.Code)
	}

	_, samples := parsePrometheusText(t, rec.Body.Bytes())
	var found bool
	for _, s := range samples {
		if s.name == "llmgateway_provider_model_attempts_total" && s.labels["provider"] == "openai" && s.labels["model"] == "gpt-test" {
			found = true
			break
		}
	}
	if !found {
		t.Fatalf("no llmgateway_provider_model_attempts_total{provider=%q,model=%q} sample found", "openai", "gpt-test")
	}
}

// --- counters reflect actual recorded usage ---

func TestMetrics_RequestTokenCostCounters_ReflectSeededUsage(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	h, gw := newMetricsGatewayHandle(t, cfg)

	fixedNow := time.Date(2026, 8, 20, 12, 30, 0, 0, time.UTC)
	gw.limiter.nowFn = func() time.Time { return fixedNow }
	gw.limiter.incrCounter("user", "alice", metricReq, windowDay, fixedNow, 7, dayWindowTTL)
	gw.limiter.account([]limitScope{{kind: "user", id: "alice"}}, usage{prompt: 40, completion: 10}, 2_500_000)

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200", rec.Code)
	}
	_, samples := parsePrometheusText(t, rec.Body.Bytes())

	assertSample(t, samples, "llmgateway_requests_total", map[string]string{"scope_kind": "user", "scope_id": "alice"}, "7")
	assertSample(t, samples, "llmgateway_tokens_total", map[string]string{"scope_kind": "user", "scope_id": "alice", "direction": "prompt"}, "40")
	assertSample(t, samples, "llmgateway_tokens_total", map[string]string{"scope_kind": "user", "scope_id": "alice", "direction": "completion"}, "10")
	assertSample(t, samples, "llmgateway_cost_micro_usd_total", map[string]string{"scope_kind": "user", "scope_id": "alice"}, "2.5e+06")
}

// assertSample fails the test unless samples contains exactly one sample
// named name whose labels are exactly want (same keys, same values), and
// whose value round-trips (as a float64) to the same value wantValue
// does. Comparing as float64 rather than as a raw string tolerates
// strconv.FormatFloat's shortest-representation choice (e.g. "2.5e+06"
// for 2,500,000) without the test hardcoding that exact spelling.
func assertSample(t *testing.T, samples []promSample, name string, want map[string]string, wantValue string) {
	t.Helper()
	wantF, err := strconv.ParseFloat(wantValue, 64)
	if err != nil {
		t.Fatalf("assertSample: wantValue %q does not parse: %v", wantValue, err)
	}
	for _, s := range samples {
		if s.name != name || len(s.labels) != len(want) {
			continue
		}
		match := true
		for k, v := range want {
			if s.labels[k] != v {
				match = false
				break
			}
		}
		if !match {
			continue
		}
		gotF, err := strconv.ParseFloat(s.value, 64)
		if err != nil {
			t.Fatalf("sample %s%v value %q does not parse: %v", name, s.labels, s.value, err)
		}
		if gotF != wantF {
			t.Errorf("sample %s%v value = %q, want %s", name, s.labels, s.value, wantValue)
		}
		return
	}
	t.Errorf("no sample %s with labels %v found", name, want)
}

// TestMetrics_ProviderCounters_ReflectRealTraffic drives one successful
// and one failing upstream call through the real unified route (not a
// direct counter seed) and asserts llmgateway_provider_attempts_total/
// _failures_total match — the same shape TestAdminOverview_
// ProviderRates_EndToEndAfterTraffic (admin_test.go) already proves for
// the JSON admin API, proving the identical underlying providerUsage read
// renders correctly here too.
func TestMetrics_ProviderCounters_ReflectRealTraffic(t *testing.T) {
	t.Parallel()
	var callN int
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		callN++
		if callN == 1 {
			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(http.StatusOK)
			_, _ = w.Write([]byte(`{"id":"c1","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`))
			return
		}
		w.WriteHeader(http.StatusInternalServerError)
		_, _ = w.Write([]byte(`{"error":"boom"}`))
	}))
	defer srv.Close()

	cfg := newMetricsTestConfig()
	cfg.Providers["openai"].BaseURL = srv.URL
	h, _ := newMetricsGatewayHandle(t, cfg)

	body := map[string]any{"model": "gpt-test", "messages": []any{map[string]any{"role": "user", "content": "hi"}}}
	for i := 0; i < 2; i++ {
		rec := httptest.NewRecorder()
		h.ServeHTTP(rec, newUnifiedRequest(t, http.MethodPost, "/v1/chat/completions", "sk-alice", body))
	}

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200", rec.Code)
	}
	_, samples := parsePrometheusText(t, rec.Body.Bytes())

	assertSample(t, samples, "llmgateway_provider_attempts_total", map[string]string{"provider": "openai"}, "2")
	assertSample(t, samples, "llmgateway_provider_failures_total", map[string]string{"provider": "openai"}, "1")
}

// TestMetrics_ProviderHealthy_ReflectsOpenBreaker proves the gauge
// follows the discovery circuit breaker's real state, not a static 1 —
// mutating registry.states directly (same package, mirrors how
// registry_health_test.go inspects breaker state) rather than driving a
// real failing discovery loop, since this test only needs to prove the
// RENDERING reads health correctly, not re-prove the breaker state
// machine itself (registry_health_test.go already does that).
func TestMetrics_ProviderHealthy_ReflectsOpenBreaker(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	h, gw := newMetricsGatewayHandle(t, cfg)

	st := gw.registry.states["openai"]
	st.mu.Lock()
	st.health = breakerOpen
	st.mu.Unlock()

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	if rec.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200", rec.Code)
	}
	_, samples := parsePrometheusText(t, rec.Body.Bytes())
	assertSample(t, samples, "llmgateway_provider_healthy", map[string]string{"provider": "openai"}, "0")
}

// --- rate-limit rejections ---

// TestMetrics_RateLimitRejections_CountedByScope drives a real 429
// (alice's RequestsPerMinute:1 exhausted by a second request) through the
// unified route, then asserts llmgateway_rate_limit_rejections_total
// carries exactly 1 for {scope_kind="user",scope_id="alice"} — the
// SAME scope checkAndCount's own violation message names.
func TestMetrics_RateLimitRejections_CountedByScope(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	cfg.Users.Inline[0].Limits = &LimitsConfig{RequestsPerMinute: 1}
	h, _ := newMetricsGatewayHandle(t, cfg)

	body := map[string]any{"model": "gpt-test", "messages": []any{map[string]any{"role": "user", "content": "hi"}}}
	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, newUnifiedRequest(t, http.MethodPost, "/v1/chat/completions", "sk-alice", body))
	rec = httptest.NewRecorder()
	h.ServeHTTP(rec, newUnifiedRequest(t, http.MethodPost, "/v1/chat/completions", "sk-alice", body))
	if rec.Code != http.StatusTooManyRequests {
		t.Fatalf("status = %d, want 429", rec.Code)
	}

	rec = httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	_, samples := parsePrometheusText(t, rec.Body.Bytes())
	assertSample(t, samples, "llmgateway_rate_limit_rejections_total", map[string]string{"scope_kind": "user", "scope_id": "alice"}, "1")
}

// TestLimiter_RejectionSnapshot_StoreDown proves checkAndCount records a
// store_down rejection (distinct from a real limit breach) when the
// configured store errors and failOpen is false — driven directly
// against a *limiter (not through Gateway/HTTP), mirroring
// TestLimiterCurrentUsage_StoreDown's own direct-limiter style
// (admin_test.go).
func TestLimiter_RejectionSnapshot_StoreDown(t *testing.T) {
	t.Parallel()
	l := newLimiter(alwaysErrStore{}, false)
	scopes := []limitScope{{kind: "user", id: "x", limits: &LimitsConfig{RequestsPerMinute: 10}}}

	v := l.checkAndCount(scopes)
	if v == nil || !v.storeDown {
		t.Fatalf("checkAndCount = %+v, want a storeDown violation", v)
	}

	snaps := l.rejectionSnapshot()
	var found bool
	for _, s := range snaps {
		if s.kind == rejectionScopeStoreDown && s.count == 1 {
			found = true
		}
	}
	if !found {
		t.Fatalf("rejectionSnapshot = %+v, want one store_down entry with count 1", snaps)
	}
}

// --- budget-consumed ratio ---

func TestMetrics_BudgetConsumedRatio_ReflectsUsageVsLimit(t *testing.T) {
	t.Parallel()
	cfg := newMetricsTestConfig()
	cfg.Users.Inline[0].Limits = &LimitsConfig{RequestsPerDay: 100}
	h, gw := newMetricsGatewayHandle(t, cfg)

	fixedNow := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC)
	gw.limiter.nowFn = func() time.Time { return fixedNow }
	gw.limiter.incrCounter("user", "alice", metricReq, windowDay, fixedNow, 25, dayWindowTTL)

	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, metricsRequest(http.MethodGet, metricsPathDefault, "sk-admin1"))
	_, samples := parsePrometheusText(t, rec.Body.Bytes())

	assertSample(t, samples, "llmgateway_budget_consumed_ratio",
		map[string]string{"scope_kind": "user", "scope_id": "alice", "budget": "requests-per-day"}, "0.25")

	// The synthetic total scope carries no limit — no budget ratio sample
	// of any kind should exist for it.
	for _, s := range samples {
		if s.name == "llmgateway_budget_consumed_ratio" && s.labels["scope_kind"] == totalScopeKind {
			t.Errorf("unexpected budget ratio sample for the unlimited total scope: %+v", s)
		}
	}
}