package traefikllmgateway import ( "encoding/json" "fmt" "math" "net/http" "net/http/httptest" "strings" "testing" "time" ) // --- histogramQuantile --- func TestHistogramQuantile_NoObservationsReturnsZeroNotOverflow(t *testing.T) { t.Parallel() bounds := []float64{1, 2, 3} counts := []int64{0, 0, 0, 0} // 3 buckets + overflow, all empty ms, overflow := histogramQuantile(bounds, counts, 0.5) if ms != 0 || overflow { t.Errorf("histogramQuantile(empty) = (%v, %v), want (0, false)", ms, overflow) } } func TestHistogramQuantile_InterpolatesWithinBucket(t *testing.T) { t.Parallel() bounds := []float64{1, 2, 3} // Every observation lands in bucket 0 (0, 1] seconds. counts := []int64{10, 0, 0, 0} ms, overflow := histogramQuantile(bounds, counts, 0.5) if overflow { t.Fatal("overflow = true, want false") } want := 500.0 // midpoint of (0, 1]s in ms if math.Abs(ms-want) > 1e-9 { t.Errorf("histogramQuantile(bucket0, q=0.5) = %v ms, want %v", ms, want) } } func TestHistogramQuantile_SpansTwoBuckets(t *testing.T) { t.Parallel() bounds := []float64{1, 2, 3} // 5 observations in (0,1], 5 in (1,2]: the p50 rank (target=5) lands // exactly at the boundary between the two buckets. counts := []int64{5, 5, 0, 0} ms, overflow := histogramQuantile(bounds, counts, 0.5) if overflow { t.Fatal("overflow = true, want false") } if math.Abs(ms-1000) > 1e-9 { t.Errorf("histogramQuantile(boundary) = %v ms, want 1000", ms) } // p90: target=9, cum after bucket0=5, falls in bucket1 (1,2], frac=(9-5)/5=0.8 p90, overflow := histogramQuantile(bounds, counts, 0.9) if overflow { t.Fatal("overflow = true, want false") } want := (1 + 0.8*(2-1)) * 1000 if math.Abs(p90-want) > 1e-9 { t.Errorf("histogramQuantile(q=0.9) = %v ms, want %v", p90, want) } } func TestHistogramQuantile_OverflowBucket(t *testing.T) { t.Parallel() bounds := []float64{1, 2, 3} // Every observation is past the largest bound. counts := []int64{0, 0, 0, 7} ms, overflow := histogramQuantile(bounds, counts, 0.99) if !overflow { t.Fatal("overflow = false, want true") } if ms != 3000 { t.Errorf("histogramQuantile(overflow) = %v ms, want 3000 (largest bound in ms)", ms) } } func TestHistogramQuantile_RealLatencyBounds(t *testing.T) { t.Parallel() n := len(latencyBucketBounds) + 1 counts := make([]int64, n) counts[3] = 1 // latencyBucketBounds[3] == 1 (index for the 1s bound) ms, overflow := histogramQuantile(latencyBucketBounds, counts, 0.5) if overflow { t.Fatal("overflow = true, want false") } // Single observation in bucket (0.5, 1]s: midpoint interpolation. wantLow, wantHigh := 500.0, 1000.0 if ms < wantLow || ms > wantHigh { t.Errorf("histogramQuantile(real bounds) = %v ms, want in [%v, %v]", ms, wantLow, wantHigh) } } // --- HTTP endpoint --- func TestAdminPerformance_GateMatrix(t *testing.T) { t.Parallel() cfg := newAdminTestConfig() h, _ := newAdminGatewayHandle(t, cfg) path := adminPerformancePath + "?kind=provider&window=day" cases := []struct { name string apiKey string want int }{ {"unauthenticated", "", http.StatusUnauthorized}, {"non-admin", "sk-alice", http.StatusForbidden}, {"admin", "sk-admin1", http.StatusOK}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, path, c.apiKey)) if rec.Code != c.want { t.Errorf("status = %d, want %d, body=%s", rec.Code, c.want, rec.Body.String()) } }) } } func TestAdminPerformance_ParamValidation(t *testing.T) { t.Parallel() cfg := newAdminTestConfig() h, _ := newAdminGatewayHandle(t, cfg) cases := []struct { name string qs string want int }{ {"unknown kind", "?kind=bogus&window=day", http.StatusBadRequest}, {"month window rejected", "?kind=provider&window=month", http.StatusBadRequest}, {"invalid series", "?kind=provider&window=day&series=2", http.StatusBadRequest}, {"series without exactly one id", "?kind=provider&window=day&series=1&id=alpha&id=zeta", http.StatusBadRequest}, {"series with zero ids", "?kind=provider&window=day&series=1", http.StatusBadRequest}, // P1 fix (admin dashboard redesign verify round): exactly one raw // id given (satisfying the len(rawIDs)!=1 check above) that does // not resolve to a real provider/model — ids ends up empty, and // ids[0] used to panic (recovered as a 500 by errors.go, but still // a logged panic) instead of this 400. {"series with unknown provider id", "?kind=provider&window=day&series=1&id=nope", http.StatusBadRequest}, {"series with unknown model id", "?kind=model&window=day&series=1&id=nope/x", http.StatusBadRequest}, {"valid provider", "?kind=provider&window=day", http.StatusOK}, {"valid model", "?kind=model&window=hour", http.StatusOK}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, adminPerformancePath+c.qs, "sk-admin1")) if rec.Code != c.want { t.Errorf("status = %d, want %d, body=%s", rec.Code, c.want, rec.Body.String()) } }) } } func TestAdminPerformance_TooManyIDsIs400(t *testing.T) { t.Parallel() cfg := newAdminTestConfig() h, _ := newAdminGatewayHandle(t, cfg) q := "?kind=provider&window=day" for i := 0; i <= adminPerfMaxIDs; i++ { q += "&id=alpha" } rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, adminPerformancePath+q, "sk-admin1")) if rec.Code != http.StatusBadRequest { t.Errorf("status = %d, want 400 (over adminPerfMaxIDs)", rec.Code) } } // TestAdminPerformance_AttemptsAlwaysLatencyGated seeds provider attempt/ // fail counters (always-on) and latency buckets (opt-in), then confirms: // with stats.latency OFF, Attempts/Failures are populated but every // latency field stays zero/nil and LatencyEnabled is false; with it ON, // the percentile fields populate too. func TestAdminPerformance_AttemptsAlwaysLatencyGated(t *testing.T) { t.Parallel() fixedNow := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) store := &countingMultiStore{values: map[string]int64{ windowKey(kindProvider, "alpha", metricProvAttempt, windowDay, fixedNow): 10, windowKey(kindProvider, "alpha", metricProvFail, windowDay, fixedNow): 2, windowKey(kindProvider, "alpha", latencyDurationMetric(3), windowDay, fixedNow): 4, // bucket for bound index 3 }} // Latency OFF (default). cfg := newAdminTestConfig() h, gw := newAdminGatewayHandle(t, cfg) gw.limiter = newLimiter(store, true) gw.limiter.nowFn = func() time.Time { return fixedNow } rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, adminPerformancePath+"?kind=provider&window=day&id=alpha", "sk-admin1")) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var got adminPerfResponse if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { t.Fatalf("decode: %v", err) } if got.LatencyEnabled { t.Error("LatencyEnabled = true, want false (stats.latency off)") } if len(got.Rows) != 1 { t.Fatalf("Rows = %+v, want exactly 1", got.Rows) } row := got.Rows[0] if row.Attempts != 10 || row.Failures != 2 { t.Errorf("Attempts/Failures = %d/%d, want 10/2 (always-on)", row.Attempts, row.Failures) } if row.P50Ms != nil || row.Count != 0 { t.Errorf("P50Ms/Count = %v/%d, want nil/0 with latency stats off", row.P50Ms, row.Count) } // Latency ON. cfg2 := newAdminTestConfig() cfg2.Admin.Stats = &AdminStatsConfig{Latency: true} h2, gw2 := newAdminGatewayHandle(t, cfg2) gw2.limiter = newLimiter(store, true) gw2.limiter.nowFn = func() time.Time { return fixedNow } rec2 := httptest.NewRecorder() h2.ServeHTTP(rec2, adminRequest(http.MethodGet, adminPerformancePath+"?kind=provider&window=day&id=alpha", "sk-admin1")) if rec2.Code != http.StatusOK { t.Fatalf("status = %d, want 200, body=%s", rec2.Code, rec2.Body.String()) } var got2 adminPerfResponse if err := json.Unmarshal(rec2.Body.Bytes(), &got2); err != nil { t.Fatalf("decode: %v", err) } if !got2.LatencyEnabled { t.Error("LatencyEnabled = false, want true (stats.latency on)") } if len(got2.Rows) != 1 { t.Fatalf("Rows = %+v, want exactly 1", got2.Rows) } row2 := got2.Rows[0] if row2.Count != 4 { t.Errorf("Count = %d, want 4", row2.Count) } if row2.P50Ms == nil { t.Error("P50Ms = nil, want populated with latency observations present") } } // TestAdminPerformance_ModelKindHasNoTimeoutsFailovers pins that // kind=model rows never populate Timeouts/Failovers (plan §1.2: no // per-model scope tracks either). func TestAdminPerformance_ModelKindHasNoTimeoutsFailovers(t *testing.T) { t.Parallel() fixedNow := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) store := &countingMultiStore{values: map[string]int64{ windowKey(kindProviderModel, "alpha/a-model-1", metricProvAttempt, windowDay, fixedNow): 3, }} cfg := newAdminTestConfig() h, gw := newAdminGatewayHandle(t, cfg) gw.limiter = newLimiter(store, true) gw.limiter.nowFn = func() time.Time { return fixedNow } rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, adminPerformancePath+"?kind=model&window=day&id=alpha/a-model-1", "sk-admin1")) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var got adminPerfResponse if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { t.Fatalf("decode: %v", err) } if len(got.Rows) != 1 { t.Fatalf("Rows = %+v, want exactly 1", got.Rows) } if got.Rows[0].Timeouts != 0 || got.Rows[0].Failovers != 0 { t.Errorf("Timeouts/Failovers = %d/%d, want 0/0 for kind=model", got.Rows[0].Timeouts, got.Rows[0].Failovers) } if got.Rows[0].Attempts != 3 { t.Errorf("Attempts = %d, want 3", got.Rows[0].Attempts) } } // TestAdminPerformance_DefaultProviderIDsAreAllConfigured pins the // "provider default = all configured" rule (plan §1.3(vi)). func TestAdminPerformance_DefaultProviderIDsAreAllConfigured(t *testing.T) { t.Parallel() cfg := newAdminTestConfig() h, _ := newAdminGatewayHandle(t, cfg) rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, adminPerformancePath+"?kind=provider&window=day", "sk-admin1")) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var got adminPerfResponse if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { t.Fatalf("decode: %v", err) } if len(got.Rows) != 2 { t.Fatalf("Rows = %+v, want 2 (both configured providers)", got.Rows) } } // TestAdminPerformance_SeriesModeBucketPerPoint pins series=1's shape: // Points has exactly `span` entries, one per bucket, ID set to the // bucket label, Rows left empty. func TestAdminPerformance_SeriesModeBucketPerPoint(t *testing.T) { t.Parallel() fixedNow := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) yesterday := fixedNow.AddDate(0, 0, -1) store := &countingMultiStore{values: map[string]int64{ windowKey(kindProvider, "alpha", metricProvAttempt, windowDay, fixedNow): 5, windowKey(kindProvider, "alpha", metricProvAttempt, windowDay, yesterday): 9, }} cfg := newAdminTestConfig() h, gw := newAdminGatewayHandle(t, cfg) gw.limiter = newLimiter(store, true) gw.limiter.nowFn = func() time.Time { return fixedNow } rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, adminPerformancePath+"?kind=provider&window=day&span=2&series=1&id=alpha", "sk-admin1")) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } var got adminPerfResponse if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { t.Fatalf("decode: %v", err) } if len(got.Rows) != 0 { t.Errorf("Rows = %+v, want empty in series mode", got.Rows) } if len(got.Points) != 2 || len(got.Buckets) != 2 { t.Fatalf("Points/Buckets = %d/%d, want 2/2", len(got.Points), len(got.Buckets)) } if got.Points[0].ID != got.Buckets[0] || got.Points[1].ID != got.Buckets[1] { t.Errorf("Points ID mismatch with Buckets: points=%+v buckets=%v", got.Points, got.Buckets) } if got.Points[0].Attempts != 9 || got.Points[1].Attempts != 5 { t.Errorf("Points attempts = [%d, %d], want [9, 5] (oldest first)", got.Points[0].Attempts, got.Points[1].Attempts) } // "rows" must always be present in the raw JSON, never omitted/null, // even in series=1 mode where it is semantically empty — a client // that always does `for (const r of resp.rows)` without a null check // must never see this one field be the exception. if !strings.Contains(rec.Body.String(), `"rows":[]`) { t.Errorf("series=1 response body does not contain a non-null empty \"rows\":[] field: %s", rec.Body.String()) } } func TestAdminPerformance_StoreDownIs503(t *testing.T) { t.Parallel() cfg := newAdminTestConfig() h, gw := newAdminGatewayHandle(t, cfg) gw.limiter = newLimiter(alwaysErrStore{}, false) rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, adminPerformancePath+"?kind=provider&window=day", "sk-admin1")) if rec.Code != http.StatusServiceUnavailable { t.Errorf("status = %d, want 503, body=%s", rec.Code, rec.Body.String()) } } // TestAdminPerformance_SeriesModeLatencyOn_BatchesIntoOneRoundTrip is P6 // (admin dashboard redesign verify round): perfSeries used to read every // metric via its OWN storeGetMulti round trip — a provider's series=1 with // latency on cost 4 core + 28 latency metrics = 32 sequential round trips. // They now batch into one chunked pipeline: (4+28)*48 = 1,536 keys, under // usageModelsChunkKeys (1,600) — exactly 1 round trip. func TestAdminPerformance_SeriesModeLatencyOn_BatchesIntoOneRoundTrip(t *testing.T) { t.Parallel() fixedNow := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) store := &countingMultiStore{values: map[string]int64{}} cfg := newAdminTestConfig() cfg.Admin.Stats = &AdminStatsConfig{Latency: true} h, gw := newAdminGatewayHandle(t, cfg) gw.limiter = newLimiter(store, true) gw.limiter.nowFn = func() time.Time { return fixedNow } rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, adminPerformancePath+"?kind=provider&window=hour&span=48&series=1&id=alpha", "sk-admin1")) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String()) } if store.getMultiCalls != 1 { t.Errorf("getMultiCalls = %d, want 1 (batched into one chunked pipeline, not 32 sequential round trips)", store.getMultiCalls) } } // TestAdminUsageModels_DetailKeyCap_AccountsForExtraMetrics is P7 (admin // dashboard redesign verify round): this cap used to check len(ids)*span // alone, regardless of detail mode — but detail mode reads up to 7 // metrics per id (req/tokin/tokout/cost + r402 + chit/csave when cache is // configured), not the 1 the plain check assumed. A catalog/span // combination whose SINGLE-metric size fits under adminMaxKeysPerRequest // but whose true 7-metric detail read does not must 400, and the // identical catalog/span in non-detail mode (truly 1 metric) must still // be allowed through. func TestAdminUsageModels_DetailKeyCap_AccountsForExtraMetrics(t *testing.T) { t.Parallel() cfg := newAdminTestConfig() models := make([]string, 200) for i := range models { models[i] = fmt.Sprintf("m%03d", i) } cfg.Providers["alpha"].Models = models cfg.Cache = CacheConfig{Enabled: true, TTL: "1m"} ln := newBehavioralRedisServer(t) cfg.Redis = &RedisConfig{Address: ln.Addr().String()} h, _ := newAdminGatewayHandle(t, cfg) // 201 catalog ids (200 alpha + 1 zeta) * span 48 = 9,648 keys for a // single metric — under adminMaxKeysPerRequest (64,000) alone, but // 9,648*7 = 67,536 once detail mode's true metric count is accounted // for, which is over it. rec := httptest.NewRecorder() h.ServeHTTP(rec, adminRequest(http.MethodGet, adminUsageModelsPath+"?metric=req&window=hour&span=48&detail=1", "sk-admin1")) if rec.Code != http.StatusBadRequest { t.Errorf("detail status = %d, want 400 (range too large once detail mode's extra metrics are accounted for)", rec.Code) } rec2 := httptest.NewRecorder() h.ServeHTTP(rec2, adminRequest(http.MethodGet, adminUsageModelsPath+"?metric=req&window=hour&span=48", "sk-admin1")) if rec2.Code != http.StatusOK { t.Errorf("non-detail status = %d, want 200 (same catalog/span, single metric, must not be rejected)", rec2.Code) } }