package traefikllmgateway import ( "context" "errors" "fmt" "math" "net/http" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestLimiterRequestWindow is the brief's Step-1 case: a per-minute // request limit blocks the third request in a window and resets on the // next window. func TestLimiterRequestWindow(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC) l := newLimiter(nil, true) // nil store → uses fallback memoryStore l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "user", id: "a", limits: &LimitsConfig{RequestsPerMinute: 2}}} if v := l.checkAndCount(scopes); v != nil { t.Fatal("1st should pass") } if v := l.checkAndCount(scopes); v != nil { t.Fatal("2nd should pass") } if v := l.checkAndCount(scopes); v == nil { t.Fatal("3rd should violate") } now = now.Add(time.Minute) // next window if v := l.checkAndCount(scopes); v != nil { t.Fatal("new window should pass") } } // TestLimiterTokenBudget is the brief's Step-1 case: accounting tokens // past a day budget makes the next check refuse. func TestLimiterTokenBudget(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerDay: 100}}} l.account(scopes, usage{prompt: 60, completion: 50}, 0) // 110 > 100 if v := l.checkAndCount(scopes); v == nil { t.Fatal("over token budget must refuse") } } // TestLimiterCostBudget mirrors TestLimiterTokenBudget for a cost budget: // accounting micro-USD cost past a day budget makes the next check refuse, // and the violation carries the day window's Retry-After. func TestLimiterCostBudget(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{CostPerDayUSD: 1.00}}} l.account(scopes, usage{}, 1_500_000) // $1.50 > $1.00 budget v := l.checkAndCount(scopes) if v == nil { t.Fatal("over cost budget must refuse") } wantRetry := int(time.Date(2026, 8, 21, 0, 0, 0, 0, time.UTC).Sub(now).Seconds()) if v.retryAfter != wantRetry { t.Errorf("retryAfter = %d, want %d (seconds to next UTC midnight)", v.retryAfter, wantRetry) } if v.storeDown { t.Error("storeDown must be false in this task") } } // TestLimiterNilLimitsUnlimited asserts a scope with limits == nil never // blocks a request, however many times it is checked — request counters // still climb unboundedly but nothing evaluates them. func TestLimiterNilLimitsUnlimited(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "user", id: "unlimited", limits: nil}} for i := 0; i < 1000; i++ { if v := l.checkAndCount(scopes); v != nil { t.Fatalf("iteration %d: nil limits must never violate, got %+v", i, v) } } } // TestLimiterUserOverridesAlongsideGroup asserts checkAndCount evaluates // every scope it is given in one call: a tight user limit blocks the // request even though the group scope in the same call is unlimited, and // an unlimited user scope still lets a tight group limit block. func TestLimiterUserOverridesAlongsideGroup(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) t.Run("tight user limit blocks despite unlimited group", func(t *testing.T) { l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: "user", id: "u1", limits: &LimitsConfig{RequestsPerMinute: 1}}, {kind: "group", id: "g1", limits: nil}, } if v := l.checkAndCount(scopes); v != nil { t.Fatal("1st request should pass") } v := l.checkAndCount(scopes) if v == nil { t.Fatal("2nd request should violate the user's tight limit") } }) t.Run("tight group limit blocks despite unlimited user", func(t *testing.T) { l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: "user", id: "u2", limits: nil}, {kind: "group", id: "g2", limits: &LimitsConfig{RequestsPerMinute: 1}}, } if v := l.checkAndCount(scopes); v != nil { t.Fatal("1st request should pass") } v := l.checkAndCount(scopes) if v == nil { t.Fatal("2nd request should violate the group's tight limit") } }) } // TestLimiterRetryAfter checks the Retry-After computation directly for // each window kind: seconds to the next minute boundary, next UTC // midnight, and the first of next month UTC. func TestLimiterRetryAfter(t *testing.T) { cases := []struct { name string now time.Time window string want int }{ {"minute boundary", time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC), windowMin, 30}, {"day boundary", time.Date(2026, 8, 20, 23, 59, 0, 0, time.UTC), windowDay, 60}, {"month boundary, 31-day month", time.Date(2026, 8, 31, 23, 59, 59, 0, time.UTC), windowMonth, 1}, {"month boundary, mid-month", time.Date(2026, 2, 15, 0, 0, 0, 0, time.UTC), windowMonth, int(14 * 24 * time.Hour / time.Second)}, // Feb 2026 (not a leap year) runs Feb1-Mar1 = 28 days; Feb15 is 14 days before Mar1 } for _, c := range cases { t.Run(c.name, func(t *testing.T) { if got := retryAfterSeconds(c.now, c.window); got != c.want { t.Errorf("retryAfterSeconds(%v, %q) = %d, want %d", c.now, c.window, got, c.want) } }) } } // TestLimiterRequestLimitRetryAfter asserts a request-limit violation // (not a budget violation) also carries the correct Retry-After for its // window. func TestLimiterRequestLimitRetryAfter(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "user", id: "a", limits: &LimitsConfig{RequestsPerMinute: 1}}} if v := l.checkAndCount(scopes); v != nil { t.Fatal("1st should pass") } v := l.checkAndCount(scopes) if v == nil { t.Fatal("2nd should violate") } if v.retryAfter != 30 { t.Errorf("retryAfter = %d, want 30", v.retryAfter) } } // TestWindowKeyFormat pins the exact key layout every store operation // depends on: llmgw:{kind}:{id}:{metric}:{window}:{bucket}, with bucket // formatted from t.UTC() at each window's granularity — including hour // (v0.2 data-layer task). func TestWindowKeyFormat(t *testing.T) { tm := time.Date(2026, 8, 20, 10, 4, 59, 0, time.FixedZone("UTC+1", 3600)) // 09:04:59 UTC cases := []struct { window string want string }{ {windowHour, "llmgw:user:a:req:hour:2026082009"}, {windowMin, "llmgw:user:a:req:min:202608200904"}, {windowDay, "llmgw:user:a:req:day:20260820"}, {windowMonth, "llmgw:user:a:req:month:202608"}, } for _, c := range cases { if got := windowKey("user", "a", "req", c.window, tm); got != c.want { t.Errorf("windowKey(..., %q, ...) = %q, want %q", c.window, got, c.want) } } } // TestWindowKeyFormat_HourUTCBoundary pins the hour bucket rolling over // exactly at the UTC hour boundary, and proves a non-UTC input is // converted to UTC first (an input at 00:04:59 in a UTC+1 zone is 23:04:59 // UTC the PREVIOUS day — a bug converting the hour field alone without // re-deriving the date would produce "...23" glued onto the wrong day). func TestWindowKeyFormat_HourUTCBoundary(t *testing.T) { justBefore := time.Date(2026, 8, 20, 8, 59, 59, 0, time.UTC) justAfter := time.Date(2026, 8, 20, 9, 0, 0, 0, time.UTC) if got := windowKey("user", "a", "req", windowHour, justBefore); got != "llmgw:user:a:req:hour:2026082008" { t.Errorf("just before the hour boundary: got %q", got) } if got := windowKey("user", "a", "req", windowHour, justAfter); got != "llmgw:user:a:req:hour:2026082009" { t.Errorf("just after the hour boundary: got %q", got) } // 00:04:59 in UTC+1 is 23:04:59 UTC the previous day. crossMidnight := time.Date(2026, 8, 21, 0, 4, 59, 0, time.FixedZone("UTC+1", 3600)) if got := windowKey("user", "a", "req", windowHour, crossMidnight); got != "llmgw:user:a:req:hour:2026082023" { t.Errorf("cross-midnight non-UTC input: got %q, want the previous UTC day's hour 23 bucket", got) } } // TestWindowEnd_HourBoundary asserts windowEnd's new hour case (v0.2 // data-layer task) returns the next UTC hour boundary, and that a value // exactly on the boundary reports zero seconds remaining in that boundary // (retryAfterSeconds rounds a zero duration up to 0, distinct from // rounding a fractional second up to 1 — see TestLimiterRetryAfter for // the day/month/min cases this mirrors). func TestWindowEnd_HourBoundary(t *testing.T) { t.Run("mid-hour", func(t *testing.T) { got := windowEnd(time.Date(2026, 8, 20, 9, 30, 0, 0, time.UTC), windowHour) want := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) if !got.Equal(want) { t.Errorf("windowEnd = %v, want %v", got, want) } }) t.Run("exactly on the boundary", func(t *testing.T) { got := windowEnd(time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC), windowHour) want := time.Date(2026, 8, 20, 11, 0, 0, 0, time.UTC) if !got.Equal(want) { t.Errorf("windowEnd = %v, want %v", got, want) } }) } // TestHistoryStepBack_HourAndDay asserts the fixed-duration step-back // cases (hour, day) land i units before now exactly, including across a // UTC day boundary. func TestHistoryStepBack_HourAndDay(t *testing.T) { now := time.Date(2026, 8, 20, 1, 30, 0, 0, time.UTC) if got := historyStepBack(now, windowHour, 0); !got.Equal(now) { t.Errorf("i=0 must be now itself, got %v", got) } wantHour := time.Date(2026, 8, 19, 23, 30, 0, 0, time.UTC) // 2 hours back, crossing midnight if got := historyStepBack(now, windowHour, 2); !got.Equal(wantHour) { t.Errorf("i=2 hours back = %v, want %v", got, wantHour) } wantDay := time.Date(2026, 8, 17, 1, 30, 0, 0, time.UTC) if got := historyStepBack(now, windowDay, 3); !got.Equal(wantDay) { t.Errorf("i=3 days back = %v, want %v", got, wantDay) } } // TestHistoryStepBack_MonthAnchorsOnDayOne is the month-arithmetic trap // TestLimiterRetryAfter's own month case does not exercise: stepping back // from a high day-of-month (the 31st) with a naive // now.AddDate(0, -i, 0) overflows a shorter target month (e.g. "Mar 31" // minus one month normalizes to "Mar 3", not "Feb 28") and would land in // the WRONG calendar month's bucket. historyStepBack avoids this by // anchoring on day 1 first — this test drives it from Mar 31 and Jan 31 // specifically because those are the inputs that would expose a // regression back to the naive form. func TestHistoryStepBack_MonthAnchorsOnDayOne(t *testing.T) { cases := []struct { now time.Time name string want string i int }{ {name: "Mar 31 minus 1 month = Feb", now: time.Date(2026, 3, 31, 12, 0, 0, 0, time.UTC), i: 1, want: "202602"}, {name: "Jan 31 minus 1 month = Dec (year rollover)", now: time.Date(2026, 1, 31, 0, 0, 0, 0, time.UTC), i: 1, want: "202512"}, {name: "May 31 minus 3 months = Feb", now: time.Date(2026, 5, 31, 0, 0, 0, 0, time.UTC), i: 3, want: "202602"}, {name: "i=0 is now's own month", now: time.Date(2026, 3, 31, 0, 0, 0, 0, time.UTC), i: 0, want: "202603"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { got := bucketFor(historyStepBack(c.now, windowMonth, c.i), windowMonth) if got != c.want { t.Errorf("bucketFor(historyStepBack(%v, month, %d)) = %q, want %q", c.now, c.i, got, c.want) } }) } } // TestHistoryBucketKeys_OldestFirstInclusiveOfCurrent asserts // historyBucketKeys returns span buckets oldest-first, with the LAST // entry being now's own (current, possibly partial) bucket — the // "inclusive of current bucket" contract GET /admin/api/usage/history // promises. func TestHistoryBucketKeys_OldestFirstInclusiveOfCurrent(t *testing.T) { now := time.Date(2026, 8, 20, 14, 0, 0, 0, time.UTC) keys, buckets := historyBucketKeys("total", "all", "req", windowHour, now, 4) wantBuckets := []string{"2026082011", "2026082012", "2026082013", "2026082014"} if len(buckets) != len(wantBuckets) { t.Fatalf("len(buckets) = %d, want %d", len(buckets), len(wantBuckets)) } for i, want := range wantBuckets { if buckets[i] != want { t.Errorf("buckets[%d] = %q, want %q", i, buckets[i], want) } wantKey := "llmgw:total:all:req:hour:" + want if keys[i] != wantKey { t.Errorf("keys[%d] = %q, want %q", i, keys[i], wantKey) } } } // TestLimiterHistory_OneBatchCallOldestFirst drives limiter.history // against a counting stub store: it must issue exactly ONE getMulti call // for the whole span, and the returned points must be oldest-first with // the seeded values in the right buckets. func TestLimiterHistory_OneBatchCallOldestFirst(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) countStore := &historyCountingStore{values: map[string]int64{ windowKey("total", "all", metricReq, windowHour, now.Add(-2*time.Hour)): 5, windowKey("total", "all", metricReq, windowHour, now): 9, }} l := newLimiter(countStore, true) l.nowFn = func() time.Time { return now } points, ok := l.history("total", "all", "req", windowHour, now, 3) if !ok { t.Fatal("want ok=true") } if countStore.getMultiCalls != 1 { t.Errorf("getMultiCalls = %d, want 1", countStore.getMultiCalls) } if len(points) != 3 { t.Fatalf("len(points) = %d, want 3", len(points)) } if points[0].value != 5 || points[0].bucket != bucketFor(now.Add(-2*time.Hour), windowHour) { t.Errorf("points[0] (oldest) = %+v, want value 5 at the -2h bucket", points[0]) } if points[1].value != 0 { t.Errorf("points[1] (unseeded -1h bucket) = %+v, want value 0", points[1]) } if points[2].value != 9 || points[2].bucket != bucketFor(now, windowHour) { t.Errorf("points[2] (newest, current bucket) = %+v, want value 9", points[2]) } } // historyCountingStore is a counterStore stub whose getMulti serves fixed // values and counts its own calls — mirrors admin_test.go's // countingMultiStore, defined separately here since limits_test.go must // not depend on admin_test.go's test-only types. type historyCountingStore struct { values map[string]int64 getMultiCalls int } func (s *historyCountingStore) getMulti(keys []string) ([]int64, error) { s.getMultiCalls++ out := make([]int64, len(keys)) for i, k := range keys { out[i] = s.values[k] } return out, nil } func (s *historyCountingStore) incrMulti(entries []counterIncr) ([]int64, error) { return make([]int64, len(entries)), nil } func (s *historyCountingStore) incrAndGetMulti(entries []counterIncr, reads []string) ([]int64, []int64, error) { readVals, err := s.getMulti(reads) return make([]int64, len(entries)), readVals, err } // TestLimiterHistory_StoreDown asserts limiter.history reports ok=false // when the configured store errors and failOpen is false — the caller // (GET /admin/api/usage/history) must answer 503, never a silently-zero // series. func TestLimiterHistory_StoreDown(t *testing.T) { l := newLimiter(alwaysErrStore{}, false) points, ok := l.history("total", "all", "req", windowDay, time.Now(), 5) if ok { t.Fatal("want ok=false when the configured store errors and failOpen is false") } if points != nil { t.Errorf("points = %+v, want nil on a storeDown read", points) } } // TestLimiterHistory_ConfiguredStoreDown_FailOpen_ReturnsNotOK is finding // F5, 2026-09 review: with the default failOpen=true, storeGetMulti's own // ok stays true during a store outage — it silently serves the in-process // fallback instead — so history must ALSO check configuredStoreDown, not // just ok, or a Redis blip would present the fallback's near-empty data // as a genuine drop on the usage-history chart instead of the 503 this // function's own doc comment promises. alwaysErrStore's first call opens // the store-down latch (recordStoreFailure) synchronously, before // configuredStoreDown is checked below, so this single call already // exercises the fail-open-AND-latched path. func TestLimiterHistory_ConfiguredStoreDown_FailOpen_ReturnsNotOK(t *testing.T) { l := newLimiter(alwaysErrStore{}, true) // failOpen: true, unlike TestLimiterHistory_StoreDown above points, ok := l.history("total", "all", "req", windowDay, time.Now(), 5) if ok { t.Fatal("want ok=false: the store is down even though failOpen silently served the fallback") } if points != nil { t.Errorf("points = %+v, want nil", points) } } // TestWithTotalScope_AppendsUnlimitedTotalScope asserts withTotalScope // appends exactly one {kind: totalScopeKind, id: totalScopeID, limits: // nil} scope after whatever buildLimitScopes already returned. func TestWithTotalScope_AppendsUnlimitedTotalScope(t *testing.T) { base := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}}} got := withTotalScope(base) if len(got) != 2 { t.Fatalf("len(got) = %d, want 2", len(got)) } if got[0].kind != "user" || got[0].id != "u" { t.Errorf("got[0] = %+v, want the original user scope untouched", got[0]) } total := got[1] if total.kind != totalScopeKind || total.id != totalScopeID || total.limits != nil { t.Errorf("got[1] = %+v, want kind=%q id=%q limits=nil", total, totalScopeKind, totalScopeID) } } // TestWithTotalScope_CountedButNeverEvaluated drives checkAndCount many // times against a scopes slice built via withTotalScope: the total // scope's own limits are always nil, so no volume of traffic can ever // make it violate — checkAndCount's own nil-limits check skips it — while its // req counter still climbs by exactly one per call, proving // checkAndCount's counting loop does not skip a nil-limits scope the way // evaluation does. func TestWithTotalScope_CountedButNeverEvaluated(t *testing.T) { l := newLimiter(nil, true) now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } scopes := withTotalScope([]limitScope{{kind: "user", id: "u", limits: nil}}) const n = 250 for i := 0; i < n; i++ { if v := l.checkAndCount(scopes); v != nil { t.Fatalf("iteration %d: total scope must never violate (limits always nil), got %+v", i, v) } } total, ok := l.getCounter(totalScopeKind, totalScopeID, metricReq, windowDay, now) if !ok || total != n { t.Errorf("total req:day = %d, ok=%v, want %d (counted every call despite never evaluating)", total, ok, n) } totalHour, ok := l.getCounter(totalScopeKind, totalScopeID, metricReq, windowHour, now) if !ok || totalHour != n { t.Errorf("total req:hour = %d, ok=%v, want %d", totalHour, ok, n) } } // TestWithTotalScope_AccountWritesHourBuckets asserts account (v0.2 // data-layer task) writes the total scope's hour-window tokin/tokout/cost // counters, not just day/month — checkAndCount's own hour write is // covered by TestWithTotalScope_CountedButNeverEvaluated above. func TestWithTotalScope_AccountWritesHourBuckets(t *testing.T) { l := newLimiter(nil, true) now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } scopes := withTotalScope([]limitScope{{kind: "user", id: "u", limits: nil}}) l.account(scopes, usage{prompt: 4, completion: 6}, 700) tokIn, ok := l.getCounter(totalScopeKind, totalScopeID, metricTokIn, windowHour, now) if !ok || tokIn != 4 { t.Errorf("total tokin:hour = %d, ok=%v, want 4", tokIn, ok) } tokOut, ok := l.getCounter(totalScopeKind, totalScopeID, metricTokOut, windowHour, now) if !ok || tokOut != 6 { t.Errorf("total tokout:hour = %d, ok=%v, want 6", tokOut, ok) } cost, ok := l.getCounter(totalScopeKind, totalScopeID, metricCost, windowHour, now) if !ok || cost != 700 { t.Errorf("total cost:hour = %d, ok=%v, want 700", cost, ok) } } // TestMemoryStoreIncrAndGet exercises the fallback store directly: fresh // keys start at n, repeated incrBy accumulates, and get on an unknown key // returns 0 with no error. func TestMemoryStoreIncrAndGet(t *testing.T) { m := newMemoryStore() v, err := m.incrBy("k", 5, time.Minute) if err != nil || v != 5 { t.Fatalf("incrBy = %d, %v, want 5, nil", v, err) } v, err = m.incrBy("k", 3, time.Minute) if err != nil || v != 8 { t.Fatalf("incrBy = %d, %v, want 8, nil", v, err) } got, err := m.get("k") if err != nil || got != 8 { t.Fatalf("get = %d, %v, want 8, nil", got, err) } got, err = m.get("missing") if err != nil || got != 0 { t.Fatalf("get missing = %d, %v, want 0, nil", got, err) } } // TestMemoryStoreExpiry asserts a counter reset to 0 (as a fresh key) once // its TTL has elapsed, rather than continuing to accumulate forever. func TestMemoryStoreExpiry(t *testing.T) { m := newMemoryStore() if _, err := m.incrBy("k", 5, time.Nanosecond); err != nil { t.Fatal(err) } time.Sleep(time.Millisecond) got, err := m.get("k") if err != nil || got != 0 { t.Fatalf("get after expiry = %d, %v, want 0, nil", got, err) } v, err := m.incrBy("k", 2, time.Minute) if err != nil || v != 2 { t.Fatalf("incrBy after expiry = %d, %v, want 2, nil (fresh counter, not 7)", v, err) } } // TestMemoryStore_TTLClampedToMax is the perf-review (2026-08-21) case: // a TTL beyond memoryStoreMaxTTL (48h) — e.g. dayWindowTTL at 35 days or // monthWindowTTL at 400 days — must be clamped down to 48h before it is // ever applied, so the fallback's own live-key count stays bounded // independent of which window a caller writes. func TestMemoryStore_TTLClampedToMax(t *testing.T) { m := newMemoryStore() fake := time.Date(2026, 8, 21, 0, 0, 0, 0, time.UTC) m.nowFn = func() time.Time { return fake } cases := []struct { name string ttl time.Duration }{ {"dayWindowTTL (35d)", dayWindowTTL}, {"monthWindowTTL (400d)", monthWindowTTL}, {"an arbitrary huge ttl", 10 * 365 * 24 * time.Hour}, } for i, c := range cases { t.Run(c.name, func(t *testing.T) { key := fmt.Sprintf("k%d", i) if _, err := m.incrBy(key, 1, c.ttl); err != nil { t.Fatal(err) } m.mu.Lock() got := m.data[key].expiry m.mu.Unlock() want := fake.Add(memoryStoreMaxTTL) if !got.Equal(want) { t.Errorf("expiry = %v, want %v (clamped to memoryStoreMaxTTL, not the requested %v)", got, want, c.ttl) } }) } } // TestMemoryStore_TTLBelowMaxIsUnaffected asserts the clamp is a ceiling, // not a rewrite: a TTL already at or under memoryStoreMaxTTL passes // through exactly as requested — hourWindowTTL (48h) sits precisely at // the clamp boundary and must round-trip unchanged, not get pushed under // it by an off-by-one comparison. func TestMemoryStore_TTLBelowMaxIsUnaffected(t *testing.T) { m := newMemoryStore() fake := time.Date(2026, 8, 21, 0, 0, 0, 0, time.UTC) m.nowFn = func() time.Time { return fake } cases := []struct { name string ttl time.Duration }{ {"minWindowTTL (2m)", minWindowTTL}, {"hourWindowTTL (48h, exactly at the clamp)", hourWindowTTL}, } for i, c := range cases { t.Run(c.name, func(t *testing.T) { key := fmt.Sprintf("k%d", i) if _, err := m.incrBy(key, 1, c.ttl); err != nil { t.Fatal(err) } m.mu.Lock() got := m.data[key].expiry m.mu.Unlock() want := fake.Add(c.ttl) if !got.Equal(want) { t.Errorf("expiry = %v, want %v (ttl under the clamp must pass through unchanged)", got, want) } }) } } // TestMemoryStoreSweepTimeGated asserts the sweep gate is purely // time-based (sweepEvery), independent of how many keys memoryStore // holds: an incrBy call before the gate elapses leaves an already-expired // key physically in place (no scan happened), and the first incrBy at or // past the gate reclaims it. Time is injected via m.nowFn so the test // needs no real sleep and no size threshold to trigger the behavior. func TestMemoryStoreSweepTimeGated(t *testing.T) { m := newMemoryStore() fake := time.Date(2026, 8, 20, 0, 0, 0, 0, time.UTC) m.nowFn = func() time.Time { return fake } // lastSweep starts at the zero time, so this first incrBy always // sweeps (a no-op on an empty store) and sets lastSweep to fake. if _, err := m.incrBy("k1", 1, time.Nanosecond); err != nil { t.Fatal(err) } // k1 is already expired (ttl was 1ns), but stay well inside the // sweepEvery gate — the next incrBy must not scan, so k1 survives as // a physical (if stale) map entry. fake = fake.Add(time.Millisecond) if _, err := m.incrBy("k2", 1, time.Minute); err != nil { t.Fatal(err) } if !memoryStoreHasKey(m, "k1") { t.Fatal("k1 must still be present: the sweep gate has not elapsed yet") } // Cross the sweepEvery boundary from the last sweep: this incrBy must // scan and reclaim k1. fake = fake.Add(sweepEvery) if _, err := m.incrBy("k3", 1, time.Minute); err != nil { t.Fatal(err) } if memoryStoreHasKey(m, "k1") { t.Error("k1 should have been swept once sweepEvery elapsed") } } // memoryStoreHasKey reports whether key is physically present in m.data, // bypassing get's expiry check — used to observe sweep timing directly. func memoryStoreHasKey(m *memoryStore, key string) bool { m.mu.Lock() defer m.mu.Unlock() _, ok := m.data[key] return ok } // TestLimiterCostBudgetRoundsUSDConversion pins usdToMicros' rounding // against a value (8.2) where float64 representation sits just under the // true value: a truncating int64(8.2*1e6) would undershoot to 8_199_999, // making a spend of exactly $8.199999 look already over an intended // $8.20 budget. With rounding, 8_199_999 must stay under budget and only // the exact $8.20 spend violates. func TestLimiterCostBudgetRoundsUSDConversion(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{CostPerDayUSD: 8.2}}} l.account(scopes, usage{}, 8_199_999) if v := l.checkAndCount(scopes); v != nil { t.Fatalf("spending $8.199999 against an $8.20 budget must not violate, got %+v", v) } l.account(scopes, usage{}, 1) // now exactly at $8.20 if v := l.checkAndCount(scopes); v == nil { t.Fatal("spending exactly $8.20 against an $8.20 budget must violate") } } // TestAccountSkipsZeroMetrics asserts account writes no tokin/tokout // counter when the matching usage direction is 0, and no cost counter // when costMicros is 0 — each metric's absence is independently // observable via get. func TestAccountSkipsZeroMetrics(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "user", id: "z", limits: nil}} l.account(scopes, usage{}, 0) tokIn, ok := l.getCounter("user", "z", metricTokIn, windowDay, now) if !ok || tokIn != 0 { t.Errorf("tokin counter = %d, ok=%v, want 0, true", tokIn, ok) } tokOut, ok := l.getCounter("user", "z", metricTokOut, windowDay, now) if !ok || tokOut != 0 { t.Errorf("tokout counter = %d, ok=%v, want 0, true", tokOut, ok) } cost, ok := l.getCounter("user", "z", metricCost, windowDay, now) if !ok || cost != 0 { t.Errorf("cost counter = %d, ok=%v, want 0, true", cost, ok) } } // TestAccountSplitsTokensIndependently asserts account writes prompt // tokens to metricTokIn and completion tokens to metricTokOut // independently — including the asymmetric case (prompt only, no // completion yet) the estimation fallback and a dropped stream both rely // on: a nonzero prompt with a zero completion must still write tokin // alone, never skip it because completion is 0. func TestAccountSplitsTokensIndependently(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "user", id: "split", limits: nil}} l.account(scopes, usage{prompt: 25}, 0) // completion left at zero tokIn, ok := l.getCounter("user", "split", metricTokIn, windowDay, now) if !ok || tokIn != 25 { t.Errorf("tokin counter = %d, ok=%v, want 25, true", tokIn, ok) } tokOut, ok := l.getCounter("user", "split", metricTokOut, windowDay, now) if !ok || tokOut != 0 { t.Errorf("tokout counter = %d, ok=%v, want 0, true (no completion tokens reported)", tokOut, ok) } l.account(scopes, usage{completion: 9}, 0) // prompt left at zero this time tokIn, ok = l.getCounter("user", "split", metricTokIn, windowDay, now) if !ok || tokIn != 25 { t.Errorf("tokin counter = %d, ok=%v, want still 25 (this call reported no prompt tokens)", tokIn, ok) } tokOut, ok = l.getCounter("user", "split", metricTokOut, windowDay, now) if !ok || tokOut != 9 { t.Errorf("tokout counter = %d, ok=%v, want 9", tokOut, ok) } } // TestTokenBudgetSumsInAndOut asserts a TokensPerDay/TokensPerMonth limit // enforces a TOTAL budget across metricTokIn and metricTokOut combined — // the operator directive's "limits stay total-token (limiter sums both at // read)": neither direction alone crosses the budget, but their sum does. func TestTokenBudgetSumsInAndOut(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerDay: 100}}} l.account(scopes, usage{prompt: 60, completion: 45}, 0) // 105 > 100, split 60/45 if v := l.checkAndCount(scopes); v == nil { t.Fatal("60 tokin + 45 tokout = 105 > 100 must refuse, even though neither direction alone exceeds 100") } } // TestLimiterRequestPerDayLimit is carried-item (b): a per-day request // limit blocks once the day's count exceeds it, independent of the // (unset, unlimited) per-minute limit. func TestLimiterRequestPerDayLimit(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "user", id: "a", limits: &LimitsConfig{RequestsPerDay: 2}}} if v := l.checkAndCount(scopes); v != nil { t.Fatal("1st should pass") } if v := l.checkAndCount(scopes); v != nil { t.Fatal("2nd should pass") } v := l.checkAndCount(scopes) if v == nil { t.Fatal("3rd should violate the day limit") } if v.storeDown { t.Error("storeDown must be false for a real limit breach") } } // TestLimiterTokensPerMonthBudget mirrors TestLimiterTokenBudget for the // month window: accounting tokens past a month budget makes the next // check refuse. func TestLimiterTokensPerMonthBudget(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerMonth: 100}}} l.account(scopes, usage{prompt: 60, completion: 50}, 0) // 110 > 100 if v := l.checkAndCount(scopes); v == nil { t.Fatal("over month token budget must refuse") } } // TestLimiterCostPerMonthBudget mirrors TestLimiterCostBudget for the // month window, and checks the violation carries the month window's // Retry-After. func TestLimiterCostPerMonthBudget(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{CostPerMonthUSD: 1.00}}} l.account(scopes, usage{}, 1_500_000) // $1.50 > $1.00 budget v := l.checkAndCount(scopes) if v == nil { t.Fatal("over month cost budget must refuse") } wantRetry := int(time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC).Sub(now).Seconds()) if v.retryAfter != wantRetry { t.Errorf("retryAfter = %d, want %d (seconds to next UTC month start)", v.retryAfter, wantRetry) } if v.storeDown { t.Error("storeDown must be false for a real limit breach") } } // erroringStore is a counterStore stub that always fails, used to test the // limiter's fail-open/fail-closed handling on a store error without a // real network dependency (see redis_store_test.go for the equivalent // end-to-end test against an unreachable redisStore). type erroringStore struct { err error } func (s *erroringStore) getMulti([]string) ([]int64, error) { return nil, s.err } func (s *erroringStore) incrMulti([]counterIncr) ([]int64, error) { return nil, s.err } func (s *erroringStore) incrAndGetMulti([]counterIncr, []string) ([]int64, []int64, error) { return nil, nil, s.err } // TestLimiter_FailOpen_StoreErrorUsesFallback is carried-item (c): a store // error with failOpen=true must not block the request — the operation // transparently falls back to the limiter's in-process memoryStore, and // the fallback counter actually advances (proving the write really // happened somewhere, not just that no violation was reported). func TestLimiter_FailOpen_StoreErrorUsesFallback(t *testing.T) { store := &erroringStore{err: errors.New("boom")} l := newLimiter(store, true) scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 100}}} if v := l.checkAndCount(scopes); v != nil { t.Fatalf("want no violation with failOpen=true on a store error, got %+v", v) } got, err := l.fallback.get(windowKey("user", "u", metricReq, windowDay, l.now())) if err != nil { t.Fatalf("fallback.get: %v", err) } if got != 1 { t.Errorf("fallback req:day counter = %d, want 1", got) } } // TestLimiter_FailClosed_StoreErrorReturnsStoreDownViolation is // carried-item (c): failOpen=false refuses the request outright on a // store error, rather than silently using the fallback. func TestLimiter_FailClosed_StoreErrorReturnsStoreDownViolation(t *testing.T) { store := &erroringStore{err: errors.New("boom")} l := newLimiter(store, false) scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 100}}} v := l.checkAndCount(scopes) if v == nil { t.Fatal("want a storeDown violation on a store error with failOpen=false") } if !v.storeDown { t.Error("v.storeDown = false, want true") } if v.message != "limit store unavailable" { t.Errorf("v.message = %q, want %q", v.message, "limit store unavailable") } } // succeedIncrFailGetStore is a counterStore stub whose incrMulti always // succeeds and whose getMulti/incrAndGetMulti always error. Its // incrMulti/getMulti stay independently overridable (used by any // OTHER caller — account, currentUsage, and so on — that still calls them // separately) even though checkAndCount itself, since the round-3 fusion // (2026-08-22), calls incrAndGetMulti exclusively: a store that cannot // serve a read reliably fails that ONE combined round trip as a whole, // there being no longer a separate "increment succeeded, read failed" // split to isolate at the store level for checkAndCount specifically. type succeedIncrFailGetStore struct { getErr error } func (s *succeedIncrFailGetStore) getMulti([]string) ([]int64, error) { return nil, s.getErr } func (s *succeedIncrFailGetStore) incrMulti(entries []counterIncr) ([]int64, error) { out := make([]int64, len(entries)) for i := range out { out[i] = 1 } return out, nil } func (s *succeedIncrFailGetStore) incrAndGetMulti([]counterIncr, []string) ([]int64, []int64, error) { return nil, nil, s.getErr } // TestLimiter_FailClosed_BudgetReadReturnsStoreDownViolation covers the // budget-read fail-closed path: a store that cannot be trusted to serve a // token/cost budget read refuses the request when failOpen is false, via // checkAndCount's own storeDownViolation short-circuit (limits.go) — the // same contract the pre-fusion budgetViolation helper enforced before // checkAndCount's admission round trip was fused (perf review round 3, // 2026-08-22). A TokensPerDay limit is what makes checkAndCount build a // budget probe/read for this scope at all. func TestLimiter_FailClosed_BudgetReadReturnsStoreDownViolation(t *testing.T) { store := &succeedIncrFailGetStore{getErr: errors.New("boom")} l := newLimiter(store, false) scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{TokensPerDay: 100}}} v := l.checkAndCount(scopes) if v == nil { t.Fatal("want a storeDown violation from the budget-read path with failOpen=false") } if !v.storeDown { t.Error("v.storeDown = false, want true") } } // TestLimiter_FailClosed_AccountDropsSampleOnStoreError asserts account // drops a sample entirely — writes it nowhere — when failOpen=false and // the store errors, rather than falling back to counting it locally (which // would double-count once the store recovers). account has no error // return to signal the drop, so this is observed via the fallback staying // at 0. func TestLimiter_FailClosed_AccountDropsSampleOnStoreError(t *testing.T) { store := &erroringStore{err: errors.New("boom")} l := newLimiter(store, false) scopes := []limitScope{{kind: "user", id: "u", limits: nil}} l.account(scopes, usage{prompt: 10, completion: 10}, 500) gotIn, err := l.fallback.get(windowKey("user", "u", metricTokIn, windowDay, l.now())) if err != nil { t.Fatalf("fallback.get: %v", err) } if gotIn != 0 { t.Errorf("fallback tokin:day counter = %d, want 0 (sample must be dropped, not counted locally)", gotIn) } gotOut, err := l.fallback.get(windowKey("user", "u", metricTokOut, windowDay, l.now())) if err != nil { t.Fatalf("fallback.get: %v", err) } if gotOut != 0 { t.Errorf("fallback tokout:day counter = %d, want 0 (sample must be dropped, not counted locally)", gotOut) } } // TestLimiter_LogsStoreErrorOncePerRateLimit asserts a store error is // logged via the limiter's injectable log func, rate-limited to once per // storeErrorLogEvery — a store outage under load must not flood the log // with one line per request, however many store operations fail within // the window. func TestLimiter_LogsStoreErrorOncePerRateLimit(t *testing.T) { store := &erroringStore{err: errors.New("boom")} l := newLimiter(store, true) now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } var logged []string l.logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) } scopes := []limitScope{{kind: "user", id: "u", limits: nil}} l.checkAndCount(scopes) // two store ops (min, day) within one call if len(logged) != 1 { t.Fatalf("logged = %d lines after the first checkAndCount, want 1 (rate-limited)", len(logged)) } l.checkAndCount(scopes) // still inside the rate-limit window if len(logged) != 1 { t.Fatalf("logged = %d lines after a second call inside the window, want 1", len(logged)) } now = now.Add(storeErrorLogEvery) l.checkAndCount(scopes) if len(logged) != 2 { t.Fatalf("logged = %d lines after the rate-limit window elapsed, want 2", len(logged)) } } // countingErrorStore is a counterStore stub that always errors and counts // how many times each method is actually invoked. It proves the // store-down latch (limits.go) skips the network call entirely during // its window, rather than merely tolerating the error each time. type countingErrorStore struct { err error incrCalls int getCalls int } func (s *countingErrorStore) getMulti([]string) ([]int64, error) { s.getCalls++ return nil, s.err } func (s *countingErrorStore) incrMulti([]counterIncr) ([]int64, error) { s.incrCalls++ return nil, s.err } // incrAndGetMulti counts against the same incrCalls field incrMulti uses: // checkAndCount's own admission round trip calls incrAndGetMulti // exclusively since the round-3 fusion (2026-08-22, perf review), so // TestLimiter_StoreDownLatch_SkipsStoreCallsWithinWindow's "the store must // not be called" assertion needs this call counted the same way. func (s *countingErrorStore) incrAndGetMulti([]counterIncr, []string) ([]int64, []int64, error) { s.incrCalls++ return nil, nil, s.err } // TestLimiter_StoreDownLatch_SkipsStoreCallsWithinWindow is review-round-3 // item 2: after a store operation fails, every operation for the next // storeDownLatchFor must short-circuit straight to the fail-open policy // without calling the store at all — a request that touches several // counters (min/day/hour, tokens/cost, user/group) must not pay a fresh // bounded wait against a store that just proved unreachable for each one. // checkAndCount batches its whole req:min/req:day/req:hour increment into // ONE incrMulti call (perf review, 2026-08-21), so "the store must not be // called" now means that single call is skipped entirely during the latch // window, not that some subset of several per-key calls is. failOpen=true // is required so every checkAndCount call below returns no violation // (ok=true via the fallback) instead of a storeDownViolation — the // incrCalls assertions only make sense if checkAndCount actually // completes normally each time. func TestLimiter_StoreDownLatch_SkipsStoreCallsWithinWindow(t *testing.T) { store := &countingErrorStore{err: errors.New("boom")} l := newLimiter(store, true) now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "user", id: "u", limits: nil}} if v := l.checkAndCount(scopes); v != nil { t.Fatalf("want no violation with failOpen=true, got %+v", v) } firstCalls := store.incrCalls if firstCalls == 0 { t.Fatal("want the first call to actually reach the store at least once") } if v := l.checkAndCount(scopes); v != nil { t.Fatalf("want no violation with failOpen=true, got %+v", v) } if store.incrCalls != firstCalls { t.Errorf("incrBy calls = %d after a second checkAndCount within the latch window, want unchanged at %d (the store must not be called)", store.incrCalls, firstCalls) } now = now.Add(storeDownLatchFor) // latch expires if v := l.checkAndCount(scopes); v != nil { t.Fatalf("want no violation with failOpen=true, got %+v", v) } if store.incrCalls <= firstCalls { t.Errorf("incrBy calls = %d after the latch window elapsed, want more than %d (the store must be probed again)", store.incrCalls, firstCalls) } } // TestLimitsConfigValidate is carried-item (a): validate rejects // NaN/±Inf/negative cost limits and negative int64 count limits, accepts // zero/positive values, and treats a nil receiver (no limits configured // at all) as valid. func TestLimitsConfigValidate(t *testing.T) { cases := []struct { lc *LimitsConfig name string wantErr bool }{ {name: "nil is valid", lc: nil, wantErr: false}, {name: "zero value is valid", lc: &LimitsConfig{}, wantErr: false}, {name: "positive values are valid", lc: &LimitsConfig{ RequestsPerMinute: 1, RequestsPerDay: 1, TokensPerDay: 1, TokensPerMonth: 1, CostPerDayUSD: 1.5, CostPerMonthUSD: 1.5, }, wantErr: false}, {name: "negative RequestsPerMinute", lc: &LimitsConfig{RequestsPerMinute: -1}, wantErr: true}, {name: "negative RequestsPerDay", lc: &LimitsConfig{RequestsPerDay: -1}, wantErr: true}, {name: "negative TokensPerDay", lc: &LimitsConfig{TokensPerDay: -1}, wantErr: true}, {name: "negative TokensPerMonth", lc: &LimitsConfig{TokensPerMonth: -1}, wantErr: true}, {name: "negative CostPerDayUSD", lc: &LimitsConfig{CostPerDayUSD: -0.01}, wantErr: true}, {name: "negative CostPerMonthUSD", lc: &LimitsConfig{CostPerMonthUSD: -0.01}, wantErr: true}, {name: "NaN CostPerDayUSD", lc: &LimitsConfig{CostPerDayUSD: math.NaN()}, wantErr: true}, {name: "+Inf CostPerMonthUSD", lc: &LimitsConfig{CostPerMonthUSD: math.Inf(1)}, wantErr: true}, {name: "-Inf CostPerDayUSD", lc: &LimitsConfig{CostPerDayUSD: math.Inf(-1)}, wantErr: true}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { err := c.lc.validate() if (err != nil) != c.wantErr { t.Errorf("validate() error = %v, wantErr %v", err, c.wantErr) } }) } } // TestLimiter_FailPolicyGet covers failPolicyGet's own fail-open/fail-closed // branching in isolation (storeGet's error/latched paths already drive it // indirectly elsewhere in this file): failOpen=true reads through to the // in-process fallback, failOpen=false refuses outright. func TestLimiter_FailPolicyGet(t *testing.T) { cases := []struct { name string failOpen bool wantOK bool }{ {name: "failOpen true reads the fallback", failOpen: true, wantOK: true}, {name: "failOpen false refuses", failOpen: false, wantOK: false}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { l := newLimiter(nil, c.failOpen) v, ok := l.failPolicyGet("k") require.Equal(t, c.wantOK, ok) if c.wantOK { assert.Equal(t, int64(0), v) } }) } } // TestLimiter_FailPolicyIncrBy mirrors TestLimiter_FailPolicyGet for the // write side: failOpen=true counts n into the in-process fallback and // returns its real post-increment value; failOpen=false refuses outright // (0, false) WITHOUT touching the fallback at all — a caller must not // double-count once the store recovers by having silently counted // locally on the refused path too. // // MUTATION PROVEN: removing the `if !l.failOpen { return 0, false }` // guard (falling straight through to the fallback increment regardless of // failOpen) made the "failOpen false" case return ok=true and leave a // nonzero fallback counter instead of refusing untouched — confirmed via // `go test -run TestLimiter_FailPolicyIncrBy`, then reverted with `git // checkout -- limits.go`. func TestLimiter_FailPolicyIncrBy(t *testing.T) { cases := []struct { name string failOpen bool wantOK bool wantV int64 }{ {name: "failOpen true counts in the fallback", failOpen: true, wantOK: true, wantV: 3}, {name: "failOpen false refuses without touching the fallback", failOpen: false, wantOK: false, wantV: 0}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { l := newLimiter(nil, c.failOpen) v, ok := l.failPolicyIncrBy("k", 3, time.Minute) require.Equal(t, c.wantOK, ok) assert.Equal(t, c.wantV, v) got, err := l.fallback.get("k") if err != nil { t.Fatalf("fallback.get: %v", err) } wantFallback := int64(0) if c.failOpen { wantFallback = 3 } if got != wantFallback { t.Errorf("fallback[k] = %d, want %d", got, wantFallback) } }) } } // fixedStore is a counterStore stub whose incrMulti/getMulti always // succeed with a fixed value per entry/key, independent of key/n/ttl — // used to prove storeIncrBy/storeGet (limits_helpers_test.go, built on // the batched storeIncrMulti/storeGetMulti production paths) return the // STORE's own reported value on success, rather than silently // substituting the in-process fallback's. type fixedStore struct { incrVal int64 getVal int64 } func (s *fixedStore) getMulti(keys []string) ([]int64, error) { out := make([]int64, len(keys)) for i := range out { out[i] = s.getVal } return out, nil } func (s *fixedStore) incrMulti(entries []counterIncr) ([]int64, error) { out := make([]int64, len(entries)) for i := range out { out[i] = s.incrVal } return out, nil } func (s *fixedStore) incrAndGetMulti([]counterIncr, []string) ([]int64, []int64, error) { return nil, nil, nil } // TestLimiter_StoreIncrBy_DirectPaths exercises storeIncrBy's four // branches directly. storeIncrBy has no production caller since // checkAndCount/account moved to the batched storeIncrMulti (its own doc // comment) — its only caller, incrCounter, is itself only called from // tests for direct counter seeding, which is why the existing suite only // ever drove it through the nil-store path (30% baseline coverage): a nil // store always uses the in-process fallback; a configured store's // successful call returns the STORE's own value; a store error records // the failure and falls through to the fail-open/fail-closed policy; and // once the store-down latch opens from a prior failure, the store is // never called again until the latch expires. // // MUTATION PROVEN: deleting the `if l.storeLatched() { return // l.failPolicyIncrBy(key, n, ttl) }` branch made the "latched skips the // network call" subtest fail (the erroring store's incrBy was invoked a // second time instead of being skipped) — confirmed via `go test -run // TestLimiter_StoreIncrBy_DirectPaths/latched`, then reverted. func TestLimiter_StoreIncrBy_DirectPaths(t *testing.T) { t.Run("nil store uses the fallback", func(t *testing.T) { l := newLimiter(nil, true) v, ok := l.storeIncrBy("k", 5, time.Minute) require.True(t, ok) assert.Equal(t, int64(5), v) got, err := l.fallback.get("k") if err != nil { t.Fatalf("fallback.get: %v", err) } if got != 5 { t.Errorf("fallback[k] = %d, want 5", got) } }) t.Run("configured store success returns the store's own value", func(t *testing.T) { store := &fixedStore{incrVal: 42} l := newLimiter(store, true) v, ok := l.storeIncrBy("k", 5, time.Minute) require.True(t, ok) assert.Equal(t, int64(42), v) }) t.Run("store error falls through to the fail-open policy", func(t *testing.T) { store := &erroringStore{err: errors.New("boom")} l := newLimiter(store, true) v, ok := l.storeIncrBy("k", 5, time.Minute) require.True(t, ok) assert.Equal(t, int64(5), v) got, err := l.fallback.get("k") if err != nil { t.Fatalf("fallback.get: %v", err) } if got != 5 { t.Errorf("fallback[k] = %d, want 5 (fail-open must count in the fallback)", got) } }) t.Run("store error with failOpen false refuses", func(t *testing.T) { store := &erroringStore{err: errors.New("boom")} l := newLimiter(store, false) _, ok := l.storeIncrBy("k", 5, time.Minute) require.False(t, ok) }) t.Run("latched skips the network call entirely", func(t *testing.T) { store := &countingErrorStore{err: errors.New("boom")} l := newLimiter(store, true) now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } l.storeIncrBy("k", 1, time.Minute) // first call: reaches the store, opens the latch if store.incrCalls != 1 { t.Fatalf("incrCalls after first call = %d, want 1", store.incrCalls) } l.storeIncrBy("k", 1, time.Minute) // second call: within the latch window if store.incrCalls != 1 { t.Errorf("incrCalls after second call = %d, want still 1 (the latch must skip the store)", store.incrCalls) } }) } // TestLimiter_StoreGet_DirectPaths mirrors // TestLimiter_StoreIncrBy_DirectPaths for the read side. The round-3 perf // fusion moved checkAndCount's own reads onto storeGetMulti/ // incrAndGetMulti, so a direct call is storeGet's only remaining exercise // of its store!=nil branches (70% baseline coverage, from other tests // that only drove it through account/currentUsage's own indirect paths). // // MUTATION PROVEN: deleting the `if l.storeLatched() { return // l.failPolicyGet(key) }` branch made the "latched skips the network // call" subtest fail (the erroring store's get was invoked a second time) // — confirmed, then reverted. func TestLimiter_StoreGet_DirectPaths(t *testing.T) { t.Run("nil store uses the fallback", func(t *testing.T) { l := newLimiter(nil, true) if _, err := l.fallback.incrBy("k", 7, time.Minute); err != nil { t.Fatalf("fallback.incrBy: %v", err) } v, ok := l.storeGet("k") require.True(t, ok) assert.Equal(t, int64(7), v) }) t.Run("configured store success returns the store's own value", func(t *testing.T) { store := &fixedStore{getVal: 99} l := newLimiter(store, true) v, ok := l.storeGet("k") require.True(t, ok) assert.Equal(t, int64(99), v) }) t.Run("store error with failOpen false refuses", func(t *testing.T) { store := &erroringStore{err: errors.New("boom")} l := newLimiter(store, false) _, ok := l.storeGet("k") require.False(t, ok) }) t.Run("latched skips the network call entirely", func(t *testing.T) { store := &countingErrorStore{err: errors.New("boom")} l := newLimiter(store, true) now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } l.storeGet("k") // first call: reaches the store, opens the latch if store.getCalls != 1 { t.Fatalf("getCalls after first call = %d, want 1", store.getCalls) } l.storeGet("k") // second call: within the latch window if store.getCalls != 1 { t.Errorf("getCalls after second call = %d, want still 1 (the latch must skip the store)", store.getCalls) } }) } // TestLimiter_LogSpawnDrop_RateLimited proves logSpawnDrop logs one line // describing the dropped telemetry write, rate-limited to once per // storeErrorLogEvery regardless of how many drops happen in between — // mirroring TestLimiter_LogsStoreErrorOncePerRateLimit, since logSpawnDrop // deliberately shares lastLogAt with logStoreError (logSpawnDrop's own // doc comment: "a stressed store... is the same 'the store is struggling' // story"). // // MUTATION PROVEN: removing the `if shouldLog { ... }` guard around the // l.logf call (logging unconditionally on every drop) made the // "still inside the window" assertion fail (2 lines logged after the // second call instead of 1) — confirmed via `go test -run // TestLimiter_LogSpawnDrop_RateLimited`, then reverted. func TestLimiter_LogSpawnDrop_RateLimited(t *testing.T) { l := newLimiter(nil, true) now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } var logged []string l.logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) } l.logSpawnDrop() if len(logged) != 1 { t.Fatalf("logged = %d lines after the first drop, want 1", len(logged)) } if !strings.Contains(logged[0], "limit telemetry write dropped") { t.Errorf("logged[0] = %q, want it to mention the telemetry write drop", logged[0]) } l.logSpawnDrop() // still inside the rate-limit window if len(logged) != 1 { t.Fatalf("logged = %d lines after a second drop inside the window, want still 1", len(logged)) } now = now.Add(storeErrorLogEvery) l.logSpawnDrop() if len(logged) != 2 { t.Fatalf("logged = %d lines after the rate-limit window elapsed, want 2", len(logged)) } } // --- perf review, 2026-08-21: single-batch counter writes --- // countingIncrStore is a counterStore stub whose incrMulti/getMulti/ // incrAndGetMulti all succeed against a real backing map (so repeated // calls actually accumulate, proving checkAndCount/account read back // correct values after batching, not just that batching happened at all), // counting how many times each was called. account still calls incrMulti // directly; checkAndCount, since the round-3 fusion (2026-08-22, perf // review), calls incrAndGetMulti exclusively — so the two counters below // track genuinely different callers, not the same round trip under two // names. type countingIncrStore struct { values map[string]int64 incrMultiCalls int incrAndGetMultiCalls int getMultiCalls int } func (s *countingIncrStore) getMulti(keys []string) ([]int64, error) { s.getMultiCalls++ out := make([]int64, len(keys)) for i, k := range keys { if s.values != nil { out[i] = s.values[k] } } return out, nil } func (s *countingIncrStore) incrMulti(entries []counterIncr) ([]int64, error) { s.incrMultiCalls++ if s.values == nil { s.values = make(map[string]int64) } out := make([]int64, len(entries)) for i, e := range entries { s.values[e.key] += e.delta out[i] = s.values[e.key] } return out, nil } func (s *countingIncrStore) incrAndGetMulti(entries []counterIncr, reads []string) ([]int64, []int64, error) { s.incrAndGetMultiCalls++ if s.values == nil { s.values = make(map[string]int64) } incrVals := make([]int64, len(entries)) for i, e := range entries { s.values[e.key] += e.delta incrVals[i] = s.values[e.key] } readVals := make([]int64, len(reads)) for i, k := range reads { readVals[i] = s.values[k] } return incrVals, readVals, nil } // TestCheckAndCount_OneIncrAndGetMultiCallRegardlessOfScopeCount is the // perf-review round-3 (2026-08-22) case for checkAndCount: a 3-scope // request (user, group, total — 9 req counters, min/day/hour x 3 scopes) // must cost exactly ONE incrAndGetMulti round trip, not one per counter // and not a separate round trip for token/cost budget reads, and the // batched reply must still drive correct evaluation — a real // requestsPerMinute violation on the 3rd call, with the exact right // counter value. Supersedes the pre-fusion incrMulti-only version of this // test (perf review, 2026-08-21): checkAndCount's own ADMISSION path no // longer calls incrMulti at all, only incrAndGetMulti (account still // does, covered separately below by // TestAccount_OneIncrMultiCallRegardlessOfScopeOrMetricCount). // // Updated (finding F1, 2026-09 review): a REJECTION now costs one further // incrMulti call — rollbackOtherScopeCounts' own compensating batch, // undoing the increments the fused round trip above already made for // every scope OTHER than the one that actually violated. See // TestCheckAndCount_RejectionRollsBackOtherScopes_MemoryFallback for the // dedicated group-lockout regression this fixes; this test's own // assertions below were updated to match the corrected counter values. func TestCheckAndCount_OneIncrAndGetMultiCallRegardlessOfScopeCount(t *testing.T) { store := &countingIncrStore{} l := newLimiter(store, true) now := time.Date(2026, 8, 21, 9, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 2}}, {kind: "group", id: "g", limits: nil}, {kind: totalScopeKind, id: totalScopeID, limits: nil}, } if v := l.checkAndCount(scopes); v != nil { t.Fatalf("1st call should pass, got %+v", v) } if store.incrAndGetMultiCalls != 1 { t.Errorf("incrAndGetMultiCalls after 1st checkAndCount = %d, want 1", store.incrAndGetMultiCalls) } if v := l.checkAndCount(scopes); v != nil { t.Fatalf("2nd call should pass, got %+v", v) } if store.incrAndGetMultiCalls != 2 { t.Errorf("incrAndGetMultiCalls after 2nd checkAndCount = %d, want 2 (one call per checkAndCount, not per scope or per counter)", store.incrAndGetMultiCalls) } if store.incrMultiCalls != 0 { t.Errorf("incrMultiCalls after 2nd checkAndCount = %d, want 0 (no rollback — nothing violated yet)", store.incrMultiCalls) } v := l.checkAndCount(scopes) if v == nil { t.Fatal("3rd call should violate u's requestsPerMinute:2") } if v.storeDown { t.Error("v.storeDown must be false — this is a real limit breach, not a store failure") } if store.incrAndGetMultiCalls != 3 { t.Errorf("incrAndGetMultiCalls after 3rd checkAndCount = %d, want 3", store.incrAndGetMultiCalls) } // Finding F1, 2026-09 review: a violation now fires ONE compensating // incrMulti call (rollbackOtherScopeCounts) to undo the OTHER // scopes' increments for this rejected request — see this test's own // updated value assertions below for why. if store.incrMultiCalls != 1 { t.Errorf("incrMultiCalls after the violation = %d, want 1 (rollbackOtherScopeCounts' own compensating batch)", store.incrMultiCalls) } // Values correct: batching three scopes into one call must not // corrupt any one scope's own counter, AND (finding F1) the group and // total scopes' req:min/day/hour counters must be rolled back to what // they were before this rejected 3rd request — only the VIOLATING // scope ("user") keeps counting a request that was never admitted. uMin, ok := l.getCounter("user", "u", metricReq, windowMin, now) if !ok || uMin != 3 { t.Errorf("user req:min = %d, ok=%v, want 3 (the violating scope's own count is not rolled back)", uMin, ok) } totalMin, ok := l.getCounter(totalScopeKind, totalScopeID, metricReq, windowMin, now) if !ok || totalMin != 2 { t.Errorf("total req:min = %d, ok=%v, want 2 (rolled back — the total scope had nothing to do with user u's own violation)", totalMin, ok) } gDay, ok := l.getCounter("group", "g", metricReq, windowDay, now) if !ok || gDay != 2 { t.Errorf("group req:day = %d, ok=%v, want 2 (rolled back)", gDay, ok) } } // TestAccount_OneIncrMultiCallRegardlessOfScopeOrMetricCount is the // perf-review (2026-08-21) case for account: a 3-scope request with all // three metrics nonzero (27 counters: 3 scopes x 3 metrics x 3 windows) // must cost exactly ONE incrMulti round trip, and the batched reply must // land every value in the right scope/metric/window slot. func TestAccount_OneIncrMultiCallRegardlessOfScopeOrMetricCount(t *testing.T) { store := &countingIncrStore{} l := newLimiter(store, true) now := time.Date(2026, 8, 21, 9, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: "user", id: "u"}, {kind: "group", id: "g"}, {kind: totalScopeKind, id: totalScopeID}, } l.account(scopes, usage{prompt: 10, completion: 5}, 700) if store.incrMultiCalls != 1 { t.Errorf("incrMultiCalls = %d, want 1 (one batch for 27 counters)", store.incrMultiCalls) } uTokInDay, ok := l.getCounter("user", "u", metricTokIn, windowDay, now) if !ok || uTokInDay != 10 { t.Errorf("user tokin:day = %d, ok=%v, want 10", uTokInDay, ok) } totalTokOutHour, ok := l.getCounter(totalScopeKind, totalScopeID, metricTokOut, windowHour, now) if !ok || totalTokOutHour != 5 { t.Errorf("total tokout:hour = %d, ok=%v, want 5", totalTokOutHour, ok) } gCostMonth, ok := l.getCounter("group", "g", metricCost, windowMonth, now) if !ok || gCostMonth != 700 { t.Errorf("group cost:month = %d, ok=%v, want 700", gCostMonth, ok) } // A second call accumulates rather than overwrites, and still costs // exactly one more round trip. l.account(scopes, usage{prompt: 2}, 0) if store.incrMultiCalls != 2 { t.Errorf("incrMultiCalls after 2nd account = %d, want 2", store.incrMultiCalls) } uTokInDay, ok = l.getCounter("user", "u", metricTokIn, windowDay, now) if !ok || uTokInDay != 12 { t.Errorf("user tokin:day after 2nd account = %d, ok=%v, want 12", uTokInDay, ok) } } // TestTokenBudget_ExactBoundaryViolates asserts budgetViolation/ // tokenBudgetViolation's ">= limit" semantics at the exact boundary: a // scope whose accumulated tokin+tokout lands EXACTLY on the limit (not // merely over it) must still violate. budgetViolation's own "used < // limit passes" check makes "used == limit" the smallest value that // violates; this pins that boundary directly rather than only exercising // values already over it (TestLimiterTokenBudget, TestTokenBudgetSumsInAndOut). func TestTokenBudget_ExactBoundaryViolates(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerDay: 100}}} l.account(scopes, usage{prompt: 63, completion: 36}, 0) // 99 < 100: must not violate yet if v := l.checkAndCount(scopes); v != nil { t.Fatalf("99 < 100 must not violate, got %+v", v) } l.account(scopes, usage{completion: 1}, 0) // pushes tokout to 37: total now exactly 100 v := l.checkAndCount(scopes) if v == nil { t.Fatal("tokin+tokout landing exactly on the 100 budget must violate (>= limit, not only > limit)") } if v.storeDown { t.Error("storeDown must be false for a real limit breach") } } // --- perf review round 3, 2026-08-22: fused admission round trip --- // TestCheckAndCount_FusedViolationPrecedence is the fusion-equivalence // case: checkAndCount's admission round trip now fuses the request-counter // increments with every scope's token/cost budget reads into ONE store // call (buildBudgetProbes, incrAndGetMulti), replacing the pre-fusion // evaluateScope/budgetViolation/tokenBudgetViolation helpers that made // those reads as separate, serial round trips. This table pins that the // VIOLATION PRECEDENCE those helpers enforced is byte-for-byte unchanged: // within one scope, requests-per-minute, then requests-per-day, then // tokens-per-day, then tokens-per-month, then cost-per-day, then // cost-per-month; across scopes, the scope earlier in the given slice // (buildLimitScopes puts a user before their group) wins. The first six // subtests each engineer TWO simultaneously-true violation conditions // (one per limit type in a precedence pair) and assert only the // higher-precedence one is ever reported; the seventh engineers only ONE // condition (the group violates, the user carries no limits at all) to // pin the separate claim that a later scope still gets evaluated at all // when an earlier one has nothing to check (review fix, 2026-08-22, round // 2: this doc comment previously claimed all seven subtests engineered // two conditions, which was not true of the seventh). func TestCheckAndCount_FusedViolationPrecedence(t *testing.T) { t.Run("requests-per-minute beats requests-per-day in the same scope", func(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1, RequestsPerDay: 1}}} if v := l.checkAndCount(scopes); v != nil { t.Fatalf("1st call must pass (both counters land exactly on their limit), got %+v", v) } v := l.checkAndCount(scopes) if v == nil { t.Fatal("2nd call must violate: both requests-per-minute and requests-per-day are now over limit") } assert.Contains(t, v.message, "requests-per-minute") assert.NotContains(t, v.message, "requests-per-day") }) t.Run("requests-per-day beats tokens-per-day in the same scope", func(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerDay: 1, TokensPerDay: 10}}} if v := l.checkAndCount(scopes); v != nil { t.Fatalf("1st call must pass (req:day lands exactly on its limit, no tokens seeded yet), got %+v", v) } l.account(scopes, usage{prompt: 20}, 0) // tokin:day now 20, over the 10 budget v := l.checkAndCount(scopes) if v == nil { t.Fatal("2nd call must violate: both requests-per-day and tokens-per-day are now over limit") } assert.Contains(t, v.message, "requests-per-day") assert.NotContains(t, v.message, "tokens-per-day") }) t.Run("tokens-per-day beats tokens-per-month in the same scope", func(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{TokensPerDay: 10, TokensPerMonth: 10}}} l.account(scopes, usage{prompt: 20}, 0) // tokin:day and tokin:month both now 20, both over budget v := l.checkAndCount(scopes) if v == nil { t.Fatal("must violate: both tokens-per-day and tokens-per-month are over limit") } assert.Contains(t, v.message, "tokens-per-day") assert.NotContains(t, v.message, "tokens-per-month") }) t.Run("tokens-per-month beats cost-per-day in the same scope", func(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{TokensPerMonth: 10, CostPerDayUSD: 0.0001}}} l.account(scopes, usage{prompt: 20}, 200) // tokin:month = 20 (>10); cost:day = 200 micros (>100 micros) v := l.checkAndCount(scopes) if v == nil { t.Fatal("must violate: both tokens-per-month and cost-per-day are over limit") } assert.Contains(t, v.message, "tokens-per-month") assert.NotContains(t, v.message, "cost-per-day") }) t.Run("cost-per-day beats cost-per-month in the same scope", func(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{CostPerDayUSD: 0.0001, CostPerMonthUSD: 0.0001}}} l.account(scopes, usage{}, 200) // cost:day and cost:month both now 200 micros, both over the 100-micro budget v := l.checkAndCount(scopes) if v == nil { t.Fatal("must violate: both cost-per-day and cost-per-month are over limit") } assert.Contains(t, v.message, "cost-per-day") assert.NotContains(t, v.message, "cost-per-month") }) t.Run("user scope beats group scope when both violate", func(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{ {kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}}, {kind: "group", id: "g", limits: &LimitsConfig{RequestsPerMinute: 1}}, } if v := l.checkAndCount(scopes); v != nil { t.Fatalf("1st call must pass, got %+v", v) } v := l.checkAndCount(scopes) if v == nil { t.Fatal("2nd call must violate: both user and group are now over their requests-per-minute limit") } assert.Contains(t, v.message, `user "u"`) }) t.Run("group scope evaluated when the earlier user scope has no violation", func(t *testing.T) { l := newLimiter(nil, true) scopes := []limitScope{ {kind: "user", id: "u", limits: nil}, {kind: "group", id: "g", limits: &LimitsConfig{RequestsPerMinute: 1}}, } if v := l.checkAndCount(scopes); v != nil { t.Fatalf("1st call must pass, got %+v", v) } v := l.checkAndCount(scopes) if v == nil { t.Fatal("2nd call must violate: group is now over its requests-per-minute limit") } assert.Contains(t, v.message, `group "g"`) }) } // TestCheckAndCount_FusedRoundTrip_MemoryFallback proves the fused // admission round trip works correctly end to end against memoryStore — // the limiter's ALWAYS-available in-process fallback (newLimiter(nil, _) // uses it directly; a configured store also falls back to it on error or // while store-latched) — covering both an increment-only scope and a // scope with every budget type configured at once, in one round trip. func TestCheckAndCount_FusedRoundTrip_MemoryFallback(t *testing.T) { l := newLimiter(nil, true) now := time.Date(2026, 8, 22, 9, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: "user", id: "u", limits: &LimitsConfig{ RequestsPerMinute: 100, RequestsPerDay: 100, TokensPerDay: 1000, TokensPerMonth: 1000, CostPerDayUSD: 1, CostPerMonthUSD: 1, }}, {kind: totalScopeKind, id: totalScopeID, limits: nil}, } // Well under every budget: must pass and must have actually // incremented every request counter (proving the fused write side // still lands correctly, not just that no violation was reported). if v := l.checkAndCount(scopes); v != nil { t.Fatalf("want no violation comfortably under every limit, got %+v", v) } uMin, ok := l.getCounter("user", "u", metricReq, windowMin, now) if !ok || uMin != 1 { t.Errorf("user req:min = %d, ok=%v, want 1", uMin, ok) } totalMin, ok := l.getCounter(totalScopeKind, totalScopeID, metricReq, windowMin, now) if !ok || totalMin != 1 { t.Errorf("total req:min = %d, ok=%v, want 1 (the unlimited total scope is still counted)", totalMin, ok) } // Push tokens over budget via account, then confirm the SAME fused // round trip catches it on the very next call. l.account(scopes, usage{prompt: 2000}, 0) v := l.checkAndCount(scopes) if v == nil { t.Fatal("want a tokens-per-day violation now that usage exceeds the 1000 budget") } if v.storeDown { t.Error("v.storeDown must be false — this is a real limit breach, not a store failure") } assert.Contains(t, v.message, "tokens-per-day") } // TestCheckAndCount_RejectionRollsBackOtherScopes_MemoryFallback is // finding F1, 2026-09 review, the group-lockout bug: a user hammering // past THEIR OWN requests-per-minute limit must not also drive up their // group's identical counter on every rejected attempt — before this fix, // checkAndCount's unconditional per-scope counting had no rollback, so // enough rejected user-scope requests could push the group's own limit // past its threshold too, locking out every OTHER member of the group // from traffic that was never actually admitted. func TestCheckAndCount_RejectionRollsBackOtherScopes_MemoryFallback(t *testing.T) { l := newLimiter(nil, true) // nil store -> every operation uses the in-process fallback now := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}}, {kind: "group", id: "g", limits: &LimitsConfig{RequestsPerMinute: 100}}, } // 1st call: user count 1 <= its own limit of 1, group count 1 <= its // own limit of 100 — admitted. if v := l.checkAndCount(scopes); v != nil { t.Fatalf("1st call: want no violation, got %+v", v) } // 2nd call: user count reaches 2, over its own limit of 1 — rejected. // Without the fix, the group's own req:min counter would also land at // 2 here and stay there, even though the group never actually // admitted a 2nd request. v := l.checkAndCount(scopes) if v == nil { t.Fatal("2nd call: want a requests-per-minute violation on the user scope") } assert.Contains(t, v.message, "requests-per-minute") groupMin, ok := l.getCounter("group", "g", metricReq, windowMin, now) if !ok || groupMin != 1 { t.Errorf("group req:min after the user's rejection = %d, ok=%v, want 1 (rolled back)", groupMin, ok) } groupDay, ok := l.getCounter("group", "g", metricReq, windowDay, now) if !ok || groupDay != 1 { t.Errorf("group req:day after the user's rejection = %d, ok=%v, want 1 (rolled back)", groupDay, ok) } userMin, ok := l.getCounter("user", "u", metricReq, windowMin, now) if !ok || userMin != 2 { t.Errorf("user req:min after its own rejection = %d, ok=%v, want 2 (the violating scope's own count is not rolled back)", userMin, ok) } // A 3rd request, from a DIFFERENT user in the SAME group, must still // be admitted — proving the group is not locked out by the first // user's rejected attempts. otherUserScopes := []limitScope{ {kind: "user", id: "other", limits: &LimitsConfig{RequestsPerMinute: 1}}, {kind: "group", id: "g", limits: &LimitsConfig{RequestsPerMinute: 100}}, } if v := l.checkAndCount(otherUserScopes); v != nil { t.Fatalf("a different user in the same group: want no violation (group must not be locked out), got %+v", v) } } // --- review round 2, 2026-08-21: enforcement-aware fallback TTL clamp --- // TestClampTTL_ExactMathTable pins clampTTL's ceiling-vs-floor precedence // for every window's real (ttl, enforceTTL) pair. Round 1's clamp applied // memoryStoreMaxTTL unconditionally; round 2 corrects it so a window's // own enforceTTL floor always wins a conflict with that ceiling — this is // the exact math that keeps min/hour/day capped at 48h while month lands // back at its own ~32-day floor. func TestClampTTL_ExactMathTable(t *testing.T) { cases := []struct { name string ttl time.Duration enforceTTL time.Duration want time.Duration }{ {"min: 2m sits under both ceiling and floor, unchanged", minWindowTTL, enforceTTLFor(windowMin), minWindowTTL}, {"hour: 48h ttl equals the 48h ceiling; its own 2h floor never applies", hourWindowTTL, enforceTTLFor(windowHour), memoryStoreMaxTTL}, {"day: 35d ttl clamped down to the 48h ceiling, still above the 25h floor", dayWindowTTL, enforceTTLFor(windowDay), memoryStoreMaxTTL}, {"month: 400d ttl would clamp to 48h, but the 32d floor wins instead", monthWindowTTL, enforceTTLFor(windowMonth), enforceTTLFor(windowMonth)}, {"enforceTTL=0 (incrBy's own single-key path): pure ceiling, no floor", 400 * 24 * time.Hour, 0, memoryStoreMaxTTL}, {"ttl already under both ceiling and floor passes through unchanged", 90 * time.Minute, time.Hour, 90 * time.Minute}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { if got := clampTTL(c.ttl, c.enforceTTL); got != c.want { t.Errorf("clampTTL(%v, %v) = %v, want %v", c.ttl, c.enforceTTL, got, c.want) } }) } } // TestEnforceTTLFor_MatchesTheOriginalPreHistoryBumpValues pins // enforceTTLFor's four values directly — they are not arbitrary, they // match dayWindowTTL/monthWindowTTL's values from before the v0.2 // data-layer task's history-retention bump (25h/32d), which were // themselves already windowLength+margin for correct single-window // enforcement. func TestEnforceTTLFor_MatchesTheOriginalPreHistoryBumpValues(t *testing.T) { cases := []struct { window string want time.Duration }{ {windowMin, 2 * time.Minute}, {windowHour, 2 * time.Hour}, {windowDay, 25 * time.Hour}, {windowMonth, 32 * 24 * time.Hour}, } for _, c := range cases { if got := enforceTTLFor(c.window); got != c.want { t.Errorf("enforceTTLFor(%q) = %v, want %v", c.window, got, c.want) } } } // TestMemoryStore_IncrMulti_MonthCounterSurvivesPast48hAndStillEnforces // is the review-round-2 regression case: a month counter written via // incrMulti — the real production path, checkAndCount/account — must // stay alive, not silently reset to 0, well past 48h, and a // TokensPerMonth limit must still enforce correctly at that point. This // is exactly what round 1's unconditional 48h ceiling broke: a month // counter's key would have expired and reset roughly every two days, // turning the budget into a rolling ~48h one instead of a true // calendar-month one whenever Redis is absent or down. func TestMemoryStore_IncrMulti_MonthCounterSurvivesPast48hAndStillEnforces(t *testing.T) { l := newLimiter(nil, true) // nil store -> every operation uses the in-process fallback now := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerMonth: 100}}} l.account(scopes, usage{prompt: 60, completion: 20}, 0) // 80 tokens on day 1 // Advance 5 days: well past memoryStoreMaxTTL's 48h ceiling, far short // of monthWindowTTL's own natural rollover. now = now.Add(5 * 24 * time.Hour) l.nowFn = func() time.Time { return now } tokIn, ok := l.getCounter("group", "g", metricTokIn, windowMonth, now) if !ok || tokIn != 60 { t.Fatalf("tokin:month after 5 days = %d, ok=%v, want 60 (the key must not have expired at 48h)", tokIn, ok) } // Push the total over budget and confirm enforcement still fires — // proves the surviving counter is actually read by // tokenBudgetViolation, not merely present. l.account(scopes, usage{completion: 21}, 0) // 60 + 41 = 101 > 100 if v := l.checkAndCount(scopes); v == nil { t.Fatal("want a TokensPerMonth violation 5 days in; the round-1 clamp bug would have reset the counter well before this") } } // TestMemoryStore_IncrMulti_DayCounterStillClampsAt48h asserts the // round-1 clamp's original goal — bounding the fallback's own live-key // growth — still holds for day, whose own enforcement floor (25h) sits // below the 48h ceiling: a day counter written via incrMulti expires and // resets once its key crosses 48h, well before dayWindowTTL's own 35-day // retention would otherwise have kept it alive. func TestMemoryStore_IncrMulti_DayCounterStillClampsAt48h(t *testing.T) { m := newMemoryStore() now := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) m.nowFn = func() time.Time { return now } entry := counterIncr{key: "k", delta: 10, ttl: dayWindowTTL, enforceTTL: enforceTTLFor(windowDay)} if _, err := m.incrMulti([]counterIncr{entry}); err != nil { t.Fatal(err) } now = now.Add(49 * time.Hour) // just past the 48h ceiling m.nowFn = func() time.Time { return now } got, err := m.get("k") if err != nil { t.Fatal(err) } if got != 0 { t.Errorf("day counter at 49h past write = %d, want 0 (expired at the 48h ceiling; day's own 25h floor never raises it)", got) } } // --- Feature A (v0.22): per-provider/per-(provider,model) success-rate accounting --- // newSyncLimiter builds a limiter exactly like newLimiter, but with spawn // overridden to run synchronously instead of in its own goroutine — every // recordProviderAttempt test in this file needs its store write to have // already landed by the time it reads the counter back. Production // spawns a real goroutine there (SHOULD-5 ruling, v0.22 review round: off // a streaming response's TTFB path); this is the deterministic-test half // of that same dependency-injection field, matching nowFn/waitFn's // existing pattern elsewhere in this package. func newSyncLimiter(store counterStore, failOpen bool) *limiter { l := newLimiter(store, failOpen) l.spawn = func(f func()) { f() } return l } // TestRecordProviderAttempt_ClassifiesUsingIsTransient proves // recordProviderAttempt's failure classification is exactly isTransient's // own — reused, not forked (spec ruling) — across the full outcome table: // success, a non-429 4xx (still a "success" for provider-health purposes: // the provider answered correctly to a request it did not like), 429, a // 5xx, a plain network error, and a context-canceled error (not a // provider fault, per isTransient's own carve-out) — PLUS the SHOULD-1 // addition (v0.22 review round): a context.DeadlineExceeded error (a // hung upstream the gateway's own deadline finally cut off) counts as a // failure even though isTransient itself excludes it, both bare and // wrapped (proving isDeadlineExceeded's manual Unwrap walk actually // walks) — while context.Canceled, wrapped the identical way, still does // not. func TestRecordProviderAttempt_ClassifiesUsingIsTransient(t *testing.T) { tests := []struct { err error resp *http.Response name string wantFail bool }{ {name: "200 OK", resp: &http.Response{StatusCode: http.StatusOK}, wantFail: false}, {name: "404 not found (provider answered)", resp: &http.Response{StatusCode: http.StatusNotFound}, wantFail: false}, {name: "429 too many requests", resp: &http.Response{StatusCode: http.StatusTooManyRequests}, wantFail: true}, {name: "500 internal server error", resp: &http.Response{StatusCode: http.StatusInternalServerError}, wantFail: true}, {name: "503 service unavailable", resp: &http.Response{StatusCode: http.StatusServiceUnavailable}, wantFail: true}, {name: "network error", err: errors.New("dial tcp: connection refused"), wantFail: true}, {name: "context canceled (not a provider fault)", err: context.Canceled, wantFail: false}, {name: "context deadline exceeded (hung upstream timed out — SHOULD-1)", err: context.DeadlineExceeded, wantFail: true}, {name: "wrapped context deadline exceeded (proves errors.Is walks the chain)", err: fmt.Errorf("%w: dial timeout", context.DeadlineExceeded), wantFail: true}, {name: "wrapped context canceled stays non-failure too", err: fmt.Errorf("%w: client hung up", context.Canceled), wantFail: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { l := newSyncLimiter(nil, true) now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } l.recordProviderAttempt("openai", "gpt-4o", tt.resp, tt.err) attempts, _ := l.getCounter(kindProvider, "openai", metricProvAttempt, windowDay, now) if attempts != 1 { t.Errorf("provider attempts/day = %d, want 1", attempts) } fails, _ := l.getCounter(kindProvider, "openai", metricProvFail, windowDay, now) wantFails := int64(0) if tt.wantFail { wantFails = 1 } if fails != wantFails { t.Errorf("provider fails/day = %d, want %d", fails, wantFails) } // Minute window and the (provider, model) scope both mirror the // provider/day counters exactly. attemptsMin, _ := l.getCounter(kindProvider, "openai", metricProvAttempt, windowMin, now) if attemptsMin != 1 { t.Errorf("provider attempts/min = %d, want 1", attemptsMin) } modelAttempts, _ := l.getCounter(kindProviderModel, "openai/gpt-4o", metricProvAttempt, windowDay, now) if modelAttempts != 1 { t.Errorf("model attempts/day = %d, want 1", modelAttempts) } modelFails, _ := l.getCounter(kindProviderModel, "openai/gpt-4o", metricProvFail, windowDay, now) if modelFails != wantFails { t.Errorf("model fails/day = %d, want %d", modelFails, wantFails) } // Dead-code sweep, 2026-09 review: recordProviderAttempt no // longer writes a (provider, model) scope's MINUTE-window // counters — providerCounterKeys has never read one // (SHOULD-2, v0.22 review round), so those writes were pure // cost with no reader anywhere in this codebase. modelAttemptsMin, _ := l.getCounter(kindProviderModel, "openai/gpt-4o", metricProvAttempt, windowMin, now) if modelAttemptsMin != 0 { t.Errorf("model attempts/min = %d, want 0 (no longer written)", modelAttemptsMin) } modelFailsMin, _ := l.getCounter(kindProviderModel, "openai/gpt-4o", metricProvFail, windowMin, now) if modelFailsMin != 0 { t.Errorf("model fails/min = %d, want 0 (no longer written)", modelFailsMin) } }) } } // TestRecordProviderAttempt_EmptyModel_ProviderScopeOnly proves a caller // that cannot cheaply know the upstream model before the attempt resolves // (native passthrough, routes_passthrough.go's handlePassthrough: the // model lives in the response body, read only afterward) gets // provider-level accounting only when it passes model="". func TestRecordProviderAttempt_EmptyModel_ProviderScopeOnly(t *testing.T) { l := newSyncLimiter(nil, true) now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } l.recordProviderAttempt("openai", "", &http.Response{StatusCode: http.StatusInternalServerError}, nil) attempts, _ := l.getCounter(kindProvider, "openai", metricProvAttempt, windowDay, now) if attempts != 1 { t.Errorf("provider attempts/day = %d, want 1", attempts) } fails, _ := l.getCounter(kindProvider, "openai", metricProvFail, windowDay, now) if fails != 1 { t.Errorf("provider fails/day = %d, want 1", fails) } } // TestRecordProviderAttempt_EmptyProvider_NoOp proves a caller with no // resolved provider name (should never happen in production traffic, but // must never panic) is a silent no-op. func TestRecordProviderAttempt_EmptyProvider_NoOp(t *testing.T) { l := newSyncLimiter(nil, true) l.recordProviderAttempt("", "some-model", &http.Response{StatusCode: http.StatusOK}, nil) // must not panic } // TestRecordProviderAttempt_ProductionSpawn_WriteEventuallyLandsAsync // leaves limiter.spawn at its real production default (newLimiter's own // bounded, goroutine-spawning closure) instead of overriding it to run // synchronously like every other recordProviderAttempt test in this file // — FOLDED-2 (v0.22 review round, round 2): every other test here proves // the WRITE is correct, but only by bypassing the actual async code path // (limiter.spawn's own doc comment) entirely; this is the one test that // exercises it for real, closing a real -race async-path gap. Polls // (waitUntil, registry_test.go) rather than sleeping a fixed duration — // a fixed sleep is either flaky (too short, especially under `-race`, // which distorts goroutine scheduling) or wastefully slow (too long); a // deadline-bounded poll is neither. func TestRecordProviderAttempt_ProductionSpawn_WriteEventuallyLandsAsync(t *testing.T) { l := newLimiter(nil, true) // production default spawn — deliberately NOT newSyncLimiter now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } l.recordProviderAttempt("openai", "gpt-4o", &http.Response{StatusCode: http.StatusOK}, nil) waitUntil(t, time.Second, func() bool { attempts, _ := l.getCounter(kindProvider, "openai", metricProvAttempt, windowDay, now) return attempts == 1 }) fails, _ := l.getCounter(kindProvider, "openai", metricProvFail, windowDay, now) if fails != 0 { t.Errorf("fails/day = %d, want 0 (a 200 response is never a failure)", fails) } } // TestIsDeadlineExceeded_Classification exercises isDeadlineExceeded's // observable classification across every wrap shape that matters: bare, // wrapped once, wrapped twice, an unrelated sentinel (context.Canceled, // which must never match), an unrelated wrapped error, a plain error // with no Unwrap method — PLUS the prod-shape double-%w/triple-%w rows a // review round-2 blocker demanded, byte-for-byte the exact format // strings providers.go's three upstreamJSON/upstreamBytes call sites use. // // A load-bearing caveat this test's own name used to hide (it was // TestIsDeadlineExceeded_YaegiSafeUnwrap): every case here runs COMPILED, // under `go test`, and compiled-Go correctness is not the same claim as // Yaegi-interpreted correctness. Round 2's original fix — a hand-rolled // Unwrap walk against two locally declared interfaces, matched with bare // comma-ok type assertions — passed every one of these exact rows // (including the double-/triple-%w ones) under `go test`, while // SHOULD-A's yaegi-check harness (tools/yaegi-check/main.go's // exerciseAttemptAccounting) proved it silently returned false for the // double-%w case under the REAL interpreter: an interpreted interface // type does not correctly match a compiled concrete value's method set // in Yaegi, the reverse direction of the already-documented errors.As- // on-interpreted-types trap. matchesSentinel now calls real errors.Is // instead (limits.go's own doc comment there has the full account of why // that is safe here specifically, unlike errors.As on an interpreted // type) — this table still pins the OBSERVABLE behavior, but // tools/yaegi-check is what actually proves it under the interpreter; // this table alone would not have caught the round-2 regression, and a // future change here must keep running yaegi-check, not just `go test`. func TestIsDeadlineExceeded_Classification(t *testing.T) { tests := []struct { err error name string want bool }{ {name: "nil error", err: nil, want: false}, {name: "bare sentinel", err: context.DeadlineExceeded, want: true}, {name: "wrapped once", err: fmt.Errorf("upstream: %w", context.DeadlineExceeded), want: true}, {name: "wrapped twice", err: fmt.Errorf("outer: %w", fmt.Errorf("upstream: %w", context.DeadlineExceeded)), want: true}, {name: "unrelated sentinel (context.Canceled)", err: context.Canceled, want: false}, {name: "unrelated wrapped error", err: fmt.Errorf("dial: %w", errors.New("connection refused")), want: false}, {name: "plain error, no Unwrap method", err: errors.New("boom"), want: false}, { name: "double-%w, upstreamBytes' client.Do shape — the blocker's own reproduction", err: fmt.Errorf("%w: %w", errUpstream, context.DeadlineExceeded), want: true, }, { name: "double-%w, upstreamJSON's encode-failure shape", err: fmt.Errorf("%w: encode request body: %w", errUpstream, context.DeadlineExceeded), want: true, }, { name: "triple-%w, upstreamBytes' build-request shape", err: fmt.Errorf("%w: %w: build request: %w", errUpstream, errRequestBuildFailed, context.DeadlineExceeded), want: true, }, { name: "double-%w wrapping context.Canceled must still NOT match — multi-wrap does not blur the two sentinels apart", err: fmt.Errorf("%w: %w", errUpstream, context.Canceled), want: false, }, { name: "double-%w wrapping an unrelated error must still NOT match", err: fmt.Errorf("%w: %w", errUpstream, errors.New("connection refused")), want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := isDeadlineExceeded(tt.err); got != tt.want { t.Errorf("isDeadlineExceeded(%v) = %v, want %v", tt.err, got, tt.want) } }) } } // TestProviderModelScopeID proves the "provider/model" id convention // providerUsage/recordProviderAttempt share with pricing.go's own // unifiedCostMicros and the admin API. func TestProviderModelScopeID(t *testing.T) { if got := providerModelScopeID("openai", "gpt-4o"); got != "openai/gpt-4o" { t.Errorf("providerModelScopeID = %q, want %q", got, "openai/gpt-4o") } } // TestProviderUsage_BatchedSingleRoundTrip drives limiter.providerUsage // against a counting stub store (historyCountingStore, shared with // TestLimiterHistory_OneBatchCallOldestFirst above): it must issue // exactly ONE getMulti call for both a provider-level and a // (provider, model) scope together, and slice the flat result back to the // right scope in order — including SHOULD-2's variable stride (v0.22 // review round): the provider scope reads its minute window too, the // model scope does not, so a stray seeded minute-window key under the // model scope's own id must never leak into its result — providerUsage // never even asks the store for it. func TestProviderUsage_BatchedSingleRoundTrip(t *testing.T) { now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC) countStore := &historyCountingStore{values: map[string]int64{ windowKey(kindProvider, "openai", metricProvAttempt, windowMin, now): 3, windowKey(kindProvider, "openai", metricProvFail, windowMin, now): 1, windowKey(kindProvider, "openai", metricProvAttempt, windowDay, now): 30, windowKey(kindProvider, "openai", metricProvFail, windowDay, now): 2, windowKey(kindProviderModel, "openai/gpt-4o", metricProvAttempt, windowDay, now): 10, // Seeded but must never be read: providerCounterKeys omits a // kindProviderModel scope's minute-window keys entirely (SHOULD-2). // A bug that started reading them again would make this key's // value leak into got[1].attemptsMinute below, catching the // regression even though allKeys's own length is never asserted // directly. windowKey(kindProviderModel, "openai/gpt-4o", metricProvAttempt, windowMin, now): 999, }} l := newLimiter(countStore, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: kindProvider, id: "openai"}, {kind: kindProviderModel, id: "openai/gpt-4o"}, } got := l.providerUsage(scopes) if countStore.getMultiCalls != 1 { t.Errorf("getMultiCalls = %d, want 1 (both scopes' keys flattened into one call)", countStore.getMultiCalls) } if len(got) != 2 { t.Fatalf("len(got) = %d, want 2", len(got)) } if got[0].attemptsMinute != 3 || got[0].failuresMinute != 1 || got[0].attemptsDay != 30 || got[0].failuresDay != 2 { t.Errorf("provider counters = %+v, want {3 1 30 2}", got[0]) } if got[1].attemptsDay != 10 || got[1].failuresDay != 0 { t.Errorf("model counters = %+v, want attemptsDay 10, failuresDay 0", got[1]) } if got[1].attemptsMinute != 0 || got[1].failuresMinute != 0 { t.Errorf("model minute counters = %+v, want the zero value (never read for a model scope — SHOULD-2)", got[1]) } } // TestProviderUsage_OffsetSlicing_ScopeOrderIndependent proves the // running-offset slicing SHOULD-2 introduced (providerUsage no longer // assumes a single fixed stride) is correct regardless of scope order — // a model scope (2 keys) followed by TWO provider scopes (4 keys each) // followed by another model scope, deliberately not the "providers then // models" order buildAdminOverview itself always builds, to prove the // slicing logic itself does not silently depend on that convention. func TestProviderUsage_OffsetSlicing_ScopeOrderIndependent(t *testing.T) { now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC) countStore := &historyCountingStore{values: map[string]int64{ windowKey(kindProviderModel, "a/m1", metricProvAttempt, windowDay, now): 1, windowKey(kindProviderModel, "a/m1", metricProvFail, windowDay, now): 0, windowKey(kindProvider, "a", metricProvAttempt, windowMin, now): 2, windowKey(kindProvider, "a", metricProvFail, windowMin, now): 0, windowKey(kindProvider, "a", metricProvAttempt, windowDay, now): 20, windowKey(kindProvider, "a", metricProvFail, windowDay, now): 1, windowKey(kindProvider, "b", metricProvAttempt, windowMin, now): 3, windowKey(kindProvider, "b", metricProvFail, windowMin, now): 1, windowKey(kindProvider, "b", metricProvAttempt, windowDay, now): 30, windowKey(kindProvider, "b", metricProvFail, windowDay, now): 2, windowKey(kindProviderModel, "b/m2", metricProvAttempt, windowDay, now): 4, windowKey(kindProviderModel, "b/m2", metricProvFail, windowDay, now): 4, }} l := newLimiter(countStore, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: kindProviderModel, id: "a/m1"}, {kind: kindProvider, id: "a"}, {kind: kindProvider, id: "b"}, {kind: kindProviderModel, id: "b/m2"}, } got := l.providerUsage(scopes) if len(got) != 4 { t.Fatalf("len(got) = %d, want 4", len(got)) } if got[0].attemptsDay != 1 || got[0].failuresDay != 0 { t.Errorf("scope[0] (model a/m1) = %+v, want attemptsDay 1, failuresDay 0", got[0]) } if got[1].attemptsMinute != 2 || got[1].attemptsDay != 20 || got[1].failuresDay != 1 { t.Errorf("scope[1] (provider a) = %+v, want attemptsMinute 2, attemptsDay 20, failuresDay 1", got[1]) } if got[2].attemptsMinute != 3 || got[2].failuresMinute != 1 || got[2].attemptsDay != 30 || got[2].failuresDay != 2 { t.Errorf("scope[2] (provider b) = %+v, want {3 1 30 2}", got[2]) } if got[3].attemptsDay != 4 || got[3].failuresDay != 4 { t.Errorf("scope[3] (model b/m2) = %+v, want attemptsDay 4, failuresDay 4", got[3]) } } // TestProviderUsage_EmptyScopes_ReturnsEmptyWithoutTouchingStore mirrors // targetUsage's own empty-input short circuit. func TestProviderUsage_EmptyScopes_ReturnsEmptyWithoutTouchingStore(t *testing.T) { countStore := &historyCountingStore{} l := newLimiter(countStore, true) got := l.providerUsage(nil) if len(got) != 0 { t.Errorf("len(got) = %d, want 0", len(got)) } if countStore.getMultiCalls != 0 { t.Errorf("getMultiCalls = %d, want 0 (empty scopes must not touch the store)", countStore.getMultiCalls) } } // TestProviderUsage_StoreDown_ReturnsZeroCounters mirrors targetUsage's // own fail-closed contract: a store error with failOpen=false must report // every scope's counters as zero, not partial or stale values, AND must // set storeDown (review fix, adversarial verification 2026-08-23): a // caller that cannot tell a real zero from a failed read would otherwise // fabricate a counter-reset spike in metrics.go's rendering. func TestProviderUsage_StoreDown_ReturnsZeroCounters(t *testing.T) { l := newLimiter(alwaysErrStore{}, false) got := l.providerUsage([]limitScope{{kind: kindProvider, id: "openai"}}) if len(got) != 1 { t.Fatalf("len(got) = %d, want 1", len(got)) } if !got[0].storeDown { t.Errorf("counters = %+v, want storeDown true on a failed-closed read", got[0]) } if got[0] != (providerCounters{storeDown: true}) { t.Errorf("counters = %+v, want every non-storeDown field at its zero value", got[0]) } } // --- rejectionCounter: bounding, rotation, pruning (review fixes, adversarial verification 2026-08-23) --- // TestRejectionCounter_BoundsMemoryViaRotation mirrors // TestAuthFailureTracker_BoundsMemoryViaRotation (auth_test.go) exactly: // filling current past rejectionCounterMapCap distinct scopes must // trigger an O(1) generation rotation, not unbounded growth — a // deployment that renames or churns users over a long process lifetime // must never grow this map (and the resulting Prometheus series count) // without bound. func TestRejectionCounter_BoundsMemoryViaRotation(t *testing.T) { var c rejectionCounter for i := 0; i < rejectionCounterMapCap+10; i++ { c.increment("user", fmt.Sprintf("user-%d", i)) } c.mu.Lock() total := len(c.current) + len(c.previous) currentLen := len(c.current) c.mu.Unlock() if total > 2*rejectionCounterMapCap { t.Errorf("tracked entries = %d, want <= %d (2x cap)", total, 2*rejectionCounterMapCap) } if currentLen > rejectionCounterMapCap { t.Errorf("current generation = %d entries, want <= %d", currentLen, rejectionCounterMapCap) } } // TestRejectionCounter_Increment_AccumulatesPerScope proves the ordinary // path: repeated increments for the same (kind, id) accumulate in one // series rather than creating a new one each time. func TestRejectionCounter_Increment_AccumulatesPerScope(t *testing.T) { var c rejectionCounter c.increment("user", "alice") c.increment("user", "alice") c.increment("user", "bob") snaps := c.snapshot() got := map[rejectionScope]int64{} for _, s := range snaps { got[rejectionScope{kind: s.kind, id: s.id}] = s.count } if got[rejectionScope{kind: "user", id: "alice"}] != 2 { t.Errorf("alice count = %d, want 2", got[rejectionScope{kind: "user", id: "alice"}]) } if got[rejectionScope{kind: "user", id: "bob"}] != 1 { t.Errorf("bob count = %d, want 1", got[rejectionScope{kind: "user", id: "bob"}]) } } // TestLimiter_PruneRejections_RemovesDeletedUserScope is the mutation- // proofing test for pruneRejections/pruneUserScopes: a user's rejection // series must disappear once that user is no longer in keep (the shape // a real file-sourced hot-reload produces, via Gateway. // pruneRejectionsAfterReload, metrics.go), while an unrelated user's // series, and every non-"user" scope kind (group, the synthetic total // and store_down entries), must survive untouched. func TestLimiter_PruneRejections_RemovesDeletedUserScope(t *testing.T) { l := newLimiter(nil, true) l.rejections.increment("user", "alice") l.rejections.increment("user", "bob") l.rejections.increment("group", "agroup") l.rejections.increment(totalScopeKind, totalScopeID) l.rejections.increment(rejectionScopeStoreDown, rejectionScopeStoreDown) l.pruneRejections(map[string]bool{"bob": true}) // alice was removed from the users file; bob stays byKey := map[rejectionScope]bool{} for _, s := range l.rejectionSnapshot() { byKey[rejectionScope{kind: s.kind, id: s.id}] = true } if byKey[rejectionScope{kind: "user", id: "alice"}] { t.Error("alice's user-scope rejection entry survived pruning, want it removed") } if !byKey[rejectionScope{kind: "user", id: "bob"}] { t.Error("bob's user-scope rejection entry was pruned, want it kept (bob is still a known user)") } if !byKey[rejectionScope{kind: "group", id: "agroup"}] { t.Error("group-scope entry was pruned — pruning must only ever touch \"user\"-kind scopes") } if !byKey[rejectionScope{kind: totalScopeKind, id: totalScopeID}] { t.Error("the synthetic total-scope entry was pruned — pruning must only ever touch \"user\"-kind scopes") } if !byKey[rejectionScope{kind: rejectionScopeStoreDown, id: rejectionScopeStoreDown}] { t.Error("the synthetic store_down entry was pruned — pruning must only ever touch \"user\"-kind scopes") } } // --- kindModel scope: per-model usage accounting --- // readCounter reads one counter's current-bucket value directly, for tests // asserting exactly what account wrote. func readCounter(t *testing.T, l *limiter, kind, id, metric, window string, now time.Time) int64 { t.Helper() vals, ok := l.storeGetMulti([]string{windowKey(kind, id, metric, window, now)}) require.True(t, ok, "storeGetMulti") require.Len(t, vals, 1) return vals[0] } // TestAccount_ModelScopeWritesRequestsAndUsage pins the one behaviour that // separates a kindModel scope from every other kind inside account: it // carries its own request counter, because checkAndCount never counted one // for it at admission (the serving model is only known post-response). // A user scope in the SAME call must still get no req from account. func TestAccount_ModelScopeWritesRequestsAndUsage(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: "user", id: "alice"}, {kind: kindModel, id: "uni/qwen3.8-flash-next"}, } l.account(scopes, usage{prompt: 40, completion: 10}, 2_500_000) // The model scope gets req at all three chart windows. for _, w := range []string{windowHour, windowDay, windowMonth} { got := readCounter(t, l, kindModel, "uni/qwen3.8-flash-next", metricReq, w, now) assert.Equal(t, int64(1), got, "model req at %s", w) } // ...and the usage metrics, same as any other scope. assert.Equal(t, int64(40), readCounter(t, l, kindModel, "uni/qwen3.8-flash-next", metricTokIn, windowDay, now)) assert.Equal(t, int64(10), readCounter(t, l, kindModel, "uni/qwen3.8-flash-next", metricTokOut, windowDay, now)) assert.Equal(t, int64(2_500_000), readCounter(t, l, kindModel, "uni/qwen3.8-flash-next", metricCost, windowDay, now)) // The user scope must NOT gain a req from account — checkAndCount owns // that at admission, and double-counting it here would inflate every // existing per-user request figure. assert.Equal(t, int64(0), readCounter(t, l, "user", "alice", metricReq, windowDay, now), "account must not write req for a non-model scope") } // TestAccount_ModelScopeRequestCountedWithZeroUsage covers the case a // token/cost-gated write would miss: an upstream that reports no usage at // all still served a request, so the model's request counter must move. func TestAccount_ModelScopeRequestCountedWithZeroUsage(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } l.account([]limitScope{{kind: kindModel, id: "p/m"}}, usage{}, 0) assert.Equal(t, int64(1), readCounter(t, l, kindModel, "p/m", metricReq, windowDay, now)) assert.Equal(t, int64(0), readCounter(t, l, kindModel, "p/m", metricTokIn, windowDay, now)) } // TestWithModelScope covers the helper's add/skip decisions and, in the // last subtest, the aliasing hazard it exists to avoid. func TestWithModelScope(t *testing.T) { base := []limitScope{{kind: "user", id: "u"}} t.Run("adds the model scope", func(t *testing.T) { got := withModelScope(base, "uni/m") require.Len(t, got, 2) assert.Equal(t, kindModel, got[1].kind) assert.Equal(t, "uni/m", got[1].id) }) t.Run("skips an unattributable canonical", func(t *testing.T) { for _, canonical := range []string{"", "provider/", "/"} { got := withModelScope(base, canonical) assert.Len(t, got, 1, "canonical %q must add no scope", canonical) } }) t.Run("does not alias the caller's backing array", func(t *testing.T) { // A slice with spare capacity is the dangerous shape: a plain // append would write into the caller's own array. shared := make([]limitScope, 1, 4) shared[0] = limitScope{kind: "user", id: "u"} a := withModelScope(shared, "p/one") b := withModelScope(shared, "p/two") assert.Equal(t, "p/one", a[1].id, "second call must not overwrite the first result") assert.Equal(t, "p/two", b[1].id) }) } // TestModelTotals covers positional results, the non-existent-counter zero, // and the empty-input short circuit. func TestModelTotals(t *testing.T) { now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC) l := newLimiter(nil, true) l.nowFn = func() time.Time { return now } l.incrCounter(kindModel, "b/two", metricCost, windowDay, now, 500, dayWindowTTL) got, ok := l.modelTotals([]string{"a/one", "b/two", "c/three"}, metricCost, windowDay) require.True(t, ok) assert.Equal(t, []int64{0, 500, 0}, got, "results must be positional, zero for an unused model") empty, ok := l.modelTotals(nil, metricCost, windowDay) assert.True(t, ok, "an empty id list is not a failed read") assert.Empty(t, empty) } // --- bucketsFor: every window's bucketFor computed once per call --- // TestBucketsFor_MatchesBucketForEachWindow pins bucketsFor's four fields // against bucketFor called directly per window. func TestBucketsFor_MatchesBucketForEachWindow(t *testing.T) { now := time.Date(2026, 8, 20, 14, 37, 9, 0, time.UTC) wb := bucketsFor(now) assert.Equal(t, bucketFor(now, windowMin), wb.min) assert.Equal(t, bucketFor(now, windowHour), wb.hour) assert.Equal(t, bucketFor(now, windowDay), wb.day) assert.Equal(t, bucketFor(now, windowMonth), wb.month) } // TestEnforceTTLsFor_MatchesEnforceTTLForEachWindow pins enforceTTLsFor's // four fields against enforceTTLFor called directly per window — the two // must never drift apart, since checkAndCount/account/countTargetRequests/ // recordProviderAttempt all build their counterIncr entries from // enforceTTLsFor's single shared value instead of calling enforceTTLFor // per entry. func TestEnforceTTLsFor_MatchesEnforceTTLForEachWindow(t *testing.T) { et := enforceTTLsFor() assert.Equal(t, enforceTTLFor(windowMin), et.min) assert.Equal(t, enforceTTLFor(windowHour), et.hour) assert.Equal(t, enforceTTLFor(windowDay), et.day) assert.Equal(t, enforceTTLFor(windowMonth), et.month) } // TestWindowKeyForBucket_MatchesWindowKey pins windowKeyForBucket (the // building block windowKey and every checkAndCount/account/ // countTargetRequests/recordProviderAttempt call site now use directly // with a bucket from their own bucketsFor(now) call) against windowKey's // own output for the equivalent bucketFor(t, window) — the two must // produce byte-identical keys, since windowKey is now defined in terms of // this function. func TestWindowKeyForBucket_MatchesWindowKey(t *testing.T) { now := time.Date(2026, 8, 20, 9, 0, 30, 0, time.UTC) for _, window := range []string{windowMin, windowHour, windowDay, windowMonth} { got := windowKeyForBucket("user", "a", metricReq, window, bucketFor(now, window)) want := windowKey("user", "a", metricReq, window, now) assert.Equal(t, want, got, "window=%s", window) } } // --- the window/bucket/TTL/enforceTTL pairing invariant the 27 inline // counterIncr{...} literals (checkAndCount/account/countTargetRequests/ // recordProviderAttempt) removed structural protection for. newCounterIncr // used to derive enforceTTL from window itself, so a mismatch between the // two was impossible by construction; the inline literals pair window, // bucket, ttl, and enforceTTL by hand at each of the 27 call sites, and // nothing in the Go type system keeps them correctly paired any more. --- // recordingStore is a counterStore that records every counterIncr entry // passed to incrMulti/incrAndGetMulti and otherwise behaves like a // trivially successful backend — used by // TestCounterIncrEntries_WindowBucketTTLInvariant to capture exactly what // each of the four methods hands the store, without a real Redis or the // in-process fallback's own bucketing in the way. type recordingStore struct { entries []counterIncr } func (s *recordingStore) getMulti(keys []string) ([]int64, error) { return make([]int64, len(keys)), nil } func (s *recordingStore) incrMulti(entries []counterIncr) ([]int64, error) { s.entries = append(s.entries, entries...) return make([]int64, len(entries)), nil } func (s *recordingStore) incrAndGetMulti(entries []counterIncr, reads []string) ([]int64, []int64, error) { s.entries = append(s.entries, entries...) return make([]int64, len(entries)), make([]int64, len(reads)), nil } // TestCounterIncrEntries_WindowBucketTTLInvariant asserts, for every entry // each of the four methods produces: its enforceTTL equals // enforceTTLFor of the WINDOW embedded in its own key (key shape // llmgw:kind:id:metric:window:bucket, so strings.Split(e.key, ":")[4] is // the window), and its key ends in the bucket that window actually has // for now (bucketFor(now, window)) — catching a hand-paired mismatch like // a windowDay entry built with et.month, which nothing else in the suite // would notice (the wrong enforceTTL only ever changes memoryStore's // clampTTL floor, never the value counted). func TestCounterIncrEntries_WindowBucketTTLInvariant(t *testing.T) { now := time.Date(2026, 8, 20, 9, 30, 0, 0, time.UTC) scopes := []limitScope{{kind: "user", id: "u1", limits: &LimitsConfig{RequestsPerMinute: 1000, RequestsPerDay: 1000}}} cases := []struct { run func(l *limiter) name string }{ {name: "checkAndCount", run: func(l *limiter) { l.checkAndCount(scopes) }}, {name: "account", run: func(l *limiter) { l.account(scopes, usage{prompt: 10, completion: 5}, 100) }}, {name: "countTargetRequests", run: func(l *limiter) { l.countTargetRequests(scopes) }}, {name: "recordProviderAttempt", run: func(l *limiter) { l.recordProviderAttempt("openai", "gpt-4o", nil, errors.New("boom")) }}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.nowFn = func() time.Time { return now } l.spawn = func(f func()) { f() } // recordProviderAttempt's write must land synchronously to be captured c.run(l) require.NotEmpty(t, store.entries, "test fixture produced no counterIncr entries to check") for _, e := range store.entries { parts := strings.Split(e.key, ":") require.GreaterOrEqualf(t, len(parts), 6, "key %q does not have the llmgw:kind:id:metric:window:bucket shape", e.key) window := parts[4] wantTTL := enforceTTLFor(window) assert.Equalf(t, wantTTL, e.enforceTTL, "entry %q: enforceTTL, want enforceTTLFor(%q)", e.key, window) wantBucket := bucketFor(now, window) assert.Truef(t, strings.HasSuffix(e.key, wantBucket), "entry %q does not end with bucketFor(now, %q) = %q", e.key, window, wantBucket) } }) } } // TestModelTotals_ConfiguredStoreDown_FailOpen_ReturnsNotOK is finding // F5, 2026-09 review — modelTotals' own counterpart of // TestLimiterHistory_ConfiguredStoreDown_FailOpen_ReturnsNotOK above: with // failOpen=true, a store outage must still report ok=false via // configuredStoreDown rather than silently presenting the empty fallback // as a genuine "these models were barely used" ranking. func TestModelTotals_ConfiguredStoreDown_FailOpen_ReturnsNotOK(t *testing.T) { l := newLimiter(alwaysErrStore{}, true) got, ok := l.modelTotals([]string{"a/one"}, metricCost, windowDay) if ok { t.Fatal("want ok=false: the store is down even though failOpen silently served the fallback") } if got != nil { t.Errorf("got = %+v, want nil", got) } } // TestModelSpanTotalsSumsBuckets is F2 (v0.3 dashboard task): // modelSpanTotals sums span buckets ending at now, per id, positionally // — the arithmetic GET /admin/api/usage/models' ?span= parameter relies // on. It also proves modelTotals (its span=1 special case) reads only // today's own bucket, byte-for-byte the pre-F2 behavior. func TestModelSpanTotalsSumsBuckets(t *testing.T) { l := newLimiter(nil, true) now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) // Fix (verify-retry.md item 5): pin the limiter's own clock to the // fixed now above — the same convention every other test in this file // that touches l.now() already follows (TestLimiterRequestWindow and // dozens more, each setting l.nowFn right after newLimiter). Without // this, modelTotals below reads l.now() (real time.Now(), limits.go's // newLimiter default) instead of this test's fixed now: the buckets // were written against 2026-09-23, so the span=1 assertion only // happened to pass on the calendar day this test was first written, // and fails deterministically on every later day — not a race, and // not flaky under -race specifically; a plain date mismatch that // -race's slower execution does not change the odds of, just the day // this was first noticed on. l.nowFn = func() time.Time { return now } id := "openai/gpt-4o" l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 2), 5, dayWindowTTL) l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 1), 7, dayWindowTTL) l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 0), 11, dayWindowTTL) got, ok := l.modelSpanTotals([]string{id}, metricReq, windowDay, now, 3) if !ok { t.Fatal("want ok=true") } if len(got) != 1 || got[0] != 23 { t.Errorf("span=3 got = %v, want [23] (5+7+11)", got) } span1, ok := l.modelTotals([]string{id}, metricReq, windowDay) if !ok || len(span1) != 1 || span1[0] != 11 { t.Errorf("modelTotals (span=1 wrapper) got = %v ok=%v, want [11] (today's own bucket only)", span1, ok) } } // TestModelSpanTotalsSumsBuckets_MultipleIDsPositional proves the // flattened storeGetMulti's reply is sliced back per id in the SAME // order ids was given, not just correct for a single id. func TestModelSpanTotalsSumsBuckets_MultipleIDsPositional(t *testing.T) { l := newLimiter(nil, true) now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) l.incrCounter(kindModel, "a/one", metricCost, windowDay, historyStepBack(now, windowDay, 1), 100, dayWindowTTL) l.incrCounter(kindModel, "a/one", metricCost, windowDay, historyStepBack(now, windowDay, 0), 50, dayWindowTTL) l.incrCounter(kindModel, "b/two", metricCost, windowDay, historyStepBack(now, windowDay, 1), 3, dayWindowTTL) l.incrCounter(kindModel, "b/two", metricCost, windowDay, historyStepBack(now, windowDay, 0), 4, dayWindowTTL) got, ok := l.modelSpanTotals([]string{"a/one", "b/two"}, metricCost, windowDay, now, 2) if !ok { t.Fatal("want ok=true") } if len(got) != 2 || got[0] != 150 || got[1] != 7 { t.Errorf("got = %v, want [150 7]", got) } } // TestModelSpanTotals_EmptyIDs_NoStoreRead proves an empty ids slice // never touches the store at all — matches modelTotals' own pre-existing // contract, unaffected by F2's span extension. func TestModelSpanTotals_EmptyIDs_NoStoreRead(t *testing.T) { l := newLimiter(alwaysErrStore{}, false) // fail-closed AND always-erroring: any store read would fail the test got, ok := l.modelSpanTotals(nil, metricReq, windowDay, time.Now(), 5) if !ok || got != nil { t.Errorf("got = %v, ok = %v, want nil, true (no store read for an empty ids slice)", got, ok) } } // TestCheckAndCountRejectionFoldedIntoRollbackBatch is F4 (v0.3 dashboard // task): the fleet-wide rejectionsPerDay counter for the violated scope // AND the synthetic total scope rides the SAME storeIncrMulti batch // settleRejection already pays to compensate every other scope's // increments (see TestCheckAndCount_OneIncrAndGetMultiCallRegardlessOfScopeCount // above) — a rejection never costs a SECOND round trip beyond that one. func TestCheckAndCountRejectionFoldedIntoRollbackBatch(t *testing.T) { store := &countingIncrStore{} l := newLimiter(store, true) now := time.Date(2026, 9, 23, 9, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } scopes := []limitScope{ {kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}}, {kind: "group", id: "g", limits: nil}, {kind: totalScopeKind, id: totalScopeID, limits: nil}, } if v := l.checkAndCount(scopes); v != nil { t.Fatalf("1st call should pass, got %+v", v) } v := l.checkAndCount(scopes) if v == nil { t.Fatal("2nd call should violate u's requestsPerMinute:1") } if v.kind != eventKindRateLimit { t.Errorf("v.kind = %q, want %q", v.kind, eventKindRateLimit) } if store.incrMultiCalls != 1 { t.Errorf("incrMultiCalls after the violation = %d, want 1 (compensations + rejection counters share one batch)", store.incrMultiCalls) } uRej, ok := l.getCounter("user", "u", metricRej, windowDay, now) if !ok || uRej != 1 { t.Errorf("user rej:day = %d, ok=%v, want 1", uRej, ok) } totalRej, ok := l.getCounter(totalScopeKind, totalScopeID, metricRej, windowDay, now) if !ok || totalRej != 1 { t.Errorf("total rej:day = %d, ok=%v, want 1 (Q6: rejections attributed to total/all too)", totalRej, ok) } gRej, ok := l.getCounter("group", "g", metricRej, windowDay, now) if ok && gRej != 0 { t.Errorf("group rej:day = %d, want 0 (group did not violate)", gRej) } } // TestCheckAndCount_StoreDownRejection_NoRejCounterWrite proves the // store-down branch stays purely in-process (limits.go's own checkAndCount // doc comment): a storeDown violation never reaches settleRejection at // all — it fires before any per-scope evaluation loop — so it costs no // further store round trip beyond the one failed incrAndGetMulti attempt. func TestCheckAndCount_StoreDownRejection_NoRejCounterWrite(t *testing.T) { store := &erroringStore{err: errStoreDownStub} l := newLimiter(store, false) // fail-closed scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}}} v := l.checkAndCount(scopes) if v == nil || !v.storeDown { t.Fatalf("v = %+v, want a storeDown violation", v) } if v.kind != eventKindStoreDown { t.Errorf("v.kind = %q, want %q", v.kind, eventKindStoreDown) } } // TestCurrentUsage_RejectionsPerDay proves currentUsage reads the // rej:day counter into scopeUsage.rejectionsPerDay — F4's usageKeysPerScope // 8->9 growth (v0.3 dashboard task). func TestCurrentUsage_RejectionsPerDay(t *testing.T) { l := newLimiter(nil, true) now := time.Date(2026, 9, 23, 9, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } l.incrCounter("user", "u", metricRej, windowDay, now, 3, dayWindowTTL) got := l.currentUsage([]limitScope{{kind: "user", id: "u"}}) if len(got) != 1 { t.Fatalf("len(got) = %d, want 1", len(got)) } if got[0].rejectionsPerDay != 3 { t.Errorf("rejectionsPerDay = %d, want 3", got[0].rejectionsPerDay) } if got[0].storeDown { t.Error("storeDown = true, want false") } } // TestTargetUsage_ConfiguredStoreDown_FailOpen_ReturnsZeroCounters is // finding F5, 2026-09 review — targetUsage's own counterpart: with // failOpen=true, a store outage must fall back to the same zero-value // targetCounters this function's own doc comment already promises for // the fail-closed case, rather than presenting the empty in-process // fallback's data as confirmed current usage. func TestTargetUsage_ConfiguredStoreDown_FailOpen_ReturnsZeroCounters(t *testing.T) { l := newLimiter(alwaysErrStore{}, true) got := l.targetUsage([]limitScope{{kind: targetKindMCP, id: "alpha"}}) if len(got) != 1 { t.Fatalf("len(got) = %d, want 1", len(got)) } if got[0] != (targetCounters{}) { t.Errorf("got[0] = %+v, want the zero value", got[0]) } } // --- admin-redesign WP-A (steps 1-8) --- // TestMemoryStore_ApplySet_OverwritesRatherThanAdds proves memoryStore's // absolute-entry path (last-seen SET, Q6 DECISIONS) always REPLACES a // key's value, never adds to it — the SET-vs-INCRBY distinction // counterIncr.absolute exists to carry. func TestMemoryStore_ApplySet_OverwritesRatherThanAdds(t *testing.T) { m := newMemoryStore() m.nowFn = func() time.Time { return time.Unix(1000, 0) } if got := m.applySet("k", 42, time.Minute); got != 42 { t.Fatalf("first applySet = %d, want 42", got) } if got := m.applySet("k", 7, time.Minute); got != 7 { t.Errorf("second applySet = %d, want 7 (overwrite, not 49)", got) } if v, err := m.get("k"); err != nil || v != 7 { t.Errorf("get(k) = %d, err=%v, want 7, nil", v, err) } } // TestMemoryStore_IncrMulti_AbsoluteEntry_DispatchesToApplySet proves // incrMulti routes an absolute counterIncr through applySet (overwrite), // never applyIncr (add) — a non-absolute entry in the same batch still // increments normally. func TestMemoryStore_IncrMulti_AbsoluteEntry_DispatchesToApplySet(t *testing.T) { m := newMemoryStore() m.nowFn = func() time.Time { return time.Unix(2000, 0) } out, err := m.incrMulti([]counterIncr{ {key: "seen", delta: 555, ttl: time.Hour, absolute: true}, {key: "count", delta: 3, ttl: time.Hour}, }) if err != nil { t.Fatalf("incrMulti: %v", err) } if out[0] != 555 { t.Errorf("absolute entry result = %d, want 555", out[0]) } if out[1] != 3 { t.Errorf("incr entry result = %d, want 3", out[1]) } out2, _ := m.incrMulti([]counterIncr{{key: "count", delta: 3, ttl: time.Hour}}) if out2[0] != 6 { t.Errorf("second incr on non-absolute key = %d, want 6 (added)", out2[0]) } out3, _ := m.incrMulti([]counterIncr{{key: "seen", delta: 999, ttl: time.Hour, absolute: true}}) if out3[0] != 999 { t.Errorf("second absolute write = %d, want 999 (overwrite, not 555+999)", out3[0]) } } // TestLastSeenGate_ThrottlesWithin60Seconds is Q6's own throttle contract: // two due() calls for the same scope inside lastSeenGateThrottle yield // exactly one true; a different scope is never throttled by another // scope's own state. func TestLastSeenGate_ThrottlesWithin60Seconds(t *testing.T) { g := &lastSeenGate{} t0 := time.Unix(10_000, 0) if !g.due("user", "alice", t0) { t.Fatal("first due() = false, want true") } if g.due("user", "alice", t0.Add(30*time.Second)) { t.Error("due() 30s later = true, want false (within throttle)") } if !g.due("user", "alice", t0.Add(61*time.Second)) { t.Error("due() 61s later = false, want true (throttle elapsed)") } if !g.due("user", "bob", t0.Add(30*time.Second)) { t.Error("due() for a different scope = false, want true") } } // TestLastSeenGate_ThrottlesPastMapCap is P8 (admin dashboard redesign // verify round): due()'s two-map rotation wrote a scope forward into // `previous` once `current` filled past lastSeenGateMapCapBase, but // nothing ever CONSULTED previous on lookup — every scope that had been // rotated out was treated as brand new on its very next call, so // throttling silently stopped working once more than // lastSeenGateMapCapBase (4,096) distinct scopes were active. 5,000 // distinct scopes, each looked up twice within the same second (round 2 // repeats round 1's exact scopes, same instant), must produce exactly // 5,000 due()=true calls total — the pre-fix code produced far more, // since round 2 mostly missed both the (rotated) previous map's own // throttle and its freshly-empty current map. 5,000 stays below the N5 // adaptive-growth threshold (it only crosses ONE rotation, and growth // only triggers on a SECOND rotation within lastSeenGateThrottle — see // TestLastSeenGate_AdaptiveCapacityHoldsPastEightThousandScopes for the // scope count that exercises growth itself), so this keeps proving the // original fixed-cap rotation path unchanged. func TestLastSeenGate_ThrottlesPastMapCap(t *testing.T) { g := &lastSeenGate{} now := time.Unix(1_800_000_000, 0) const scopeCount = 5000 if scopeCount <= lastSeenGateMapCapBase { t.Fatalf("scopeCount (%d) must exceed lastSeenGateMapCapBase (%d) to exercise rotation", scopeCount, lastSeenGateMapCapBase) } writes := 0 for round := 0; round < 2; round++ { for i := 0; i < scopeCount; i++ { if g.due("user", fmt.Sprintf("u%d", i), now) { writes++ } } } if writes != scopeCount { t.Errorf("due()=true across 2 rounds of %d scopes within the same second = %d, want %d (round 2 must be fully throttled)", scopeCount, writes, scopeCount) } } // TestLastSeenGate_AdaptiveCapacityHoldsPastEightThousandScopes is N5 // (verify-redesign-final.md): a fixed 4,096-entry cap only throttled // correctly up to ~2x itself (current+previous both full, ~8,192 live // scopes) — beyond that, current rotated out before lastSeenGateThrottle // could elapse for most of what it held, discarding still-fresh writes // on every rotation, which degraded round 2 of a repeat to one write per // admission instead of zero. 20,000 distinct scopes (well past the old // ~8,192 ceiling), each looked up twice within the same second, must // still produce close to 20,000 due()=true calls total (round 1 only; // round 2 must be (almost) fully throttled) — proving due()'s adaptive // capacity (grows on a same-window second rotation, capped at // lastSeenGateMapCapMax) keeps the whole live population inside // current+previous instead of thrashing it. func TestLastSeenGate_AdaptiveCapacityHoldsPastEightThousandScopes(t *testing.T) { g := &lastSeenGate{} now := time.Unix(1_800_000_000, 0) const scopeCount = 20000 if scopeCount <= 2*lastSeenGateMapCapBase { t.Fatalf("scopeCount (%d) must exceed the old 2x-cap ceiling (%d) to exercise adaptive growth", scopeCount, 2*lastSeenGateMapCapBase) } writes := 0 for round := 0; round < 2; round++ { for i := 0; i < scopeCount; i++ { if g.due("user", fmt.Sprintf("u%d", i), now) { writes++ } } } // "~20000": round 1 always writes scopeCount times (every scope is // new). Round 2 may add a handful more only if a scope's write fell // right before the growth threshold was reached, never a large // fraction of scopeCount — the old, pre-fix behaviour wrote roughly // scopeCount MORE on round 2 alone (P8's "far more" regression, at // this population size), so a generous 1% slack still tells the two // behaviours apart with room to spare. maxWant := scopeCount + scopeCount/100 if writes < scopeCount || writes > maxWant { t.Errorf("due()=true across 2 rounds of %d scopes within the same second = %d, want in [%d, %d]", scopeCount, writes, scopeCount, maxWant) } if g.capacity > lastSeenGateMapCapMax { t.Errorf("g.capacity = %d, want <= lastSeenGateMapCapMax (%d)", g.capacity, lastSeenGateMapCapMax) } total := len(g.current) + len(g.previous) if bound := 2 * lastSeenGateMapCapMax; total > bound { t.Errorf("len(current)+len(previous) = %d, want <= 2*lastSeenGateMapCapMax (%d) — memory bound violated", total, bound) } } // TestCheckAndCount_NeverWritesLastSeen is the P9-fix replacement for this // test's own pre-fix name (AppendedAfterScopeStride): last-seen no longer // rides checkAndCount's batch AT ALL (admin dashboard redesign verify // round) — it used to be appended after every scope's own three stride // entries unconditionally, which meant a REJECTED request still moved a // scope's last-seen timestamp forward, since checkAndCount's whole batch, // last-seen tail included, was already sent to the store before the // violation check even ran. checkAndCount's own entry count is now // EXACTLY 3*checkAndCountKeysPerScope regardless of l.statsAdmin — see // TestRecordLastSeen_WritesTailEntriesForUserAndGroupScopesOnly, below, // for the tail-entry coverage this test used to carry, now exercised // against recordLastSeen directly (the function real admission call sites // invoke instead, only once admission is confirmed). func TestCheckAndCount_NeverWritesLastSeen(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } l.statsAdmin = true scopes := []limitScope{ {kind: "user", id: "alice", limits: &LimitsConfig{RequestsPerMinute: 1000}}, {kind: "group", id: "friends", limits: &LimitsConfig{RequestsPerMinute: 1000}}, {kind: totalScopeKind, id: totalScopeID}, } if v := l.checkAndCount(scopes); v != nil { t.Fatalf("checkAndCount violation = %+v, want nil", v) } if len(store.entries) != 3*checkAndCountKeysPerScope { t.Fatalf("len(store.entries) = %d, want %d (no last-seen tail)", len(store.entries), 3*checkAndCountKeysPerScope) } for i, e := range store.entries { if e.absolute { t.Errorf("entries[%d] (%q) is absolute, want checkAndCount to never write an absolute entry any more", i, e.key) } } } // TestRecordLastSeen_WritesTailEntriesForUserAndGroupScopesOnly proves // recordLastSeen (limits.go; called by every admission call site's own // "violation == nil" branch, P9 fix) writes exactly one absolute SET per // "user"/"group" scope, keyed by lastSeenKey, and never one for the // synthetic total scope. func TestRecordLastSeen_WritesTailEntriesForUserAndGroupScopesOnly(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } l.statsAdmin = true l.spawn = func(f func()) { f() } // synchronous for the test to observe entries scopes := []limitScope{ {kind: "user", id: "alice"}, {kind: "group", id: "friends"}, {kind: totalScopeKind, id: totalScopeID}, } l.recordLastSeen(scopes, now) if len(store.entries) != 2 { t.Fatalf("len(store.entries) = %d, want 2 (user + group, never total)", len(store.entries)) } seenKeys := map[string]bool{} for _, e := range store.entries { if !e.absolute { t.Errorf("entry %q is not absolute", e.key) } seenKeys[e.key] = true } if !seenKeys[lastSeenKey("user", "alice")] || !seenKeys[lastSeenKey("group", "friends")] { t.Errorf("entries = %v, want lastSeenKey for user alice and group friends", store.entries) } if seenKeys[lastSeenKey(totalScopeKind, totalScopeID)] { t.Error("total scope must never get a last-seen entry") } } // TestRecordLastSeen_OffWithoutStatsAdmin proves recordLastSeen (P9 fix's // own call site, invoked separately from checkAndCount now) builds zero // entries and spawns nothing when statsAdmin is false (the default for // every limiter built directly via newLimiter, unless a test opts in) — // the same "additive and default-preserving" guarantee every other family // in this task carries. spawn is deliberately left at its real (async, // goroutine-based) default: if this ever regressed to spawning an empty // batch, len(store.entries) staying 0 synchronously would not catch it, // so failing to compile a nil/empty entries slice at all (countAsync's own // "nil/empty entries slice spawns nothing" contract, limits.go) is what // this test actually pins. func TestRecordLastSeen_OffWithoutStatsAdmin(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.recordLastSeen([]limitScope{{kind: "user", id: "alice"}}, l.now()) if len(store.entries) != 0 { t.Errorf("entries = %d, want 0 (no last-seen without statsAdmin)", len(store.entries)) } } // TestCheckAndCount_LastSeen_OffWithoutStatsAdmin proves checkAndCount // itself never writes a last-seen entry, with or without statsAdmin — P9 // fix, admin dashboard redesign verify round: last-seen no longer rides // checkAndCount's batch at all (TestCheckAndCount_NeverWritesLastSeen, // above, pins the statsAdmin=true case; this pins the default-off case // the pre-fix name of this test originally covered). func TestCheckAndCount_LastSeen_OffWithoutStatsAdmin(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.checkAndCount([]limitScope{{kind: "user", id: "alice"}}) if len(store.entries) != checkAndCountKeysPerScope { t.Errorf("entries = %d, want %d (checkAndCount never writes last-seen any more)", len(store.entries), checkAndCountKeysPerScope) } } // TestRecordLastSeen_ThrottledSecondCallWithinWindowSkipsEntry proves the // SAME replica calling recordLastSeen twice in immediate succession (the // P9-fix call site, admin dashboard redesign verify round — every // admission call site invokes it once per admitted request) writes the // last-seen SET only once, via lastSeenGate.due's own throttle. func TestRecordLastSeen_ThrottledSecondCallWithinWindowSkipsEntry(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) l.nowFn = func() time.Time { return now } l.statsAdmin = true l.spawn = func(f func()) { f() } scopes := []limitScope{{kind: "user", id: "alice"}} l.recordLastSeen(scopes, now) if len(store.entries) != 1 { t.Fatalf("first call entries = %d, want 1", len(store.entries)) } store.entries = nil l.recordLastSeen(scopes, now) if len(store.entries) != 0 { t.Errorf("second call (same instant) entries = %d, want 0 (throttled, no last-seen entry)", len(store.entries)) } } // TestSettleRejection_RejHour_OnlyWhenStatsAdmin proves settleRejection's // rej:hour addition (rej/hour, always-on-with-admin) rides the same // compensation batch only when statsAdmin is true, alongside the // pre-existing, unconditional rej:day entry — and that the compensation // entries for OTHER scopes (the rollback half) are unaffected either way. func TestSettleRejection_RejHour_OnlyWhenStatsAdmin(t *testing.T) { now := time.Date(2026, 9, 23, 10, 30, 0, 0, time.UTC) wb := bucketsFor(now) et := enforceTTLsFor() scopes := []limitScope{ {kind: "user", id: "alice"}, {kind: totalScopeKind, id: totalScopeID}, } entries := []counterIncr{ {key: windowKeyForBucket("user", "alice", metricReq, windowMin, wb.min), delta: 1, ttl: minWindowTTL, enforceTTL: et.min}, {key: windowKeyForBucket("user", "alice", metricReq, windowDay, wb.day), delta: 1, ttl: dayWindowTTL, enforceTTL: et.day}, {key: windowKeyForBucket("user", "alice", metricReq, windowHour, wb.hour), delta: 1, ttl: hourWindowTTL, enforceTTL: et.hour}, {key: windowKeyForBucket(totalScopeKind, totalScopeID, metricReq, windowMin, wb.min), delta: 1, ttl: minWindowTTL, enforceTTL: et.min}, {key: windowKeyForBucket(totalScopeKind, totalScopeID, metricReq, windowDay, wb.day), delta: 1, ttl: dayWindowTTL, enforceTTL: et.day}, {key: windowKeyForBucket(totalScopeKind, totalScopeID, metricReq, windowHour, wb.hour), delta: 1, ttl: hourWindowTTL, enforceTTL: et.hour}, } for _, statsAdmin := range []bool{false, true} { store := &recordingStore{} l := newLimiter(store, true) l.nowFn = func() time.Time { return now } l.statsAdmin = statsAdmin l.settleRejection(entries, 0, scopes, wb, et) rejHourKey := windowKeyForBucket("user", "alice", metricRej, windowHour, wb.hour) var gotHour bool for _, e := range store.entries { if e.key == rejHourKey { gotHour = true } } if gotHour != statsAdmin { t.Errorf("statsAdmin=%v: rej:hour entry present=%v, want %v", statsAdmin, gotHour, statsAdmin) } } } // TestAccountWith_UserModel_RequiresOneUserScopeAndModelScope is (a), // plan §2: the opt-in per-(user, model) breakdown fires only when // statsUserModel is on, exactly one "user" scope is present, and a // kindModel scope is present — writing <=8 entries (req always, // tokin/tokout/cost only when nonzero) at day+month, never hour. func TestAccountWith_UserModel_RequiresOneUserScopeAndModelScope(t *testing.T) { id := userModelScopeID("alice", "openai/gpt-4o") cases := []struct { name string scopes []limitScope statsUserModel bool wantEntries bool }{ {name: "on, one user scope, model scope", statsUserModel: true, wantEntries: true, scopes: withModelScope([]limitScope{{kind: "user", id: "alice"}}, "openai/gpt-4o")}, {name: "flag off", statsUserModel: false, wantEntries: false, scopes: withModelScope([]limitScope{{kind: "user", id: "alice"}}, "openai/gpt-4o")}, {name: "no model scope", statsUserModel: true, wantEntries: false, scopes: []limitScope{{kind: "user", id: "alice"}}}, {name: "multi-group principal: two user-kind scopes never happens, but two USER scopes must still refuse", statsUserModel: true, wantEntries: false, scopes: withModelScope([]limitScope{{kind: "user", id: "alice"}, {kind: "user", id: "bob"}}, "openai/gpt-4o")}, {name: "media: no user scope at all", statsUserModel: true, wantEntries: false, scopes: withModelScope(nil, "openai/gpt-4o")}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.statsAdmin = true l.statsUserModel = c.statsUserModel l.accountWith(c.scopes, usage{prompt: 10, completion: 5}, 1000, accountExtras{}) var got int for _, e := range store.entries { if strings.HasPrefix(e.key, "llmgw:"+kindUserModel+":"+id+":") { got++ } } if c.wantEntries && got != 8 { t.Errorf("umodel entries = %d, want 8", got) } if !c.wantEntries && got != 0 { t.Errorf("umodel entries = %d, want 0", got) } }) } } // TestAccountWith_UserModel_ReqAlwaysEvenWithZeroUsage proves the req pair // is written unconditionally (mirroring kindModel's own unconditional req // write), even for a served request with zero reported usage. func TestAccountWith_UserModel_ReqAlwaysEvenWithZeroUsage(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.statsAdmin = true l.statsUserModel = true scopes := withModelScope([]limitScope{{kind: "user", id: "alice"}}, "openai/gpt-4o") l.accountWith(scopes, usage{}, 0, accountExtras{}) id := userModelScopeID("alice", "openai/gpt-4o") var got int for _, e := range store.entries { if strings.HasPrefix(e.key, "llmgw:"+kindUserModel+":"+id+":") && strings.Contains(e.key, ":"+metricReq+":") { got++ } } if got != 2 { t.Errorf("umodel req entries = %d, want 2 (day+month)", got) } } // TestAccountWith_Failover_IncrementsFromProvider is (c), plan §2: a // non-empty accountExtras.failoverFrom writes prov/{from}:fover at // hour+day when statsAdmin is on, never when it is off. func TestAccountWith_Failover_IncrementsFromProvider(t *testing.T) { for _, statsAdmin := range []bool{false, true} { store := &recordingStore{} l := newLimiter(store, true) l.statsAdmin = statsAdmin l.accountWith(nil, usage{}, 0, accountExtras{failoverFrom: "openai"}) var got int for _, e := range store.entries { if strings.HasPrefix(e.key, "llmgw:"+kindProvider+":openai:"+metricFover+":") { got++ } } want := 0 if statsAdmin { want = 2 } if got != want { t.Errorf("statsAdmin=%v: fover entries = %d, want %d", statsAdmin, got, want) } } } // TestLatencyBucketIndex_MatchesObserveLatencyBucket proves accountWith's // own bucket-index helper (limits.go) never disagrees with // observeLatencyBucket's in-process histogram (metrics.go) about which // bucket a duration belongs to — both must derive from the identical // latencyBucketIndex. func TestLatencyBucketIndex_MatchesObserveLatencyBucket(t *testing.T) { durations := []time.Duration{ 0, 50 * time.Millisecond, 100 * time.Millisecond, 999 * time.Millisecond, 2 * time.Second, 30 * time.Second, 600 * time.Second, 3600 * time.Second, } for _, d := range durations { idx, overflow := latencyBucketIndex(d) var buckets [1]int64 full := make([]int64, len(latencyBucketBounds)) var overflowCount int64 var sum float64 var count int64 observeLatencyBucket(full, &overflowCount, &sum, &count, d) _ = buckets if overflow { if overflowCount != 1 { t.Errorf("d=%v: latencyBucketIndex says overflow, observeLatencyBucket overflow count = %d, want 1", d, overflowCount) } if idx != len(latencyBucketBounds) { t.Errorf("d=%v: overflow idx = %d, want %d", d, idx, len(latencyBucketBounds)) } continue } if full[idx] != 1 { t.Errorf("d=%v: latencyBucketIndex says bucket %d, observeLatencyBucket did not increment it: buckets=%v", d, idx, full) } } } // TestAccountWith_Latency_WritesProviderAndModelBuckets is (c), plan §2: // hasLat && statsLatency writes the serving provider's own duration // bucket (hour+day) plus the served model's own (when a kindModel scope // is present), and TTFB buckets too when the sample carries one — riding // the SAME storeIncrMulti batch account already builds (never a second // round trip: this test's own recordingStore records exactly one // incrMulti call for the whole accountWith invocation). func TestAccountWith_Latency_WritesProviderAndModelBuckets(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.statsAdmin = true l.statsLatency = true scopes := withModelScope(nil, "openai/gpt-4o") lat := latencySample{duration: 2 * time.Second, ttfb: 100 * time.Millisecond, hasTTFB: true} l.accountWith(scopes, usage{}, 0, accountExtras{provider: "openai", lat: lat, hasLat: true}) durIdx, _ := latencyBucketIndex(lat.duration) ttfbIdx, _ := latencyBucketIndex(lat.ttfb) wantDur := latencyDurationMetric(durIdx) wantTTFB := latencyTTFBMetric(ttfbIdx) counts := map[string]int{} for _, e := range store.entries { counts[e.key]++ } for _, kind := range []string{kindProvider, kindModel} { id := "openai" if kind == kindModel { id = "openai/gpt-4o" } for _, window := range []string{windowHour, windowDay} { for _, metric := range []string{wantDur, wantTTFB} { key := windowKeyForBucket(kind, id, metric, window, "x") // windowKeyForBucket's own bucket arg is fixed to "x" here // only to build a comparable prefix; match by prefix minus // bucket instead of exact key. prefix := key[:len(key)-1] var found bool for k := range counts { if strings.HasPrefix(k, prefix) { found = true } } if !found { t.Errorf("missing latency entry for kind=%s id=%s metric=%s window=%s", kind, id, metric, window) } } } } } // TestAccountWith_CacheMiss_TotalHourDayAndModelDayOnly is (d), plan §2: // cacheMiss writes total cmiss at hour+day, and the served model's own // cmiss at day ONLY (never hour) — gated on statsAdmin. func TestAccountWith_CacheMiss_TotalHourDayAndModelDayOnly(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.statsAdmin = true scopes := withModelScope([]limitScope{{kind: totalScopeKind, id: totalScopeID}}, "openai/gpt-4o") l.accountWith(scopes, usage{}, 0, accountExtras{cacheMiss: true}) totalHour := windowKeyForBucket(totalScopeKind, totalScopeID, metricCmiss, windowHour, bucketFor(l.now(), windowHour)) totalDay := windowKeyForBucket(totalScopeKind, totalScopeID, metricCmiss, windowDay, bucketFor(l.now(), windowDay)) modelDay := windowKeyForBucket(kindModel, "openai/gpt-4o", metricCmiss, windowDay, bucketFor(l.now(), windowDay)) modelHour := windowKeyForBucket(kindModel, "openai/gpt-4o", metricCmiss, windowHour, bucketFor(l.now(), windowHour)) present := map[string]bool{} for _, e := range store.entries { present[e.key] = true } if !present[totalHour] || !present[totalDay] { t.Errorf("missing total cmiss hour/day entries: %v", store.entries) } if !present[modelDay] { t.Error("missing model cmiss day entry") } if present[modelHour] { t.Error("model cmiss hour entry present, want day only") } } // TestRecordCacheHit_ChitAlwaysCsaveSkippedWhenZero is recordCacheHit's // own contract (Q7, DECISIONS): total chit hour+day always, csave // hour+day only when savedMicros != 0, model chit/csave at day only. func TestRecordCacheHit_ChitAlwaysCsaveSkippedWhenZero(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.statsAdmin = true l.spawn = func(f func()) { f() } // synchronous for the test to observe entries l.recordCacheHit("openai/gpt-4o", 0) var chit, csave int for _, e := range store.entries { switch { case strings.Contains(e.key, ":"+metricChit+":"): chit++ case strings.Contains(e.key, ":"+metricCsave+":"): csave++ } } if chit != 3 { // total hour, total day, model day t.Errorf("chit entries = %d, want 3", chit) } if csave != 0 { t.Errorf("csave entries = %d, want 0 (saved=0)", csave) } store.entries = nil l.recordCacheHit("openai/gpt-4o", 500) csave = 0 for _, e := range store.entries { if strings.Contains(e.key, ":"+metricCsave+":") { csave++ } } if csave != 3 { // total hour, total day, model day t.Errorf("csave entries (saved=500) = %d, want 3", csave) } } // TestRecordCacheHit_OffWithoutStatsAdmin proves recordCacheHit spawns // nothing at all when admin stats are off. func TestRecordCacheHit_OffWithoutStatsAdmin(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.spawn = func(f func()) { f() } l.recordCacheHit("openai/gpt-4o", 500) if len(store.entries) != 0 { t.Errorf("entries = %d, want 0 (statsAdmin off)", len(store.entries)) } } // TestRecordUnpriced402_TotalAndModelHourDay proves recordUnpriced402 // writes total r402 hour+day plus the refused model's own r402 hour+day, // gated on statsAdmin — the counter events.go's recordUnpricedRefusalEvent // drives from its own single chokepoint (both routes_unified.go's and // routes_passthrough.go's identical 402 guard). func TestRecordUnpriced402_TotalAndModelHourDay(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.statsAdmin = true l.spawn = func(f func()) { f() } l.recordUnpriced402("openai/free-model") present := map[string]bool{} for _, e := range store.entries { present[e.key] = true } now := l.now() wantKeys := []string{ windowKeyForBucket(totalScopeKind, totalScopeID, metricR402, windowHour, bucketFor(now, windowHour)), windowKeyForBucket(totalScopeKind, totalScopeID, metricR402, windowDay, bucketFor(now, windowDay)), windowKeyForBucket(kindModel, "openai/free-model", metricR402, windowHour, bucketFor(now, windowHour)), windowKeyForBucket(kindModel, "openai/free-model", metricR402, windowDay, bucketFor(now, windowDay)), } for _, k := range wantKeys { if !present[k] { t.Errorf("missing r402 entry %q", k) } } if len(store.entries) != 4 { t.Errorf("total entries = %d, want 4", len(store.entries)) } } // TestRecordProviderAttempt_HourlyAndTimeout_OnlyWithStatsAdmin proves // step 3: prov/provmodel attempt+fail at HOUR granularity, and prov's own // timeout hour+day, are added only when statsAdmin is on — riding // recordProviderAttempt's SAME async batch (one spawn call either way, // never a second round trip). func TestRecordProviderAttempt_HourlyAndTimeout_OnlyWithStatsAdmin(t *testing.T) { for _, statsAdmin := range []bool{false, true} { store := &recordingStore{} l := newLimiter(store, true) l.spawn = func(f func()) { f() } l.statsAdmin = statsAdmin l.recordProviderAttempt("openai", "gpt-4o", nil, errProviderTimeout) var hourAttempt, hourFail, hourTimeout, dayTimeout, modelHourAttempt int for _, e := range store.entries { switch { case e.key == windowKeyForBucket(kindProvider, "openai", metricProvAttempt, windowHour, bucketFor(l.now(), windowHour)): hourAttempt++ case e.key == windowKeyForBucket(kindProvider, "openai", metricProvFail, windowHour, bucketFor(l.now(), windowHour)): hourFail++ case e.key == windowKeyForBucket(kindProvider, "openai", metricProvTimeout, windowHour, bucketFor(l.now(), windowHour)): hourTimeout++ case e.key == windowKeyForBucket(kindProvider, "openai", metricProvTimeout, windowDay, bucketFor(l.now(), windowDay)): dayTimeout++ case e.key == windowKeyForBucket(kindProviderModel, providerModelScopeID("openai", "gpt-4o"), metricProvAttempt, windowHour, bucketFor(l.now(), windowHour)): modelHourAttempt++ } } want := 0 if statsAdmin { want = 1 } if hourAttempt != want || hourFail != want || hourTimeout != want || dayTimeout != want || modelHourAttempt != want { t.Errorf("statsAdmin=%v: hourAttempt=%d hourFail=%d hourTimeout=%d dayTimeout=%d modelHourAttempt=%d, want %d each", statsAdmin, hourAttempt, hourFail, hourTimeout, dayTimeout, modelHourAttempt, want) } } } // TestCountTargetRequestsBy_CallerID_OnlyWithStatsAdmin proves step 8: // a non-empty caller writes kindTargetCaller req entries at day+month // (targetCallerScopeID("mcp", target, caller)) only when statsAdmin is // on, in the SAME batch countTargetRequests already builds. func TestCountTargetRequestsBy_CallerID_OnlyWithStatsAdmin(t *testing.T) { for _, statsAdmin := range []bool{false, true} { store := &recordingStore{} l := newLimiter(store, true) l.statsAdmin = statsAdmin l.countTargetRequestBy("alice", targetKindMCP, "fetch") id := targetCallerScopeID(targetKindMCP, "fetch", "alice") var got int for _, e := range store.entries { if strings.HasPrefix(e.key, "llmgw:"+kindTargetCaller+":"+id+":") { got++ } } want := 0 if statsAdmin { want = 2 // day + month } if got != want { t.Errorf("statsAdmin=%v: tcaller entries = %d, want %d", statsAdmin, got, want) } } } // TestCountTargetRequestsBy_EmptyCaller_MatchesCountTargetRequests proves // countTargetRequests (no caller) is byte-for-byte countTargetRequestsBy's // own caller="" special case — no behavior change for any pre-existing // caller of the old name. func TestCountTargetRequestsBy_EmptyCaller_MatchesCountTargetRequests(t *testing.T) { scopes := []limitScope{{kind: targetKindMCP, id: "fetch"}} storeA := &recordingStore{} lA := newLimiter(storeA, true) lA.statsAdmin = true lA.nowFn = func() time.Time { return time.Unix(0, 0) } lA.countTargetRequests(scopes) storeB := &recordingStore{} lB := newLimiter(storeB, true) lB.statsAdmin = true lB.nowFn = func() time.Time { return time.Unix(0, 0) } lB.countTargetRequestsBy("", scopes) if len(storeA.entries) != len(storeB.entries) { t.Fatalf("len(A)=%d len(B)=%d, want equal", len(storeA.entries), len(storeB.entries)) } for i := range storeA.entries { if storeA.entries[i].key != storeB.entries[i].key || storeA.entries[i].delta != storeB.entries[i].delta { t.Errorf("entry %d differs: A=%+v B=%+v", i, storeA.entries[i], storeB.entries[i]) } } } // TestHistoryBucketKeysAt_OffsetZero_MatchesHistoryBucketKeys proves // offset=0 is byte-for-byte historyBucketKeys' own pre-existing output. func TestHistoryBucketKeysAt_OffsetZero_MatchesHistoryBucketKeys(t *testing.T) { now := time.Date(2026, 9, 23, 10, 30, 0, 0, time.UTC) wantKeys, wantBuckets := historyBucketKeys(kindModel, "openai/gpt-4o", metricReq, windowDay, now, 5) gotKeys, gotBuckets := historyBucketKeysAt(kindModel, "openai/gpt-4o", metricReq, windowDay, now, 5, 0) assert.Equal(t, wantKeys, gotKeys) assert.Equal(t, wantBuckets, gotBuckets) } // TestHistoryBucketKeysAt_Offset_ShiftsWholeSpanBack proves an offset // shifts the ENTIRE span back that many window units — the span=3, // offset=2 window ends exactly 2 days before the span=3, offset=0 // window's own oldest bucket. func TestHistoryBucketKeysAt_Offset_ShiftsWholeSpanBack(t *testing.T) { now := time.Date(2026, 9, 23, 10, 30, 0, 0, time.UTC) _, baseBuckets := historyBucketKeysAt(kindModel, "m", metricReq, windowDay, now, 3, 0) _, offsetBuckets := historyBucketKeysAt(kindModel, "m", metricReq, windowDay, now, 3, 2) // offsetBuckets' newest bucket must equal baseBuckets' oldest bucket // shifted back one more day (span=3 starting 2 days earlier ends 2 // days earlier too). wantNewest := bucketFor(historyStepBack(now, windowDay, 2), windowDay) if offsetBuckets[len(offsetBuckets)-1] != wantNewest { t.Errorf("offset span's newest bucket = %s, want %s", offsetBuckets[len(offsetBuckets)-1], wantNewest) } if offsetBuckets[0] == baseBuckets[0] { t.Error("offset span's oldest bucket must differ from the unshifted span's") } } // TestSpanTotalsMulti_Offset_ReadsAnEarlierSpan proves spanTotalsMulti's // offset argument reads a DIFFERENT (earlier) set of buckets than // offset=0 (plan §1.3(iii)) — exercised directly against the PRODUCTION // reader (limits.go), the same one admin.go's chunkedModelSpanTotals/ // chunkedModelSpanTotalsMulti and stats_read.go's readSpanTotals now all // share (P12 fix, admin dashboard redesign verify round). func TestSpanTotalsMulti_Offset_ReadsAnEarlierSpan(t *testing.T) { l := newLimiter(nil, true) now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC) id := "openai/gpt-4o" l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 3), 100, dayWindowTTL) // 3 days ago l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 0), 7, dayWindowTTL) // today gotNow, ok := l.spanTotalsMulti(kindModel, []string{id}, []string{metricReq}, windowDay, now, 1, 0) if !ok || gotNow[0][0] != 7 { t.Errorf("offset=0 = %v, ok=%v, want [[7]]", gotNow, ok) } gotOffset, ok := l.spanTotalsMulti(kindModel, []string{id}, []string{metricReq}, windowDay, now, 1, 3) if !ok || gotOffset[0][0] != 100 { t.Errorf("offset=3 = %v, ok=%v, want [[100]]", gotOffset, ok) } } // TestAccountWith_ZeroValueExtras_AddsNoFailoverLatencyOrCacheEntries is a // smoke test that account (accountWith with a zero-value accountExtras) // never adds a fover/latency/cmiss entry — those three are gated purely by // accountExtras fields, unlike umodel (gated by limiter flags + scope // shape, exercised separately above), so they must stay silent regardless // of statsAdmin/statsLatency when the caller passes no extras at all. func TestAccountWith_ZeroValueExtras_AddsNoFailoverLatencyOrCacheEntries(t *testing.T) { store := &recordingStore{} l := newLimiter(store, true) l.statsAdmin = true l.statsLatency = true scopes := withModelScope([]limitScope{{kind: "user", id: "alice"}}, "openai/gpt-4o") l.account(scopes, usage{prompt: 10, completion: 5}, 1000) for _, e := range store.entries { if strings.Contains(e.key, ":"+metricFover+":") || strings.Contains(e.key, ":ld") || strings.Contains(e.key, ":lt") || strings.Contains(e.key, ":"+metricCmiss+":") { t.Errorf("unexpected extras-driven entry from account() with zero-value extras: %q", e.key) } } }