Files
lukaszraczylo 8807f4d27d fix(test): pin the limiter clock in TestModelSpanTotalsSumsBuckets
The test wrote counters for a fixed date, 2026-09-23, but never pinned
the limiter's clock. modelTotals therefore read the real time. The test
passed only on the date it was written and fails on every later day.
It now sets l.nowFn to the fixed time, as the other span tests do. This
is a test-only fix; production code is unchanged.

Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
2026-09-24 06:04:40 +01:00

3623 lines
151 KiB
Go

package traefikllmgateway
import (
"context"
"errors"
"fmt"
"math"
"net/http"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestLimiterRequestWindow is the brief's Step-1 case: a per-minute
// request limit blocks the third request in a window and resets on the
// next window.
func TestLimiterRequestWindow(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC)
l := newLimiter(nil, true) // nil store → uses fallback memoryStore
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "user", id: "a", limits: &LimitsConfig{RequestsPerMinute: 2}}}
if v := l.checkAndCount(scopes); v != nil {
t.Fatal("1st should pass")
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatal("2nd should pass")
}
if v := l.checkAndCount(scopes); v == nil {
t.Fatal("3rd should violate")
}
now = now.Add(time.Minute) // next window
if v := l.checkAndCount(scopes); v != nil {
t.Fatal("new window should pass")
}
}
// TestLimiterTokenBudget is the brief's Step-1 case: accounting tokens
// past a day budget makes the next check refuse.
func TestLimiterTokenBudget(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerDay: 100}}}
l.account(scopes, usage{prompt: 60, completion: 50}, 0) // 110 > 100
if v := l.checkAndCount(scopes); v == nil {
t.Fatal("over token budget must refuse")
}
}
// TestLimiterCostBudget mirrors TestLimiterTokenBudget for a cost budget:
// accounting micro-USD cost past a day budget makes the next check refuse,
// and the violation carries the day window's Retry-After.
func TestLimiterCostBudget(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{CostPerDayUSD: 1.00}}}
l.account(scopes, usage{}, 1_500_000) // $1.50 > $1.00 budget
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("over cost budget must refuse")
}
wantRetry := int(time.Date(2026, 8, 21, 0, 0, 0, 0, time.UTC).Sub(now).Seconds())
if v.retryAfter != wantRetry {
t.Errorf("retryAfter = %d, want %d (seconds to next UTC midnight)", v.retryAfter, wantRetry)
}
if v.storeDown {
t.Error("storeDown must be false in this task")
}
}
// TestLimiterNilLimitsUnlimited asserts a scope with limits == nil never
// blocks a request, however many times it is checked — request counters
// still climb unboundedly but nothing evaluates them.
func TestLimiterNilLimitsUnlimited(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "user", id: "unlimited", limits: nil}}
for i := 0; i < 1000; i++ {
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("iteration %d: nil limits must never violate, got %+v", i, v)
}
}
}
// TestLimiterUserOverridesAlongsideGroup asserts checkAndCount evaluates
// every scope it is given in one call: a tight user limit blocks the
// request even though the group scope in the same call is unlimited, and
// an unlimited user scope still lets a tight group limit block.
func TestLimiterUserOverridesAlongsideGroup(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
t.Run("tight user limit blocks despite unlimited group", func(t *testing.T) {
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: "user", id: "u1", limits: &LimitsConfig{RequestsPerMinute: 1}},
{kind: "group", id: "g1", limits: nil},
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatal("1st request should pass")
}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("2nd request should violate the user's tight limit")
}
})
t.Run("tight group limit blocks despite unlimited user", func(t *testing.T) {
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: "user", id: "u2", limits: nil},
{kind: "group", id: "g2", limits: &LimitsConfig{RequestsPerMinute: 1}},
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatal("1st request should pass")
}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("2nd request should violate the group's tight limit")
}
})
}
// TestLimiterRetryAfter checks the Retry-After computation directly for
// each window kind: seconds to the next minute boundary, next UTC
// midnight, and the first of next month UTC.
func TestLimiterRetryAfter(t *testing.T) {
cases := []struct {
name string
now time.Time
window string
want int
}{
{"minute boundary", time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC), windowMin, 30},
{"day boundary", time.Date(2026, 8, 20, 23, 59, 0, 0, time.UTC), windowDay, 60},
{"month boundary, 31-day month", time.Date(2026, 8, 31, 23, 59, 59, 0, time.UTC), windowMonth, 1},
{"month boundary, mid-month", time.Date(2026, 2, 15, 0, 0, 0, 0, time.UTC), windowMonth, int(14 * 24 * time.Hour / time.Second)}, // Feb 2026 (not a leap year) runs Feb1-Mar1 = 28 days; Feb15 is 14 days before Mar1
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := retryAfterSeconds(c.now, c.window); got != c.want {
t.Errorf("retryAfterSeconds(%v, %q) = %d, want %d", c.now, c.window, got, c.want)
}
})
}
}
// TestLimiterRequestLimitRetryAfter asserts a request-limit violation
// (not a budget violation) also carries the correct Retry-After for its
// window.
func TestLimiterRequestLimitRetryAfter(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "user", id: "a", limits: &LimitsConfig{RequestsPerMinute: 1}}}
if v := l.checkAndCount(scopes); v != nil {
t.Fatal("1st should pass")
}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("2nd should violate")
}
if v.retryAfter != 30 {
t.Errorf("retryAfter = %d, want 30", v.retryAfter)
}
}
// TestWindowKeyFormat pins the exact key layout every store operation
// depends on: llmgw:{kind}:{id}:{metric}:{window}:{bucket}, with bucket
// formatted from t.UTC() at each window's granularity — including hour
// (v0.2 data-layer task).
func TestWindowKeyFormat(t *testing.T) {
tm := time.Date(2026, 8, 20, 10, 4, 59, 0, time.FixedZone("UTC+1", 3600)) // 09:04:59 UTC
cases := []struct {
window string
want string
}{
{windowHour, "llmgw:user:a:req:hour:2026082009"},
{windowMin, "llmgw:user:a:req:min:202608200904"},
{windowDay, "llmgw:user:a:req:day:20260820"},
{windowMonth, "llmgw:user:a:req:month:202608"},
}
for _, c := range cases {
if got := windowKey("user", "a", "req", c.window, tm); got != c.want {
t.Errorf("windowKey(..., %q, ...) = %q, want %q", c.window, got, c.want)
}
}
}
// TestWindowKeyFormat_HourUTCBoundary pins the hour bucket rolling over
// exactly at the UTC hour boundary, and proves a non-UTC input is
// converted to UTC first (an input at 00:04:59 in a UTC+1 zone is 23:04:59
// UTC the PREVIOUS day — a bug converting the hour field alone without
// re-deriving the date would produce "...23" glued onto the wrong day).
func TestWindowKeyFormat_HourUTCBoundary(t *testing.T) {
justBefore := time.Date(2026, 8, 20, 8, 59, 59, 0, time.UTC)
justAfter := time.Date(2026, 8, 20, 9, 0, 0, 0, time.UTC)
if got := windowKey("user", "a", "req", windowHour, justBefore); got != "llmgw:user:a:req:hour:2026082008" {
t.Errorf("just before the hour boundary: got %q", got)
}
if got := windowKey("user", "a", "req", windowHour, justAfter); got != "llmgw:user:a:req:hour:2026082009" {
t.Errorf("just after the hour boundary: got %q", got)
}
// 00:04:59 in UTC+1 is 23:04:59 UTC the previous day.
crossMidnight := time.Date(2026, 8, 21, 0, 4, 59, 0, time.FixedZone("UTC+1", 3600))
if got := windowKey("user", "a", "req", windowHour, crossMidnight); got != "llmgw:user:a:req:hour:2026082023" {
t.Errorf("cross-midnight non-UTC input: got %q, want the previous UTC day's hour 23 bucket", got)
}
}
// TestWindowEnd_HourBoundary asserts windowEnd's new hour case (v0.2
// data-layer task) returns the next UTC hour boundary, and that a value
// exactly on the boundary reports zero seconds remaining in that boundary
// (retryAfterSeconds rounds a zero duration up to 0, distinct from
// rounding a fractional second up to 1 — see TestLimiterRetryAfter for
// the day/month/min cases this mirrors).
func TestWindowEnd_HourBoundary(t *testing.T) {
t.Run("mid-hour", func(t *testing.T) {
got := windowEnd(time.Date(2026, 8, 20, 9, 30, 0, 0, time.UTC), windowHour)
want := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
if !got.Equal(want) {
t.Errorf("windowEnd = %v, want %v", got, want)
}
})
t.Run("exactly on the boundary", func(t *testing.T) {
got := windowEnd(time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC), windowHour)
want := time.Date(2026, 8, 20, 11, 0, 0, 0, time.UTC)
if !got.Equal(want) {
t.Errorf("windowEnd = %v, want %v", got, want)
}
})
}
// TestHistoryStepBack_HourAndDay asserts the fixed-duration step-back
// cases (hour, day) land i units before now exactly, including across a
// UTC day boundary.
func TestHistoryStepBack_HourAndDay(t *testing.T) {
now := time.Date(2026, 8, 20, 1, 30, 0, 0, time.UTC)
if got := historyStepBack(now, windowHour, 0); !got.Equal(now) {
t.Errorf("i=0 must be now itself, got %v", got)
}
wantHour := time.Date(2026, 8, 19, 23, 30, 0, 0, time.UTC) // 2 hours back, crossing midnight
if got := historyStepBack(now, windowHour, 2); !got.Equal(wantHour) {
t.Errorf("i=2 hours back = %v, want %v", got, wantHour)
}
wantDay := time.Date(2026, 8, 17, 1, 30, 0, 0, time.UTC)
if got := historyStepBack(now, windowDay, 3); !got.Equal(wantDay) {
t.Errorf("i=3 days back = %v, want %v", got, wantDay)
}
}
// TestHistoryStepBack_MonthAnchorsOnDayOne is the month-arithmetic trap
// TestLimiterRetryAfter's own month case does not exercise: stepping back
// from a high day-of-month (the 31st) with a naive
// now.AddDate(0, -i, 0) overflows a shorter target month (e.g. "Mar 31"
// minus one month normalizes to "Mar 3", not "Feb 28") and would land in
// the WRONG calendar month's bucket. historyStepBack avoids this by
// anchoring on day 1 first — this test drives it from Mar 31 and Jan 31
// specifically because those are the inputs that would expose a
// regression back to the naive form.
func TestHistoryStepBack_MonthAnchorsOnDayOne(t *testing.T) {
cases := []struct {
now time.Time
name string
want string
i int
}{
{name: "Mar 31 minus 1 month = Feb", now: time.Date(2026, 3, 31, 12, 0, 0, 0, time.UTC), i: 1, want: "202602"},
{name: "Jan 31 minus 1 month = Dec (year rollover)", now: time.Date(2026, 1, 31, 0, 0, 0, 0, time.UTC), i: 1, want: "202512"},
{name: "May 31 minus 3 months = Feb", now: time.Date(2026, 5, 31, 0, 0, 0, 0, time.UTC), i: 3, want: "202602"},
{name: "i=0 is now's own month", now: time.Date(2026, 3, 31, 0, 0, 0, 0, time.UTC), i: 0, want: "202603"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := bucketFor(historyStepBack(c.now, windowMonth, c.i), windowMonth)
if got != c.want {
t.Errorf("bucketFor(historyStepBack(%v, month, %d)) = %q, want %q", c.now, c.i, got, c.want)
}
})
}
}
// TestHistoryBucketKeys_OldestFirstInclusiveOfCurrent asserts
// historyBucketKeys returns span buckets oldest-first, with the LAST
// entry being now's own (current, possibly partial) bucket — the
// "inclusive of current bucket" contract GET /admin/api/usage/history
// promises.
func TestHistoryBucketKeys_OldestFirstInclusiveOfCurrent(t *testing.T) {
now := time.Date(2026, 8, 20, 14, 0, 0, 0, time.UTC)
keys, buckets := historyBucketKeys("total", "all", "req", windowHour, now, 4)
wantBuckets := []string{"2026082011", "2026082012", "2026082013", "2026082014"}
if len(buckets) != len(wantBuckets) {
t.Fatalf("len(buckets) = %d, want %d", len(buckets), len(wantBuckets))
}
for i, want := range wantBuckets {
if buckets[i] != want {
t.Errorf("buckets[%d] = %q, want %q", i, buckets[i], want)
}
wantKey := "llmgw:total:all:req:hour:" + want
if keys[i] != wantKey {
t.Errorf("keys[%d] = %q, want %q", i, keys[i], wantKey)
}
}
}
// TestLimiterHistory_OneBatchCallOldestFirst drives limiter.history
// against a counting stub store: it must issue exactly ONE getMulti call
// for the whole span, and the returned points must be oldest-first with
// the seeded values in the right buckets.
func TestLimiterHistory_OneBatchCallOldestFirst(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
countStore := &historyCountingStore{values: map[string]int64{
windowKey("total", "all", metricReq, windowHour, now.Add(-2*time.Hour)): 5,
windowKey("total", "all", metricReq, windowHour, now): 9,
}}
l := newLimiter(countStore, true)
l.nowFn = func() time.Time { return now }
points, ok := l.history("total", "all", "req", windowHour, now, 3)
if !ok {
t.Fatal("want ok=true")
}
if countStore.getMultiCalls != 1 {
t.Errorf("getMultiCalls = %d, want 1", countStore.getMultiCalls)
}
if len(points) != 3 {
t.Fatalf("len(points) = %d, want 3", len(points))
}
if points[0].value != 5 || points[0].bucket != bucketFor(now.Add(-2*time.Hour), windowHour) {
t.Errorf("points[0] (oldest) = %+v, want value 5 at the -2h bucket", points[0])
}
if points[1].value != 0 {
t.Errorf("points[1] (unseeded -1h bucket) = %+v, want value 0", points[1])
}
if points[2].value != 9 || points[2].bucket != bucketFor(now, windowHour) {
t.Errorf("points[2] (newest, current bucket) = %+v, want value 9", points[2])
}
}
// historyCountingStore is a counterStore stub whose getMulti serves fixed
// values and counts its own calls — mirrors admin_test.go's
// countingMultiStore, defined separately here since limits_test.go must
// not depend on admin_test.go's test-only types.
type historyCountingStore struct {
values map[string]int64
getMultiCalls int
}
func (s *historyCountingStore) getMulti(keys []string) ([]int64, error) {
s.getMultiCalls++
out := make([]int64, len(keys))
for i, k := range keys {
out[i] = s.values[k]
}
return out, nil
}
func (s *historyCountingStore) incrMulti(entries []counterIncr) ([]int64, error) {
return make([]int64, len(entries)), nil
}
func (s *historyCountingStore) incrAndGetMulti(entries []counterIncr, reads []string) ([]int64, []int64, error) {
readVals, err := s.getMulti(reads)
return make([]int64, len(entries)), readVals, err
}
// TestLimiterHistory_StoreDown asserts limiter.history reports ok=false
// when the configured store errors and failOpen is false — the caller
// (GET /admin/api/usage/history) must answer 503, never a silently-zero
// series.
func TestLimiterHistory_StoreDown(t *testing.T) {
l := newLimiter(alwaysErrStore{}, false)
points, ok := l.history("total", "all", "req", windowDay, time.Now(), 5)
if ok {
t.Fatal("want ok=false when the configured store errors and failOpen is false")
}
if points != nil {
t.Errorf("points = %+v, want nil on a storeDown read", points)
}
}
// TestLimiterHistory_ConfiguredStoreDown_FailOpen_ReturnsNotOK is finding
// F5, 2026-09 review: with the default failOpen=true, storeGetMulti's own
// ok stays true during a store outage — it silently serves the in-process
// fallback instead — so history must ALSO check configuredStoreDown, not
// just ok, or a Redis blip would present the fallback's near-empty data
// as a genuine drop on the usage-history chart instead of the 503 this
// function's own doc comment promises. alwaysErrStore's first call opens
// the store-down latch (recordStoreFailure) synchronously, before
// configuredStoreDown is checked below, so this single call already
// exercises the fail-open-AND-latched path.
func TestLimiterHistory_ConfiguredStoreDown_FailOpen_ReturnsNotOK(t *testing.T) {
l := newLimiter(alwaysErrStore{}, true) // failOpen: true, unlike TestLimiterHistory_StoreDown above
points, ok := l.history("total", "all", "req", windowDay, time.Now(), 5)
if ok {
t.Fatal("want ok=false: the store is down even though failOpen silently served the fallback")
}
if points != nil {
t.Errorf("points = %+v, want nil", points)
}
}
// TestWithTotalScope_AppendsUnlimitedTotalScope asserts withTotalScope
// appends exactly one {kind: totalScopeKind, id: totalScopeID, limits:
// nil} scope after whatever buildLimitScopes already returned.
func TestWithTotalScope_AppendsUnlimitedTotalScope(t *testing.T) {
base := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}}}
got := withTotalScope(base)
if len(got) != 2 {
t.Fatalf("len(got) = %d, want 2", len(got))
}
if got[0].kind != "user" || got[0].id != "u" {
t.Errorf("got[0] = %+v, want the original user scope untouched", got[0])
}
total := got[1]
if total.kind != totalScopeKind || total.id != totalScopeID || total.limits != nil {
t.Errorf("got[1] = %+v, want kind=%q id=%q limits=nil", total, totalScopeKind, totalScopeID)
}
}
// TestWithTotalScope_CountedButNeverEvaluated drives checkAndCount many
// times against a scopes slice built via withTotalScope: the total
// scope's own limits are always nil, so no volume of traffic can ever
// make it violate — checkAndCount's own nil-limits check skips it — while its
// req counter still climbs by exactly one per call, proving
// checkAndCount's counting loop does not skip a nil-limits scope the way
// evaluation does.
func TestWithTotalScope_CountedButNeverEvaluated(t *testing.T) {
l := newLimiter(nil, true)
now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
scopes := withTotalScope([]limitScope{{kind: "user", id: "u", limits: nil}})
const n = 250
for i := 0; i < n; i++ {
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("iteration %d: total scope must never violate (limits always nil), got %+v", i, v)
}
}
total, ok := l.getCounter(totalScopeKind, totalScopeID, metricReq, windowDay, now)
if !ok || total != n {
t.Errorf("total req:day = %d, ok=%v, want %d (counted every call despite never evaluating)", total, ok, n)
}
totalHour, ok := l.getCounter(totalScopeKind, totalScopeID, metricReq, windowHour, now)
if !ok || totalHour != n {
t.Errorf("total req:hour = %d, ok=%v, want %d", totalHour, ok, n)
}
}
// TestWithTotalScope_AccountWritesHourBuckets asserts account (v0.2
// data-layer task) writes the total scope's hour-window tokin/tokout/cost
// counters, not just day/month — checkAndCount's own hour write is
// covered by TestWithTotalScope_CountedButNeverEvaluated above.
func TestWithTotalScope_AccountWritesHourBuckets(t *testing.T) {
l := newLimiter(nil, true)
now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
scopes := withTotalScope([]limitScope{{kind: "user", id: "u", limits: nil}})
l.account(scopes, usage{prompt: 4, completion: 6}, 700)
tokIn, ok := l.getCounter(totalScopeKind, totalScopeID, metricTokIn, windowHour, now)
if !ok || tokIn != 4 {
t.Errorf("total tokin:hour = %d, ok=%v, want 4", tokIn, ok)
}
tokOut, ok := l.getCounter(totalScopeKind, totalScopeID, metricTokOut, windowHour, now)
if !ok || tokOut != 6 {
t.Errorf("total tokout:hour = %d, ok=%v, want 6", tokOut, ok)
}
cost, ok := l.getCounter(totalScopeKind, totalScopeID, metricCost, windowHour, now)
if !ok || cost != 700 {
t.Errorf("total cost:hour = %d, ok=%v, want 700", cost, ok)
}
}
// TestMemoryStoreIncrAndGet exercises the fallback store directly: fresh
// keys start at n, repeated incrBy accumulates, and get on an unknown key
// returns 0 with no error.
func TestMemoryStoreIncrAndGet(t *testing.T) {
m := newMemoryStore()
v, err := m.incrBy("k", 5, time.Minute)
if err != nil || v != 5 {
t.Fatalf("incrBy = %d, %v, want 5, nil", v, err)
}
v, err = m.incrBy("k", 3, time.Minute)
if err != nil || v != 8 {
t.Fatalf("incrBy = %d, %v, want 8, nil", v, err)
}
got, err := m.get("k")
if err != nil || got != 8 {
t.Fatalf("get = %d, %v, want 8, nil", got, err)
}
got, err = m.get("missing")
if err != nil || got != 0 {
t.Fatalf("get missing = %d, %v, want 0, nil", got, err)
}
}
// TestMemoryStoreExpiry asserts a counter reset to 0 (as a fresh key) once
// its TTL has elapsed, rather than continuing to accumulate forever.
func TestMemoryStoreExpiry(t *testing.T) {
m := newMemoryStore()
if _, err := m.incrBy("k", 5, time.Nanosecond); err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond)
got, err := m.get("k")
if err != nil || got != 0 {
t.Fatalf("get after expiry = %d, %v, want 0, nil", got, err)
}
v, err := m.incrBy("k", 2, time.Minute)
if err != nil || v != 2 {
t.Fatalf("incrBy after expiry = %d, %v, want 2, nil (fresh counter, not 7)", v, err)
}
}
// TestMemoryStore_TTLClampedToMax is the perf-review (2026-08-21) case:
// a TTL beyond memoryStoreMaxTTL (48h) — e.g. dayWindowTTL at 35 days or
// monthWindowTTL at 400 days — must be clamped down to 48h before it is
// ever applied, so the fallback's own live-key count stays bounded
// independent of which window a caller writes.
func TestMemoryStore_TTLClampedToMax(t *testing.T) {
m := newMemoryStore()
fake := time.Date(2026, 8, 21, 0, 0, 0, 0, time.UTC)
m.nowFn = func() time.Time { return fake }
cases := []struct {
name string
ttl time.Duration
}{
{"dayWindowTTL (35d)", dayWindowTTL},
{"monthWindowTTL (400d)", monthWindowTTL},
{"an arbitrary huge ttl", 10 * 365 * 24 * time.Hour},
}
for i, c := range cases {
t.Run(c.name, func(t *testing.T) {
key := fmt.Sprintf("k%d", i)
if _, err := m.incrBy(key, 1, c.ttl); err != nil {
t.Fatal(err)
}
m.mu.Lock()
got := m.data[key].expiry
m.mu.Unlock()
want := fake.Add(memoryStoreMaxTTL)
if !got.Equal(want) {
t.Errorf("expiry = %v, want %v (clamped to memoryStoreMaxTTL, not the requested %v)", got, want, c.ttl)
}
})
}
}
// TestMemoryStore_TTLBelowMaxIsUnaffected asserts the clamp is a ceiling,
// not a rewrite: a TTL already at or under memoryStoreMaxTTL passes
// through exactly as requested — hourWindowTTL (48h) sits precisely at
// the clamp boundary and must round-trip unchanged, not get pushed under
// it by an off-by-one comparison.
func TestMemoryStore_TTLBelowMaxIsUnaffected(t *testing.T) {
m := newMemoryStore()
fake := time.Date(2026, 8, 21, 0, 0, 0, 0, time.UTC)
m.nowFn = func() time.Time { return fake }
cases := []struct {
name string
ttl time.Duration
}{
{"minWindowTTL (2m)", minWindowTTL},
{"hourWindowTTL (48h, exactly at the clamp)", hourWindowTTL},
}
for i, c := range cases {
t.Run(c.name, func(t *testing.T) {
key := fmt.Sprintf("k%d", i)
if _, err := m.incrBy(key, 1, c.ttl); err != nil {
t.Fatal(err)
}
m.mu.Lock()
got := m.data[key].expiry
m.mu.Unlock()
want := fake.Add(c.ttl)
if !got.Equal(want) {
t.Errorf("expiry = %v, want %v (ttl under the clamp must pass through unchanged)", got, want)
}
})
}
}
// TestMemoryStoreSweepTimeGated asserts the sweep gate is purely
// time-based (sweepEvery), independent of how many keys memoryStore
// holds: an incrBy call before the gate elapses leaves an already-expired
// key physically in place (no scan happened), and the first incrBy at or
// past the gate reclaims it. Time is injected via m.nowFn so the test
// needs no real sleep and no size threshold to trigger the behavior.
func TestMemoryStoreSweepTimeGated(t *testing.T) {
m := newMemoryStore()
fake := time.Date(2026, 8, 20, 0, 0, 0, 0, time.UTC)
m.nowFn = func() time.Time { return fake }
// lastSweep starts at the zero time, so this first incrBy always
// sweeps (a no-op on an empty store) and sets lastSweep to fake.
if _, err := m.incrBy("k1", 1, time.Nanosecond); err != nil {
t.Fatal(err)
}
// k1 is already expired (ttl was 1ns), but stay well inside the
// sweepEvery gate — the next incrBy must not scan, so k1 survives as
// a physical (if stale) map entry.
fake = fake.Add(time.Millisecond)
if _, err := m.incrBy("k2", 1, time.Minute); err != nil {
t.Fatal(err)
}
if !memoryStoreHasKey(m, "k1") {
t.Fatal("k1 must still be present: the sweep gate has not elapsed yet")
}
// Cross the sweepEvery boundary from the last sweep: this incrBy must
// scan and reclaim k1.
fake = fake.Add(sweepEvery)
if _, err := m.incrBy("k3", 1, time.Minute); err != nil {
t.Fatal(err)
}
if memoryStoreHasKey(m, "k1") {
t.Error("k1 should have been swept once sweepEvery elapsed")
}
}
// memoryStoreHasKey reports whether key is physically present in m.data,
// bypassing get's expiry check — used to observe sweep timing directly.
func memoryStoreHasKey(m *memoryStore, key string) bool {
m.mu.Lock()
defer m.mu.Unlock()
_, ok := m.data[key]
return ok
}
// TestLimiterCostBudgetRoundsUSDConversion pins usdToMicros' rounding
// against a value (8.2) where float64 representation sits just under the
// true value: a truncating int64(8.2*1e6) would undershoot to 8_199_999,
// making a spend of exactly $8.199999 look already over an intended
// $8.20 budget. With rounding, 8_199_999 must stay under budget and only
// the exact $8.20 spend violates.
func TestLimiterCostBudgetRoundsUSDConversion(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{CostPerDayUSD: 8.2}}}
l.account(scopes, usage{}, 8_199_999)
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("spending $8.199999 against an $8.20 budget must not violate, got %+v", v)
}
l.account(scopes, usage{}, 1) // now exactly at $8.20
if v := l.checkAndCount(scopes); v == nil {
t.Fatal("spending exactly $8.20 against an $8.20 budget must violate")
}
}
// TestAccountSkipsZeroMetrics asserts account writes no tokin/tokout
// counter when the matching usage direction is 0, and no cost counter
// when costMicros is 0 — each metric's absence is independently
// observable via get.
func TestAccountSkipsZeroMetrics(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "user", id: "z", limits: nil}}
l.account(scopes, usage{}, 0)
tokIn, ok := l.getCounter("user", "z", metricTokIn, windowDay, now)
if !ok || tokIn != 0 {
t.Errorf("tokin counter = %d, ok=%v, want 0, true", tokIn, ok)
}
tokOut, ok := l.getCounter("user", "z", metricTokOut, windowDay, now)
if !ok || tokOut != 0 {
t.Errorf("tokout counter = %d, ok=%v, want 0, true", tokOut, ok)
}
cost, ok := l.getCounter("user", "z", metricCost, windowDay, now)
if !ok || cost != 0 {
t.Errorf("cost counter = %d, ok=%v, want 0, true", cost, ok)
}
}
// TestAccountSplitsTokensIndependently asserts account writes prompt
// tokens to metricTokIn and completion tokens to metricTokOut
// independently — including the asymmetric case (prompt only, no
// completion yet) the estimation fallback and a dropped stream both rely
// on: a nonzero prompt with a zero completion must still write tokin
// alone, never skip it because completion is 0.
func TestAccountSplitsTokensIndependently(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "user", id: "split", limits: nil}}
l.account(scopes, usage{prompt: 25}, 0) // completion left at zero
tokIn, ok := l.getCounter("user", "split", metricTokIn, windowDay, now)
if !ok || tokIn != 25 {
t.Errorf("tokin counter = %d, ok=%v, want 25, true", tokIn, ok)
}
tokOut, ok := l.getCounter("user", "split", metricTokOut, windowDay, now)
if !ok || tokOut != 0 {
t.Errorf("tokout counter = %d, ok=%v, want 0, true (no completion tokens reported)", tokOut, ok)
}
l.account(scopes, usage{completion: 9}, 0) // prompt left at zero this time
tokIn, ok = l.getCounter("user", "split", metricTokIn, windowDay, now)
if !ok || tokIn != 25 {
t.Errorf("tokin counter = %d, ok=%v, want still 25 (this call reported no prompt tokens)", tokIn, ok)
}
tokOut, ok = l.getCounter("user", "split", metricTokOut, windowDay, now)
if !ok || tokOut != 9 {
t.Errorf("tokout counter = %d, ok=%v, want 9", tokOut, ok)
}
}
// TestTokenBudgetSumsInAndOut asserts a TokensPerDay/TokensPerMonth limit
// enforces a TOTAL budget across metricTokIn and metricTokOut combined —
// the operator directive's "limits stay total-token (limiter sums both at
// read)": neither direction alone crosses the budget, but their sum does.
func TestTokenBudgetSumsInAndOut(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerDay: 100}}}
l.account(scopes, usage{prompt: 60, completion: 45}, 0) // 105 > 100, split 60/45
if v := l.checkAndCount(scopes); v == nil {
t.Fatal("60 tokin + 45 tokout = 105 > 100 must refuse, even though neither direction alone exceeds 100")
}
}
// TestLimiterRequestPerDayLimit is carried-item (b): a per-day request
// limit blocks once the day's count exceeds it, independent of the
// (unset, unlimited) per-minute limit.
func TestLimiterRequestPerDayLimit(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "user", id: "a", limits: &LimitsConfig{RequestsPerDay: 2}}}
if v := l.checkAndCount(scopes); v != nil {
t.Fatal("1st should pass")
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatal("2nd should pass")
}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("3rd should violate the day limit")
}
if v.storeDown {
t.Error("storeDown must be false for a real limit breach")
}
}
// TestLimiterTokensPerMonthBudget mirrors TestLimiterTokenBudget for the
// month window: accounting tokens past a month budget makes the next
// check refuse.
func TestLimiterTokensPerMonthBudget(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerMonth: 100}}}
l.account(scopes, usage{prompt: 60, completion: 50}, 0) // 110 > 100
if v := l.checkAndCount(scopes); v == nil {
t.Fatal("over month token budget must refuse")
}
}
// TestLimiterCostPerMonthBudget mirrors TestLimiterCostBudget for the
// month window, and checks the violation carries the month window's
// Retry-After.
func TestLimiterCostPerMonthBudget(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{CostPerMonthUSD: 1.00}}}
l.account(scopes, usage{}, 1_500_000) // $1.50 > $1.00 budget
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("over month cost budget must refuse")
}
wantRetry := int(time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC).Sub(now).Seconds())
if v.retryAfter != wantRetry {
t.Errorf("retryAfter = %d, want %d (seconds to next UTC month start)", v.retryAfter, wantRetry)
}
if v.storeDown {
t.Error("storeDown must be false for a real limit breach")
}
}
// erroringStore is a counterStore stub that always fails, used to test the
// limiter's fail-open/fail-closed handling on a store error without a
// real network dependency (see redis_store_test.go for the equivalent
// end-to-end test against an unreachable redisStore).
type erroringStore struct {
err error
}
func (s *erroringStore) getMulti([]string) ([]int64, error) { return nil, s.err }
func (s *erroringStore) incrMulti([]counterIncr) ([]int64, error) { return nil, s.err }
func (s *erroringStore) incrAndGetMulti([]counterIncr, []string) ([]int64, []int64, error) {
return nil, nil, s.err
}
// TestLimiter_FailOpen_StoreErrorUsesFallback is carried-item (c): a store
// error with failOpen=true must not block the request — the operation
// transparently falls back to the limiter's in-process memoryStore, and
// the fallback counter actually advances (proving the write really
// happened somewhere, not just that no violation was reported).
func TestLimiter_FailOpen_StoreErrorUsesFallback(t *testing.T) {
store := &erroringStore{err: errors.New("boom")}
l := newLimiter(store, true)
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 100}}}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("want no violation with failOpen=true on a store error, got %+v", v)
}
got, err := l.fallback.get(windowKey("user", "u", metricReq, windowDay, l.now()))
if err != nil {
t.Fatalf("fallback.get: %v", err)
}
if got != 1 {
t.Errorf("fallback req:day counter = %d, want 1", got)
}
}
// TestLimiter_FailClosed_StoreErrorReturnsStoreDownViolation is
// carried-item (c): failOpen=false refuses the request outright on a
// store error, rather than silently using the fallback.
func TestLimiter_FailClosed_StoreErrorReturnsStoreDownViolation(t *testing.T) {
store := &erroringStore{err: errors.New("boom")}
l := newLimiter(store, false)
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 100}}}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("want a storeDown violation on a store error with failOpen=false")
}
if !v.storeDown {
t.Error("v.storeDown = false, want true")
}
if v.message != "limit store unavailable" {
t.Errorf("v.message = %q, want %q", v.message, "limit store unavailable")
}
}
// succeedIncrFailGetStore is a counterStore stub whose incrMulti always
// succeeds and whose getMulti/incrAndGetMulti always error. Its
// incrMulti/getMulti stay independently overridable (used by any
// OTHER caller — account, currentUsage, and so on — that still calls them
// separately) even though checkAndCount itself, since the round-3 fusion
// (2026-08-22), calls incrAndGetMulti exclusively: a store that cannot
// serve a read reliably fails that ONE combined round trip as a whole,
// there being no longer a separate "increment succeeded, read failed"
// split to isolate at the store level for checkAndCount specifically.
type succeedIncrFailGetStore struct {
getErr error
}
func (s *succeedIncrFailGetStore) getMulti([]string) ([]int64, error) {
return nil, s.getErr
}
func (s *succeedIncrFailGetStore) incrMulti(entries []counterIncr) ([]int64, error) {
out := make([]int64, len(entries))
for i := range out {
out[i] = 1
}
return out, nil
}
func (s *succeedIncrFailGetStore) incrAndGetMulti([]counterIncr, []string) ([]int64, []int64, error) {
return nil, nil, s.getErr
}
// TestLimiter_FailClosed_BudgetReadReturnsStoreDownViolation covers the
// budget-read fail-closed path: a store that cannot be trusted to serve a
// token/cost budget read refuses the request when failOpen is false, via
// checkAndCount's own storeDownViolation short-circuit (limits.go) — the
// same contract the pre-fusion budgetViolation helper enforced before
// checkAndCount's admission round trip was fused (perf review round 3,
// 2026-08-22). A TokensPerDay limit is what makes checkAndCount build a
// budget probe/read for this scope at all.
func TestLimiter_FailClosed_BudgetReadReturnsStoreDownViolation(t *testing.T) {
store := &succeedIncrFailGetStore{getErr: errors.New("boom")}
l := newLimiter(store, false)
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{TokensPerDay: 100}}}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("want a storeDown violation from the budget-read path with failOpen=false")
}
if !v.storeDown {
t.Error("v.storeDown = false, want true")
}
}
// TestLimiter_FailClosed_AccountDropsSampleOnStoreError asserts account
// drops a sample entirely — writes it nowhere — when failOpen=false and
// the store errors, rather than falling back to counting it locally (which
// would double-count once the store recovers). account has no error
// return to signal the drop, so this is observed via the fallback staying
// at 0.
func TestLimiter_FailClosed_AccountDropsSampleOnStoreError(t *testing.T) {
store := &erroringStore{err: errors.New("boom")}
l := newLimiter(store, false)
scopes := []limitScope{{kind: "user", id: "u", limits: nil}}
l.account(scopes, usage{prompt: 10, completion: 10}, 500)
gotIn, err := l.fallback.get(windowKey("user", "u", metricTokIn, windowDay, l.now()))
if err != nil {
t.Fatalf("fallback.get: %v", err)
}
if gotIn != 0 {
t.Errorf("fallback tokin:day counter = %d, want 0 (sample must be dropped, not counted locally)", gotIn)
}
gotOut, err := l.fallback.get(windowKey("user", "u", metricTokOut, windowDay, l.now()))
if err != nil {
t.Fatalf("fallback.get: %v", err)
}
if gotOut != 0 {
t.Errorf("fallback tokout:day counter = %d, want 0 (sample must be dropped, not counted locally)", gotOut)
}
}
// TestLimiter_LogsStoreErrorOncePerRateLimit asserts a store error is
// logged via the limiter's injectable log func, rate-limited to once per
// storeErrorLogEvery — a store outage under load must not flood the log
// with one line per request, however many store operations fail within
// the window.
func TestLimiter_LogsStoreErrorOncePerRateLimit(t *testing.T) {
store := &erroringStore{err: errors.New("boom")}
l := newLimiter(store, true)
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
var logged []string
l.logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }
scopes := []limitScope{{kind: "user", id: "u", limits: nil}}
l.checkAndCount(scopes) // two store ops (min, day) within one call
if len(logged) != 1 {
t.Fatalf("logged = %d lines after the first checkAndCount, want 1 (rate-limited)", len(logged))
}
l.checkAndCount(scopes) // still inside the rate-limit window
if len(logged) != 1 {
t.Fatalf("logged = %d lines after a second call inside the window, want 1", len(logged))
}
now = now.Add(storeErrorLogEvery)
l.checkAndCount(scopes)
if len(logged) != 2 {
t.Fatalf("logged = %d lines after the rate-limit window elapsed, want 2", len(logged))
}
}
// countingErrorStore is a counterStore stub that always errors and counts
// how many times each method is actually invoked. It proves the
// store-down latch (limits.go) skips the network call entirely during
// its window, rather than merely tolerating the error each time.
type countingErrorStore struct {
err error
incrCalls int
getCalls int
}
func (s *countingErrorStore) getMulti([]string) ([]int64, error) {
s.getCalls++
return nil, s.err
}
func (s *countingErrorStore) incrMulti([]counterIncr) ([]int64, error) {
s.incrCalls++
return nil, s.err
}
// incrAndGetMulti counts against the same incrCalls field incrMulti uses:
// checkAndCount's own admission round trip calls incrAndGetMulti
// exclusively since the round-3 fusion (2026-08-22, perf review), so
// TestLimiter_StoreDownLatch_SkipsStoreCallsWithinWindow's "the store must
// not be called" assertion needs this call counted the same way.
func (s *countingErrorStore) incrAndGetMulti([]counterIncr, []string) ([]int64, []int64, error) {
s.incrCalls++
return nil, nil, s.err
}
// TestLimiter_StoreDownLatch_SkipsStoreCallsWithinWindow is review-round-3
// item 2: after a store operation fails, every operation for the next
// storeDownLatchFor must short-circuit straight to the fail-open policy
// without calling the store at all — a request that touches several
// counters (min/day/hour, tokens/cost, user/group) must not pay a fresh
// bounded wait against a store that just proved unreachable for each one.
// checkAndCount batches its whole req:min/req:day/req:hour increment into
// ONE incrMulti call (perf review, 2026-08-21), so "the store must not be
// called" now means that single call is skipped entirely during the latch
// window, not that some subset of several per-key calls is. failOpen=true
// is required so every checkAndCount call below returns no violation
// (ok=true via the fallback) instead of a storeDownViolation — the
// incrCalls assertions only make sense if checkAndCount actually
// completes normally each time.
func TestLimiter_StoreDownLatch_SkipsStoreCallsWithinWindow(t *testing.T) {
store := &countingErrorStore{err: errors.New("boom")}
l := newLimiter(store, true)
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "user", id: "u", limits: nil}}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("want no violation with failOpen=true, got %+v", v)
}
firstCalls := store.incrCalls
if firstCalls == 0 {
t.Fatal("want the first call to actually reach the store at least once")
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("want no violation with failOpen=true, got %+v", v)
}
if store.incrCalls != firstCalls {
t.Errorf("incrBy calls = %d after a second checkAndCount within the latch window, want unchanged at %d (the store must not be called)", store.incrCalls, firstCalls)
}
now = now.Add(storeDownLatchFor) // latch expires
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("want no violation with failOpen=true, got %+v", v)
}
if store.incrCalls <= firstCalls {
t.Errorf("incrBy calls = %d after the latch window elapsed, want more than %d (the store must be probed again)", store.incrCalls, firstCalls)
}
}
// TestLimitsConfigValidate is carried-item (a): validate rejects
// NaN/±Inf/negative cost limits and negative int64 count limits, accepts
// zero/positive values, and treats a nil receiver (no limits configured
// at all) as valid.
func TestLimitsConfigValidate(t *testing.T) {
cases := []struct {
lc *LimitsConfig
name string
wantErr bool
}{
{name: "nil is valid", lc: nil, wantErr: false},
{name: "zero value is valid", lc: &LimitsConfig{}, wantErr: false},
{name: "positive values are valid", lc: &LimitsConfig{
RequestsPerMinute: 1, RequestsPerDay: 1, TokensPerDay: 1, TokensPerMonth: 1,
CostPerDayUSD: 1.5, CostPerMonthUSD: 1.5,
}, wantErr: false},
{name: "negative RequestsPerMinute", lc: &LimitsConfig{RequestsPerMinute: -1}, wantErr: true},
{name: "negative RequestsPerDay", lc: &LimitsConfig{RequestsPerDay: -1}, wantErr: true},
{name: "negative TokensPerDay", lc: &LimitsConfig{TokensPerDay: -1}, wantErr: true},
{name: "negative TokensPerMonth", lc: &LimitsConfig{TokensPerMonth: -1}, wantErr: true},
{name: "negative CostPerDayUSD", lc: &LimitsConfig{CostPerDayUSD: -0.01}, wantErr: true},
{name: "negative CostPerMonthUSD", lc: &LimitsConfig{CostPerMonthUSD: -0.01}, wantErr: true},
{name: "NaN CostPerDayUSD", lc: &LimitsConfig{CostPerDayUSD: math.NaN()}, wantErr: true},
{name: "+Inf CostPerMonthUSD", lc: &LimitsConfig{CostPerMonthUSD: math.Inf(1)}, wantErr: true},
{name: "-Inf CostPerDayUSD", lc: &LimitsConfig{CostPerDayUSD: math.Inf(-1)}, wantErr: true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
err := c.lc.validate()
if (err != nil) != c.wantErr {
t.Errorf("validate() error = %v, wantErr %v", err, c.wantErr)
}
})
}
}
// TestLimiter_FailPolicyGet covers failPolicyGet's own fail-open/fail-closed
// branching in isolation (storeGet's error/latched paths already drive it
// indirectly elsewhere in this file): failOpen=true reads through to the
// in-process fallback, failOpen=false refuses outright.
func TestLimiter_FailPolicyGet(t *testing.T) {
cases := []struct {
name string
failOpen bool
wantOK bool
}{
{name: "failOpen true reads the fallback", failOpen: true, wantOK: true},
{name: "failOpen false refuses", failOpen: false, wantOK: false},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
l := newLimiter(nil, c.failOpen)
v, ok := l.failPolicyGet("k")
require.Equal(t, c.wantOK, ok)
if c.wantOK {
assert.Equal(t, int64(0), v)
}
})
}
}
// TestLimiter_FailPolicyIncrBy mirrors TestLimiter_FailPolicyGet for the
// write side: failOpen=true counts n into the in-process fallback and
// returns its real post-increment value; failOpen=false refuses outright
// (0, false) WITHOUT touching the fallback at all — a caller must not
// double-count once the store recovers by having silently counted
// locally on the refused path too.
//
// MUTATION PROVEN: removing the `if !l.failOpen { return 0, false }`
// guard (falling straight through to the fallback increment regardless of
// failOpen) made the "failOpen false" case return ok=true and leave a
// nonzero fallback counter instead of refusing untouched — confirmed via
// `go test -run TestLimiter_FailPolicyIncrBy`, then reverted with `git
// checkout -- limits.go`.
func TestLimiter_FailPolicyIncrBy(t *testing.T) {
cases := []struct {
name string
failOpen bool
wantOK bool
wantV int64
}{
{name: "failOpen true counts in the fallback", failOpen: true, wantOK: true, wantV: 3},
{name: "failOpen false refuses without touching the fallback", failOpen: false, wantOK: false, wantV: 0},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
l := newLimiter(nil, c.failOpen)
v, ok := l.failPolicyIncrBy("k", 3, time.Minute)
require.Equal(t, c.wantOK, ok)
assert.Equal(t, c.wantV, v)
got, err := l.fallback.get("k")
if err != nil {
t.Fatalf("fallback.get: %v", err)
}
wantFallback := int64(0)
if c.failOpen {
wantFallback = 3
}
if got != wantFallback {
t.Errorf("fallback[k] = %d, want %d", got, wantFallback)
}
})
}
}
// fixedStore is a counterStore stub whose incrMulti/getMulti always
// succeed with a fixed value per entry/key, independent of key/n/ttl —
// used to prove storeIncrBy/storeGet (limits_helpers_test.go, built on
// the batched storeIncrMulti/storeGetMulti production paths) return the
// STORE's own reported value on success, rather than silently
// substituting the in-process fallback's.
type fixedStore struct {
incrVal int64
getVal int64
}
func (s *fixedStore) getMulti(keys []string) ([]int64, error) {
out := make([]int64, len(keys))
for i := range out {
out[i] = s.getVal
}
return out, nil
}
func (s *fixedStore) incrMulti(entries []counterIncr) ([]int64, error) {
out := make([]int64, len(entries))
for i := range out {
out[i] = s.incrVal
}
return out, nil
}
func (s *fixedStore) incrAndGetMulti([]counterIncr, []string) ([]int64, []int64, error) {
return nil, nil, nil
}
// TestLimiter_StoreIncrBy_DirectPaths exercises storeIncrBy's four
// branches directly. storeIncrBy has no production caller since
// checkAndCount/account moved to the batched storeIncrMulti (its own doc
// comment) — its only caller, incrCounter, is itself only called from
// tests for direct counter seeding, which is why the existing suite only
// ever drove it through the nil-store path (30% baseline coverage): a nil
// store always uses the in-process fallback; a configured store's
// successful call returns the STORE's own value; a store error records
// the failure and falls through to the fail-open/fail-closed policy; and
// once the store-down latch opens from a prior failure, the store is
// never called again until the latch expires.
//
// MUTATION PROVEN: deleting the `if l.storeLatched() { return
// l.failPolicyIncrBy(key, n, ttl) }` branch made the "latched skips the
// network call" subtest fail (the erroring store's incrBy was invoked a
// second time instead of being skipped) — confirmed via `go test -run
// TestLimiter_StoreIncrBy_DirectPaths/latched`, then reverted.
func TestLimiter_StoreIncrBy_DirectPaths(t *testing.T) {
t.Run("nil store uses the fallback", func(t *testing.T) {
l := newLimiter(nil, true)
v, ok := l.storeIncrBy("k", 5, time.Minute)
require.True(t, ok)
assert.Equal(t, int64(5), v)
got, err := l.fallback.get("k")
if err != nil {
t.Fatalf("fallback.get: %v", err)
}
if got != 5 {
t.Errorf("fallback[k] = %d, want 5", got)
}
})
t.Run("configured store success returns the store's own value", func(t *testing.T) {
store := &fixedStore{incrVal: 42}
l := newLimiter(store, true)
v, ok := l.storeIncrBy("k", 5, time.Minute)
require.True(t, ok)
assert.Equal(t, int64(42), v)
})
t.Run("store error falls through to the fail-open policy", func(t *testing.T) {
store := &erroringStore{err: errors.New("boom")}
l := newLimiter(store, true)
v, ok := l.storeIncrBy("k", 5, time.Minute)
require.True(t, ok)
assert.Equal(t, int64(5), v)
got, err := l.fallback.get("k")
if err != nil {
t.Fatalf("fallback.get: %v", err)
}
if got != 5 {
t.Errorf("fallback[k] = %d, want 5 (fail-open must count in the fallback)", got)
}
})
t.Run("store error with failOpen false refuses", func(t *testing.T) {
store := &erroringStore{err: errors.New("boom")}
l := newLimiter(store, false)
_, ok := l.storeIncrBy("k", 5, time.Minute)
require.False(t, ok)
})
t.Run("latched skips the network call entirely", func(t *testing.T) {
store := &countingErrorStore{err: errors.New("boom")}
l := newLimiter(store, true)
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
l.storeIncrBy("k", 1, time.Minute) // first call: reaches the store, opens the latch
if store.incrCalls != 1 {
t.Fatalf("incrCalls after first call = %d, want 1", store.incrCalls)
}
l.storeIncrBy("k", 1, time.Minute) // second call: within the latch window
if store.incrCalls != 1 {
t.Errorf("incrCalls after second call = %d, want still 1 (the latch must skip the store)", store.incrCalls)
}
})
}
// TestLimiter_StoreGet_DirectPaths mirrors
// TestLimiter_StoreIncrBy_DirectPaths for the read side. The round-3 perf
// fusion moved checkAndCount's own reads onto storeGetMulti/
// incrAndGetMulti, so a direct call is storeGet's only remaining exercise
// of its store!=nil branches (70% baseline coverage, from other tests
// that only drove it through account/currentUsage's own indirect paths).
//
// MUTATION PROVEN: deleting the `if l.storeLatched() { return
// l.failPolicyGet(key) }` branch made the "latched skips the network
// call" subtest fail (the erroring store's get was invoked a second time)
// — confirmed, then reverted.
func TestLimiter_StoreGet_DirectPaths(t *testing.T) {
t.Run("nil store uses the fallback", func(t *testing.T) {
l := newLimiter(nil, true)
if _, err := l.fallback.incrBy("k", 7, time.Minute); err != nil {
t.Fatalf("fallback.incrBy: %v", err)
}
v, ok := l.storeGet("k")
require.True(t, ok)
assert.Equal(t, int64(7), v)
})
t.Run("configured store success returns the store's own value", func(t *testing.T) {
store := &fixedStore{getVal: 99}
l := newLimiter(store, true)
v, ok := l.storeGet("k")
require.True(t, ok)
assert.Equal(t, int64(99), v)
})
t.Run("store error with failOpen false refuses", func(t *testing.T) {
store := &erroringStore{err: errors.New("boom")}
l := newLimiter(store, false)
_, ok := l.storeGet("k")
require.False(t, ok)
})
t.Run("latched skips the network call entirely", func(t *testing.T) {
store := &countingErrorStore{err: errors.New("boom")}
l := newLimiter(store, true)
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
l.storeGet("k") // first call: reaches the store, opens the latch
if store.getCalls != 1 {
t.Fatalf("getCalls after first call = %d, want 1", store.getCalls)
}
l.storeGet("k") // second call: within the latch window
if store.getCalls != 1 {
t.Errorf("getCalls after second call = %d, want still 1 (the latch must skip the store)", store.getCalls)
}
})
}
// TestLimiter_LogSpawnDrop_RateLimited proves logSpawnDrop logs one line
// describing the dropped telemetry write, rate-limited to once per
// storeErrorLogEvery regardless of how many drops happen in between —
// mirroring TestLimiter_LogsStoreErrorOncePerRateLimit, since logSpawnDrop
// deliberately shares lastLogAt with logStoreError (logSpawnDrop's own
// doc comment: "a stressed store... is the same 'the store is struggling'
// story").
//
// MUTATION PROVEN: removing the `if shouldLog { ... }` guard around the
// l.logf call (logging unconditionally on every drop) made the
// "still inside the window" assertion fail (2 lines logged after the
// second call instead of 1) — confirmed via `go test -run
// TestLimiter_LogSpawnDrop_RateLimited`, then reverted.
func TestLimiter_LogSpawnDrop_RateLimited(t *testing.T) {
l := newLimiter(nil, true)
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
var logged []string
l.logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }
l.logSpawnDrop()
if len(logged) != 1 {
t.Fatalf("logged = %d lines after the first drop, want 1", len(logged))
}
if !strings.Contains(logged[0], "limit telemetry write dropped") {
t.Errorf("logged[0] = %q, want it to mention the telemetry write drop", logged[0])
}
l.logSpawnDrop() // still inside the rate-limit window
if len(logged) != 1 {
t.Fatalf("logged = %d lines after a second drop inside the window, want still 1", len(logged))
}
now = now.Add(storeErrorLogEvery)
l.logSpawnDrop()
if len(logged) != 2 {
t.Fatalf("logged = %d lines after the rate-limit window elapsed, want 2", len(logged))
}
}
// --- perf review, 2026-08-21: single-batch counter writes ---
// countingIncrStore is a counterStore stub whose incrMulti/getMulti/
// incrAndGetMulti all succeed against a real backing map (so repeated
// calls actually accumulate, proving checkAndCount/account read back
// correct values after batching, not just that batching happened at all),
// counting how many times each was called. account still calls incrMulti
// directly; checkAndCount, since the round-3 fusion (2026-08-22, perf
// review), calls incrAndGetMulti exclusively — so the two counters below
// track genuinely different callers, not the same round trip under two
// names.
type countingIncrStore struct {
values map[string]int64
incrMultiCalls int
incrAndGetMultiCalls int
getMultiCalls int
}
func (s *countingIncrStore) getMulti(keys []string) ([]int64, error) {
s.getMultiCalls++
out := make([]int64, len(keys))
for i, k := range keys {
if s.values != nil {
out[i] = s.values[k]
}
}
return out, nil
}
func (s *countingIncrStore) incrMulti(entries []counterIncr) ([]int64, error) {
s.incrMultiCalls++
if s.values == nil {
s.values = make(map[string]int64)
}
out := make([]int64, len(entries))
for i, e := range entries {
s.values[e.key] += e.delta
out[i] = s.values[e.key]
}
return out, nil
}
func (s *countingIncrStore) incrAndGetMulti(entries []counterIncr, reads []string) ([]int64, []int64, error) {
s.incrAndGetMultiCalls++
if s.values == nil {
s.values = make(map[string]int64)
}
incrVals := make([]int64, len(entries))
for i, e := range entries {
s.values[e.key] += e.delta
incrVals[i] = s.values[e.key]
}
readVals := make([]int64, len(reads))
for i, k := range reads {
readVals[i] = s.values[k]
}
return incrVals, readVals, nil
}
// TestCheckAndCount_OneIncrAndGetMultiCallRegardlessOfScopeCount is the
// perf-review round-3 (2026-08-22) case for checkAndCount: a 3-scope
// request (user, group, total — 9 req counters, min/day/hour x 3 scopes)
// must cost exactly ONE incrAndGetMulti round trip, not one per counter
// and not a separate round trip for token/cost budget reads, and the
// batched reply must still drive correct evaluation — a real
// requestsPerMinute violation on the 3rd call, with the exact right
// counter value. Supersedes the pre-fusion incrMulti-only version of this
// test (perf review, 2026-08-21): checkAndCount's own ADMISSION path no
// longer calls incrMulti at all, only incrAndGetMulti (account still
// does, covered separately below by
// TestAccount_OneIncrMultiCallRegardlessOfScopeOrMetricCount).
//
// Updated (finding F1, 2026-09 review): a REJECTION now costs one further
// incrMulti call — rollbackOtherScopeCounts' own compensating batch,
// undoing the increments the fused round trip above already made for
// every scope OTHER than the one that actually violated. See
// TestCheckAndCount_RejectionRollsBackOtherScopes_MemoryFallback for the
// dedicated group-lockout regression this fixes; this test's own
// assertions below were updated to match the corrected counter values.
func TestCheckAndCount_OneIncrAndGetMultiCallRegardlessOfScopeCount(t *testing.T) {
store := &countingIncrStore{}
l := newLimiter(store, true)
now := time.Date(2026, 8, 21, 9, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 2}},
{kind: "group", id: "g", limits: nil},
{kind: totalScopeKind, id: totalScopeID, limits: nil},
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("1st call should pass, got %+v", v)
}
if store.incrAndGetMultiCalls != 1 {
t.Errorf("incrAndGetMultiCalls after 1st checkAndCount = %d, want 1", store.incrAndGetMultiCalls)
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("2nd call should pass, got %+v", v)
}
if store.incrAndGetMultiCalls != 2 {
t.Errorf("incrAndGetMultiCalls after 2nd checkAndCount = %d, want 2 (one call per checkAndCount, not per scope or per counter)", store.incrAndGetMultiCalls)
}
if store.incrMultiCalls != 0 {
t.Errorf("incrMultiCalls after 2nd checkAndCount = %d, want 0 (no rollback — nothing violated yet)", store.incrMultiCalls)
}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("3rd call should violate u's requestsPerMinute:2")
}
if v.storeDown {
t.Error("v.storeDown must be false — this is a real limit breach, not a store failure")
}
if store.incrAndGetMultiCalls != 3 {
t.Errorf("incrAndGetMultiCalls after 3rd checkAndCount = %d, want 3", store.incrAndGetMultiCalls)
}
// Finding F1, 2026-09 review: a violation now fires ONE compensating
// incrMulti call (rollbackOtherScopeCounts) to undo the OTHER
// scopes' increments for this rejected request — see this test's own
// updated value assertions below for why.
if store.incrMultiCalls != 1 {
t.Errorf("incrMultiCalls after the violation = %d, want 1 (rollbackOtherScopeCounts' own compensating batch)", store.incrMultiCalls)
}
// Values correct: batching three scopes into one call must not
// corrupt any one scope's own counter, AND (finding F1) the group and
// total scopes' req:min/day/hour counters must be rolled back to what
// they were before this rejected 3rd request — only the VIOLATING
// scope ("user") keeps counting a request that was never admitted.
uMin, ok := l.getCounter("user", "u", metricReq, windowMin, now)
if !ok || uMin != 3 {
t.Errorf("user req:min = %d, ok=%v, want 3 (the violating scope's own count is not rolled back)", uMin, ok)
}
totalMin, ok := l.getCounter(totalScopeKind, totalScopeID, metricReq, windowMin, now)
if !ok || totalMin != 2 {
t.Errorf("total req:min = %d, ok=%v, want 2 (rolled back — the total scope had nothing to do with user u's own violation)", totalMin, ok)
}
gDay, ok := l.getCounter("group", "g", metricReq, windowDay, now)
if !ok || gDay != 2 {
t.Errorf("group req:day = %d, ok=%v, want 2 (rolled back)", gDay, ok)
}
}
// TestAccount_OneIncrMultiCallRegardlessOfScopeOrMetricCount is the
// perf-review (2026-08-21) case for account: a 3-scope request with all
// three metrics nonzero (27 counters: 3 scopes x 3 metrics x 3 windows)
// must cost exactly ONE incrMulti round trip, and the batched reply must
// land every value in the right scope/metric/window slot.
func TestAccount_OneIncrMultiCallRegardlessOfScopeOrMetricCount(t *testing.T) {
store := &countingIncrStore{}
l := newLimiter(store, true)
now := time.Date(2026, 8, 21, 9, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: "user", id: "u"},
{kind: "group", id: "g"},
{kind: totalScopeKind, id: totalScopeID},
}
l.account(scopes, usage{prompt: 10, completion: 5}, 700)
if store.incrMultiCalls != 1 {
t.Errorf("incrMultiCalls = %d, want 1 (one batch for 27 counters)", store.incrMultiCalls)
}
uTokInDay, ok := l.getCounter("user", "u", metricTokIn, windowDay, now)
if !ok || uTokInDay != 10 {
t.Errorf("user tokin:day = %d, ok=%v, want 10", uTokInDay, ok)
}
totalTokOutHour, ok := l.getCounter(totalScopeKind, totalScopeID, metricTokOut, windowHour, now)
if !ok || totalTokOutHour != 5 {
t.Errorf("total tokout:hour = %d, ok=%v, want 5", totalTokOutHour, ok)
}
gCostMonth, ok := l.getCounter("group", "g", metricCost, windowMonth, now)
if !ok || gCostMonth != 700 {
t.Errorf("group cost:month = %d, ok=%v, want 700", gCostMonth, ok)
}
// A second call accumulates rather than overwrites, and still costs
// exactly one more round trip.
l.account(scopes, usage{prompt: 2}, 0)
if store.incrMultiCalls != 2 {
t.Errorf("incrMultiCalls after 2nd account = %d, want 2", store.incrMultiCalls)
}
uTokInDay, ok = l.getCounter("user", "u", metricTokIn, windowDay, now)
if !ok || uTokInDay != 12 {
t.Errorf("user tokin:day after 2nd account = %d, ok=%v, want 12", uTokInDay, ok)
}
}
// TestTokenBudget_ExactBoundaryViolates asserts budgetViolation/
// tokenBudgetViolation's ">= limit" semantics at the exact boundary: a
// scope whose accumulated tokin+tokout lands EXACTLY on the limit (not
// merely over it) must still violate. budgetViolation's own "used <
// limit passes" check makes "used == limit" the smallest value that
// violates; this pins that boundary directly rather than only exercising
// values already over it (TestLimiterTokenBudget, TestTokenBudgetSumsInAndOut).
func TestTokenBudget_ExactBoundaryViolates(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerDay: 100}}}
l.account(scopes, usage{prompt: 63, completion: 36}, 0) // 99 < 100: must not violate yet
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("99 < 100 must not violate, got %+v", v)
}
l.account(scopes, usage{completion: 1}, 0) // pushes tokout to 37: total now exactly 100
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("tokin+tokout landing exactly on the 100 budget must violate (>= limit, not only > limit)")
}
if v.storeDown {
t.Error("storeDown must be false for a real limit breach")
}
}
// --- perf review round 3, 2026-08-22: fused admission round trip ---
// TestCheckAndCount_FusedViolationPrecedence is the fusion-equivalence
// case: checkAndCount's admission round trip now fuses the request-counter
// increments with every scope's token/cost budget reads into ONE store
// call (buildBudgetProbes, incrAndGetMulti), replacing the pre-fusion
// evaluateScope/budgetViolation/tokenBudgetViolation helpers that made
// those reads as separate, serial round trips. This table pins that the
// VIOLATION PRECEDENCE those helpers enforced is byte-for-byte unchanged:
// within one scope, requests-per-minute, then requests-per-day, then
// tokens-per-day, then tokens-per-month, then cost-per-day, then
// cost-per-month; across scopes, the scope earlier in the given slice
// (buildLimitScopes puts a user before their group) wins. The first six
// subtests each engineer TWO simultaneously-true violation conditions
// (one per limit type in a precedence pair) and assert only the
// higher-precedence one is ever reported; the seventh engineers only ONE
// condition (the group violates, the user carries no limits at all) to
// pin the separate claim that a later scope still gets evaluated at all
// when an earlier one has nothing to check (review fix, 2026-08-22, round
// 2: this doc comment previously claimed all seven subtests engineered
// two conditions, which was not true of the seventh).
func TestCheckAndCount_FusedViolationPrecedence(t *testing.T) {
t.Run("requests-per-minute beats requests-per-day in the same scope", func(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1, RequestsPerDay: 1}}}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("1st call must pass (both counters land exactly on their limit), got %+v", v)
}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("2nd call must violate: both requests-per-minute and requests-per-day are now over limit")
}
assert.Contains(t, v.message, "requests-per-minute")
assert.NotContains(t, v.message, "requests-per-day")
})
t.Run("requests-per-day beats tokens-per-day in the same scope", func(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerDay: 1, TokensPerDay: 10}}}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("1st call must pass (req:day lands exactly on its limit, no tokens seeded yet), got %+v", v)
}
l.account(scopes, usage{prompt: 20}, 0) // tokin:day now 20, over the 10 budget
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("2nd call must violate: both requests-per-day and tokens-per-day are now over limit")
}
assert.Contains(t, v.message, "requests-per-day")
assert.NotContains(t, v.message, "tokens-per-day")
})
t.Run("tokens-per-day beats tokens-per-month in the same scope", func(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{TokensPerDay: 10, TokensPerMonth: 10}}}
l.account(scopes, usage{prompt: 20}, 0) // tokin:day and tokin:month both now 20, both over budget
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("must violate: both tokens-per-day and tokens-per-month are over limit")
}
assert.Contains(t, v.message, "tokens-per-day")
assert.NotContains(t, v.message, "tokens-per-month")
})
t.Run("tokens-per-month beats cost-per-day in the same scope", func(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{TokensPerMonth: 10, CostPerDayUSD: 0.0001}}}
l.account(scopes, usage{prompt: 20}, 200) // tokin:month = 20 (>10); cost:day = 200 micros (>100 micros)
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("must violate: both tokens-per-month and cost-per-day are over limit")
}
assert.Contains(t, v.message, "tokens-per-month")
assert.NotContains(t, v.message, "cost-per-day")
})
t.Run("cost-per-day beats cost-per-month in the same scope", func(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{CostPerDayUSD: 0.0001, CostPerMonthUSD: 0.0001}}}
l.account(scopes, usage{}, 200) // cost:day and cost:month both now 200 micros, both over the 100-micro budget
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("must violate: both cost-per-day and cost-per-month are over limit")
}
assert.Contains(t, v.message, "cost-per-day")
assert.NotContains(t, v.message, "cost-per-month")
})
t.Run("user scope beats group scope when both violate", func(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{
{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}},
{kind: "group", id: "g", limits: &LimitsConfig{RequestsPerMinute: 1}},
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("1st call must pass, got %+v", v)
}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("2nd call must violate: both user and group are now over their requests-per-minute limit")
}
assert.Contains(t, v.message, `user "u"`)
})
t.Run("group scope evaluated when the earlier user scope has no violation", func(t *testing.T) {
l := newLimiter(nil, true)
scopes := []limitScope{
{kind: "user", id: "u", limits: nil},
{kind: "group", id: "g", limits: &LimitsConfig{RequestsPerMinute: 1}},
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("1st call must pass, got %+v", v)
}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("2nd call must violate: group is now over its requests-per-minute limit")
}
assert.Contains(t, v.message, `group "g"`)
})
}
// TestCheckAndCount_FusedRoundTrip_MemoryFallback proves the fused
// admission round trip works correctly end to end against memoryStore —
// the limiter's ALWAYS-available in-process fallback (newLimiter(nil, _)
// uses it directly; a configured store also falls back to it on error or
// while store-latched) — covering both an increment-only scope and a
// scope with every budget type configured at once, in one round trip.
func TestCheckAndCount_FusedRoundTrip_MemoryFallback(t *testing.T) {
l := newLimiter(nil, true)
now := time.Date(2026, 8, 22, 9, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: "user", id: "u", limits: &LimitsConfig{
RequestsPerMinute: 100, RequestsPerDay: 100,
TokensPerDay: 1000, TokensPerMonth: 1000,
CostPerDayUSD: 1, CostPerMonthUSD: 1,
}},
{kind: totalScopeKind, id: totalScopeID, limits: nil},
}
// Well under every budget: must pass and must have actually
// incremented every request counter (proving the fused write side
// still lands correctly, not just that no violation was reported).
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("want no violation comfortably under every limit, got %+v", v)
}
uMin, ok := l.getCounter("user", "u", metricReq, windowMin, now)
if !ok || uMin != 1 {
t.Errorf("user req:min = %d, ok=%v, want 1", uMin, ok)
}
totalMin, ok := l.getCounter(totalScopeKind, totalScopeID, metricReq, windowMin, now)
if !ok || totalMin != 1 {
t.Errorf("total req:min = %d, ok=%v, want 1 (the unlimited total scope is still counted)", totalMin, ok)
}
// Push tokens over budget via account, then confirm the SAME fused
// round trip catches it on the very next call.
l.account(scopes, usage{prompt: 2000}, 0)
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("want a tokens-per-day violation now that usage exceeds the 1000 budget")
}
if v.storeDown {
t.Error("v.storeDown must be false — this is a real limit breach, not a store failure")
}
assert.Contains(t, v.message, "tokens-per-day")
}
// TestCheckAndCount_RejectionRollsBackOtherScopes_MemoryFallback is
// finding F1, 2026-09 review, the group-lockout bug: a user hammering
// past THEIR OWN requests-per-minute limit must not also drive up their
// group's identical counter on every rejected attempt — before this fix,
// checkAndCount's unconditional per-scope counting had no rollback, so
// enough rejected user-scope requests could push the group's own limit
// past its threshold too, locking out every OTHER member of the group
// from traffic that was never actually admitted.
func TestCheckAndCount_RejectionRollsBackOtherScopes_MemoryFallback(t *testing.T) {
l := newLimiter(nil, true) // nil store -> every operation uses the in-process fallback
now := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}},
{kind: "group", id: "g", limits: &LimitsConfig{RequestsPerMinute: 100}},
}
// 1st call: user count 1 <= its own limit of 1, group count 1 <= its
// own limit of 100 — admitted.
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("1st call: want no violation, got %+v", v)
}
// 2nd call: user count reaches 2, over its own limit of 1 — rejected.
// Without the fix, the group's own req:min counter would also land at
// 2 here and stay there, even though the group never actually
// admitted a 2nd request.
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("2nd call: want a requests-per-minute violation on the user scope")
}
assert.Contains(t, v.message, "requests-per-minute")
groupMin, ok := l.getCounter("group", "g", metricReq, windowMin, now)
if !ok || groupMin != 1 {
t.Errorf("group req:min after the user's rejection = %d, ok=%v, want 1 (rolled back)", groupMin, ok)
}
groupDay, ok := l.getCounter("group", "g", metricReq, windowDay, now)
if !ok || groupDay != 1 {
t.Errorf("group req:day after the user's rejection = %d, ok=%v, want 1 (rolled back)", groupDay, ok)
}
userMin, ok := l.getCounter("user", "u", metricReq, windowMin, now)
if !ok || userMin != 2 {
t.Errorf("user req:min after its own rejection = %d, ok=%v, want 2 (the violating scope's own count is not rolled back)", userMin, ok)
}
// A 3rd request, from a DIFFERENT user in the SAME group, must still
// be admitted — proving the group is not locked out by the first
// user's rejected attempts.
otherUserScopes := []limitScope{
{kind: "user", id: "other", limits: &LimitsConfig{RequestsPerMinute: 1}},
{kind: "group", id: "g", limits: &LimitsConfig{RequestsPerMinute: 100}},
}
if v := l.checkAndCount(otherUserScopes); v != nil {
t.Fatalf("a different user in the same group: want no violation (group must not be locked out), got %+v", v)
}
}
// --- review round 2, 2026-08-21: enforcement-aware fallback TTL clamp ---
// TestClampTTL_ExactMathTable pins clampTTL's ceiling-vs-floor precedence
// for every window's real (ttl, enforceTTL) pair. Round 1's clamp applied
// memoryStoreMaxTTL unconditionally; round 2 corrects it so a window's
// own enforceTTL floor always wins a conflict with that ceiling — this is
// the exact math that keeps min/hour/day capped at 48h while month lands
// back at its own ~32-day floor.
func TestClampTTL_ExactMathTable(t *testing.T) {
cases := []struct {
name string
ttl time.Duration
enforceTTL time.Duration
want time.Duration
}{
{"min: 2m sits under both ceiling and floor, unchanged", minWindowTTL, enforceTTLFor(windowMin), minWindowTTL},
{"hour: 48h ttl equals the 48h ceiling; its own 2h floor never applies", hourWindowTTL, enforceTTLFor(windowHour), memoryStoreMaxTTL},
{"day: 35d ttl clamped down to the 48h ceiling, still above the 25h floor", dayWindowTTL, enforceTTLFor(windowDay), memoryStoreMaxTTL},
{"month: 400d ttl would clamp to 48h, but the 32d floor wins instead", monthWindowTTL, enforceTTLFor(windowMonth), enforceTTLFor(windowMonth)},
{"enforceTTL=0 (incrBy's own single-key path): pure ceiling, no floor", 400 * 24 * time.Hour, 0, memoryStoreMaxTTL},
{"ttl already under both ceiling and floor passes through unchanged", 90 * time.Minute, time.Hour, 90 * time.Minute},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := clampTTL(c.ttl, c.enforceTTL); got != c.want {
t.Errorf("clampTTL(%v, %v) = %v, want %v", c.ttl, c.enforceTTL, got, c.want)
}
})
}
}
// TestEnforceTTLFor_MatchesTheOriginalPreHistoryBumpValues pins
// enforceTTLFor's four values directly — they are not arbitrary, they
// match dayWindowTTL/monthWindowTTL's values from before the v0.2
// data-layer task's history-retention bump (25h/32d), which were
// themselves already windowLength+margin for correct single-window
// enforcement.
func TestEnforceTTLFor_MatchesTheOriginalPreHistoryBumpValues(t *testing.T) {
cases := []struct {
window string
want time.Duration
}{
{windowMin, 2 * time.Minute},
{windowHour, 2 * time.Hour},
{windowDay, 25 * time.Hour},
{windowMonth, 32 * 24 * time.Hour},
}
for _, c := range cases {
if got := enforceTTLFor(c.window); got != c.want {
t.Errorf("enforceTTLFor(%q) = %v, want %v", c.window, got, c.want)
}
}
}
// TestMemoryStore_IncrMulti_MonthCounterSurvivesPast48hAndStillEnforces
// is the review-round-2 regression case: a month counter written via
// incrMulti — the real production path, checkAndCount/account — must
// stay alive, not silently reset to 0, well past 48h, and a
// TokensPerMonth limit must still enforce correctly at that point. This
// is exactly what round 1's unconditional 48h ceiling broke: a month
// counter's key would have expired and reset roughly every two days,
// turning the budget into a rolling ~48h one instead of a true
// calendar-month one whenever Redis is absent or down.
func TestMemoryStore_IncrMulti_MonthCounterSurvivesPast48hAndStillEnforces(t *testing.T) {
l := newLimiter(nil, true) // nil store -> every operation uses the in-process fallback
now := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{{kind: "group", id: "g", limits: &LimitsConfig{TokensPerMonth: 100}}}
l.account(scopes, usage{prompt: 60, completion: 20}, 0) // 80 tokens on day 1
// Advance 5 days: well past memoryStoreMaxTTL's 48h ceiling, far short
// of monthWindowTTL's own natural rollover.
now = now.Add(5 * 24 * time.Hour)
l.nowFn = func() time.Time { return now }
tokIn, ok := l.getCounter("group", "g", metricTokIn, windowMonth, now)
if !ok || tokIn != 60 {
t.Fatalf("tokin:month after 5 days = %d, ok=%v, want 60 (the key must not have expired at 48h)", tokIn, ok)
}
// Push the total over budget and confirm enforcement still fires —
// proves the surviving counter is actually read by
// tokenBudgetViolation, not merely present.
l.account(scopes, usage{completion: 21}, 0) // 60 + 41 = 101 > 100
if v := l.checkAndCount(scopes); v == nil {
t.Fatal("want a TokensPerMonth violation 5 days in; the round-1 clamp bug would have reset the counter well before this")
}
}
// TestMemoryStore_IncrMulti_DayCounterStillClampsAt48h asserts the
// round-1 clamp's original goal — bounding the fallback's own live-key
// growth — still holds for day, whose own enforcement floor (25h) sits
// below the 48h ceiling: a day counter written via incrMulti expires and
// resets once its key crosses 48h, well before dayWindowTTL's own 35-day
// retention would otherwise have kept it alive.
func TestMemoryStore_IncrMulti_DayCounterStillClampsAt48h(t *testing.T) {
m := newMemoryStore()
now := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC)
m.nowFn = func() time.Time { return now }
entry := counterIncr{key: "k", delta: 10, ttl: dayWindowTTL, enforceTTL: enforceTTLFor(windowDay)}
if _, err := m.incrMulti([]counterIncr{entry}); err != nil {
t.Fatal(err)
}
now = now.Add(49 * time.Hour) // just past the 48h ceiling
m.nowFn = func() time.Time { return now }
got, err := m.get("k")
if err != nil {
t.Fatal(err)
}
if got != 0 {
t.Errorf("day counter at 49h past write = %d, want 0 (expired at the 48h ceiling; day's own 25h floor never raises it)", got)
}
}
// --- Feature A (v0.22): per-provider/per-(provider,model) success-rate accounting ---
// newSyncLimiter builds a limiter exactly like newLimiter, but with spawn
// overridden to run synchronously instead of in its own goroutine — every
// recordProviderAttempt test in this file needs its store write to have
// already landed by the time it reads the counter back. Production
// spawns a real goroutine there (SHOULD-5 ruling, v0.22 review round: off
// a streaming response's TTFB path); this is the deterministic-test half
// of that same dependency-injection field, matching nowFn/waitFn's
// existing pattern elsewhere in this package.
func newSyncLimiter(store counterStore, failOpen bool) *limiter {
l := newLimiter(store, failOpen)
l.spawn = func(f func()) { f() }
return l
}
// TestRecordProviderAttempt_ClassifiesUsingIsTransient proves
// recordProviderAttempt's failure classification is exactly isTransient's
// own — reused, not forked (spec ruling) — across the full outcome table:
// success, a non-429 4xx (still a "success" for provider-health purposes:
// the provider answered correctly to a request it did not like), 429, a
// 5xx, a plain network error, and a context-canceled error (not a
// provider fault, per isTransient's own carve-out) — PLUS the SHOULD-1
// addition (v0.22 review round): a context.DeadlineExceeded error (a
// hung upstream the gateway's own deadline finally cut off) counts as a
// failure even though isTransient itself excludes it, both bare and
// wrapped (proving isDeadlineExceeded's manual Unwrap walk actually
// walks) — while context.Canceled, wrapped the identical way, still does
// not.
func TestRecordProviderAttempt_ClassifiesUsingIsTransient(t *testing.T) {
tests := []struct {
err error
resp *http.Response
name string
wantFail bool
}{
{name: "200 OK", resp: &http.Response{StatusCode: http.StatusOK}, wantFail: false},
{name: "404 not found (provider answered)", resp: &http.Response{StatusCode: http.StatusNotFound}, wantFail: false},
{name: "429 too many requests", resp: &http.Response{StatusCode: http.StatusTooManyRequests}, wantFail: true},
{name: "500 internal server error", resp: &http.Response{StatusCode: http.StatusInternalServerError}, wantFail: true},
{name: "503 service unavailable", resp: &http.Response{StatusCode: http.StatusServiceUnavailable}, wantFail: true},
{name: "network error", err: errors.New("dial tcp: connection refused"), wantFail: true},
{name: "context canceled (not a provider fault)", err: context.Canceled, wantFail: false},
{name: "context deadline exceeded (hung upstream timed out — SHOULD-1)", err: context.DeadlineExceeded, wantFail: true},
{name: "wrapped context deadline exceeded (proves errors.Is walks the chain)", err: fmt.Errorf("%w: dial timeout", context.DeadlineExceeded), wantFail: true},
{name: "wrapped context canceled stays non-failure too", err: fmt.Errorf("%w: client hung up", context.Canceled), wantFail: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
l := newSyncLimiter(nil, true)
now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
l.recordProviderAttempt("openai", "gpt-4o", tt.resp, tt.err)
attempts, _ := l.getCounter(kindProvider, "openai", metricProvAttempt, windowDay, now)
if attempts != 1 {
t.Errorf("provider attempts/day = %d, want 1", attempts)
}
fails, _ := l.getCounter(kindProvider, "openai", metricProvFail, windowDay, now)
wantFails := int64(0)
if tt.wantFail {
wantFails = 1
}
if fails != wantFails {
t.Errorf("provider fails/day = %d, want %d", fails, wantFails)
}
// Minute window and the (provider, model) scope both mirror the
// provider/day counters exactly.
attemptsMin, _ := l.getCounter(kindProvider, "openai", metricProvAttempt, windowMin, now)
if attemptsMin != 1 {
t.Errorf("provider attempts/min = %d, want 1", attemptsMin)
}
modelAttempts, _ := l.getCounter(kindProviderModel, "openai/gpt-4o", metricProvAttempt, windowDay, now)
if modelAttempts != 1 {
t.Errorf("model attempts/day = %d, want 1", modelAttempts)
}
modelFails, _ := l.getCounter(kindProviderModel, "openai/gpt-4o", metricProvFail, windowDay, now)
if modelFails != wantFails {
t.Errorf("model fails/day = %d, want %d", modelFails, wantFails)
}
// Dead-code sweep, 2026-09 review: recordProviderAttempt no
// longer writes a (provider, model) scope's MINUTE-window
// counters — providerCounterKeys has never read one
// (SHOULD-2, v0.22 review round), so those writes were pure
// cost with no reader anywhere in this codebase.
modelAttemptsMin, _ := l.getCounter(kindProviderModel, "openai/gpt-4o", metricProvAttempt, windowMin, now)
if modelAttemptsMin != 0 {
t.Errorf("model attempts/min = %d, want 0 (no longer written)", modelAttemptsMin)
}
modelFailsMin, _ := l.getCounter(kindProviderModel, "openai/gpt-4o", metricProvFail, windowMin, now)
if modelFailsMin != 0 {
t.Errorf("model fails/min = %d, want 0 (no longer written)", modelFailsMin)
}
})
}
}
// TestRecordProviderAttempt_EmptyModel_ProviderScopeOnly proves a caller
// that cannot cheaply know the upstream model before the attempt resolves
// (native passthrough, routes_passthrough.go's handlePassthrough: the
// model lives in the response body, read only afterward) gets
// provider-level accounting only when it passes model="".
func TestRecordProviderAttempt_EmptyModel_ProviderScopeOnly(t *testing.T) {
l := newSyncLimiter(nil, true)
now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
l.recordProviderAttempt("openai", "", &http.Response{StatusCode: http.StatusInternalServerError}, nil)
attempts, _ := l.getCounter(kindProvider, "openai", metricProvAttempt, windowDay, now)
if attempts != 1 {
t.Errorf("provider attempts/day = %d, want 1", attempts)
}
fails, _ := l.getCounter(kindProvider, "openai", metricProvFail, windowDay, now)
if fails != 1 {
t.Errorf("provider fails/day = %d, want 1", fails)
}
}
// TestRecordProviderAttempt_EmptyProvider_NoOp proves a caller with no
// resolved provider name (should never happen in production traffic, but
// must never panic) is a silent no-op.
func TestRecordProviderAttempt_EmptyProvider_NoOp(t *testing.T) {
l := newSyncLimiter(nil, true)
l.recordProviderAttempt("", "some-model", &http.Response{StatusCode: http.StatusOK}, nil) // must not panic
}
// TestRecordProviderAttempt_ProductionSpawn_WriteEventuallyLandsAsync
// leaves limiter.spawn at its real production default (newLimiter's own
// bounded, goroutine-spawning closure) instead of overriding it to run
// synchronously like every other recordProviderAttempt test in this file
// — FOLDED-2 (v0.22 review round, round 2): every other test here proves
// the WRITE is correct, but only by bypassing the actual async code path
// (limiter.spawn's own doc comment) entirely; this is the one test that
// exercises it for real, closing a real -race async-path gap. Polls
// (waitUntil, registry_test.go) rather than sleeping a fixed duration —
// a fixed sleep is either flaky (too short, especially under `-race`,
// which distorts goroutine scheduling) or wastefully slow (too long); a
// deadline-bounded poll is neither.
func TestRecordProviderAttempt_ProductionSpawn_WriteEventuallyLandsAsync(t *testing.T) {
l := newLimiter(nil, true) // production default spawn — deliberately NOT newSyncLimiter
now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
l.recordProviderAttempt("openai", "gpt-4o", &http.Response{StatusCode: http.StatusOK}, nil)
waitUntil(t, time.Second, func() bool {
attempts, _ := l.getCounter(kindProvider, "openai", metricProvAttempt, windowDay, now)
return attempts == 1
})
fails, _ := l.getCounter(kindProvider, "openai", metricProvFail, windowDay, now)
if fails != 0 {
t.Errorf("fails/day = %d, want 0 (a 200 response is never a failure)", fails)
}
}
// TestIsDeadlineExceeded_Classification exercises isDeadlineExceeded's
// observable classification across every wrap shape that matters: bare,
// wrapped once, wrapped twice, an unrelated sentinel (context.Canceled,
// which must never match), an unrelated wrapped error, a plain error
// with no Unwrap method — PLUS the prod-shape double-%w/triple-%w rows a
// review round-2 blocker demanded, byte-for-byte the exact format
// strings providers.go's three upstreamJSON/upstreamBytes call sites use.
//
// A load-bearing caveat this test's own name used to hide (it was
// TestIsDeadlineExceeded_YaegiSafeUnwrap): every case here runs COMPILED,
// under `go test`, and compiled-Go correctness is not the same claim as
// Yaegi-interpreted correctness. Round 2's original fix — a hand-rolled
// Unwrap walk against two locally declared interfaces, matched with bare
// comma-ok type assertions — passed every one of these exact rows
// (including the double-/triple-%w ones) under `go test`, while
// SHOULD-A's yaegi-check harness (tools/yaegi-check/main.go's
// exerciseAttemptAccounting) proved it silently returned false for the
// double-%w case under the REAL interpreter: an interpreted interface
// type does not correctly match a compiled concrete value's method set
// in Yaegi, the reverse direction of the already-documented errors.As-
// on-interpreted-types trap. matchesSentinel now calls real errors.Is
// instead (limits.go's own doc comment there has the full account of why
// that is safe here specifically, unlike errors.As on an interpreted
// type) — this table still pins the OBSERVABLE behavior, but
// tools/yaegi-check is what actually proves it under the interpreter;
// this table alone would not have caught the round-2 regression, and a
// future change here must keep running yaegi-check, not just `go test`.
func TestIsDeadlineExceeded_Classification(t *testing.T) {
tests := []struct {
err error
name string
want bool
}{
{name: "nil error", err: nil, want: false},
{name: "bare sentinel", err: context.DeadlineExceeded, want: true},
{name: "wrapped once", err: fmt.Errorf("upstream: %w", context.DeadlineExceeded), want: true},
{name: "wrapped twice", err: fmt.Errorf("outer: %w", fmt.Errorf("upstream: %w", context.DeadlineExceeded)), want: true},
{name: "unrelated sentinel (context.Canceled)", err: context.Canceled, want: false},
{name: "unrelated wrapped error", err: fmt.Errorf("dial: %w", errors.New("connection refused")), want: false},
{name: "plain error, no Unwrap method", err: errors.New("boom"), want: false},
{
name: "double-%w, upstreamBytes' client.Do shape — the blocker's own reproduction",
err: fmt.Errorf("%w: %w", errUpstream, context.DeadlineExceeded),
want: true,
},
{
name: "double-%w, upstreamJSON's encode-failure shape",
err: fmt.Errorf("%w: encode request body: %w", errUpstream, context.DeadlineExceeded),
want: true,
},
{
name: "triple-%w, upstreamBytes' build-request shape",
err: fmt.Errorf("%w: %w: build request: %w", errUpstream, errRequestBuildFailed, context.DeadlineExceeded),
want: true,
},
{
name: "double-%w wrapping context.Canceled must still NOT match — multi-wrap does not blur the two sentinels apart",
err: fmt.Errorf("%w: %w", errUpstream, context.Canceled),
want: false,
},
{
name: "double-%w wrapping an unrelated error must still NOT match",
err: fmt.Errorf("%w: %w", errUpstream, errors.New("connection refused")),
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isDeadlineExceeded(tt.err); got != tt.want {
t.Errorf("isDeadlineExceeded(%v) = %v, want %v", tt.err, got, tt.want)
}
})
}
}
// TestProviderModelScopeID proves the "provider/model" id convention
// providerUsage/recordProviderAttempt share with pricing.go's own
// unifiedCostMicros and the admin API.
func TestProviderModelScopeID(t *testing.T) {
if got := providerModelScopeID("openai", "gpt-4o"); got != "openai/gpt-4o" {
t.Errorf("providerModelScopeID = %q, want %q", got, "openai/gpt-4o")
}
}
// TestProviderUsage_BatchedSingleRoundTrip drives limiter.providerUsage
// against a counting stub store (historyCountingStore, shared with
// TestLimiterHistory_OneBatchCallOldestFirst above): it must issue
// exactly ONE getMulti call for both a provider-level and a
// (provider, model) scope together, and slice the flat result back to the
// right scope in order — including SHOULD-2's variable stride (v0.22
// review round): the provider scope reads its minute window too, the
// model scope does not, so a stray seeded minute-window key under the
// model scope's own id must never leak into its result — providerUsage
// never even asks the store for it.
func TestProviderUsage_BatchedSingleRoundTrip(t *testing.T) {
now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC)
countStore := &historyCountingStore{values: map[string]int64{
windowKey(kindProvider, "openai", metricProvAttempt, windowMin, now): 3,
windowKey(kindProvider, "openai", metricProvFail, windowMin, now): 1,
windowKey(kindProvider, "openai", metricProvAttempt, windowDay, now): 30,
windowKey(kindProvider, "openai", metricProvFail, windowDay, now): 2,
windowKey(kindProviderModel, "openai/gpt-4o", metricProvAttempt, windowDay, now): 10,
// Seeded but must never be read: providerCounterKeys omits a
// kindProviderModel scope's minute-window keys entirely (SHOULD-2).
// A bug that started reading them again would make this key's
// value leak into got[1].attemptsMinute below, catching the
// regression even though allKeys's own length is never asserted
// directly.
windowKey(kindProviderModel, "openai/gpt-4o", metricProvAttempt, windowMin, now): 999,
}}
l := newLimiter(countStore, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: kindProvider, id: "openai"},
{kind: kindProviderModel, id: "openai/gpt-4o"},
}
got := l.providerUsage(scopes)
if countStore.getMultiCalls != 1 {
t.Errorf("getMultiCalls = %d, want 1 (both scopes' keys flattened into one call)", countStore.getMultiCalls)
}
if len(got) != 2 {
t.Fatalf("len(got) = %d, want 2", len(got))
}
if got[0].attemptsMinute != 3 || got[0].failuresMinute != 1 || got[0].attemptsDay != 30 || got[0].failuresDay != 2 {
t.Errorf("provider counters = %+v, want {3 1 30 2}", got[0])
}
if got[1].attemptsDay != 10 || got[1].failuresDay != 0 {
t.Errorf("model counters = %+v, want attemptsDay 10, failuresDay 0", got[1])
}
if got[1].attemptsMinute != 0 || got[1].failuresMinute != 0 {
t.Errorf("model minute counters = %+v, want the zero value (never read for a model scope — SHOULD-2)", got[1])
}
}
// TestProviderUsage_OffsetSlicing_ScopeOrderIndependent proves the
// running-offset slicing SHOULD-2 introduced (providerUsage no longer
// assumes a single fixed stride) is correct regardless of scope order —
// a model scope (2 keys) followed by TWO provider scopes (4 keys each)
// followed by another model scope, deliberately not the "providers then
// models" order buildAdminOverview itself always builds, to prove the
// slicing logic itself does not silently depend on that convention.
func TestProviderUsage_OffsetSlicing_ScopeOrderIndependent(t *testing.T) {
now := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC)
countStore := &historyCountingStore{values: map[string]int64{
windowKey(kindProviderModel, "a/m1", metricProvAttempt, windowDay, now): 1,
windowKey(kindProviderModel, "a/m1", metricProvFail, windowDay, now): 0,
windowKey(kindProvider, "a", metricProvAttempt, windowMin, now): 2,
windowKey(kindProvider, "a", metricProvFail, windowMin, now): 0,
windowKey(kindProvider, "a", metricProvAttempt, windowDay, now): 20,
windowKey(kindProvider, "a", metricProvFail, windowDay, now): 1,
windowKey(kindProvider, "b", metricProvAttempt, windowMin, now): 3,
windowKey(kindProvider, "b", metricProvFail, windowMin, now): 1,
windowKey(kindProvider, "b", metricProvAttempt, windowDay, now): 30,
windowKey(kindProvider, "b", metricProvFail, windowDay, now): 2,
windowKey(kindProviderModel, "b/m2", metricProvAttempt, windowDay, now): 4,
windowKey(kindProviderModel, "b/m2", metricProvFail, windowDay, now): 4,
}}
l := newLimiter(countStore, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: kindProviderModel, id: "a/m1"},
{kind: kindProvider, id: "a"},
{kind: kindProvider, id: "b"},
{kind: kindProviderModel, id: "b/m2"},
}
got := l.providerUsage(scopes)
if len(got) != 4 {
t.Fatalf("len(got) = %d, want 4", len(got))
}
if got[0].attemptsDay != 1 || got[0].failuresDay != 0 {
t.Errorf("scope[0] (model a/m1) = %+v, want attemptsDay 1, failuresDay 0", got[0])
}
if got[1].attemptsMinute != 2 || got[1].attemptsDay != 20 || got[1].failuresDay != 1 {
t.Errorf("scope[1] (provider a) = %+v, want attemptsMinute 2, attemptsDay 20, failuresDay 1", got[1])
}
if got[2].attemptsMinute != 3 || got[2].failuresMinute != 1 || got[2].attemptsDay != 30 || got[2].failuresDay != 2 {
t.Errorf("scope[2] (provider b) = %+v, want {3 1 30 2}", got[2])
}
if got[3].attemptsDay != 4 || got[3].failuresDay != 4 {
t.Errorf("scope[3] (model b/m2) = %+v, want attemptsDay 4, failuresDay 4", got[3])
}
}
// TestProviderUsage_EmptyScopes_ReturnsEmptyWithoutTouchingStore mirrors
// targetUsage's own empty-input short circuit.
func TestProviderUsage_EmptyScopes_ReturnsEmptyWithoutTouchingStore(t *testing.T) {
countStore := &historyCountingStore{}
l := newLimiter(countStore, true)
got := l.providerUsage(nil)
if len(got) != 0 {
t.Errorf("len(got) = %d, want 0", len(got))
}
if countStore.getMultiCalls != 0 {
t.Errorf("getMultiCalls = %d, want 0 (empty scopes must not touch the store)", countStore.getMultiCalls)
}
}
// TestProviderUsage_StoreDown_ReturnsZeroCounters mirrors targetUsage's
// own fail-closed contract: a store error with failOpen=false must report
// every scope's counters as zero, not partial or stale values, AND must
// set storeDown (review fix, adversarial verification 2026-08-23): a
// caller that cannot tell a real zero from a failed read would otherwise
// fabricate a counter-reset spike in metrics.go's rendering.
func TestProviderUsage_StoreDown_ReturnsZeroCounters(t *testing.T) {
l := newLimiter(alwaysErrStore{}, false)
got := l.providerUsage([]limitScope{{kind: kindProvider, id: "openai"}})
if len(got) != 1 {
t.Fatalf("len(got) = %d, want 1", len(got))
}
if !got[0].storeDown {
t.Errorf("counters = %+v, want storeDown true on a failed-closed read", got[0])
}
if got[0] != (providerCounters{storeDown: true}) {
t.Errorf("counters = %+v, want every non-storeDown field at its zero value", got[0])
}
}
// --- rejectionCounter: bounding, rotation, pruning (review fixes, adversarial verification 2026-08-23) ---
// TestRejectionCounter_BoundsMemoryViaRotation mirrors
// TestAuthFailureTracker_BoundsMemoryViaRotation (auth_test.go) exactly:
// filling current past rejectionCounterMapCap distinct scopes must
// trigger an O(1) generation rotation, not unbounded growth — a
// deployment that renames or churns users over a long process lifetime
// must never grow this map (and the resulting Prometheus series count)
// without bound.
func TestRejectionCounter_BoundsMemoryViaRotation(t *testing.T) {
var c rejectionCounter
for i := 0; i < rejectionCounterMapCap+10; i++ {
c.increment("user", fmt.Sprintf("user-%d", i))
}
c.mu.Lock()
total := len(c.current) + len(c.previous)
currentLen := len(c.current)
c.mu.Unlock()
if total > 2*rejectionCounterMapCap {
t.Errorf("tracked entries = %d, want <= %d (2x cap)", total, 2*rejectionCounterMapCap)
}
if currentLen > rejectionCounterMapCap {
t.Errorf("current generation = %d entries, want <= %d", currentLen, rejectionCounterMapCap)
}
}
// TestRejectionCounter_Increment_AccumulatesPerScope proves the ordinary
// path: repeated increments for the same (kind, id) accumulate in one
// series rather than creating a new one each time.
func TestRejectionCounter_Increment_AccumulatesPerScope(t *testing.T) {
var c rejectionCounter
c.increment("user", "alice")
c.increment("user", "alice")
c.increment("user", "bob")
snaps := c.snapshot()
got := map[rejectionScope]int64{}
for _, s := range snaps {
got[rejectionScope{kind: s.kind, id: s.id}] = s.count
}
if got[rejectionScope{kind: "user", id: "alice"}] != 2 {
t.Errorf("alice count = %d, want 2", got[rejectionScope{kind: "user", id: "alice"}])
}
if got[rejectionScope{kind: "user", id: "bob"}] != 1 {
t.Errorf("bob count = %d, want 1", got[rejectionScope{kind: "user", id: "bob"}])
}
}
// TestLimiter_PruneRejections_RemovesDeletedUserScope is the mutation-
// proofing test for pruneRejections/pruneUserScopes: a user's rejection
// series must disappear once that user is no longer in keep (the shape
// a real file-sourced hot-reload produces, via Gateway.
// pruneRejectionsAfterReload, metrics.go), while an unrelated user's
// series, and every non-"user" scope kind (group, the synthetic total
// and store_down entries), must survive untouched.
func TestLimiter_PruneRejections_RemovesDeletedUserScope(t *testing.T) {
l := newLimiter(nil, true)
l.rejections.increment("user", "alice")
l.rejections.increment("user", "bob")
l.rejections.increment("group", "agroup")
l.rejections.increment(totalScopeKind, totalScopeID)
l.rejections.increment(rejectionScopeStoreDown, rejectionScopeStoreDown)
l.pruneRejections(map[string]bool{"bob": true}) // alice was removed from the users file; bob stays
byKey := map[rejectionScope]bool{}
for _, s := range l.rejectionSnapshot() {
byKey[rejectionScope{kind: s.kind, id: s.id}] = true
}
if byKey[rejectionScope{kind: "user", id: "alice"}] {
t.Error("alice's user-scope rejection entry survived pruning, want it removed")
}
if !byKey[rejectionScope{kind: "user", id: "bob"}] {
t.Error("bob's user-scope rejection entry was pruned, want it kept (bob is still a known user)")
}
if !byKey[rejectionScope{kind: "group", id: "agroup"}] {
t.Error("group-scope entry was pruned — pruning must only ever touch \"user\"-kind scopes")
}
if !byKey[rejectionScope{kind: totalScopeKind, id: totalScopeID}] {
t.Error("the synthetic total-scope entry was pruned — pruning must only ever touch \"user\"-kind scopes")
}
if !byKey[rejectionScope{kind: rejectionScopeStoreDown, id: rejectionScopeStoreDown}] {
t.Error("the synthetic store_down entry was pruned — pruning must only ever touch \"user\"-kind scopes")
}
}
// --- kindModel scope: per-model usage accounting ---
// readCounter reads one counter's current-bucket value directly, for tests
// asserting exactly what account wrote.
func readCounter(t *testing.T, l *limiter, kind, id, metric, window string, now time.Time) int64 {
t.Helper()
vals, ok := l.storeGetMulti([]string{windowKey(kind, id, metric, window, now)})
require.True(t, ok, "storeGetMulti")
require.Len(t, vals, 1)
return vals[0]
}
// TestAccount_ModelScopeWritesRequestsAndUsage pins the one behaviour that
// separates a kindModel scope from every other kind inside account: it
// carries its own request counter, because checkAndCount never counted one
// for it at admission (the serving model is only known post-response).
// A user scope in the SAME call must still get no req from account.
func TestAccount_ModelScopeWritesRequestsAndUsage(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: "user", id: "alice"},
{kind: kindModel, id: "uni/qwen3.8-flash-next"},
}
l.account(scopes, usage{prompt: 40, completion: 10}, 2_500_000)
// The model scope gets req at all three chart windows.
for _, w := range []string{windowHour, windowDay, windowMonth} {
got := readCounter(t, l, kindModel, "uni/qwen3.8-flash-next", metricReq, w, now)
assert.Equal(t, int64(1), got, "model req at %s", w)
}
// ...and the usage metrics, same as any other scope.
assert.Equal(t, int64(40), readCounter(t, l, kindModel, "uni/qwen3.8-flash-next", metricTokIn, windowDay, now))
assert.Equal(t, int64(10), readCounter(t, l, kindModel, "uni/qwen3.8-flash-next", metricTokOut, windowDay, now))
assert.Equal(t, int64(2_500_000), readCounter(t, l, kindModel, "uni/qwen3.8-flash-next", metricCost, windowDay, now))
// The user scope must NOT gain a req from account — checkAndCount owns
// that at admission, and double-counting it here would inflate every
// existing per-user request figure.
assert.Equal(t, int64(0), readCounter(t, l, "user", "alice", metricReq, windowDay, now),
"account must not write req for a non-model scope")
}
// TestAccount_ModelScopeRequestCountedWithZeroUsage covers the case a
// token/cost-gated write would miss: an upstream that reports no usage at
// all still served a request, so the model's request counter must move.
func TestAccount_ModelScopeRequestCountedWithZeroUsage(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
l.account([]limitScope{{kind: kindModel, id: "p/m"}}, usage{}, 0)
assert.Equal(t, int64(1), readCounter(t, l, kindModel, "p/m", metricReq, windowDay, now))
assert.Equal(t, int64(0), readCounter(t, l, kindModel, "p/m", metricTokIn, windowDay, now))
}
// TestWithModelScope covers the helper's add/skip decisions and, in the
// last subtest, the aliasing hazard it exists to avoid.
func TestWithModelScope(t *testing.T) {
base := []limitScope{{kind: "user", id: "u"}}
t.Run("adds the model scope", func(t *testing.T) {
got := withModelScope(base, "uni/m")
require.Len(t, got, 2)
assert.Equal(t, kindModel, got[1].kind)
assert.Equal(t, "uni/m", got[1].id)
})
t.Run("skips an unattributable canonical", func(t *testing.T) {
for _, canonical := range []string{"", "provider/", "/"} {
got := withModelScope(base, canonical)
assert.Len(t, got, 1, "canonical %q must add no scope", canonical)
}
})
t.Run("does not alias the caller's backing array", func(t *testing.T) {
// A slice with spare capacity is the dangerous shape: a plain
// append would write into the caller's own array.
shared := make([]limitScope, 1, 4)
shared[0] = limitScope{kind: "user", id: "u"}
a := withModelScope(shared, "p/one")
b := withModelScope(shared, "p/two")
assert.Equal(t, "p/one", a[1].id, "second call must not overwrite the first result")
assert.Equal(t, "p/two", b[1].id)
})
}
// TestModelTotals covers positional results, the non-existent-counter zero,
// and the empty-input short circuit.
func TestModelTotals(t *testing.T) {
now := time.Date(2026, 8, 20, 10, 0, 30, 0, time.UTC)
l := newLimiter(nil, true)
l.nowFn = func() time.Time { return now }
l.incrCounter(kindModel, "b/two", metricCost, windowDay, now, 500, dayWindowTTL)
got, ok := l.modelTotals([]string{"a/one", "b/two", "c/three"}, metricCost, windowDay)
require.True(t, ok)
assert.Equal(t, []int64{0, 500, 0}, got, "results must be positional, zero for an unused model")
empty, ok := l.modelTotals(nil, metricCost, windowDay)
assert.True(t, ok, "an empty id list is not a failed read")
assert.Empty(t, empty)
}
// --- bucketsFor: every window's bucketFor computed once per call ---
// TestBucketsFor_MatchesBucketForEachWindow pins bucketsFor's four fields
// against bucketFor called directly per window.
func TestBucketsFor_MatchesBucketForEachWindow(t *testing.T) {
now := time.Date(2026, 8, 20, 14, 37, 9, 0, time.UTC)
wb := bucketsFor(now)
assert.Equal(t, bucketFor(now, windowMin), wb.min)
assert.Equal(t, bucketFor(now, windowHour), wb.hour)
assert.Equal(t, bucketFor(now, windowDay), wb.day)
assert.Equal(t, bucketFor(now, windowMonth), wb.month)
}
// TestEnforceTTLsFor_MatchesEnforceTTLForEachWindow pins enforceTTLsFor's
// four fields against enforceTTLFor called directly per window — the two
// must never drift apart, since checkAndCount/account/countTargetRequests/
// recordProviderAttempt all build their counterIncr entries from
// enforceTTLsFor's single shared value instead of calling enforceTTLFor
// per entry.
func TestEnforceTTLsFor_MatchesEnforceTTLForEachWindow(t *testing.T) {
et := enforceTTLsFor()
assert.Equal(t, enforceTTLFor(windowMin), et.min)
assert.Equal(t, enforceTTLFor(windowHour), et.hour)
assert.Equal(t, enforceTTLFor(windowDay), et.day)
assert.Equal(t, enforceTTLFor(windowMonth), et.month)
}
// TestWindowKeyForBucket_MatchesWindowKey pins windowKeyForBucket (the
// building block windowKey and every checkAndCount/account/
// countTargetRequests/recordProviderAttempt call site now use directly
// with a bucket from their own bucketsFor(now) call) against windowKey's
// own output for the equivalent bucketFor(t, window) — the two must
// produce byte-identical keys, since windowKey is now defined in terms of
// this function.
func TestWindowKeyForBucket_MatchesWindowKey(t *testing.T) {
now := time.Date(2026, 8, 20, 9, 0, 30, 0, time.UTC)
for _, window := range []string{windowMin, windowHour, windowDay, windowMonth} {
got := windowKeyForBucket("user", "a", metricReq, window, bucketFor(now, window))
want := windowKey("user", "a", metricReq, window, now)
assert.Equal(t, want, got, "window=%s", window)
}
}
// --- the window/bucket/TTL/enforceTTL pairing invariant the 27 inline
// counterIncr{...} literals (checkAndCount/account/countTargetRequests/
// recordProviderAttempt) removed structural protection for. newCounterIncr
// used to derive enforceTTL from window itself, so a mismatch between the
// two was impossible by construction; the inline literals pair window,
// bucket, ttl, and enforceTTL by hand at each of the 27 call sites, and
// nothing in the Go type system keeps them correctly paired any more. ---
// recordingStore is a counterStore that records every counterIncr entry
// passed to incrMulti/incrAndGetMulti and otherwise behaves like a
// trivially successful backend — used by
// TestCounterIncrEntries_WindowBucketTTLInvariant to capture exactly what
// each of the four methods hands the store, without a real Redis or the
// in-process fallback's own bucketing in the way.
type recordingStore struct {
entries []counterIncr
}
func (s *recordingStore) getMulti(keys []string) ([]int64, error) {
return make([]int64, len(keys)), nil
}
func (s *recordingStore) incrMulti(entries []counterIncr) ([]int64, error) {
s.entries = append(s.entries, entries...)
return make([]int64, len(entries)), nil
}
func (s *recordingStore) incrAndGetMulti(entries []counterIncr, reads []string) ([]int64, []int64, error) {
s.entries = append(s.entries, entries...)
return make([]int64, len(entries)), make([]int64, len(reads)), nil
}
// TestCounterIncrEntries_WindowBucketTTLInvariant asserts, for every entry
// each of the four methods produces: its enforceTTL equals
// enforceTTLFor of the WINDOW embedded in its own key (key shape
// llmgw:kind:id:metric:window:bucket, so strings.Split(e.key, ":")[4] is
// the window), and its key ends in the bucket that window actually has
// for now (bucketFor(now, window)) — catching a hand-paired mismatch like
// a windowDay entry built with et.month, which nothing else in the suite
// would notice (the wrong enforceTTL only ever changes memoryStore's
// clampTTL floor, never the value counted).
func TestCounterIncrEntries_WindowBucketTTLInvariant(t *testing.T) {
now := time.Date(2026, 8, 20, 9, 30, 0, 0, time.UTC)
scopes := []limitScope{{kind: "user", id: "u1", limits: &LimitsConfig{RequestsPerMinute: 1000, RequestsPerDay: 1000}}}
cases := []struct {
run func(l *limiter)
name string
}{
{name: "checkAndCount", run: func(l *limiter) { l.checkAndCount(scopes) }},
{name: "account", run: func(l *limiter) { l.account(scopes, usage{prompt: 10, completion: 5}, 100) }},
{name: "countTargetRequests", run: func(l *limiter) { l.countTargetRequests(scopes) }},
{name: "recordProviderAttempt", run: func(l *limiter) {
l.recordProviderAttempt("openai", "gpt-4o", nil, errors.New("boom"))
}},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.nowFn = func() time.Time { return now }
l.spawn = func(f func()) { f() } // recordProviderAttempt's write must land synchronously to be captured
c.run(l)
require.NotEmpty(t, store.entries, "test fixture produced no counterIncr entries to check")
for _, e := range store.entries {
parts := strings.Split(e.key, ":")
require.GreaterOrEqualf(t, len(parts), 6, "key %q does not have the llmgw:kind:id:metric:window:bucket shape", e.key)
window := parts[4]
wantTTL := enforceTTLFor(window)
assert.Equalf(t, wantTTL, e.enforceTTL, "entry %q: enforceTTL, want enforceTTLFor(%q)", e.key, window)
wantBucket := bucketFor(now, window)
assert.Truef(t, strings.HasSuffix(e.key, wantBucket), "entry %q does not end with bucketFor(now, %q) = %q", e.key, window, wantBucket)
}
})
}
}
// TestModelTotals_ConfiguredStoreDown_FailOpen_ReturnsNotOK is finding
// F5, 2026-09 review — modelTotals' own counterpart of
// TestLimiterHistory_ConfiguredStoreDown_FailOpen_ReturnsNotOK above: with
// failOpen=true, a store outage must still report ok=false via
// configuredStoreDown rather than silently presenting the empty fallback
// as a genuine "these models were barely used" ranking.
func TestModelTotals_ConfiguredStoreDown_FailOpen_ReturnsNotOK(t *testing.T) {
l := newLimiter(alwaysErrStore{}, true)
got, ok := l.modelTotals([]string{"a/one"}, metricCost, windowDay)
if ok {
t.Fatal("want ok=false: the store is down even though failOpen silently served the fallback")
}
if got != nil {
t.Errorf("got = %+v, want nil", got)
}
}
// TestModelSpanTotalsSumsBuckets is F2 (v0.3 dashboard task):
// modelSpanTotals sums span buckets ending at now, per id, positionally
// — the arithmetic GET /admin/api/usage/models' ?span= parameter relies
// on. It also proves modelTotals (its span=1 special case) reads only
// today's own bucket, byte-for-byte the pre-F2 behavior.
func TestModelSpanTotalsSumsBuckets(t *testing.T) {
l := newLimiter(nil, true)
now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC)
// Fix (verify-retry.md item 5): pin the limiter's own clock to the
// fixed now above — the same convention every other test in this file
// that touches l.now() already follows (TestLimiterRequestWindow and
// dozens more, each setting l.nowFn right after newLimiter). Without
// this, modelTotals below reads l.now() (real time.Now(), limits.go's
// newLimiter default) instead of this test's fixed now: the buckets
// were written against 2026-09-23, so the span=1 assertion only
// happened to pass on the calendar day this test was first written,
// and fails deterministically on every later day — not a race, and
// not flaky under -race specifically; a plain date mismatch that
// -race's slower execution does not change the odds of, just the day
// this was first noticed on.
l.nowFn = func() time.Time { return now }
id := "openai/gpt-4o"
l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 2), 5, dayWindowTTL)
l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 1), 7, dayWindowTTL)
l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 0), 11, dayWindowTTL)
got, ok := l.modelSpanTotals([]string{id}, metricReq, windowDay, now, 3)
if !ok {
t.Fatal("want ok=true")
}
if len(got) != 1 || got[0] != 23 {
t.Errorf("span=3 got = %v, want [23] (5+7+11)", got)
}
span1, ok := l.modelTotals([]string{id}, metricReq, windowDay)
if !ok || len(span1) != 1 || span1[0] != 11 {
t.Errorf("modelTotals (span=1 wrapper) got = %v ok=%v, want [11] (today's own bucket only)", span1, ok)
}
}
// TestModelSpanTotalsSumsBuckets_MultipleIDsPositional proves the
// flattened storeGetMulti's reply is sliced back per id in the SAME
// order ids was given, not just correct for a single id.
func TestModelSpanTotalsSumsBuckets_MultipleIDsPositional(t *testing.T) {
l := newLimiter(nil, true)
now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC)
l.incrCounter(kindModel, "a/one", metricCost, windowDay, historyStepBack(now, windowDay, 1), 100, dayWindowTTL)
l.incrCounter(kindModel, "a/one", metricCost, windowDay, historyStepBack(now, windowDay, 0), 50, dayWindowTTL)
l.incrCounter(kindModel, "b/two", metricCost, windowDay, historyStepBack(now, windowDay, 1), 3, dayWindowTTL)
l.incrCounter(kindModel, "b/two", metricCost, windowDay, historyStepBack(now, windowDay, 0), 4, dayWindowTTL)
got, ok := l.modelSpanTotals([]string{"a/one", "b/two"}, metricCost, windowDay, now, 2)
if !ok {
t.Fatal("want ok=true")
}
if len(got) != 2 || got[0] != 150 || got[1] != 7 {
t.Errorf("got = %v, want [150 7]", got)
}
}
// TestModelSpanTotals_EmptyIDs_NoStoreRead proves an empty ids slice
// never touches the store at all — matches modelTotals' own pre-existing
// contract, unaffected by F2's span extension.
func TestModelSpanTotals_EmptyIDs_NoStoreRead(t *testing.T) {
l := newLimiter(alwaysErrStore{}, false) // fail-closed AND always-erroring: any store read would fail the test
got, ok := l.modelSpanTotals(nil, metricReq, windowDay, time.Now(), 5)
if !ok || got != nil {
t.Errorf("got = %v, ok = %v, want nil, true (no store read for an empty ids slice)", got, ok)
}
}
// TestCheckAndCountRejectionFoldedIntoRollbackBatch is F4 (v0.3 dashboard
// task): the fleet-wide rejectionsPerDay counter for the violated scope
// AND the synthetic total scope rides the SAME storeIncrMulti batch
// settleRejection already pays to compensate every other scope's
// increments (see TestCheckAndCount_OneIncrAndGetMultiCallRegardlessOfScopeCount
// above) — a rejection never costs a SECOND round trip beyond that one.
func TestCheckAndCountRejectionFoldedIntoRollbackBatch(t *testing.T) {
store := &countingIncrStore{}
l := newLimiter(store, true)
now := time.Date(2026, 9, 23, 9, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
scopes := []limitScope{
{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}},
{kind: "group", id: "g", limits: nil},
{kind: totalScopeKind, id: totalScopeID, limits: nil},
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("1st call should pass, got %+v", v)
}
v := l.checkAndCount(scopes)
if v == nil {
t.Fatal("2nd call should violate u's requestsPerMinute:1")
}
if v.kind != eventKindRateLimit {
t.Errorf("v.kind = %q, want %q", v.kind, eventKindRateLimit)
}
if store.incrMultiCalls != 1 {
t.Errorf("incrMultiCalls after the violation = %d, want 1 (compensations + rejection counters share one batch)", store.incrMultiCalls)
}
uRej, ok := l.getCounter("user", "u", metricRej, windowDay, now)
if !ok || uRej != 1 {
t.Errorf("user rej:day = %d, ok=%v, want 1", uRej, ok)
}
totalRej, ok := l.getCounter(totalScopeKind, totalScopeID, metricRej, windowDay, now)
if !ok || totalRej != 1 {
t.Errorf("total rej:day = %d, ok=%v, want 1 (Q6: rejections attributed to total/all too)", totalRej, ok)
}
gRej, ok := l.getCounter("group", "g", metricRej, windowDay, now)
if ok && gRej != 0 {
t.Errorf("group rej:day = %d, want 0 (group did not violate)", gRej)
}
}
// TestCheckAndCount_StoreDownRejection_NoRejCounterWrite proves the
// store-down branch stays purely in-process (limits.go's own checkAndCount
// doc comment): a storeDown violation never reaches settleRejection at
// all — it fires before any per-scope evaluation loop — so it costs no
// further store round trip beyond the one failed incrAndGetMulti attempt.
func TestCheckAndCount_StoreDownRejection_NoRejCounterWrite(t *testing.T) {
store := &erroringStore{err: errStoreDownStub}
l := newLimiter(store, false) // fail-closed
scopes := []limitScope{{kind: "user", id: "u", limits: &LimitsConfig{RequestsPerMinute: 1}}}
v := l.checkAndCount(scopes)
if v == nil || !v.storeDown {
t.Fatalf("v = %+v, want a storeDown violation", v)
}
if v.kind != eventKindStoreDown {
t.Errorf("v.kind = %q, want %q", v.kind, eventKindStoreDown)
}
}
// TestCurrentUsage_RejectionsPerDay proves currentUsage reads the
// rej:day counter into scopeUsage.rejectionsPerDay — F4's usageKeysPerScope
// 8->9 growth (v0.3 dashboard task).
func TestCurrentUsage_RejectionsPerDay(t *testing.T) {
l := newLimiter(nil, true)
now := time.Date(2026, 9, 23, 9, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
l.incrCounter("user", "u", metricRej, windowDay, now, 3, dayWindowTTL)
got := l.currentUsage([]limitScope{{kind: "user", id: "u"}})
if len(got) != 1 {
t.Fatalf("len(got) = %d, want 1", len(got))
}
if got[0].rejectionsPerDay != 3 {
t.Errorf("rejectionsPerDay = %d, want 3", got[0].rejectionsPerDay)
}
if got[0].storeDown {
t.Error("storeDown = true, want false")
}
}
// TestTargetUsage_ConfiguredStoreDown_FailOpen_ReturnsZeroCounters is
// finding F5, 2026-09 review — targetUsage's own counterpart: with
// failOpen=true, a store outage must fall back to the same zero-value
// targetCounters this function's own doc comment already promises for
// the fail-closed case, rather than presenting the empty in-process
// fallback's data as confirmed current usage.
func TestTargetUsage_ConfiguredStoreDown_FailOpen_ReturnsZeroCounters(t *testing.T) {
l := newLimiter(alwaysErrStore{}, true)
got := l.targetUsage([]limitScope{{kind: targetKindMCP, id: "alpha"}})
if len(got) != 1 {
t.Fatalf("len(got) = %d, want 1", len(got))
}
if got[0] != (targetCounters{}) {
t.Errorf("got[0] = %+v, want the zero value", got[0])
}
}
// --- admin-redesign WP-A (steps 1-8) ---
// TestMemoryStore_ApplySet_OverwritesRatherThanAdds proves memoryStore's
// absolute-entry path (last-seen SET, Q6 DECISIONS) always REPLACES a
// key's value, never adds to it — the SET-vs-INCRBY distinction
// counterIncr.absolute exists to carry.
func TestMemoryStore_ApplySet_OverwritesRatherThanAdds(t *testing.T) {
m := newMemoryStore()
m.nowFn = func() time.Time { return time.Unix(1000, 0) }
if got := m.applySet("k", 42, time.Minute); got != 42 {
t.Fatalf("first applySet = %d, want 42", got)
}
if got := m.applySet("k", 7, time.Minute); got != 7 {
t.Errorf("second applySet = %d, want 7 (overwrite, not 49)", got)
}
if v, err := m.get("k"); err != nil || v != 7 {
t.Errorf("get(k) = %d, err=%v, want 7, nil", v, err)
}
}
// TestMemoryStore_IncrMulti_AbsoluteEntry_DispatchesToApplySet proves
// incrMulti routes an absolute counterIncr through applySet (overwrite),
// never applyIncr (add) — a non-absolute entry in the same batch still
// increments normally.
func TestMemoryStore_IncrMulti_AbsoluteEntry_DispatchesToApplySet(t *testing.T) {
m := newMemoryStore()
m.nowFn = func() time.Time { return time.Unix(2000, 0) }
out, err := m.incrMulti([]counterIncr{
{key: "seen", delta: 555, ttl: time.Hour, absolute: true},
{key: "count", delta: 3, ttl: time.Hour},
})
if err != nil {
t.Fatalf("incrMulti: %v", err)
}
if out[0] != 555 {
t.Errorf("absolute entry result = %d, want 555", out[0])
}
if out[1] != 3 {
t.Errorf("incr entry result = %d, want 3", out[1])
}
out2, _ := m.incrMulti([]counterIncr{{key: "count", delta: 3, ttl: time.Hour}})
if out2[0] != 6 {
t.Errorf("second incr on non-absolute key = %d, want 6 (added)", out2[0])
}
out3, _ := m.incrMulti([]counterIncr{{key: "seen", delta: 999, ttl: time.Hour, absolute: true}})
if out3[0] != 999 {
t.Errorf("second absolute write = %d, want 999 (overwrite, not 555+999)", out3[0])
}
}
// TestLastSeenGate_ThrottlesWithin60Seconds is Q6's own throttle contract:
// two due() calls for the same scope inside lastSeenGateThrottle yield
// exactly one true; a different scope is never throttled by another
// scope's own state.
func TestLastSeenGate_ThrottlesWithin60Seconds(t *testing.T) {
g := &lastSeenGate{}
t0 := time.Unix(10_000, 0)
if !g.due("user", "alice", t0) {
t.Fatal("first due() = false, want true")
}
if g.due("user", "alice", t0.Add(30*time.Second)) {
t.Error("due() 30s later = true, want false (within throttle)")
}
if !g.due("user", "alice", t0.Add(61*time.Second)) {
t.Error("due() 61s later = false, want true (throttle elapsed)")
}
if !g.due("user", "bob", t0.Add(30*time.Second)) {
t.Error("due() for a different scope = false, want true")
}
}
// TestLastSeenGate_ThrottlesPastMapCap is P8 (admin dashboard redesign
// verify round): due()'s two-map rotation wrote a scope forward into
// `previous` once `current` filled past lastSeenGateMapCapBase, but
// nothing ever CONSULTED previous on lookup — every scope that had been
// rotated out was treated as brand new on its very next call, so
// throttling silently stopped working once more than
// lastSeenGateMapCapBase (4,096) distinct scopes were active. 5,000
// distinct scopes, each looked up twice within the same second (round 2
// repeats round 1's exact scopes, same instant), must produce exactly
// 5,000 due()=true calls total — the pre-fix code produced far more,
// since round 2 mostly missed both the (rotated) previous map's own
// throttle and its freshly-empty current map. 5,000 stays below the N5
// adaptive-growth threshold (it only crosses ONE rotation, and growth
// only triggers on a SECOND rotation within lastSeenGateThrottle — see
// TestLastSeenGate_AdaptiveCapacityHoldsPastEightThousandScopes for the
// scope count that exercises growth itself), so this keeps proving the
// original fixed-cap rotation path unchanged.
func TestLastSeenGate_ThrottlesPastMapCap(t *testing.T) {
g := &lastSeenGate{}
now := time.Unix(1_800_000_000, 0)
const scopeCount = 5000
if scopeCount <= lastSeenGateMapCapBase {
t.Fatalf("scopeCount (%d) must exceed lastSeenGateMapCapBase (%d) to exercise rotation", scopeCount, lastSeenGateMapCapBase)
}
writes := 0
for round := 0; round < 2; round++ {
for i := 0; i < scopeCount; i++ {
if g.due("user", fmt.Sprintf("u%d", i), now) {
writes++
}
}
}
if writes != scopeCount {
t.Errorf("due()=true across 2 rounds of %d scopes within the same second = %d, want %d (round 2 must be fully throttled)", scopeCount, writes, scopeCount)
}
}
// TestLastSeenGate_AdaptiveCapacityHoldsPastEightThousandScopes is N5
// (verify-redesign-final.md): a fixed 4,096-entry cap only throttled
// correctly up to ~2x itself (current+previous both full, ~8,192 live
// scopes) — beyond that, current rotated out before lastSeenGateThrottle
// could elapse for most of what it held, discarding still-fresh writes
// on every rotation, which degraded round 2 of a repeat to one write per
// admission instead of zero. 20,000 distinct scopes (well past the old
// ~8,192 ceiling), each looked up twice within the same second, must
// still produce close to 20,000 due()=true calls total (round 1 only;
// round 2 must be (almost) fully throttled) — proving due()'s adaptive
// capacity (grows on a same-window second rotation, capped at
// lastSeenGateMapCapMax) keeps the whole live population inside
// current+previous instead of thrashing it.
func TestLastSeenGate_AdaptiveCapacityHoldsPastEightThousandScopes(t *testing.T) {
g := &lastSeenGate{}
now := time.Unix(1_800_000_000, 0)
const scopeCount = 20000
if scopeCount <= 2*lastSeenGateMapCapBase {
t.Fatalf("scopeCount (%d) must exceed the old 2x-cap ceiling (%d) to exercise adaptive growth", scopeCount, 2*lastSeenGateMapCapBase)
}
writes := 0
for round := 0; round < 2; round++ {
for i := 0; i < scopeCount; i++ {
if g.due("user", fmt.Sprintf("u%d", i), now) {
writes++
}
}
}
// "~20000": round 1 always writes scopeCount times (every scope is
// new). Round 2 may add a handful more only if a scope's write fell
// right before the growth threshold was reached, never a large
// fraction of scopeCount — the old, pre-fix behaviour wrote roughly
// scopeCount MORE on round 2 alone (P8's "far more" regression, at
// this population size), so a generous 1% slack still tells the two
// behaviours apart with room to spare.
maxWant := scopeCount + scopeCount/100
if writes < scopeCount || writes > maxWant {
t.Errorf("due()=true across 2 rounds of %d scopes within the same second = %d, want in [%d, %d]", scopeCount, writes, scopeCount, maxWant)
}
if g.capacity > lastSeenGateMapCapMax {
t.Errorf("g.capacity = %d, want <= lastSeenGateMapCapMax (%d)", g.capacity, lastSeenGateMapCapMax)
}
total := len(g.current) + len(g.previous)
if bound := 2 * lastSeenGateMapCapMax; total > bound {
t.Errorf("len(current)+len(previous) = %d, want <= 2*lastSeenGateMapCapMax (%d) — memory bound violated", total, bound)
}
}
// TestCheckAndCount_NeverWritesLastSeen is the P9-fix replacement for this
// test's own pre-fix name (AppendedAfterScopeStride): last-seen no longer
// rides checkAndCount's batch AT ALL (admin dashboard redesign verify
// round) — it used to be appended after every scope's own three stride
// entries unconditionally, which meant a REJECTED request still moved a
// scope's last-seen timestamp forward, since checkAndCount's whole batch,
// last-seen tail included, was already sent to the store before the
// violation check even ran. checkAndCount's own entry count is now
// EXACTLY 3*checkAndCountKeysPerScope regardless of l.statsAdmin — see
// TestRecordLastSeen_WritesTailEntriesForUserAndGroupScopesOnly, below,
// for the tail-entry coverage this test used to carry, now exercised
// against recordLastSeen directly (the function real admission call sites
// invoke instead, only once admission is confirmed).
func TestCheckAndCount_NeverWritesLastSeen(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
l.statsAdmin = true
scopes := []limitScope{
{kind: "user", id: "alice", limits: &LimitsConfig{RequestsPerMinute: 1000}},
{kind: "group", id: "friends", limits: &LimitsConfig{RequestsPerMinute: 1000}},
{kind: totalScopeKind, id: totalScopeID},
}
if v := l.checkAndCount(scopes); v != nil {
t.Fatalf("checkAndCount violation = %+v, want nil", v)
}
if len(store.entries) != 3*checkAndCountKeysPerScope {
t.Fatalf("len(store.entries) = %d, want %d (no last-seen tail)", len(store.entries), 3*checkAndCountKeysPerScope)
}
for i, e := range store.entries {
if e.absolute {
t.Errorf("entries[%d] (%q) is absolute, want checkAndCount to never write an absolute entry any more", i, e.key)
}
}
}
// TestRecordLastSeen_WritesTailEntriesForUserAndGroupScopesOnly proves
// recordLastSeen (limits.go; called by every admission call site's own
// "violation == nil" branch, P9 fix) writes exactly one absolute SET per
// "user"/"group" scope, keyed by lastSeenKey, and never one for the
// synthetic total scope.
func TestRecordLastSeen_WritesTailEntriesForUserAndGroupScopesOnly(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
l.statsAdmin = true
l.spawn = func(f func()) { f() } // synchronous for the test to observe entries
scopes := []limitScope{
{kind: "user", id: "alice"},
{kind: "group", id: "friends"},
{kind: totalScopeKind, id: totalScopeID},
}
l.recordLastSeen(scopes, now)
if len(store.entries) != 2 {
t.Fatalf("len(store.entries) = %d, want 2 (user + group, never total)", len(store.entries))
}
seenKeys := map[string]bool{}
for _, e := range store.entries {
if !e.absolute {
t.Errorf("entry %q is not absolute", e.key)
}
seenKeys[e.key] = true
}
if !seenKeys[lastSeenKey("user", "alice")] || !seenKeys[lastSeenKey("group", "friends")] {
t.Errorf("entries = %v, want lastSeenKey for user alice and group friends", store.entries)
}
if seenKeys[lastSeenKey(totalScopeKind, totalScopeID)] {
t.Error("total scope must never get a last-seen entry")
}
}
// TestRecordLastSeen_OffWithoutStatsAdmin proves recordLastSeen (P9 fix's
// own call site, invoked separately from checkAndCount now) builds zero
// entries and spawns nothing when statsAdmin is false (the default for
// every limiter built directly via newLimiter, unless a test opts in) —
// the same "additive and default-preserving" guarantee every other family
// in this task carries. spawn is deliberately left at its real (async,
// goroutine-based) default: if this ever regressed to spawning an empty
// batch, len(store.entries) staying 0 synchronously would not catch it,
// so failing to compile a nil/empty entries slice at all (countAsync's own
// "nil/empty entries slice spawns nothing" contract, limits.go) is what
// this test actually pins.
func TestRecordLastSeen_OffWithoutStatsAdmin(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.recordLastSeen([]limitScope{{kind: "user", id: "alice"}}, l.now())
if len(store.entries) != 0 {
t.Errorf("entries = %d, want 0 (no last-seen without statsAdmin)", len(store.entries))
}
}
// TestCheckAndCount_LastSeen_OffWithoutStatsAdmin proves checkAndCount
// itself never writes a last-seen entry, with or without statsAdmin — P9
// fix, admin dashboard redesign verify round: last-seen no longer rides
// checkAndCount's batch at all (TestCheckAndCount_NeverWritesLastSeen,
// above, pins the statsAdmin=true case; this pins the default-off case
// the pre-fix name of this test originally covered).
func TestCheckAndCount_LastSeen_OffWithoutStatsAdmin(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.checkAndCount([]limitScope{{kind: "user", id: "alice"}})
if len(store.entries) != checkAndCountKeysPerScope {
t.Errorf("entries = %d, want %d (checkAndCount never writes last-seen any more)", len(store.entries), checkAndCountKeysPerScope)
}
}
// TestRecordLastSeen_ThrottledSecondCallWithinWindowSkipsEntry proves the
// SAME replica calling recordLastSeen twice in immediate succession (the
// P9-fix call site, admin dashboard redesign verify round — every
// admission call site invokes it once per admitted request) writes the
// last-seen SET only once, via lastSeenGate.due's own throttle.
func TestRecordLastSeen_ThrottledSecondCallWithinWindowSkipsEntry(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC)
l.nowFn = func() time.Time { return now }
l.statsAdmin = true
l.spawn = func(f func()) { f() }
scopes := []limitScope{{kind: "user", id: "alice"}}
l.recordLastSeen(scopes, now)
if len(store.entries) != 1 {
t.Fatalf("first call entries = %d, want 1", len(store.entries))
}
store.entries = nil
l.recordLastSeen(scopes, now)
if len(store.entries) != 0 {
t.Errorf("second call (same instant) entries = %d, want 0 (throttled, no last-seen entry)", len(store.entries))
}
}
// TestSettleRejection_RejHour_OnlyWhenStatsAdmin proves settleRejection's
// rej:hour addition (rej/hour, always-on-with-admin) rides the same
// compensation batch only when statsAdmin is true, alongside the
// pre-existing, unconditional rej:day entry — and that the compensation
// entries for OTHER scopes (the rollback half) are unaffected either way.
func TestSettleRejection_RejHour_OnlyWhenStatsAdmin(t *testing.T) {
now := time.Date(2026, 9, 23, 10, 30, 0, 0, time.UTC)
wb := bucketsFor(now)
et := enforceTTLsFor()
scopes := []limitScope{
{kind: "user", id: "alice"},
{kind: totalScopeKind, id: totalScopeID},
}
entries := []counterIncr{
{key: windowKeyForBucket("user", "alice", metricReq, windowMin, wb.min), delta: 1, ttl: minWindowTTL, enforceTTL: et.min},
{key: windowKeyForBucket("user", "alice", metricReq, windowDay, wb.day), delta: 1, ttl: dayWindowTTL, enforceTTL: et.day},
{key: windowKeyForBucket("user", "alice", metricReq, windowHour, wb.hour), delta: 1, ttl: hourWindowTTL, enforceTTL: et.hour},
{key: windowKeyForBucket(totalScopeKind, totalScopeID, metricReq, windowMin, wb.min), delta: 1, ttl: minWindowTTL, enforceTTL: et.min},
{key: windowKeyForBucket(totalScopeKind, totalScopeID, metricReq, windowDay, wb.day), delta: 1, ttl: dayWindowTTL, enforceTTL: et.day},
{key: windowKeyForBucket(totalScopeKind, totalScopeID, metricReq, windowHour, wb.hour), delta: 1, ttl: hourWindowTTL, enforceTTL: et.hour},
}
for _, statsAdmin := range []bool{false, true} {
store := &recordingStore{}
l := newLimiter(store, true)
l.nowFn = func() time.Time { return now }
l.statsAdmin = statsAdmin
l.settleRejection(entries, 0, scopes, wb, et)
rejHourKey := windowKeyForBucket("user", "alice", metricRej, windowHour, wb.hour)
var gotHour bool
for _, e := range store.entries {
if e.key == rejHourKey {
gotHour = true
}
}
if gotHour != statsAdmin {
t.Errorf("statsAdmin=%v: rej:hour entry present=%v, want %v", statsAdmin, gotHour, statsAdmin)
}
}
}
// TestAccountWith_UserModel_RequiresOneUserScopeAndModelScope is (a),
// plan §2: the opt-in per-(user, model) breakdown fires only when
// statsUserModel is on, exactly one "user" scope is present, and a
// kindModel scope is present — writing <=8 entries (req always,
// tokin/tokout/cost only when nonzero) at day+month, never hour.
func TestAccountWith_UserModel_RequiresOneUserScopeAndModelScope(t *testing.T) {
id := userModelScopeID("alice", "openai/gpt-4o")
cases := []struct {
name string
scopes []limitScope
statsUserModel bool
wantEntries bool
}{
{name: "on, one user scope, model scope", statsUserModel: true, wantEntries: true,
scopes: withModelScope([]limitScope{{kind: "user", id: "alice"}}, "openai/gpt-4o")},
{name: "flag off", statsUserModel: false, wantEntries: false,
scopes: withModelScope([]limitScope{{kind: "user", id: "alice"}}, "openai/gpt-4o")},
{name: "no model scope", statsUserModel: true, wantEntries: false,
scopes: []limitScope{{kind: "user", id: "alice"}}},
{name: "multi-group principal: two user-kind scopes never happens, but two USER scopes must still refuse",
statsUserModel: true, wantEntries: false,
scopes: withModelScope([]limitScope{{kind: "user", id: "alice"}, {kind: "user", id: "bob"}}, "openai/gpt-4o")},
{name: "media: no user scope at all", statsUserModel: true, wantEntries: false,
scopes: withModelScope(nil, "openai/gpt-4o")},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.statsAdmin = true
l.statsUserModel = c.statsUserModel
l.accountWith(c.scopes, usage{prompt: 10, completion: 5}, 1000, accountExtras{})
var got int
for _, e := range store.entries {
if strings.HasPrefix(e.key, "llmgw:"+kindUserModel+":"+id+":") {
got++
}
}
if c.wantEntries && got != 8 {
t.Errorf("umodel entries = %d, want 8", got)
}
if !c.wantEntries && got != 0 {
t.Errorf("umodel entries = %d, want 0", got)
}
})
}
}
// TestAccountWith_UserModel_ReqAlwaysEvenWithZeroUsage proves the req pair
// is written unconditionally (mirroring kindModel's own unconditional req
// write), even for a served request with zero reported usage.
func TestAccountWith_UserModel_ReqAlwaysEvenWithZeroUsage(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.statsAdmin = true
l.statsUserModel = true
scopes := withModelScope([]limitScope{{kind: "user", id: "alice"}}, "openai/gpt-4o")
l.accountWith(scopes, usage{}, 0, accountExtras{})
id := userModelScopeID("alice", "openai/gpt-4o")
var got int
for _, e := range store.entries {
if strings.HasPrefix(e.key, "llmgw:"+kindUserModel+":"+id+":") && strings.Contains(e.key, ":"+metricReq+":") {
got++
}
}
if got != 2 {
t.Errorf("umodel req entries = %d, want 2 (day+month)", got)
}
}
// TestAccountWith_Failover_IncrementsFromProvider is (c), plan §2: a
// non-empty accountExtras.failoverFrom writes prov/{from}:fover at
// hour+day when statsAdmin is on, never when it is off.
func TestAccountWith_Failover_IncrementsFromProvider(t *testing.T) {
for _, statsAdmin := range []bool{false, true} {
store := &recordingStore{}
l := newLimiter(store, true)
l.statsAdmin = statsAdmin
l.accountWith(nil, usage{}, 0, accountExtras{failoverFrom: "openai"})
var got int
for _, e := range store.entries {
if strings.HasPrefix(e.key, "llmgw:"+kindProvider+":openai:"+metricFover+":") {
got++
}
}
want := 0
if statsAdmin {
want = 2
}
if got != want {
t.Errorf("statsAdmin=%v: fover entries = %d, want %d", statsAdmin, got, want)
}
}
}
// TestLatencyBucketIndex_MatchesObserveLatencyBucket proves accountWith's
// own bucket-index helper (limits.go) never disagrees with
// observeLatencyBucket's in-process histogram (metrics.go) about which
// bucket a duration belongs to — both must derive from the identical
// latencyBucketIndex.
func TestLatencyBucketIndex_MatchesObserveLatencyBucket(t *testing.T) {
durations := []time.Duration{
0, 50 * time.Millisecond, 100 * time.Millisecond, 999 * time.Millisecond,
2 * time.Second, 30 * time.Second, 600 * time.Second, 3600 * time.Second,
}
for _, d := range durations {
idx, overflow := latencyBucketIndex(d)
var buckets [1]int64
full := make([]int64, len(latencyBucketBounds))
var overflowCount int64
var sum float64
var count int64
observeLatencyBucket(full, &overflowCount, &sum, &count, d)
_ = buckets
if overflow {
if overflowCount != 1 {
t.Errorf("d=%v: latencyBucketIndex says overflow, observeLatencyBucket overflow count = %d, want 1", d, overflowCount)
}
if idx != len(latencyBucketBounds) {
t.Errorf("d=%v: overflow idx = %d, want %d", d, idx, len(latencyBucketBounds))
}
continue
}
if full[idx] != 1 {
t.Errorf("d=%v: latencyBucketIndex says bucket %d, observeLatencyBucket did not increment it: buckets=%v", d, idx, full)
}
}
}
// TestAccountWith_Latency_WritesProviderAndModelBuckets is (c), plan §2:
// hasLat && statsLatency writes the serving provider's own duration
// bucket (hour+day) plus the served model's own (when a kindModel scope
// is present), and TTFB buckets too when the sample carries one — riding
// the SAME storeIncrMulti batch account already builds (never a second
// round trip: this test's own recordingStore records exactly one
// incrMulti call for the whole accountWith invocation).
func TestAccountWith_Latency_WritesProviderAndModelBuckets(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.statsAdmin = true
l.statsLatency = true
scopes := withModelScope(nil, "openai/gpt-4o")
lat := latencySample{duration: 2 * time.Second, ttfb: 100 * time.Millisecond, hasTTFB: true}
l.accountWith(scopes, usage{}, 0, accountExtras{provider: "openai", lat: lat, hasLat: true})
durIdx, _ := latencyBucketIndex(lat.duration)
ttfbIdx, _ := latencyBucketIndex(lat.ttfb)
wantDur := latencyDurationMetric(durIdx)
wantTTFB := latencyTTFBMetric(ttfbIdx)
counts := map[string]int{}
for _, e := range store.entries {
counts[e.key]++
}
for _, kind := range []string{kindProvider, kindModel} {
id := "openai"
if kind == kindModel {
id = "openai/gpt-4o"
}
for _, window := range []string{windowHour, windowDay} {
for _, metric := range []string{wantDur, wantTTFB} {
key := windowKeyForBucket(kind, id, metric, window, "x")
// windowKeyForBucket's own bucket arg is fixed to "x" here
// only to build a comparable prefix; match by prefix minus
// bucket instead of exact key.
prefix := key[:len(key)-1]
var found bool
for k := range counts {
if strings.HasPrefix(k, prefix) {
found = true
}
}
if !found {
t.Errorf("missing latency entry for kind=%s id=%s metric=%s window=%s", kind, id, metric, window)
}
}
}
}
}
// TestAccountWith_CacheMiss_TotalHourDayAndModelDayOnly is (d), plan §2:
// cacheMiss writes total cmiss at hour+day, and the served model's own
// cmiss at day ONLY (never hour) — gated on statsAdmin.
func TestAccountWith_CacheMiss_TotalHourDayAndModelDayOnly(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.statsAdmin = true
scopes := withModelScope([]limitScope{{kind: totalScopeKind, id: totalScopeID}}, "openai/gpt-4o")
l.accountWith(scopes, usage{}, 0, accountExtras{cacheMiss: true})
totalHour := windowKeyForBucket(totalScopeKind, totalScopeID, metricCmiss, windowHour, bucketFor(l.now(), windowHour))
totalDay := windowKeyForBucket(totalScopeKind, totalScopeID, metricCmiss, windowDay, bucketFor(l.now(), windowDay))
modelDay := windowKeyForBucket(kindModel, "openai/gpt-4o", metricCmiss, windowDay, bucketFor(l.now(), windowDay))
modelHour := windowKeyForBucket(kindModel, "openai/gpt-4o", metricCmiss, windowHour, bucketFor(l.now(), windowHour))
present := map[string]bool{}
for _, e := range store.entries {
present[e.key] = true
}
if !present[totalHour] || !present[totalDay] {
t.Errorf("missing total cmiss hour/day entries: %v", store.entries)
}
if !present[modelDay] {
t.Error("missing model cmiss day entry")
}
if present[modelHour] {
t.Error("model cmiss hour entry present, want day only")
}
}
// TestRecordCacheHit_ChitAlwaysCsaveSkippedWhenZero is recordCacheHit's
// own contract (Q7, DECISIONS): total chit hour+day always, csave
// hour+day only when savedMicros != 0, model chit/csave at day only.
func TestRecordCacheHit_ChitAlwaysCsaveSkippedWhenZero(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.statsAdmin = true
l.spawn = func(f func()) { f() } // synchronous for the test to observe entries
l.recordCacheHit("openai/gpt-4o", 0)
var chit, csave int
for _, e := range store.entries {
switch {
case strings.Contains(e.key, ":"+metricChit+":"):
chit++
case strings.Contains(e.key, ":"+metricCsave+":"):
csave++
}
}
if chit != 3 { // total hour, total day, model day
t.Errorf("chit entries = %d, want 3", chit)
}
if csave != 0 {
t.Errorf("csave entries = %d, want 0 (saved=0)", csave)
}
store.entries = nil
l.recordCacheHit("openai/gpt-4o", 500)
csave = 0
for _, e := range store.entries {
if strings.Contains(e.key, ":"+metricCsave+":") {
csave++
}
}
if csave != 3 { // total hour, total day, model day
t.Errorf("csave entries (saved=500) = %d, want 3", csave)
}
}
// TestRecordCacheHit_OffWithoutStatsAdmin proves recordCacheHit spawns
// nothing at all when admin stats are off.
func TestRecordCacheHit_OffWithoutStatsAdmin(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.spawn = func(f func()) { f() }
l.recordCacheHit("openai/gpt-4o", 500)
if len(store.entries) != 0 {
t.Errorf("entries = %d, want 0 (statsAdmin off)", len(store.entries))
}
}
// TestRecordUnpriced402_TotalAndModelHourDay proves recordUnpriced402
// writes total r402 hour+day plus the refused model's own r402 hour+day,
// gated on statsAdmin — the counter events.go's recordUnpricedRefusalEvent
// drives from its own single chokepoint (both routes_unified.go's and
// routes_passthrough.go's identical 402 guard).
func TestRecordUnpriced402_TotalAndModelHourDay(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.statsAdmin = true
l.spawn = func(f func()) { f() }
l.recordUnpriced402("openai/free-model")
present := map[string]bool{}
for _, e := range store.entries {
present[e.key] = true
}
now := l.now()
wantKeys := []string{
windowKeyForBucket(totalScopeKind, totalScopeID, metricR402, windowHour, bucketFor(now, windowHour)),
windowKeyForBucket(totalScopeKind, totalScopeID, metricR402, windowDay, bucketFor(now, windowDay)),
windowKeyForBucket(kindModel, "openai/free-model", metricR402, windowHour, bucketFor(now, windowHour)),
windowKeyForBucket(kindModel, "openai/free-model", metricR402, windowDay, bucketFor(now, windowDay)),
}
for _, k := range wantKeys {
if !present[k] {
t.Errorf("missing r402 entry %q", k)
}
}
if len(store.entries) != 4 {
t.Errorf("total entries = %d, want 4", len(store.entries))
}
}
// TestRecordProviderAttempt_HourlyAndTimeout_OnlyWithStatsAdmin proves
// step 3: prov/provmodel attempt+fail at HOUR granularity, and prov's own
// timeout hour+day, are added only when statsAdmin is on — riding
// recordProviderAttempt's SAME async batch (one spawn call either way,
// never a second round trip).
func TestRecordProviderAttempt_HourlyAndTimeout_OnlyWithStatsAdmin(t *testing.T) {
for _, statsAdmin := range []bool{false, true} {
store := &recordingStore{}
l := newLimiter(store, true)
l.spawn = func(f func()) { f() }
l.statsAdmin = statsAdmin
l.recordProviderAttempt("openai", "gpt-4o", nil, errProviderTimeout)
var hourAttempt, hourFail, hourTimeout, dayTimeout, modelHourAttempt int
for _, e := range store.entries {
switch {
case e.key == windowKeyForBucket(kindProvider, "openai", metricProvAttempt, windowHour, bucketFor(l.now(), windowHour)):
hourAttempt++
case e.key == windowKeyForBucket(kindProvider, "openai", metricProvFail, windowHour, bucketFor(l.now(), windowHour)):
hourFail++
case e.key == windowKeyForBucket(kindProvider, "openai", metricProvTimeout, windowHour, bucketFor(l.now(), windowHour)):
hourTimeout++
case e.key == windowKeyForBucket(kindProvider, "openai", metricProvTimeout, windowDay, bucketFor(l.now(), windowDay)):
dayTimeout++
case e.key == windowKeyForBucket(kindProviderModel, providerModelScopeID("openai", "gpt-4o"), metricProvAttempt, windowHour, bucketFor(l.now(), windowHour)):
modelHourAttempt++
}
}
want := 0
if statsAdmin {
want = 1
}
if hourAttempt != want || hourFail != want || hourTimeout != want || dayTimeout != want || modelHourAttempt != want {
t.Errorf("statsAdmin=%v: hourAttempt=%d hourFail=%d hourTimeout=%d dayTimeout=%d modelHourAttempt=%d, want %d each",
statsAdmin, hourAttempt, hourFail, hourTimeout, dayTimeout, modelHourAttempt, want)
}
}
}
// TestCountTargetRequestsBy_CallerID_OnlyWithStatsAdmin proves step 8:
// a non-empty caller writes kindTargetCaller req entries at day+month
// (targetCallerScopeID("mcp", target, caller)) only when statsAdmin is
// on, in the SAME batch countTargetRequests already builds.
func TestCountTargetRequestsBy_CallerID_OnlyWithStatsAdmin(t *testing.T) {
for _, statsAdmin := range []bool{false, true} {
store := &recordingStore{}
l := newLimiter(store, true)
l.statsAdmin = statsAdmin
l.countTargetRequestBy("alice", targetKindMCP, "fetch")
id := targetCallerScopeID(targetKindMCP, "fetch", "alice")
var got int
for _, e := range store.entries {
if strings.HasPrefix(e.key, "llmgw:"+kindTargetCaller+":"+id+":") {
got++
}
}
want := 0
if statsAdmin {
want = 2 // day + month
}
if got != want {
t.Errorf("statsAdmin=%v: tcaller entries = %d, want %d", statsAdmin, got, want)
}
}
}
// TestCountTargetRequestsBy_EmptyCaller_MatchesCountTargetRequests proves
// countTargetRequests (no caller) is byte-for-byte countTargetRequestsBy's
// own caller="" special case — no behavior change for any pre-existing
// caller of the old name.
func TestCountTargetRequestsBy_EmptyCaller_MatchesCountTargetRequests(t *testing.T) {
scopes := []limitScope{{kind: targetKindMCP, id: "fetch"}}
storeA := &recordingStore{}
lA := newLimiter(storeA, true)
lA.statsAdmin = true
lA.nowFn = func() time.Time { return time.Unix(0, 0) }
lA.countTargetRequests(scopes)
storeB := &recordingStore{}
lB := newLimiter(storeB, true)
lB.statsAdmin = true
lB.nowFn = func() time.Time { return time.Unix(0, 0) }
lB.countTargetRequestsBy("", scopes)
if len(storeA.entries) != len(storeB.entries) {
t.Fatalf("len(A)=%d len(B)=%d, want equal", len(storeA.entries), len(storeB.entries))
}
for i := range storeA.entries {
if storeA.entries[i].key != storeB.entries[i].key || storeA.entries[i].delta != storeB.entries[i].delta {
t.Errorf("entry %d differs: A=%+v B=%+v", i, storeA.entries[i], storeB.entries[i])
}
}
}
// TestHistoryBucketKeysAt_OffsetZero_MatchesHistoryBucketKeys proves
// offset=0 is byte-for-byte historyBucketKeys' own pre-existing output.
func TestHistoryBucketKeysAt_OffsetZero_MatchesHistoryBucketKeys(t *testing.T) {
now := time.Date(2026, 9, 23, 10, 30, 0, 0, time.UTC)
wantKeys, wantBuckets := historyBucketKeys(kindModel, "openai/gpt-4o", metricReq, windowDay, now, 5)
gotKeys, gotBuckets := historyBucketKeysAt(kindModel, "openai/gpt-4o", metricReq, windowDay, now, 5, 0)
assert.Equal(t, wantKeys, gotKeys)
assert.Equal(t, wantBuckets, gotBuckets)
}
// TestHistoryBucketKeysAt_Offset_ShiftsWholeSpanBack proves an offset
// shifts the ENTIRE span back that many window units — the span=3,
// offset=2 window ends exactly 2 days before the span=3, offset=0
// window's own oldest bucket.
func TestHistoryBucketKeysAt_Offset_ShiftsWholeSpanBack(t *testing.T) {
now := time.Date(2026, 9, 23, 10, 30, 0, 0, time.UTC)
_, baseBuckets := historyBucketKeysAt(kindModel, "m", metricReq, windowDay, now, 3, 0)
_, offsetBuckets := historyBucketKeysAt(kindModel, "m", metricReq, windowDay, now, 3, 2)
// offsetBuckets' newest bucket must equal baseBuckets' oldest bucket
// shifted back one more day (span=3 starting 2 days earlier ends 2
// days earlier too).
wantNewest := bucketFor(historyStepBack(now, windowDay, 2), windowDay)
if offsetBuckets[len(offsetBuckets)-1] != wantNewest {
t.Errorf("offset span's newest bucket = %s, want %s", offsetBuckets[len(offsetBuckets)-1], wantNewest)
}
if offsetBuckets[0] == baseBuckets[0] {
t.Error("offset span's oldest bucket must differ from the unshifted span's")
}
}
// TestSpanTotalsMulti_Offset_ReadsAnEarlierSpan proves spanTotalsMulti's
// offset argument reads a DIFFERENT (earlier) set of buckets than
// offset=0 (plan §1.3(iii)) — exercised directly against the PRODUCTION
// reader (limits.go), the same one admin.go's chunkedModelSpanTotals/
// chunkedModelSpanTotalsMulti and stats_read.go's readSpanTotals now all
// share (P12 fix, admin dashboard redesign verify round).
func TestSpanTotalsMulti_Offset_ReadsAnEarlierSpan(t *testing.T) {
l := newLimiter(nil, true)
now := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC)
id := "openai/gpt-4o"
l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 3), 100, dayWindowTTL) // 3 days ago
l.incrCounter(kindModel, id, metricReq, windowDay, historyStepBack(now, windowDay, 0), 7, dayWindowTTL) // today
gotNow, ok := l.spanTotalsMulti(kindModel, []string{id}, []string{metricReq}, windowDay, now, 1, 0)
if !ok || gotNow[0][0] != 7 {
t.Errorf("offset=0 = %v, ok=%v, want [[7]]", gotNow, ok)
}
gotOffset, ok := l.spanTotalsMulti(kindModel, []string{id}, []string{metricReq}, windowDay, now, 1, 3)
if !ok || gotOffset[0][0] != 100 {
t.Errorf("offset=3 = %v, ok=%v, want [[100]]", gotOffset, ok)
}
}
// TestAccountWith_ZeroValueExtras_AddsNoFailoverLatencyOrCacheEntries is a
// smoke test that account (accountWith with a zero-value accountExtras)
// never adds a fover/latency/cmiss entry — those three are gated purely by
// accountExtras fields, unlike umodel (gated by limiter flags + scope
// shape, exercised separately above), so they must stay silent regardless
// of statsAdmin/statsLatency when the caller passes no extras at all.
func TestAccountWith_ZeroValueExtras_AddsNoFailoverLatencyOrCacheEntries(t *testing.T) {
store := &recordingStore{}
l := newLimiter(store, true)
l.statsAdmin = true
l.statsLatency = true
scopes := withModelScope([]limitScope{{kind: "user", id: "alice"}}, "openai/gpt-4o")
l.account(scopes, usage{prompt: 10, completion: 5}, 1000)
for _, e := range store.entries {
if strings.Contains(e.key, ":"+metricFover+":") || strings.Contains(e.key, ":ld") || strings.Contains(e.key, ":lt") || strings.Contains(e.key, ":"+metricCmiss+":") {
t.Errorf("unexpected extras-driven entry from account() with zero-value extras: %q", e.key)
}
}
}