mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-14 05:18:54 +00:00
feat: per-group cache TTL
GroupConfig gains CacheTTL (json cacheTTL, empty inherits the global cache.ttl). Parsed and validated at construction (newAuthStore, groupCacheTTL in cache.go): a malformed duration, a duration <= 0, or CacheTTL set while the global cache block is not configured is a constructor error naming the group, mirroring Cache:true's existing "nothing to inherit from" check. group carries the resolved time.Duration (0 means inherit). responseCache.store gains an explicit ttl parameter; routes_unified.go computes effectiveTTL(g.cache, grp) immediately before each store call, so setEx's Redis-native EX seconds reflect the group's own override. Tests: constructor-error table, empty-inherits and valid-override resolution, effectiveTTL inheritance/override, a scripted-RESP-server proof that store's SET...EX carries a different value per group, and an end-to-end Gateway.ServeHTTP test with two groups asserting the actual wire EX value (30 vs 60) differs per group. README: GroupConfig.cacheTTL config row and a Caching-section paragraph.
This commit is contained in:
@@ -211,6 +211,7 @@ also a construction error, never a panic (`providers.go`, `mcp_a2a.go`,
|
||||
| `providers` | `[]string` | `[]` (all) | Glob-matched (`path.Match`) against a provider's configured name. |
|
||||
| `models` | `[]string` | `[]` (all) | Glob-matched, exactly as given — no automatic `provider/` prefix stripping. Both the bare and provider-prefixed forms of a model id are checked, so a pattern like `gpt-*` matches a client request for either `gpt-5-mini` or `openai/gpt-5-mini`. |
|
||||
| `cache` | `*bool` | `nil` (inherit) | Overrides the top-level `cache.enabled` setting for this group's requests — see [Caching](#caching). `nil` inherits the global setting; `false` opts the group out even when caching is globally on; `true` opts the group in, but only when the top-level `cache` block is actually configured (a construction error otherwise — there is nothing to inherit `ttl`/`maxBodyBytes` from). |
|
||||
| `cacheTTL` | `string` | `""` (inherit) | Overrides the top-level `cache.ttl` for this group's cache entries — see [Caching](#caching). Empty inherits the global TTL. A set value must be a valid, positive Go duration (`"30s"`, `"5m"`), and requires the top-level `cache` block to be configured — a construction error otherwise, the same "nothing to inherit from" reasoning as `cache: true` above. |
|
||||
| `mcpServers` | `[]string` | `[]` (all) | Glob-matched against a configured MCP server name. |
|
||||
| `agents` | `[]string` | `[]` (all) | Glob-matched against a configured agent name. |
|
||||
|
||||
@@ -584,6 +585,11 @@ cached bodies for the same request.
|
||||
enabled — the block omitted entirely, or present with `enabled: false`,
|
||||
both read the same way — is a construction error: there is nothing
|
||||
configured to inherit `ttl`/`maxBodyBytes` from.
|
||||
- **Group TTL override**: `GroupConfig.cacheTTL` overrides the top-level
|
||||
`cache.ttl` for one group's cache entries only, independently of that
|
||||
group's own `cache` setting — an empty value inherits the global TTL, a
|
||||
set value must parse as a positive Go duration and requires the
|
||||
top-level `cache` block to be configured, or construction fails.
|
||||
- **Storage**: values are stored via the same hand-rolled RESP2 client
|
||||
(`resp.go`) the limiter's distributed counters use — the identical
|
||||
Redis/Valkey/Dragonfly connection, not a second one. A Redis error
|
||||
|
||||
@@ -29,9 +29,19 @@ type group struct {
|
||||
// cache is GroupConfig.Cache carried through unchanged: nil inherits
|
||||
// the global cache.enabled setting, non-nil overrides it for this
|
||||
// group's requests. Resolved by groupCacheEnabled (cache.go).
|
||||
cache *bool
|
||||
name string
|
||||
providers, models, mcpServers, agents []string
|
||||
cache *bool
|
||||
name string
|
||||
providers []string
|
||||
models []string
|
||||
mcpServers []string
|
||||
agents []string
|
||||
// cacheTTL is GroupConfig.CacheTTL parsed and validated at construction
|
||||
// (newAuthStore below): 0 inherits the global responseCache's TTL
|
||||
// (effectiveTTL, cache.go), a positive value overrides it for this
|
||||
// group's cache entries only. GroupConfig.CacheTTL == "" is the only
|
||||
// input that produces 0 here — every other value either becomes a
|
||||
// positive duration or fails newAuthStore as a constructor error.
|
||||
cacheTTL time.Duration
|
||||
}
|
||||
|
||||
// allowsProvider reports whether name matches one of the group's provider
|
||||
@@ -143,9 +153,22 @@ func newAuthStore(cfg *Config) (*authStore, error) {
|
||||
if gc.Cache != nil && *gc.Cache && !cfg.Cache.Enabled {
|
||||
return nil, fmt.Errorf("llmgateway: group %q enables cache but global cache is not configured", name)
|
||||
}
|
||||
// CacheTTL follows the identical "nothing to inherit from" reasoning
|
||||
// as Cache:true above: overriding a TTL only means something when
|
||||
// there is a global cache.ttl to override, so CacheTTL set while
|
||||
// the global cache block itself is not configured is a constructor
|
||||
// error, independent of this group's own Cache setting (a group
|
||||
// may set CacheTTL for when caching later becomes enabled globally
|
||||
// is NOT supported — cfg.Cache.Enabled is checked here, not
|
||||
// per-group, matching Cache:true's own check above).
|
||||
cacheTTL, err := groupCacheTTL(name, gc.CacheTTL, cfg.Cache.Enabled)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.groups[name] = &group{
|
||||
limits: gc.Limits,
|
||||
cache: gc.Cache,
|
||||
cacheTTL: cacheTTL,
|
||||
name: name,
|
||||
providers: gc.Providers,
|
||||
models: gc.Models,
|
||||
|
||||
@@ -190,6 +190,74 @@ func TestNewAuthStore_NoUsers_OK(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// --- GroupConfig.CacheTTL: constructor validation (item B) ---
|
||||
|
||||
// TestNewAuthStore_GroupCacheTTL_ConstructorErrors is the constructor-error
|
||||
// table for GroupConfig.CacheTTL: a malformed duration, a zero duration, a
|
||||
// negative duration, and CacheTTL set while the global cache block is not
|
||||
// configured (nothing to inherit ttl from, the same reasoning
|
||||
// newAuthStore already applies to Cache:true) must all fail construction.
|
||||
func TestNewAuthStore_GroupCacheTTL_ConstructorErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cacheTTL string
|
||||
globalCache bool
|
||||
}{
|
||||
{name: "malformed duration", cacheTTL: "not-a-duration", globalCache: true},
|
||||
{name: "zero duration", cacheTTL: "0s", globalCache: true},
|
||||
{name: "negative duration", cacheTTL: "-5s", globalCache: true},
|
||||
{name: "set without global cache enabled", cacheTTL: "5m", globalCache: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Providers: map[string]*ProviderConfig{"openai": {Type: "openai", APIKey: "k"}},
|
||||
Groups: map[string]*GroupConfig{"default": {CacheTTL: tt.cacheTTL}},
|
||||
Cache: CacheConfig{Enabled: tt.globalCache},
|
||||
}
|
||||
if _, err := newAuthStore(cfg); err == nil {
|
||||
t.Fatalf("newAuthStore: want a constructor error for cacheTTL=%q (globalCache=%v)", tt.cacheTTL, tt.globalCache)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewAuthStore_GroupCacheTTL_EmptyInheritsZero proves an omitted
|
||||
// CacheTTL resolves to group.cacheTTL == 0 — effectiveTTL's (cache.go)
|
||||
// signal to inherit the global cache TTL rather than override it.
|
||||
func TestNewAuthStore_GroupCacheTTL_EmptyInheritsZero(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Providers: map[string]*ProviderConfig{"openai": {Type: "openai", APIKey: "k"}},
|
||||
Groups: map[string]*GroupConfig{"default": {}},
|
||||
}
|
||||
a, err := newAuthStore(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newAuthStore: %v", err)
|
||||
}
|
||||
if got := a.groups["default"].cacheTTL; got != 0 {
|
||||
t.Errorf("groups[default].cacheTTL = %v, want 0 (inherit)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewAuthStore_GroupCacheTTL_ValidOverride_ResolvedOnGroup proves a
|
||||
// well-formed, positive CacheTTL with the global cache block configured
|
||||
// both constructs cleanly and resolves onto the group struct exactly,
|
||||
// ready for effectiveTTL to read.
|
||||
func TestNewAuthStore_GroupCacheTTL_ValidOverride_ResolvedOnGroup(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Providers: map[string]*ProviderConfig{"openai": {Type: "openai", APIKey: "k"}},
|
||||
Groups: map[string]*GroupConfig{"default": {CacheTTL: "30s"}},
|
||||
Cache: CacheConfig{Enabled: true},
|
||||
}
|
||||
a, err := newAuthStore(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newAuthStore: %v", err)
|
||||
}
|
||||
if got := a.groups["default"].cacheTTL; got != 30*time.Second {
|
||||
t.Errorf("groups[default].cacheTTL = %v, want 30s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroup_AllowsProvider_EmptyListAllowsAll(t *testing.T) {
|
||||
grp := &group{name: "eng"}
|
||||
if !grp.allowsProvider("openai") {
|
||||
|
||||
@@ -107,6 +107,46 @@ func groupCacheEnabled(grp *group) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// groupCacheTTL parses and validates a GroupConfig.CacheTTL string for
|
||||
// group name, returning the resolved time.Duration — 0 when raw is empty,
|
||||
// meaning "inherit the global cache TTL" (effectiveTTL below). cacheEnabled
|
||||
// is cfg.Cache.Enabled: a non-empty raw while the global cache block is not
|
||||
// configured is a constructor error, since there is then no global TTL to
|
||||
// override — the identical "nothing to inherit from" reasoning newAuthStore
|
||||
// already applies to Cache:true (auth.go). Called once per group by
|
||||
// newAuthStore (auth.go), independent of that group's own Cache setting.
|
||||
func groupCacheTTL(name, raw string, cacheEnabled bool) (time.Duration, error) {
|
||||
if raw == "" {
|
||||
return 0, nil
|
||||
}
|
||||
if !cacheEnabled {
|
||||
return 0, fmt.Errorf("llmgateway: group %q sets cacheTTL but global cache is not configured", name)
|
||||
}
|
||||
ttl, err := time.ParseDuration(raw)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("llmgateway: group %q: cacheTTL: %w", name, err)
|
||||
}
|
||||
if ttl <= 0 {
|
||||
return 0, fmt.Errorf("llmgateway: group %q: cacheTTL must be positive, got %q", name, raw)
|
||||
}
|
||||
return ttl, nil
|
||||
}
|
||||
|
||||
// effectiveTTL resolves the TTL to use for grp's cache entries: grp's own
|
||||
// resolved cacheTTL when set (non-zero — GroupConfig.CacheTTL parsed and
|
||||
// validated at construction, groupCacheTTL above), otherwise c's global TTL
|
||||
// (buildResponseCache's cc.TTL, validateCacheConfig). Called by
|
||||
// routes_unified.go's runUnified immediately before store, so every stored
|
||||
// entry's Redis-native expiry (setEx's ttl argument, resp.go) already
|
||||
// reflects the group-specific override — no separate "shrink the group's
|
||||
// TTL later" step exists.
|
||||
func effectiveTTL(c *responseCache, grp *group) time.Duration {
|
||||
if grp.cacheTTL > 0 {
|
||||
return grp.cacheTTL
|
||||
}
|
||||
return c.ttl
|
||||
}
|
||||
|
||||
// cachedResponse is the value responseCache stores per key: enough to
|
||||
// replay a cached response verbatim. json.Marshal/Unmarshal base64-encode
|
||||
// Body automatically (it is a []byte field), matching spec §2's "value:
|
||||
@@ -245,14 +285,16 @@ func (c *responseCache) lookup(key string) (*cachedResponse, bool) {
|
||||
return &cr, true
|
||||
}
|
||||
|
||||
// store saves status/contentType/body under key with the cache's
|
||||
// configured TTL. A body exceeding maxBodyBytes is skipped (logged, not
|
||||
// treated as an error) before ever touching the network; a down-latch
|
||||
// store saves status/contentType/body under key with ttl (the caller's
|
||||
// resolved TTL — effectiveTTL above; every existing caller before the
|
||||
// per-group override passed c.ttl itself, so behavior for a group with no
|
||||
// override is unchanged). A body exceeding maxBodyBytes is skipped (logged,
|
||||
// not treated as an error) before ever touching the network; a down-latch
|
||||
// already open (latched) also skips the network entirely; and a Redis
|
||||
// error writing it is logged (rate-limited) and opens the latch — none of
|
||||
// which surface to the caller: a cache write must never fail a request
|
||||
// that already succeeded and was already written to the client.
|
||||
func (c *responseCache) store(key string, status int, contentType string, body []byte) {
|
||||
func (c *responseCache) store(key string, status int, contentType string, body []byte, ttl time.Duration) {
|
||||
if len(body) > c.maxBodyBytes {
|
||||
// Pre-formatted with fmt.Sprintf, then logged as a single "%s"
|
||||
// argument, not c.logf(format, key, len(body), c.maxBodyBytes)
|
||||
@@ -277,7 +319,7 @@ func (c *responseCache) store(key string, status int, contentType string, body [
|
||||
panic(fmt.Sprintf("llmgateway: cache store: marshal cached response: %v", err))
|
||||
}
|
||||
|
||||
if err := c.client.setEx(key, val, c.ttl); err != nil {
|
||||
if err := c.client.setEx(key, val, ttl); err != nil {
|
||||
c.recordFailure(err)
|
||||
}
|
||||
}
|
||||
|
||||
+64
-5
@@ -425,7 +425,7 @@ func TestResponseCache_Store_SendsSETWithTTL(t *testing.T) {
|
||||
c := newResponseCache(client, 90*time.Second, defaultCacheMaxBodyBytes, noop, noop)
|
||||
c.nowFn = time.Now
|
||||
|
||||
c.store("llmgw:cache:k", 200, "application/json", []byte(`{"ok":true}`))
|
||||
c.store("llmgw:cache:k", 200, "application/json", []byte(`{"ok":true}`), 90*time.Second)
|
||||
}
|
||||
|
||||
func TestResponseCache_Store_OversizeBody_SkipsSilentlyLogged(t *testing.T) {
|
||||
@@ -443,7 +443,7 @@ func TestResponseCache_Store_OversizeBody_SkipsSilentlyLogged(t *testing.T) {
|
||||
c := newResponseCache(client, time.Minute, 4, logf, errorf) // maxBodyBytes=4
|
||||
c.nowFn = time.Now
|
||||
|
||||
c.store("k", 200, "text/plain", []byte("way too big"))
|
||||
c.store("k", 200, "text/plain", []byte("way too big"), time.Minute)
|
||||
|
||||
if len(logged) != 1 {
|
||||
t.Fatalf("logged = %d lines, want 1 (oversize skip is logged via logf, not errorf)", len(logged))
|
||||
@@ -464,12 +464,12 @@ func TestResponseCache_Store_RedisDown_LoggedRateLimited(t *testing.T) {
|
||||
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
|
||||
c.nowFn = func() time.Time { return now }
|
||||
|
||||
c.store("k", 200, "application/json", []byte(`{}`))
|
||||
c.store("k", 200, "application/json", []byte(`{}`), time.Minute)
|
||||
if len(logged) != 1 {
|
||||
t.Fatalf("logged = %d lines after the first failed store, want 1", len(logged))
|
||||
}
|
||||
|
||||
c.store("k", 200, "application/json", []byte(`{}`))
|
||||
c.store("k", 200, "application/json", []byte(`{}`), time.Minute)
|
||||
if len(logged) != 1 {
|
||||
t.Fatalf("logged = %d lines still inside the rate-limit window, want 1", len(logged))
|
||||
}
|
||||
@@ -526,7 +526,7 @@ func TestResponseCache_DownLatch_SkipsNetworkCallsWithinWindow(t *testing.T) {
|
||||
}
|
||||
|
||||
// store while latched: also zero network calls.
|
||||
c.store("k", 200, "application/json", []byte(`{}`))
|
||||
c.store("k", 200, "application/json", []byte(`{}`), time.Minute)
|
||||
if stub.setCalls != 0 {
|
||||
t.Fatalf("setCalls = %d while latched, want 0 (no network call attempted)", stub.setCalls)
|
||||
}
|
||||
@@ -630,3 +630,62 @@ func TestNewAuthStore_GroupCacheTrueWithGlobalCacheEnabled_NoError(t *testing.T)
|
||||
t.Fatalf("newAuthStore: %v, want no error when the global cache block is configured", err)
|
||||
}
|
||||
}
|
||||
|
||||
// --- effectiveTTL: per-group cache TTL (item B) ---
|
||||
|
||||
// TestEffectiveTTL_NilInheritsGlobal proves a group with no CacheTTL
|
||||
// override (cacheTTL == 0, GroupConfig.CacheTTL left empty) resolves to
|
||||
// the cache's own global TTL.
|
||||
func TestEffectiveTTL_NilInheritsGlobal(t *testing.T) {
|
||||
c := &responseCache{ttl: 5 * time.Minute}
|
||||
grp := &group{name: "g"}
|
||||
if got := effectiveTTL(c, grp); got != 5*time.Minute {
|
||||
t.Errorf("effectiveTTL = %v, want the global TTL 5m (group has no override)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEffectiveTTL_GroupOverrideHonored proves a group's own resolved
|
||||
// cacheTTL wins over the cache's global TTL.
|
||||
func TestEffectiveTTL_GroupOverrideHonored(t *testing.T) {
|
||||
c := &responseCache{ttl: 5 * time.Minute}
|
||||
grp := &group{name: "g", cacheTTL: 30 * time.Second}
|
||||
if got := effectiveTTL(c, grp); got != 30*time.Second {
|
||||
t.Errorf("effectiveTTL = %v, want the group override 30s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResponseCache_Store_EffectiveTTLDiffersPerGroup_SETCarriesGroupTTL is
|
||||
// item B's "override honored" proof against a real (scripted) RESP wire:
|
||||
// store's ttl argument — effectiveTTL's per-group resolution, computed by
|
||||
// the caller (routes_unified.go) before store is ever invoked — reaches
|
||||
// the SET...EX command as a genuinely different EX value for a group that
|
||||
// overrides the global TTL (30) versus one that inherits it (60, the
|
||||
// cache's own 1m global TTL). runFakeRESPServer fails the test outright if
|
||||
// either SET carries the wrong EX value.
|
||||
func TestResponseCache_Store_EffectiveTTLDiffersPerGroup_SETCarriesGroupTTL(t *testing.T) {
|
||||
val1, err := json.Marshal(cachedResponse{Status: 200, ContentType: "application/json", Body: []byte(`{"a":1}`)})
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal: %v", err)
|
||||
}
|
||||
val2, err := json.Marshal(cachedResponse{Status: 200, ContentType: "application/json", Body: []byte(`{"b":2}`)})
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal: %v", err)
|
||||
}
|
||||
|
||||
ln := newFakeListener(t)
|
||||
runFakeRESPServer(t, ln, []respStep{
|
||||
{wantArgs: []string{"SELECT", "0"}, reply: []byte("+OK\r\n")},
|
||||
{wantArgs: []string{"SET", "k-override", string(val1), "EX", "30"}, reply: []byte("+OK\r\n")},
|
||||
{wantArgs: []string{"SET", "k-inherit", string(val2), "EX", "60"}, reply: []byte("+OK\r\n")},
|
||||
})
|
||||
client := newRESPClient(ln.Addr().String(), "", 0)
|
||||
noop := func(string, ...any) {}
|
||||
c := newResponseCache(client, time.Minute, defaultCacheMaxBodyBytes, noop, noop) // global ttl = 60s
|
||||
c.nowFn = time.Now
|
||||
|
||||
overrideGrp := &group{name: "override", cacheTTL: 30 * time.Second}
|
||||
inheritGrp := &group{name: "inherit"} // cacheTTL 0 → inherits c.ttl (60s)
|
||||
|
||||
c.store("k-override", 200, "application/json", []byte(`{"a":1}`), effectiveTTL(c, overrideGrp))
|
||||
c.store("k-inherit", 200, "application/json", []byte(`{"b":2}`), effectiveTTL(c, inheritGrp))
|
||||
}
|
||||
|
||||
+12
-1
@@ -71,7 +71,18 @@ type GroupConfig struct {
|
||||
// group in — but only when the global cache block is actually
|
||||
// configured (newAuthStore rejects true otherwise, a constructor
|
||||
// error, since there is nothing to inherit ttl/maxBodyBytes from).
|
||||
Cache *bool `json:"cache,omitempty"`
|
||||
Cache *bool `json:"cache,omitempty"`
|
||||
// CacheTTL overrides the global cache.ttl (a Go duration string, e.g.
|
||||
// "5m") for this group's cache entries only. Empty (the default)
|
||||
// inherits the global TTL. Set means: parsed via time.ParseDuration at
|
||||
// construction (newAuthStore, auth.go); a malformed duration, a
|
||||
// duration <= 0, or CacheTTL set while the global cache block is not
|
||||
// configured (cfg.Cache.Enabled false — there is no global TTL to
|
||||
// override) is a constructor error naming the group. CacheTTL is
|
||||
// independent of Cache above: it names the TTL to use WHEN this
|
||||
// group's requests are cached, whether that caching comes from the
|
||||
// global default or from this group's own Cache:true.
|
||||
CacheTTL string `json:"cacheTTL,omitempty"`
|
||||
MCPServers []string `json:"mcpServers,omitempty"`
|
||||
Agents []string `json:"agents,omitempty"`
|
||||
}
|
||||
|
||||
+1
-1
@@ -210,7 +210,7 @@ func (g *Gateway) runUnified(w http.ResponseWriter, r *http.Request, u *user, gr
|
||||
// but only after being handed a silently truncated body — skip the
|
||||
// call outright instead of ever constructing a corrupt cache entry.
|
||||
if cacheable && callErr == nil && capture.status == http.StatusOK && !capture.oversize {
|
||||
g.cache.store(cacheKeyStr, capture.status, capture.contentType, capture.buf.Bytes())
|
||||
g.cache.store(cacheKeyStr, capture.status, capture.contentType, capture.buf.Bytes(), effectiveTTL(g.cache, grp))
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
|
||||
+105
-2
@@ -1002,18 +1002,45 @@ func TestHandleChat_MidStreamDropAfterUsageChunk_AccountsPartialUsage(t *testing
|
||||
// resp_test.go's scripted tests already pin.
|
||||
type behavioralRedisServer struct {
|
||||
data map[string]string
|
||||
mu sync.Mutex
|
||||
// lastSetEX is the EX seconds argument from the most recent SET...EX
|
||||
// command this server has handled — item B's per-group cache TTL
|
||||
// override needs a way to observe the actual wire value a real
|
||||
// Gateway.ServeHTTP request produced, not just the reply data's
|
||||
// content. Empty until the first such SET arrives.
|
||||
lastSetEX string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// newBehavioralRedisServer starts the server on an OS-assigned port,
|
||||
// closed automatically at test cleanup (via newFakeListener), and returns
|
||||
// its listener.
|
||||
func newBehavioralRedisServer(t *testing.T) net.Listener {
|
||||
t.Helper()
|
||||
ln, _ := newBehavioralRedisServerAndHandle(t)
|
||||
return ln
|
||||
}
|
||||
|
||||
// newBehavioralRedisServerAndHandle is newBehavioralRedisServer, additionally
|
||||
// returning the *behavioralRedisServer itself so a caller can inspect state
|
||||
// the wire protocol alone does not expose — namely lastSetEX (see setEX
|
||||
// below), used by TestHandleChat_CachePerGroupTTL_EndToEnd_DifferentEXPerGroup
|
||||
// to prove effectiveTTL's per-group override (cache.go) actually reaches
|
||||
// the SET...EX command a real Gateway.ServeHTTP request issues.
|
||||
func newBehavioralRedisServerAndHandle(t *testing.T) (net.Listener, *behavioralRedisServer) {
|
||||
t.Helper()
|
||||
ln := newFakeListener(t)
|
||||
srv := &behavioralRedisServer{data: make(map[string]string)}
|
||||
go srv.acceptLoop(ln)
|
||||
return ln
|
||||
return ln, srv
|
||||
}
|
||||
|
||||
// setEX returns the EX seconds argument from the most recent SET...EX
|
||||
// command this server has handled, and whether any such SET has arrived
|
||||
// yet.
|
||||
func (s *behavioralRedisServer) setEX() (string, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.lastSetEX, s.lastSetEX != ""
|
||||
}
|
||||
|
||||
func (s *behavioralRedisServer) acceptLoop(ln net.Listener) {
|
||||
@@ -1062,6 +1089,9 @@ func (s *behavioralRedisServer) handle(args []string) []byte {
|
||||
case "SET":
|
||||
s.mu.Lock()
|
||||
s.data[args[1]] = args[2]
|
||||
if len(args) >= 5 && strings.EqualFold(args[3], "EX") {
|
||||
s.lastSetEX = args[4]
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return []byte("+OK\r\n")
|
||||
case "INCRBY":
|
||||
@@ -1522,6 +1552,79 @@ func TestHandleChat_CacheableRequest_UpstreamDown_502HasNoCacheHeader(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
// --- per-group cache TTL: end-to-end through a real Gateway (item B) ---
|
||||
|
||||
// TestHandleChat_CachePerGroupTTL_EndToEnd_DifferentEXPerGroup is item B's
|
||||
// e2e proof: two groups share one global cache.ttl ("1m" = EX 60), one
|
||||
// ("override") overrides it to "30s", the other ("inherit") leaves it
|
||||
// unset. A cache-miss request from each group's own user must produce a
|
||||
// SET...EX command carrying that group's own resolved TTL — 30 for
|
||||
// "override", 60 for "inherit" — proving effectiveTTL's per-group
|
||||
// resolution (cache.go) actually reaches the wire through the full
|
||||
// ServeHTTP path, not just the unit level
|
||||
// (TestResponseCache_Store_EffectiveTTLDiffersPerGroup_SETCarriesGroupTTL,
|
||||
// cache_test.go). The two requests carry deliberately different bodies —
|
||||
// cacheKey has no group/user component (routes_unified.go), so identical
|
||||
// bodies from different groups would collide into one cache entry and the
|
||||
// second request would be a hit, never issuing a second SET at all.
|
||||
func TestHandleChat_CachePerGroupTTL_EndToEnd_DifferentEXPerGroup(t *testing.T) {
|
||||
const respBody = `{"id":"c1","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(respBody))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
redisLn, redisSrv := newBehavioralRedisServerAndHandle(t)
|
||||
cfg := CreateConfig()
|
||||
cfg.Providers = map[string]*ProviderConfig{
|
||||
"openai": {Type: "openai", BaseURL: srv.URL, APIKey: "k", Models: []string{"gpt-test"}},
|
||||
}
|
||||
cfg.Groups = map[string]*GroupConfig{
|
||||
"override": {CacheTTL: "30s"},
|
||||
"inherit": {},
|
||||
}
|
||||
cfg.Users = &UsersConfig{Inline: []*UserConfig{
|
||||
{Name: "bob", Group: "override", APIKey: "sk-bob", Limits: &LimitsConfig{}},
|
||||
{Name: "alice", Group: "inherit", APIKey: "sk-alice", Limits: &LimitsConfig{}},
|
||||
}}
|
||||
cfg.Redis = &RedisConfig{Address: redisLn.Addr().String()}
|
||||
cfg.Cache = CacheConfig{Enabled: true, TTL: "1m", MaxBodyBytes: defaultCacheMaxBodyBytes}
|
||||
|
||||
next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})
|
||||
h, err := New(context.Background(), next, cfg, "llmgw")
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
|
||||
// "override" group's user (bob): expect EX 30.
|
||||
overrideBody := map[string]any{"model": "gpt-test", "messages": []any{map[string]any{"role": "user", "content": "override-group"}}}
|
||||
reqOverride := newUnifiedRequest(t, http.MethodPost, "/v1/chat/completions", "sk-bob", overrideBody)
|
||||
recOverride := httptest.NewRecorder()
|
||||
h.ServeHTTP(recOverride, reqOverride)
|
||||
if recOverride.Code != http.StatusOK {
|
||||
t.Fatalf("override-group request: status = %d, want 200", recOverride.Code)
|
||||
}
|
||||
if ex, ok := redisSrv.setEX(); !ok || ex != "30" {
|
||||
t.Errorf("override-group SET...EX = %q (ok=%v), want %q", ex, ok, "30")
|
||||
}
|
||||
|
||||
// "inherit" group's user (alice), a genuinely different body so it
|
||||
// misses (and issues its own SET) rather than hitting bob's entry:
|
||||
// expect EX 60 (the global 1m).
|
||||
inheritBody := map[string]any{"model": "gpt-test", "messages": []any{map[string]any{"role": "user", "content": "inherit-group"}}}
|
||||
reqInherit := newUnifiedRequest(t, http.MethodPost, "/v1/chat/completions", "sk-alice", inheritBody)
|
||||
recInherit := httptest.NewRecorder()
|
||||
h.ServeHTTP(recInherit, reqInherit)
|
||||
if recInherit.Code != http.StatusOK {
|
||||
t.Fatalf("inherit-group request: status = %d, want 200", recInherit.Code)
|
||||
}
|
||||
if ex, ok := redisSrv.setEX(); !ok || ex != "60" {
|
||||
t.Errorf("inherit-group SET...EX = %q (ok=%v), want %q", ex, ok, "60")
|
||||
}
|
||||
}
|
||||
|
||||
// --- cacheCaptureWriter: bounded capture (review finding 2) ---
|
||||
|
||||
// TestCacheCaptureWriter_BoundsBufferAtMaxBodyBytes_StillWritesFullBodyThrough
|
||||
|
||||
Reference in New Issue
Block a user