mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-21 05:57:53 +00:00
feat: per-group cache TTL
GroupConfig gains CacheTTL (json cacheTTL, empty inherits the global cache.ttl). Parsed and validated at construction (newAuthStore, groupCacheTTL in cache.go): a malformed duration, a duration <= 0, or CacheTTL set while the global cache block is not configured is a constructor error naming the group, mirroring Cache:true's existing "nothing to inherit from" check. group carries the resolved time.Duration (0 means inherit). responseCache.store gains an explicit ttl parameter; routes_unified.go computes effectiveTTL(g.cache, grp) immediately before each store call, so setEx's Redis-native EX seconds reflect the group's own override. Tests: constructor-error table, empty-inherits and valid-override resolution, effectiveTTL inheritance/override, a scripted-RESP-server proof that store's SET...EX carries a different value per group, and an end-to-end Gateway.ServeHTTP test with two groups asserting the actual wire EX value (30 vs 60) differs per group. README: GroupConfig.cacheTTL config row and a Caching-section paragraph.
This commit is contained in:
@@ -211,6 +211,7 @@ also a construction error, never a panic (`providers.go`, `mcp_a2a.go`,
|
|||||||
| `providers` | `[]string` | `[]` (all) | Glob-matched (`path.Match`) against a provider's configured name. |
|
| `providers` | `[]string` | `[]` (all) | Glob-matched (`path.Match`) against a provider's configured name. |
|
||||||
| `models` | `[]string` | `[]` (all) | Glob-matched, exactly as given — no automatic `provider/` prefix stripping. Both the bare and provider-prefixed forms of a model id are checked, so a pattern like `gpt-*` matches a client request for either `gpt-5-mini` or `openai/gpt-5-mini`. |
|
| `models` | `[]string` | `[]` (all) | Glob-matched, exactly as given — no automatic `provider/` prefix stripping. Both the bare and provider-prefixed forms of a model id are checked, so a pattern like `gpt-*` matches a client request for either `gpt-5-mini` or `openai/gpt-5-mini`. |
|
||||||
| `cache` | `*bool` | `nil` (inherit) | Overrides the top-level `cache.enabled` setting for this group's requests — see [Caching](#caching). `nil` inherits the global setting; `false` opts the group out even when caching is globally on; `true` opts the group in, but only when the top-level `cache` block is actually configured (a construction error otherwise — there is nothing to inherit `ttl`/`maxBodyBytes` from). |
|
| `cache` | `*bool` | `nil` (inherit) | Overrides the top-level `cache.enabled` setting for this group's requests — see [Caching](#caching). `nil` inherits the global setting; `false` opts the group out even when caching is globally on; `true` opts the group in, but only when the top-level `cache` block is actually configured (a construction error otherwise — there is nothing to inherit `ttl`/`maxBodyBytes` from). |
|
||||||
|
| `cacheTTL` | `string` | `""` (inherit) | Overrides the top-level `cache.ttl` for this group's cache entries — see [Caching](#caching). Empty inherits the global TTL. A set value must be a valid, positive Go duration (`"30s"`, `"5m"`), and requires the top-level `cache` block to be configured — a construction error otherwise, the same "nothing to inherit from" reasoning as `cache: true` above. |
|
||||||
| `mcpServers` | `[]string` | `[]` (all) | Glob-matched against a configured MCP server name. |
|
| `mcpServers` | `[]string` | `[]` (all) | Glob-matched against a configured MCP server name. |
|
||||||
| `agents` | `[]string` | `[]` (all) | Glob-matched against a configured agent name. |
|
| `agents` | `[]string` | `[]` (all) | Glob-matched against a configured agent name. |
|
||||||
|
|
||||||
@@ -584,6 +585,11 @@ cached bodies for the same request.
|
|||||||
enabled — the block omitted entirely, or present with `enabled: false`,
|
enabled — the block omitted entirely, or present with `enabled: false`,
|
||||||
both read the same way — is a construction error: there is nothing
|
both read the same way — is a construction error: there is nothing
|
||||||
configured to inherit `ttl`/`maxBodyBytes` from.
|
configured to inherit `ttl`/`maxBodyBytes` from.
|
||||||
|
- **Group TTL override**: `GroupConfig.cacheTTL` overrides the top-level
|
||||||
|
`cache.ttl` for one group's cache entries only, independently of that
|
||||||
|
group's own `cache` setting — an empty value inherits the global TTL, a
|
||||||
|
set value must parse as a positive Go duration and requires the
|
||||||
|
top-level `cache` block to be configured, or construction fails.
|
||||||
- **Storage**: values are stored via the same hand-rolled RESP2 client
|
- **Storage**: values are stored via the same hand-rolled RESP2 client
|
||||||
(`resp.go`) the limiter's distributed counters use — the identical
|
(`resp.go`) the limiter's distributed counters use — the identical
|
||||||
Redis/Valkey/Dragonfly connection, not a second one. A Redis error
|
Redis/Valkey/Dragonfly connection, not a second one. A Redis error
|
||||||
|
|||||||
@@ -29,9 +29,19 @@ type group struct {
|
|||||||
// cache is GroupConfig.Cache carried through unchanged: nil inherits
|
// cache is GroupConfig.Cache carried through unchanged: nil inherits
|
||||||
// the global cache.enabled setting, non-nil overrides it for this
|
// the global cache.enabled setting, non-nil overrides it for this
|
||||||
// group's requests. Resolved by groupCacheEnabled (cache.go).
|
// group's requests. Resolved by groupCacheEnabled (cache.go).
|
||||||
cache *bool
|
cache *bool
|
||||||
name string
|
name string
|
||||||
providers, models, mcpServers, agents []string
|
providers []string
|
||||||
|
models []string
|
||||||
|
mcpServers []string
|
||||||
|
agents []string
|
||||||
|
// cacheTTL is GroupConfig.CacheTTL parsed and validated at construction
|
||||||
|
// (newAuthStore below): 0 inherits the global responseCache's TTL
|
||||||
|
// (effectiveTTL, cache.go), a positive value overrides it for this
|
||||||
|
// group's cache entries only. GroupConfig.CacheTTL == "" is the only
|
||||||
|
// input that produces 0 here — every other value either becomes a
|
||||||
|
// positive duration or fails newAuthStore as a constructor error.
|
||||||
|
cacheTTL time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
// allowsProvider reports whether name matches one of the group's provider
|
// allowsProvider reports whether name matches one of the group's provider
|
||||||
@@ -143,9 +153,22 @@ func newAuthStore(cfg *Config) (*authStore, error) {
|
|||||||
if gc.Cache != nil && *gc.Cache && !cfg.Cache.Enabled {
|
if gc.Cache != nil && *gc.Cache && !cfg.Cache.Enabled {
|
||||||
return nil, fmt.Errorf("llmgateway: group %q enables cache but global cache is not configured", name)
|
return nil, fmt.Errorf("llmgateway: group %q enables cache but global cache is not configured", name)
|
||||||
}
|
}
|
||||||
|
// CacheTTL follows the identical "nothing to inherit from" reasoning
|
||||||
|
// as Cache:true above: overriding a TTL only means something when
|
||||||
|
// there is a global cache.ttl to override, so CacheTTL set while
|
||||||
|
// the global cache block itself is not configured is a constructor
|
||||||
|
// error, independent of this group's own Cache setting (a group
|
||||||
|
// may set CacheTTL for when caching later becomes enabled globally
|
||||||
|
// is NOT supported — cfg.Cache.Enabled is checked here, not
|
||||||
|
// per-group, matching Cache:true's own check above).
|
||||||
|
cacheTTL, err := groupCacheTTL(name, gc.CacheTTL, cfg.Cache.Enabled)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
a.groups[name] = &group{
|
a.groups[name] = &group{
|
||||||
limits: gc.Limits,
|
limits: gc.Limits,
|
||||||
cache: gc.Cache,
|
cache: gc.Cache,
|
||||||
|
cacheTTL: cacheTTL,
|
||||||
name: name,
|
name: name,
|
||||||
providers: gc.Providers,
|
providers: gc.Providers,
|
||||||
models: gc.Models,
|
models: gc.Models,
|
||||||
|
|||||||
@@ -190,6 +190,74 @@ func TestNewAuthStore_NoUsers_OK(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- GroupConfig.CacheTTL: constructor validation (item B) ---
|
||||||
|
|
||||||
|
// TestNewAuthStore_GroupCacheTTL_ConstructorErrors is the constructor-error
|
||||||
|
// table for GroupConfig.CacheTTL: a malformed duration, a zero duration, a
|
||||||
|
// negative duration, and CacheTTL set while the global cache block is not
|
||||||
|
// configured (nothing to inherit ttl from, the same reasoning
|
||||||
|
// newAuthStore already applies to Cache:true) must all fail construction.
|
||||||
|
func TestNewAuthStore_GroupCacheTTL_ConstructorErrors(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
cacheTTL string
|
||||||
|
globalCache bool
|
||||||
|
}{
|
||||||
|
{name: "malformed duration", cacheTTL: "not-a-duration", globalCache: true},
|
||||||
|
{name: "zero duration", cacheTTL: "0s", globalCache: true},
|
||||||
|
{name: "negative duration", cacheTTL: "-5s", globalCache: true},
|
||||||
|
{name: "set without global cache enabled", cacheTTL: "5m", globalCache: false},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
cfg := &Config{
|
||||||
|
Providers: map[string]*ProviderConfig{"openai": {Type: "openai", APIKey: "k"}},
|
||||||
|
Groups: map[string]*GroupConfig{"default": {CacheTTL: tt.cacheTTL}},
|
||||||
|
Cache: CacheConfig{Enabled: tt.globalCache},
|
||||||
|
}
|
||||||
|
if _, err := newAuthStore(cfg); err == nil {
|
||||||
|
t.Fatalf("newAuthStore: want a constructor error for cacheTTL=%q (globalCache=%v)", tt.cacheTTL, tt.globalCache)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNewAuthStore_GroupCacheTTL_EmptyInheritsZero proves an omitted
|
||||||
|
// CacheTTL resolves to group.cacheTTL == 0 — effectiveTTL's (cache.go)
|
||||||
|
// signal to inherit the global cache TTL rather than override it.
|
||||||
|
func TestNewAuthStore_GroupCacheTTL_EmptyInheritsZero(t *testing.T) {
|
||||||
|
cfg := &Config{
|
||||||
|
Providers: map[string]*ProviderConfig{"openai": {Type: "openai", APIKey: "k"}},
|
||||||
|
Groups: map[string]*GroupConfig{"default": {}},
|
||||||
|
}
|
||||||
|
a, err := newAuthStore(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newAuthStore: %v", err)
|
||||||
|
}
|
||||||
|
if got := a.groups["default"].cacheTTL; got != 0 {
|
||||||
|
t.Errorf("groups[default].cacheTTL = %v, want 0 (inherit)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNewAuthStore_GroupCacheTTL_ValidOverride_ResolvedOnGroup proves a
|
||||||
|
// well-formed, positive CacheTTL with the global cache block configured
|
||||||
|
// both constructs cleanly and resolves onto the group struct exactly,
|
||||||
|
// ready for effectiveTTL to read.
|
||||||
|
func TestNewAuthStore_GroupCacheTTL_ValidOverride_ResolvedOnGroup(t *testing.T) {
|
||||||
|
cfg := &Config{
|
||||||
|
Providers: map[string]*ProviderConfig{"openai": {Type: "openai", APIKey: "k"}},
|
||||||
|
Groups: map[string]*GroupConfig{"default": {CacheTTL: "30s"}},
|
||||||
|
Cache: CacheConfig{Enabled: true},
|
||||||
|
}
|
||||||
|
a, err := newAuthStore(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newAuthStore: %v", err)
|
||||||
|
}
|
||||||
|
if got := a.groups["default"].cacheTTL; got != 30*time.Second {
|
||||||
|
t.Errorf("groups[default].cacheTTL = %v, want 30s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGroup_AllowsProvider_EmptyListAllowsAll(t *testing.T) {
|
func TestGroup_AllowsProvider_EmptyListAllowsAll(t *testing.T) {
|
||||||
grp := &group{name: "eng"}
|
grp := &group{name: "eng"}
|
||||||
if !grp.allowsProvider("openai") {
|
if !grp.allowsProvider("openai") {
|
||||||
|
|||||||
@@ -107,6 +107,46 @@ func groupCacheEnabled(grp *group) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// groupCacheTTL parses and validates a GroupConfig.CacheTTL string for
|
||||||
|
// group name, returning the resolved time.Duration — 0 when raw is empty,
|
||||||
|
// meaning "inherit the global cache TTL" (effectiveTTL below). cacheEnabled
|
||||||
|
// is cfg.Cache.Enabled: a non-empty raw while the global cache block is not
|
||||||
|
// configured is a constructor error, since there is then no global TTL to
|
||||||
|
// override — the identical "nothing to inherit from" reasoning newAuthStore
|
||||||
|
// already applies to Cache:true (auth.go). Called once per group by
|
||||||
|
// newAuthStore (auth.go), independent of that group's own Cache setting.
|
||||||
|
func groupCacheTTL(name, raw string, cacheEnabled bool) (time.Duration, error) {
|
||||||
|
if raw == "" {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
if !cacheEnabled {
|
||||||
|
return 0, fmt.Errorf("llmgateway: group %q sets cacheTTL but global cache is not configured", name)
|
||||||
|
}
|
||||||
|
ttl, err := time.ParseDuration(raw)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("llmgateway: group %q: cacheTTL: %w", name, err)
|
||||||
|
}
|
||||||
|
if ttl <= 0 {
|
||||||
|
return 0, fmt.Errorf("llmgateway: group %q: cacheTTL must be positive, got %q", name, raw)
|
||||||
|
}
|
||||||
|
return ttl, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// effectiveTTL resolves the TTL to use for grp's cache entries: grp's own
|
||||||
|
// resolved cacheTTL when set (non-zero — GroupConfig.CacheTTL parsed and
|
||||||
|
// validated at construction, groupCacheTTL above), otherwise c's global TTL
|
||||||
|
// (buildResponseCache's cc.TTL, validateCacheConfig). Called by
|
||||||
|
// routes_unified.go's runUnified immediately before store, so every stored
|
||||||
|
// entry's Redis-native expiry (setEx's ttl argument, resp.go) already
|
||||||
|
// reflects the group-specific override — no separate "shrink the group's
|
||||||
|
// TTL later" step exists.
|
||||||
|
func effectiveTTL(c *responseCache, grp *group) time.Duration {
|
||||||
|
if grp.cacheTTL > 0 {
|
||||||
|
return grp.cacheTTL
|
||||||
|
}
|
||||||
|
return c.ttl
|
||||||
|
}
|
||||||
|
|
||||||
// cachedResponse is the value responseCache stores per key: enough to
|
// cachedResponse is the value responseCache stores per key: enough to
|
||||||
// replay a cached response verbatim. json.Marshal/Unmarshal base64-encode
|
// replay a cached response verbatim. json.Marshal/Unmarshal base64-encode
|
||||||
// Body automatically (it is a []byte field), matching spec §2's "value:
|
// Body automatically (it is a []byte field), matching spec §2's "value:
|
||||||
@@ -245,14 +285,16 @@ func (c *responseCache) lookup(key string) (*cachedResponse, bool) {
|
|||||||
return &cr, true
|
return &cr, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// store saves status/contentType/body under key with the cache's
|
// store saves status/contentType/body under key with ttl (the caller's
|
||||||
// configured TTL. A body exceeding maxBodyBytes is skipped (logged, not
|
// resolved TTL — effectiveTTL above; every existing caller before the
|
||||||
// treated as an error) before ever touching the network; a down-latch
|
// per-group override passed c.ttl itself, so behavior for a group with no
|
||||||
|
// override is unchanged). A body exceeding maxBodyBytes is skipped (logged,
|
||||||
|
// not treated as an error) before ever touching the network; a down-latch
|
||||||
// already open (latched) also skips the network entirely; and a Redis
|
// already open (latched) also skips the network entirely; and a Redis
|
||||||
// error writing it is logged (rate-limited) and opens the latch — none of
|
// error writing it is logged (rate-limited) and opens the latch — none of
|
||||||
// which surface to the caller: a cache write must never fail a request
|
// which surface to the caller: a cache write must never fail a request
|
||||||
// that already succeeded and was already written to the client.
|
// that already succeeded and was already written to the client.
|
||||||
func (c *responseCache) store(key string, status int, contentType string, body []byte) {
|
func (c *responseCache) store(key string, status int, contentType string, body []byte, ttl time.Duration) {
|
||||||
if len(body) > c.maxBodyBytes {
|
if len(body) > c.maxBodyBytes {
|
||||||
// Pre-formatted with fmt.Sprintf, then logged as a single "%s"
|
// Pre-formatted with fmt.Sprintf, then logged as a single "%s"
|
||||||
// argument, not c.logf(format, key, len(body), c.maxBodyBytes)
|
// argument, not c.logf(format, key, len(body), c.maxBodyBytes)
|
||||||
@@ -277,7 +319,7 @@ func (c *responseCache) store(key string, status int, contentType string, body [
|
|||||||
panic(fmt.Sprintf("llmgateway: cache store: marshal cached response: %v", err))
|
panic(fmt.Sprintf("llmgateway: cache store: marshal cached response: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.client.setEx(key, val, c.ttl); err != nil {
|
if err := c.client.setEx(key, val, ttl); err != nil {
|
||||||
c.recordFailure(err)
|
c.recordFailure(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+64
-5
@@ -425,7 +425,7 @@ func TestResponseCache_Store_SendsSETWithTTL(t *testing.T) {
|
|||||||
c := newResponseCache(client, 90*time.Second, defaultCacheMaxBodyBytes, noop, noop)
|
c := newResponseCache(client, 90*time.Second, defaultCacheMaxBodyBytes, noop, noop)
|
||||||
c.nowFn = time.Now
|
c.nowFn = time.Now
|
||||||
|
|
||||||
c.store("llmgw:cache:k", 200, "application/json", []byte(`{"ok":true}`))
|
c.store("llmgw:cache:k", 200, "application/json", []byte(`{"ok":true}`), 90*time.Second)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResponseCache_Store_OversizeBody_SkipsSilentlyLogged(t *testing.T) {
|
func TestResponseCache_Store_OversizeBody_SkipsSilentlyLogged(t *testing.T) {
|
||||||
@@ -443,7 +443,7 @@ func TestResponseCache_Store_OversizeBody_SkipsSilentlyLogged(t *testing.T) {
|
|||||||
c := newResponseCache(client, time.Minute, 4, logf, errorf) // maxBodyBytes=4
|
c := newResponseCache(client, time.Minute, 4, logf, errorf) // maxBodyBytes=4
|
||||||
c.nowFn = time.Now
|
c.nowFn = time.Now
|
||||||
|
|
||||||
c.store("k", 200, "text/plain", []byte("way too big"))
|
c.store("k", 200, "text/plain", []byte("way too big"), time.Minute)
|
||||||
|
|
||||||
if len(logged) != 1 {
|
if len(logged) != 1 {
|
||||||
t.Fatalf("logged = %d lines, want 1 (oversize skip is logged via logf, not errorf)", len(logged))
|
t.Fatalf("logged = %d lines, want 1 (oversize skip is logged via logf, not errorf)", len(logged))
|
||||||
@@ -464,12 +464,12 @@ func TestResponseCache_Store_RedisDown_LoggedRateLimited(t *testing.T) {
|
|||||||
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 8, 20, 10, 0, 0, 0, time.UTC)
|
||||||
c.nowFn = func() time.Time { return now }
|
c.nowFn = func() time.Time { return now }
|
||||||
|
|
||||||
c.store("k", 200, "application/json", []byte(`{}`))
|
c.store("k", 200, "application/json", []byte(`{}`), time.Minute)
|
||||||
if len(logged) != 1 {
|
if len(logged) != 1 {
|
||||||
t.Fatalf("logged = %d lines after the first failed store, want 1", len(logged))
|
t.Fatalf("logged = %d lines after the first failed store, want 1", len(logged))
|
||||||
}
|
}
|
||||||
|
|
||||||
c.store("k", 200, "application/json", []byte(`{}`))
|
c.store("k", 200, "application/json", []byte(`{}`), time.Minute)
|
||||||
if len(logged) != 1 {
|
if len(logged) != 1 {
|
||||||
t.Fatalf("logged = %d lines still inside the rate-limit window, want 1", len(logged))
|
t.Fatalf("logged = %d lines still inside the rate-limit window, want 1", len(logged))
|
||||||
}
|
}
|
||||||
@@ -526,7 +526,7 @@ func TestResponseCache_DownLatch_SkipsNetworkCallsWithinWindow(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// store while latched: also zero network calls.
|
// store while latched: also zero network calls.
|
||||||
c.store("k", 200, "application/json", []byte(`{}`))
|
c.store("k", 200, "application/json", []byte(`{}`), time.Minute)
|
||||||
if stub.setCalls != 0 {
|
if stub.setCalls != 0 {
|
||||||
t.Fatalf("setCalls = %d while latched, want 0 (no network call attempted)", stub.setCalls)
|
t.Fatalf("setCalls = %d while latched, want 0 (no network call attempted)", stub.setCalls)
|
||||||
}
|
}
|
||||||
@@ -630,3 +630,62 @@ func TestNewAuthStore_GroupCacheTrueWithGlobalCacheEnabled_NoError(t *testing.T)
|
|||||||
t.Fatalf("newAuthStore: %v, want no error when the global cache block is configured", err)
|
t.Fatalf("newAuthStore: %v, want no error when the global cache block is configured", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- effectiveTTL: per-group cache TTL (item B) ---
|
||||||
|
|
||||||
|
// TestEffectiveTTL_NilInheritsGlobal proves a group with no CacheTTL
|
||||||
|
// override (cacheTTL == 0, GroupConfig.CacheTTL left empty) resolves to
|
||||||
|
// the cache's own global TTL.
|
||||||
|
func TestEffectiveTTL_NilInheritsGlobal(t *testing.T) {
|
||||||
|
c := &responseCache{ttl: 5 * time.Minute}
|
||||||
|
grp := &group{name: "g"}
|
||||||
|
if got := effectiveTTL(c, grp); got != 5*time.Minute {
|
||||||
|
t.Errorf("effectiveTTL = %v, want the global TTL 5m (group has no override)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEffectiveTTL_GroupOverrideHonored proves a group's own resolved
|
||||||
|
// cacheTTL wins over the cache's global TTL.
|
||||||
|
func TestEffectiveTTL_GroupOverrideHonored(t *testing.T) {
|
||||||
|
c := &responseCache{ttl: 5 * time.Minute}
|
||||||
|
grp := &group{name: "g", cacheTTL: 30 * time.Second}
|
||||||
|
if got := effectiveTTL(c, grp); got != 30*time.Second {
|
||||||
|
t.Errorf("effectiveTTL = %v, want the group override 30s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResponseCache_Store_EffectiveTTLDiffersPerGroup_SETCarriesGroupTTL is
|
||||||
|
// item B's "override honored" proof against a real (scripted) RESP wire:
|
||||||
|
// store's ttl argument — effectiveTTL's per-group resolution, computed by
|
||||||
|
// the caller (routes_unified.go) before store is ever invoked — reaches
|
||||||
|
// the SET...EX command as a genuinely different EX value for a group that
|
||||||
|
// overrides the global TTL (30) versus one that inherits it (60, the
|
||||||
|
// cache's own 1m global TTL). runFakeRESPServer fails the test outright if
|
||||||
|
// either SET carries the wrong EX value.
|
||||||
|
func TestResponseCache_Store_EffectiveTTLDiffersPerGroup_SETCarriesGroupTTL(t *testing.T) {
|
||||||
|
val1, err := json.Marshal(cachedResponse{Status: 200, ContentType: "application/json", Body: []byte(`{"a":1}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal: %v", err)
|
||||||
|
}
|
||||||
|
val2, err := json.Marshal(cachedResponse{Status: 200, ContentType: "application/json", Body: []byte(`{"b":2}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ln := newFakeListener(t)
|
||||||
|
runFakeRESPServer(t, ln, []respStep{
|
||||||
|
{wantArgs: []string{"SELECT", "0"}, reply: []byte("+OK\r\n")},
|
||||||
|
{wantArgs: []string{"SET", "k-override", string(val1), "EX", "30"}, reply: []byte("+OK\r\n")},
|
||||||
|
{wantArgs: []string{"SET", "k-inherit", string(val2), "EX", "60"}, reply: []byte("+OK\r\n")},
|
||||||
|
})
|
||||||
|
client := newRESPClient(ln.Addr().String(), "", 0)
|
||||||
|
noop := func(string, ...any) {}
|
||||||
|
c := newResponseCache(client, time.Minute, defaultCacheMaxBodyBytes, noop, noop) // global ttl = 60s
|
||||||
|
c.nowFn = time.Now
|
||||||
|
|
||||||
|
overrideGrp := &group{name: "override", cacheTTL: 30 * time.Second}
|
||||||
|
inheritGrp := &group{name: "inherit"} // cacheTTL 0 → inherits c.ttl (60s)
|
||||||
|
|
||||||
|
c.store("k-override", 200, "application/json", []byte(`{"a":1}`), effectiveTTL(c, overrideGrp))
|
||||||
|
c.store("k-inherit", 200, "application/json", []byte(`{"b":2}`), effectiveTTL(c, inheritGrp))
|
||||||
|
}
|
||||||
|
|||||||
+12
-1
@@ -71,7 +71,18 @@ type GroupConfig struct {
|
|||||||
// group in — but only when the global cache block is actually
|
// group in — but only when the global cache block is actually
|
||||||
// configured (newAuthStore rejects true otherwise, a constructor
|
// configured (newAuthStore rejects true otherwise, a constructor
|
||||||
// error, since there is nothing to inherit ttl/maxBodyBytes from).
|
// error, since there is nothing to inherit ttl/maxBodyBytes from).
|
||||||
Cache *bool `json:"cache,omitempty"`
|
Cache *bool `json:"cache,omitempty"`
|
||||||
|
// CacheTTL overrides the global cache.ttl (a Go duration string, e.g.
|
||||||
|
// "5m") for this group's cache entries only. Empty (the default)
|
||||||
|
// inherits the global TTL. Set means: parsed via time.ParseDuration at
|
||||||
|
// construction (newAuthStore, auth.go); a malformed duration, a
|
||||||
|
// duration <= 0, or CacheTTL set while the global cache block is not
|
||||||
|
// configured (cfg.Cache.Enabled false — there is no global TTL to
|
||||||
|
// override) is a constructor error naming the group. CacheTTL is
|
||||||
|
// independent of Cache above: it names the TTL to use WHEN this
|
||||||
|
// group's requests are cached, whether that caching comes from the
|
||||||
|
// global default or from this group's own Cache:true.
|
||||||
|
CacheTTL string `json:"cacheTTL,omitempty"`
|
||||||
MCPServers []string `json:"mcpServers,omitempty"`
|
MCPServers []string `json:"mcpServers,omitempty"`
|
||||||
Agents []string `json:"agents,omitempty"`
|
Agents []string `json:"agents,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -210,7 +210,7 @@ func (g *Gateway) runUnified(w http.ResponseWriter, r *http.Request, u *user, gr
|
|||||||
// but only after being handed a silently truncated body — skip the
|
// but only after being handed a silently truncated body — skip the
|
||||||
// call outright instead of ever constructing a corrupt cache entry.
|
// call outright instead of ever constructing a corrupt cache entry.
|
||||||
if cacheable && callErr == nil && capture.status == http.StatusOK && !capture.oversize {
|
if cacheable && callErr == nil && capture.status == http.StatusOK && !capture.oversize {
|
||||||
g.cache.store(cacheKeyStr, capture.status, capture.contentType, capture.buf.Bytes())
|
g.cache.store(cacheKeyStr, capture.status, capture.contentType, capture.buf.Bytes(), effectiveTTL(g.cache, grp))
|
||||||
}
|
}
|
||||||
|
|
||||||
if callErr != nil {
|
if callErr != nil {
|
||||||
|
|||||||
+105
-2
@@ -1002,18 +1002,45 @@ func TestHandleChat_MidStreamDropAfterUsageChunk_AccountsPartialUsage(t *testing
|
|||||||
// resp_test.go's scripted tests already pin.
|
// resp_test.go's scripted tests already pin.
|
||||||
type behavioralRedisServer struct {
|
type behavioralRedisServer struct {
|
||||||
data map[string]string
|
data map[string]string
|
||||||
mu sync.Mutex
|
// lastSetEX is the EX seconds argument from the most recent SET...EX
|
||||||
|
// command this server has handled — item B's per-group cache TTL
|
||||||
|
// override needs a way to observe the actual wire value a real
|
||||||
|
// Gateway.ServeHTTP request produced, not just the reply data's
|
||||||
|
// content. Empty until the first such SET arrives.
|
||||||
|
lastSetEX string
|
||||||
|
mu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// newBehavioralRedisServer starts the server on an OS-assigned port,
|
// newBehavioralRedisServer starts the server on an OS-assigned port,
|
||||||
// closed automatically at test cleanup (via newFakeListener), and returns
|
// closed automatically at test cleanup (via newFakeListener), and returns
|
||||||
// its listener.
|
// its listener.
|
||||||
func newBehavioralRedisServer(t *testing.T) net.Listener {
|
func newBehavioralRedisServer(t *testing.T) net.Listener {
|
||||||
|
t.Helper()
|
||||||
|
ln, _ := newBehavioralRedisServerAndHandle(t)
|
||||||
|
return ln
|
||||||
|
}
|
||||||
|
|
||||||
|
// newBehavioralRedisServerAndHandle is newBehavioralRedisServer, additionally
|
||||||
|
// returning the *behavioralRedisServer itself so a caller can inspect state
|
||||||
|
// the wire protocol alone does not expose — namely lastSetEX (see setEX
|
||||||
|
// below), used by TestHandleChat_CachePerGroupTTL_EndToEnd_DifferentEXPerGroup
|
||||||
|
// to prove effectiveTTL's per-group override (cache.go) actually reaches
|
||||||
|
// the SET...EX command a real Gateway.ServeHTTP request issues.
|
||||||
|
func newBehavioralRedisServerAndHandle(t *testing.T) (net.Listener, *behavioralRedisServer) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
ln := newFakeListener(t)
|
ln := newFakeListener(t)
|
||||||
srv := &behavioralRedisServer{data: make(map[string]string)}
|
srv := &behavioralRedisServer{data: make(map[string]string)}
|
||||||
go srv.acceptLoop(ln)
|
go srv.acceptLoop(ln)
|
||||||
return ln
|
return ln, srv
|
||||||
|
}
|
||||||
|
|
||||||
|
// setEX returns the EX seconds argument from the most recent SET...EX
|
||||||
|
// command this server has handled, and whether any such SET has arrived
|
||||||
|
// yet.
|
||||||
|
func (s *behavioralRedisServer) setEX() (string, bool) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return s.lastSetEX, s.lastSetEX != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *behavioralRedisServer) acceptLoop(ln net.Listener) {
|
func (s *behavioralRedisServer) acceptLoop(ln net.Listener) {
|
||||||
@@ -1062,6 +1089,9 @@ func (s *behavioralRedisServer) handle(args []string) []byte {
|
|||||||
case "SET":
|
case "SET":
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
s.data[args[1]] = args[2]
|
s.data[args[1]] = args[2]
|
||||||
|
if len(args) >= 5 && strings.EqualFold(args[3], "EX") {
|
||||||
|
s.lastSetEX = args[4]
|
||||||
|
}
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
return []byte("+OK\r\n")
|
return []byte("+OK\r\n")
|
||||||
case "INCRBY":
|
case "INCRBY":
|
||||||
@@ -1522,6 +1552,79 @@ func TestHandleChat_CacheableRequest_UpstreamDown_502HasNoCacheHeader(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- per-group cache TTL: end-to-end through a real Gateway (item B) ---
|
||||||
|
|
||||||
|
// TestHandleChat_CachePerGroupTTL_EndToEnd_DifferentEXPerGroup is item B's
|
||||||
|
// e2e proof: two groups share one global cache.ttl ("1m" = EX 60), one
|
||||||
|
// ("override") overrides it to "30s", the other ("inherit") leaves it
|
||||||
|
// unset. A cache-miss request from each group's own user must produce a
|
||||||
|
// SET...EX command carrying that group's own resolved TTL — 30 for
|
||||||
|
// "override", 60 for "inherit" — proving effectiveTTL's per-group
|
||||||
|
// resolution (cache.go) actually reaches the wire through the full
|
||||||
|
// ServeHTTP path, not just the unit level
|
||||||
|
// (TestResponseCache_Store_EffectiveTTLDiffersPerGroup_SETCarriesGroupTTL,
|
||||||
|
// cache_test.go). The two requests carry deliberately different bodies —
|
||||||
|
// cacheKey has no group/user component (routes_unified.go), so identical
|
||||||
|
// bodies from different groups would collide into one cache entry and the
|
||||||
|
// second request would be a hit, never issuing a second SET at all.
|
||||||
|
func TestHandleChat_CachePerGroupTTL_EndToEnd_DifferentEXPerGroup(t *testing.T) {
|
||||||
|
const respBody = `{"id":"c1","object":"chat.completion","model":"gpt-test","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
_, _ = w.Write([]byte(respBody))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
redisLn, redisSrv := newBehavioralRedisServerAndHandle(t)
|
||||||
|
cfg := CreateConfig()
|
||||||
|
cfg.Providers = map[string]*ProviderConfig{
|
||||||
|
"openai": {Type: "openai", BaseURL: srv.URL, APIKey: "k", Models: []string{"gpt-test"}},
|
||||||
|
}
|
||||||
|
cfg.Groups = map[string]*GroupConfig{
|
||||||
|
"override": {CacheTTL: "30s"},
|
||||||
|
"inherit": {},
|
||||||
|
}
|
||||||
|
cfg.Users = &UsersConfig{Inline: []*UserConfig{
|
||||||
|
{Name: "bob", Group: "override", APIKey: "sk-bob", Limits: &LimitsConfig{}},
|
||||||
|
{Name: "alice", Group: "inherit", APIKey: "sk-alice", Limits: &LimitsConfig{}},
|
||||||
|
}}
|
||||||
|
cfg.Redis = &RedisConfig{Address: redisLn.Addr().String()}
|
||||||
|
cfg.Cache = CacheConfig{Enabled: true, TTL: "1m", MaxBodyBytes: defaultCacheMaxBodyBytes}
|
||||||
|
|
||||||
|
next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})
|
||||||
|
h, err := New(context.Background(), next, cfg, "llmgw")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// "override" group's user (bob): expect EX 30.
|
||||||
|
overrideBody := map[string]any{"model": "gpt-test", "messages": []any{map[string]any{"role": "user", "content": "override-group"}}}
|
||||||
|
reqOverride := newUnifiedRequest(t, http.MethodPost, "/v1/chat/completions", "sk-bob", overrideBody)
|
||||||
|
recOverride := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(recOverride, reqOverride)
|
||||||
|
if recOverride.Code != http.StatusOK {
|
||||||
|
t.Fatalf("override-group request: status = %d, want 200", recOverride.Code)
|
||||||
|
}
|
||||||
|
if ex, ok := redisSrv.setEX(); !ok || ex != "30" {
|
||||||
|
t.Errorf("override-group SET...EX = %q (ok=%v), want %q", ex, ok, "30")
|
||||||
|
}
|
||||||
|
|
||||||
|
// "inherit" group's user (alice), a genuinely different body so it
|
||||||
|
// misses (and issues its own SET) rather than hitting bob's entry:
|
||||||
|
// expect EX 60 (the global 1m).
|
||||||
|
inheritBody := map[string]any{"model": "gpt-test", "messages": []any{map[string]any{"role": "user", "content": "inherit-group"}}}
|
||||||
|
reqInherit := newUnifiedRequest(t, http.MethodPost, "/v1/chat/completions", "sk-alice", inheritBody)
|
||||||
|
recInherit := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(recInherit, reqInherit)
|
||||||
|
if recInherit.Code != http.StatusOK {
|
||||||
|
t.Fatalf("inherit-group request: status = %d, want 200", recInherit.Code)
|
||||||
|
}
|
||||||
|
if ex, ok := redisSrv.setEX(); !ok || ex != "60" {
|
||||||
|
t.Errorf("inherit-group SET...EX = %q (ok=%v), want %q", ex, ok, "60")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// --- cacheCaptureWriter: bounded capture (review finding 2) ---
|
// --- cacheCaptureWriter: bounded capture (review finding 2) ---
|
||||||
|
|
||||||
// TestCacheCaptureWriter_BoundsBufferAtMaxBodyBytes_StillWritesFullBodyThrough
|
// TestCacheCaptureWriter_BoundsBufferAtMaxBodyBytes_StillWritesFullBodyThrough
|
||||||
|
|||||||
Reference in New Issue
Block a user