mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-17 05:37:43 +00:00
feat: opt-in redis response cache for unified non-streaming routes
This commit is contained in:
@@ -0,0 +1,259 @@
|
||||
package traefikllmgateway
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// cacheKeyPrefix prefixes every response-cache key (spec §2).
|
||||
const cacheKeyPrefix = "llmgw:cache:"
|
||||
|
||||
// defaultCacheTTL and defaultCacheMaxBodyBytes are the values
|
||||
// validateCacheConfig applies when a Cache config block is Enabled but
|
||||
// leaves TTL or MaxBodyBytes at its zero value — spec §2's stated
|
||||
// defaults.
|
||||
const (
|
||||
defaultCacheTTL = "5m"
|
||||
defaultCacheMaxBodyBytes = 1 << 20 // 1MiB
|
||||
)
|
||||
|
||||
// maxCacheMaxBodyBytes is the highest CacheConfig.MaxBodyBytes
|
||||
// validateCacheConfig accepts — spec §2's "maxBodyBytes ... max 8MB".
|
||||
const maxCacheMaxBodyBytes = 8 << 20 // 8MiB
|
||||
|
||||
// validateCacheConfig validates cc and returns the ttl and maxBodyBytes
|
||||
// buildResponseCache passes to newResponseCache. Called only when
|
||||
// cc.Enabled — an unset TTL/MaxBodyBytes on a disabled block is never an
|
||||
// error, since neither value is ever read, matching newRetryPolicy's
|
||||
// identical disabled-skips-validation shape (retry.go).
|
||||
func validateCacheConfig(cc CacheConfig) (time.Duration, int, error) {
|
||||
ttlStr := cc.TTL
|
||||
if ttlStr == "" {
|
||||
ttlStr = defaultCacheTTL
|
||||
}
|
||||
ttl, err := time.ParseDuration(ttlStr)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("llmgateway: cache.ttl: %w", err)
|
||||
}
|
||||
|
||||
maxBody := cc.MaxBodyBytes
|
||||
if maxBody == 0 {
|
||||
maxBody = defaultCacheMaxBodyBytes
|
||||
}
|
||||
if maxBody < 0 {
|
||||
return 0, 0, fmt.Errorf("llmgateway: cache.maxBodyBytes must be positive, got %d", maxBody)
|
||||
}
|
||||
if maxBody > maxCacheMaxBodyBytes {
|
||||
return 0, 0, fmt.Errorf("llmgateway: cache.maxBodyBytes must not exceed %d, got %d", maxCacheMaxBodyBytes, maxBody)
|
||||
}
|
||||
return ttl, maxBody, nil
|
||||
}
|
||||
|
||||
// buildResponseCache constructs the *responseCache newGateway attaches to
|
||||
// its Gateway, or nil when caching is not configured or not usable:
|
||||
//
|
||||
// - cc.Enabled is false (the default): nil, no error — v0.1 behavior.
|
||||
// - cc.Enabled is true but its TTL/MaxBodyBytes fail validation: a
|
||||
// construction error, matching every other malformed-config path.
|
||||
// - cc.Enabled is true and valid, but client is nil (config.Redis was
|
||||
// absent): nil, with one warning logged via logf — per spec §2,
|
||||
// "requires Redis... no Redis configured → caching silently disabled
|
||||
// with one warn at construction (memory fallback is NOT used for the
|
||||
// cache: per-replica caches would serve divergent responses)". This
|
||||
// is deliberately NOT a construction error, unlike the case above.
|
||||
// - cc.Enabled is true and valid and client is non-nil: a working
|
||||
// *responseCache over client.
|
||||
func buildResponseCache(cc CacheConfig, client *respClient, logf, errorf func(format string, args ...any)) (*responseCache, error) {
|
||||
if !cc.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
ttl, maxBody, err := validateCacheConfig(cc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if client == nil {
|
||||
logf("cache: enabled but redis is not configured; response caching disabled")
|
||||
return nil, nil
|
||||
}
|
||||
return newResponseCache(client, ttl, maxBody, logf, errorf), nil
|
||||
}
|
||||
|
||||
// groupCacheEnabled resolves whether grp's requests are cacheable, given
|
||||
// that caching is already known to be globally enabled and usable — every
|
||||
// call site checks g.cache != nil first (routes_unified.go), and g.cache
|
||||
// is non-nil only when buildResponseCache's cc.Enabled was true, so
|
||||
// "inherit the global setting" here is unconditionally true rather than a
|
||||
// second config lookup. grp.cache is GroupConfig.Cache carried onto the
|
||||
// group by newAuthStore (auth.go): nil inherits (true), non-nil overrides.
|
||||
func groupCacheEnabled(grp *group) bool {
|
||||
if grp.cache != nil {
|
||||
return *grp.cache
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// cachedResponse is the value responseCache stores per key: enough to
|
||||
// replay a cached response verbatim. json.Marshal/Unmarshal base64-encode
|
||||
// Body automatically (it is a []byte field), matching spec §2's "value:
|
||||
// JSON {status, contentType, body(base64)}".
|
||||
type cachedResponse struct {
|
||||
ContentType string `json:"contentType"`
|
||||
Body []byte `json:"body"`
|
||||
Status int `json:"status"`
|
||||
}
|
||||
|
||||
// responseCache is the opt-in, Redis-backed cache for unified
|
||||
// non-streaming chat/embeddings responses (spec §2). A nil *responseCache
|
||||
// on *Gateway means caching is off; every method below assumes a non-nil
|
||||
// receiver, so every call site (routes_unified.go) checks g.cache != nil
|
||||
// first rather than this type having its own always-disabled behavior.
|
||||
type responseCache struct {
|
||||
client *respClient
|
||||
// logf is informational — construction warnings (buildResponseCache)
|
||||
// and store's oversize-skip notice — matching spec §2's "construction-
|
||||
// time WARN via logf" and "skip store silently-with-debug-log": both
|
||||
// are routine, expected outcomes, not failures.
|
||||
logf func(format string, args ...any)
|
||||
// errorf is for logCacheError only: a genuine Redis failure, logged
|
||||
// at error level and rate-limited, mirroring limiter.logStoreError's
|
||||
// "existing pattern" (limits.go) — g.errorf there, g.errorf here.
|
||||
errorf func(format string, args ...any)
|
||||
nowFn func() time.Time // injected for tests; defaults to time.Now
|
||||
// lastLogAt and logMu rate-limit logCacheError to once per
|
||||
// storeErrorLogEvery (limits.go), mirroring limiter.logStoreError —
|
||||
// the same constant, so a Redis outage affecting both the limiter and
|
||||
// the cache does not double the log volume a limiter-only outage
|
||||
// would produce.
|
||||
lastLogAt time.Time
|
||||
ttl time.Duration
|
||||
maxBodyBytes int
|
||||
logMu sync.Mutex
|
||||
}
|
||||
|
||||
// newResponseCache returns a responseCache backed by client, storing
|
||||
// entries for ttl and refusing to store a body larger than maxBodyBytes.
|
||||
func newResponseCache(client *respClient, ttl time.Duration, maxBodyBytes int, logf, errorf func(format string, args ...any)) *responseCache {
|
||||
return &responseCache{client: client, ttl: ttl, maxBodyBytes: maxBodyBytes, logf: logf, errorf: errorf, nowFn: time.Now}
|
||||
}
|
||||
|
||||
// logCacheError logs err via c.errorf, rate-limited to once per
|
||||
// storeErrorLogEvery regardless of how many cache operations fail in
|
||||
// between — a Redis outage under load must not flood the log with one
|
||||
// line per request, the same reasoning limiter.logStoreError (limits.go)
|
||||
// applies to store errors.
|
||||
func (c *responseCache) logCacheError(err error) {
|
||||
now := c.nowFn()
|
||||
c.logMu.Lock()
|
||||
shouldLog := now.Sub(c.lastLogAt) >= storeErrorLogEvery
|
||||
if shouldLog {
|
||||
c.lastLogAt = now
|
||||
}
|
||||
c.logMu.Unlock()
|
||||
if shouldLog {
|
||||
c.errorf("response cache: redis error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// lookup returns the cached response for key. ok is false on a cache
|
||||
// miss — a real miss (key absent), a Redis error (logged, rate-limited),
|
||||
// or a corrupted stored value (also logged) — none of which distinguish
|
||||
// themselves to the caller: per spec §2, "redis errors during cache ops →
|
||||
// treat as miss ... never fail the request because the cache is down",
|
||||
// and a corrupted entry is handled the same way for the same reason.
|
||||
func (c *responseCache) lookup(key string) (*cachedResponse, bool) {
|
||||
raw, found, err := c.client.getBytes(key)
|
||||
if err != nil {
|
||||
c.logCacheError(err)
|
||||
return nil, false
|
||||
}
|
||||
if !found {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var cr cachedResponse
|
||||
if err := json.Unmarshal(raw, &cr); err != nil {
|
||||
c.logCacheError(fmt.Errorf("decode cached value for %q: %w", key, err))
|
||||
return nil, false
|
||||
}
|
||||
return &cr, true
|
||||
}
|
||||
|
||||
// store saves status/contentType/body under key with the cache's
|
||||
// configured TTL. A body exceeding maxBodyBytes is skipped (logged, not
|
||||
// treated as an error), and a Redis error writing it is logged
|
||||
// (rate-limited via logCacheError) rather than surfaced — a cache write
|
||||
// must never fail a request that already succeeded and was already
|
||||
// written to the client.
|
||||
func (c *responseCache) store(key string, status int, contentType string, body []byte) {
|
||||
if len(body) > c.maxBodyBytes {
|
||||
// Pre-formatted with fmt.Sprintf, then logged as a single "%s"
|
||||
// argument, not c.logf(format, key, len(body), c.maxBodyBytes)
|
||||
// directly: yaegi v0.16.1's CFG builder panics ("index out of
|
||||
// range") compiling a call to a struct FIELD of variadic func
|
||||
// type with 2+ variadic arguments (see modelRegistry.log's
|
||||
// identical note, registry.go) — c.logf is exactly such a field.
|
||||
msg := fmt.Sprintf("response cache: skipping store for %q: body %d bytes exceeds maxBodyBytes %d", key, len(body), c.maxBodyBytes)
|
||||
c.logf("%s", msg)
|
||||
return
|
||||
}
|
||||
|
||||
val, err := json.Marshal(cachedResponse{Status: status, ContentType: contentType, Body: body})
|
||||
if err != nil {
|
||||
// cachedResponse's fields are a plain int, string, and []byte —
|
||||
// every one of json.Marshal's own always-succeeds types. Reaching
|
||||
// here would be a programming error, not a runtime condition to
|
||||
// recover from.
|
||||
panic(fmt.Sprintf("llmgateway: cache store: marshal cached response: %v", err))
|
||||
}
|
||||
|
||||
if err := c.client.setEx(key, val, c.ttl); err != nil {
|
||||
c.logCacheError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// cacheKey derives the response-cache key for one request: spec §2's
|
||||
// "llmgw:cache: + hex SHA-256 of provider \x00 upstreamModel \x00
|
||||
// canonical-JSON(request body minus stream_options and user)". req is
|
||||
// never mutated — stripping builds a fresh copy — and canonical-JSON is
|
||||
// plain json.Marshal of that copy: encoding/json always marshals a
|
||||
// map[string]any's keys in sorted order, which is what makes the result
|
||||
// deterministic across requests carrying the same fields in different
|
||||
// insertion order.
|
||||
//
|
||||
// Callers must pass req before injecting gatewayAliasKey
|
||||
// (routes_unified.go): that key is a purely internal echo-back mechanism
|
||||
// with no bearing on upstream request equivalence, so two different
|
||||
// aliases resolving to the same upstream model must hash identically —
|
||||
// stripping stream_options and user, as this function does, is not
|
||||
// enough on its own if the alias key were still present.
|
||||
func cacheKey(provider, upstreamModel string, req map[string]any) string {
|
||||
stripped := make(map[string]any, len(req))
|
||||
for k, v := range req {
|
||||
if k == "stream_options" || k == "user" {
|
||||
continue
|
||||
}
|
||||
stripped[k] = v
|
||||
}
|
||||
|
||||
canonical, err := json.Marshal(stripped)
|
||||
if err != nil {
|
||||
// stripped holds only values encoding/json.Unmarshal ever
|
||||
// produces into a map[string]any (nil, bool, float64, string,
|
||||
// []any, map[string]any) — every one of Marshal's own
|
||||
// always-succeeds types. Reaching here would be a programming
|
||||
// error, not a runtime condition to recover from.
|
||||
panic(fmt.Sprintf("llmgateway: cacheKey: marshal stripped request: %v", err))
|
||||
}
|
||||
|
||||
h := sha256.New() // hash.Hash.Write never returns an error, per its doc
|
||||
_, _ = h.Write([]byte(provider))
|
||||
_, _ = h.Write([]byte{0})
|
||||
_, _ = h.Write([]byte(upstreamModel))
|
||||
_, _ = h.Write([]byte{0})
|
||||
_, _ = h.Write(canonical)
|
||||
return cacheKeyPrefix + hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
Reference in New Issue
Block a user