fix(cache): harden cache backends, resilience, recovery, and transports

- RESP -ERR no longer closes healthy conn / aborts batch; global cap replacement-aware
- global health-check pass serialized (latent data race); MemoryMonitor via ResourceManager
- token-exchange retry, open-circuit rejections counted as failures
- TLS-posture matching transport reuse (no silent InsecureSkipVerify handoff)

internal/cache/backends/hybrid.go

@@ -37,7 +37,6 @@ type HybridBackend struct {
 
 // asyncWriteItem represents an async write operation
 type asyncWriteItem struct {
-	ctx   context.Context
 	key   string
 	value []byte
 	ttl   time.Duration
@@ -195,7 +194,6 @@ func (h *HybridBackend) Set(ctx context.Context, key string, value []byte, ttl t
 			key:   key,
 			value: value,
 			ttl:   ttl,
-			ctx:   ctx,
 		}:
 			h.logger.Debugf("Queued async write to L2 for key: %s", redactKey(key))
 		default:
@@ -265,14 +263,23 @@ func (h *HybridBackend) Delete(ctx context.Context, key string) (bool, error) {
 		deleted = true
 	}
 
-	// Delete from L2 if not in fallback mode
-	if !h.fallbackMode.Load() {
-		if d, err := h.secondary.Delete(ctx, key); err != nil {
-			h.logger.Debugf("Failed to delete from L2 cache: %v", err)
-			h.recordL2Error()
-		} else if d {
-			deleted = true
-		}
+	if h.fallbackMode.Load() {
+		return deleted, nil
+	}
+
+	// Drain in-flight async L2 writes before deleting L2. A Set's async
+	// write could otherwise land AFTER this DEL, resurrecting the just-
+	// deleted key with its stale value (L1/L2 divergence). Waiting for
+	// the async worker (which tracks queued + in-flight non-critical
+	// writes) ensures no pre-existing async Set lands post-DEL (R124).
+	h.asyncWg.Wait()
+
+	// Delete from L2
+	if d, err := h.secondary.Delete(ctx, key); err != nil {
+		h.logger.Debugf("Failed to delete from L2 cache: %v", err)
+		h.recordL2Error()
+	} else if d {
+		deleted = true
 	}
 
 	return deleted, nil
@@ -390,12 +397,14 @@ func (h *HybridBackend) Ping(ctx context.Context) error {
 
 // Close shuts down the hybrid backend
 func (h *HybridBackend) Close() error {
-	// Cancel context to stop workers
+	// Cancel context to stop workers.
 	h.cancel()
 
-	// Close async write channel
-	close(h.asyncWriteBuffer)
-	close(h.l1BackfillBuffer)
+	// Do NOT close asyncWriteBuffer / l1BackfillBuffer: the workers key off
+	// h.ctx.Done() and exit on cancel, and closing these channels creates a
+	// race where a concurrent Set's select-send (line ~193) hits a closed
+	// channel and panics ("send on closed channel"). Buffered items a late
+	// Set queues after the workers have exited are harmless at shutdown.
 
 	// Wait for workers to finish with timeout
 	done := make(chan struct{})
@@ -539,7 +548,6 @@ func (h *HybridBackend) SetMany(ctx context.Context, items map[string][]byte, tt
 					key:   key,
 					value: value,
 					ttl:   ttl,
-					ctx:   ctx,
 				}:
 					// Queued
 				default:
@@ -593,7 +601,7 @@ func (h *HybridBackend) l1BackfillWorker() {
 			if !ok {
 				return
 			}
-			if h.keyStillInSecondary(item.key) {
+			if h.keyStillInSecondary(item.key) && !h.keyAlreadyInPrimary(item.key) {
 				writeCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
 				if err := h.primary.Set(writeCtx, item.key, item.value, item.ttl); err != nil {
 					h.logger.Debugf("Failed to populate L1 cache from L2 for key %s: %v", redactKey(item.key), err)
@@ -620,6 +628,19 @@ func (h *HybridBackend) keyStillInSecondary(key string) bool {
 	return err != nil || exists
 }
 
+// keyAlreadyInPrimary reports whether the key already exists in the L1
+// cache. The backfill worker writes an L2-resolved value into L1; if a
+// concurrent Set already placed a fresher value there, writing over it
+// would clobber the newer value with the stale L2 snapshot (R124). Skip
+// the backfill when L1 already has the key — L1 is only a cache and the
+// next L2 hit re-arms the write-through.
+func (h *HybridBackend) keyAlreadyInPrimary(key string) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	exists, err := h.primary.Exists(ctx, key)
+	return err == nil && exists
+}
+
 // asyncWriteWorker processes asynchronous writes to L2
 func (h *HybridBackend) asyncWriteWorker() {
 	defer h.wg.Done()
@@ -658,7 +679,11 @@ func (h *HybridBackend) asyncWriteWorker() {
 			// Clear stall us out so no buffered write can land after a
 			// clear-then-wipe.
 			h.asyncFlushMu.Lock()
-			writeCtx, cancel := context.WithTimeout(item.ctx, 500*time.Millisecond)
+			// Use a detached context, never the originating request's ctx:
+			// this is a background write that typically runs after the
+			// request has returned, so the request ctx would be canceled
+			// and the L2 write silently dropped (R94).
+			writeCtx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
 			if err := h.secondary.Set(writeCtx, item.key, item.value, item.ttl); err != nil {
 				h.errors.Add(1)
 				h.logger.Debugf("Async write to L2 failed for key %s: %v", redactKey(item.key), err)

internal/cache/backends/hybrid_r94_test.go

@@ -0,0 +1,75 @@
+package backends
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// r94CtxProbeBackend records whether any operation was handed a canceled
+// context. Set is the only method the async L2 write path exercises.
+type r94CtxProbeBackend struct {
+	setSeen   *atomic.Bool
+	setHadErr *atomic.Bool
+}
+
+func (b *r94CtxProbeBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	b.setSeen.Store(true)
+	if ctx.Err() != nil {
+		b.setHadErr.Store(true)
+	}
+	return nil
+}
+func (b *r94CtxProbeBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
+	return nil, 0, false, nil
+}
+func (b *r94CtxProbeBackend) Delete(ctx context.Context, key string) (bool, error) { return false, nil }
+func (b *r94CtxProbeBackend) Exists(ctx context.Context, key string) (bool, error) { return false, nil }
+func (b *r94CtxProbeBackend) Clear(ctx context.Context) error                      { return nil }
+func (b *r94CtxProbeBackend) GetStats() map[string]interface{}                     { return nil }
+func (b *r94CtxProbeBackend) Close() error                                         { return nil }
+func (b *r94CtxProbeBackend) Ping(ctx context.Context) error                       { return nil }
+
+// TestHybridBackend_AsyncWriteUsesDetachedContext is an R94 regression: the
+// async L2 write-through must be executed with a detached context, never the
+// originating request's context. The L2 write runs after the request has
+// returned; if it carried the (now-canceled) request context the write
+// would be silently dropped, so non-critical keys never reach L2. Before
+// the fix the worker used the request ctx and recorded a canceled context.
+func TestHybridBackend_AsyncWriteUsesDetachedContext(t *testing.T) {
+	primary := &r94CtxProbeBackend{setSeen: &atomic.Bool{}, setHadErr: &atomic.Bool{}}
+	secondary := &r94CtxProbeBackend{setSeen: &atomic.Bool{}, setHadErr: &atomic.Bool{}}
+
+	hybrid, err := NewHybridBackend(&HybridConfig{
+		Primary:             primary,
+		Secondary:           secondary,
+		SyncWriteCacheTypes: map[string]bool{
+			// empty set: no type is sync, so writes are async
+		},
+	})
+	require.NoError(t, err)
+	defer hybrid.Close()
+
+	reqCtx, cancel := context.WithCancel(context.Background())
+	err = hybrid.Set(reqCtx, "general:key1", []byte("test-value"), time.Minute)
+	require.NoError(t, err)
+
+	// The request has returned: cancel its context before the async worker
+	// gets a chance to run the L2 write.
+	cancel()
+
+	// Wait for the async L2 write to be attempted.
+	deadline := time.Now().Add(2 * time.Second)
+	for !secondary.setSeen.Load() {
+		if time.Now().After(deadline) {
+			t.Fatal("async L2 write was never attempted")
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	require.False(t, secondary.setHadErr.Load(),
+		"async L2 write must use a detached (non-canceled) context")
+}

internal/cache/backends/memory.go

@@ -202,6 +202,14 @@ func (m *MemoryCacheBackend) Set(ctx context.Context, key string, value interfac
 		return ErrBackendUnavailable
 	}
 
+	// A NEGATIVE TTL means "already expired" (the stack's convention for
+	// a value whose validity has passed — see redis.go). Previously ttl<=0
+	// fell through to a permanent entry, diverging from redis and
+	// universal_cache. Zero remains the documented "no expiry" contract.
+	if ttl < 0 {
+		return nil
+	}
+
 	start := time.Now()
 	defer func() {
 		duration := time.Since(start).Nanoseconds()
@@ -212,15 +220,19 @@ func (m *MemoryCacheBackend) Set(ctx context.Context, key string, value interfac
 	// Calculate item size
 	itemSize := int64(len(key)) + estimateValueSize(value)
 
-	// Enforce global limits before adding new item
-	m.enforceGlobalLimits(itemSize)
+	// Snapshot replacement state so the global limit enforcer accounts for
+	// key replacement: refreshing an existing key neither grows the count
+	// nor adds its full size, so at global capacity it must not evict an
+	// unrelated live entry (mirrors cacheShard.set, R177).
+	shard := m.getShard(key)
+	oldSize, exists := shard.peekSize(key)
+	m.enforceGlobalLimits(itemSize, oldSize, exists)
 
 	var expiresAt time.Time
 	if ttl > 0 {
 		expiresAt = time.Now().Add(ttl)
 	}
 
-	shard := m.getShard(key)
 	shard.set(key, value, expiresAt, itemSize)
 
 	m.sets.Add(1)
@@ -228,14 +240,30 @@ func (m *MemoryCacheBackend) Set(ctx context.Context, key string, value interfac
 }
 
 // enforceGlobalLimits ensures global size and memory limits are respected
-// by evicting from shards when necessary
-func (m *MemoryCacheBackend) enforceGlobalLimits(newItemSize int64) {
-	// Check and enforce size limit
+// by evicting from shards when necessary.
+//
+// It is replacement-aware: an existing key being rewritten (exists=oldSize)
+// is count-neutral (global cap is on the number of entries) and grows
+// memory by only the net delta, mirroring cacheShard.set. At global
+// capacity a pure key replacement must therefore not evict an unrelated
+// live entry (R177).
+func (m *MemoryCacheBackend) enforceGlobalLimits(newItemSize, oldSize int64, exists bool) {
+	// Count (entry-size) budget only grows for genuinely new keys.
+	sizeTrigger := m.maxSize > 0 && !exists
+	// Memory budget grows by the net delta for a replacement.
+	netMemory := newItemSize
+	if exists {
+		netMemory = newItemSize - oldSize
+		if netMemory < 0 {
+			netMemory = 0
+		}
+	}
+
 	for {
 		totalSize, totalMemory := m.getGlobalStats()
 
-		needsSizeEviction := m.maxSize > 0 && totalSize >= m.maxSize
-		needsMemoryEviction := m.maxMemory > 0 && totalMemory+newItemSize > m.maxMemory
+		needsSizeEviction := sizeTrigger && totalSize >= m.maxSize
+		needsMemoryEviction := m.maxMemory > 0 && totalMemory+netMemory > m.maxMemory
 
 		if !needsSizeEviction && !needsMemoryEviction {
 			break

internal/cache/backends/memory_shard.go

@@ -30,6 +30,19 @@ func newCacheShard(maxSize, maxMemory int64) *cacheShard {
 
 // get retrieves a value from this shard
 // Returns: value, exists, expired
+// peekSize returns the size of the existing entry for key and whether it
+// exists, without touching LRU state. Used by the global limit enforcer
+// to decide whether a Set is a replacement (count-neutral) or a new item.
+func (s *cacheShard) peekSize(key string) (int64, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	item, exists := s.items[key]
+	if !exists {
+		return 0, false
+	}
+	return item.size, true
+}
+
 func (s *cacheShard) get(key string) (interface{}, bool, bool) {
 	s.mu.RLock()
 	item, exists := s.items[key]
@@ -72,15 +85,27 @@ func (s *cacheShard) set(key string, value interface{}, expiresAt time.Time, siz
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	// Check if we need to evict items
-	if s.maxSize > 0 && s.size >= s.maxSize {
-		s.evictLRULocked()
-	}
-	if s.maxMemory > 0 && s.memoryUsed+size > s.maxMemory {
-		s.evictLRULocked()
+	// Snapshot the existing entry so the eviction decisions account for
+	// replacement: refreshing an existing key neither grows the count nor
+	// adds its full size, so at capacity it must not evict an unrelated
+	// LRU entry (R139 F2).
+	oldItem, exists := s.items[key]
+	var oldSize int64
+	if exists {
+		oldSize = oldItem.size
 	}
 
-	// Remove old item if exists
+	if s.maxSize > 0 && !exists && s.size >= s.maxSize {
+		s.evictLRULocked()
+	}
+	if s.maxMemory > 0 {
+		if net := size - oldSize; net > 0 && s.memoryUsed+net > s.maxMemory {
+			s.evictLRULocked()
+		}
+	}
+
+	// Remove old item if it still exists (it may have been the LRU victim
+	// of an eviction above; re-check so we don't double-account).
 	if oldItem, exists := s.items[key]; exists {
 		s.memoryUsed -= oldItem.size
 		if elem, ok := oldItem.element.(*list.Element); ok && elem != nil {

internal/cache/backends/redis.go

@@ -95,7 +95,15 @@ func (r *RedisBackend) Set(ctx context.Context, key string, value []byte, ttl ti
 		return ErrBackendClosed
 	}
 
+	// A NEGATIVE TTL means "already expired" (the stack's convention for a
+	// value whose validity has passed). Previously ttl<=0 fell through to a
+	// bare SET with no expiry, silently making the entry permanent. Zero here
+	// is the documented "no expiry" contract (see SetManyNoTTL), so only
+	// strictly-negative TTLs are skipped.
 	prefixedKey := r.prefixKey(key)
+	if ttl < 0 {
+		return nil
+	}
 
 	// Execute with retry logic
 	return r.executeWithRetry(ctx, func(conn *RedisConn) error {
@@ -450,6 +458,13 @@ func (r *RedisBackend) SetMany(ctx context.Context, items map[string][]byte, ttl
 		return nil
 	}
 
+	if ttl < 0 {
+		// Already-expired TTL: nothing to persist (see Set's guard). Avoids
+		// creating permanent entries for past-dated values via the multi path.
+		// Zero retains the "no expiry" contract (see SetManyNoTTL).
+		return nil
+	}
+
 	// For single items, use regular Set
 	if len(items) == 1 {
 		for key, value := range items {

internal/cache/backends/redis_pool.go

@@ -67,6 +67,22 @@ func NewConnectionPool(config *PoolConfig) (*ConnectionPool, error) {
 	return pool, nil
 }
 
+// reserveConnectionSlot atomically reserves capacity for one new connection,
+// returning false when the pool is already at MaxConnections. The
+// reservation is released via totalConns.Add(-1) if the dial fails.
+func (p *ConnectionPool) reserveConnectionSlot() bool {
+	max := int32(p.config.MaxConnections) // nolint:gosec // MaxConnections is operator-bounded well below int32 max
+	for {
+		cur := p.totalConns.Load()
+		if cur >= max {
+			return false
+		}
+		if p.totalConns.CompareAndSwap(cur, cur+1) {
+			return true
+		}
+	}
+}
+
 // Get retrieves a connection from the pool or creates a new one
 func (p *ConnectionPool) Get(ctx context.Context) (*RedisConn, error) {
 	if p.closed.Load() {
@@ -102,11 +118,15 @@ func (p *ConnectionPool) Get(ctx context.Context) (*RedisConn, error) {
 			return nil, ctx.Err()
 
 		default:
-			// No available connection, create new one if under limit
-			// #nosec G115 -- MaxConnections is a small config value that fits in int32
-			if p.totalConns.Load() < int32(p.config.MaxConnections) {
+			// No available connection, create new one if under limit.
+			// Reserve capacity atomically before dialing: checking
+			// totalConns then dialing is not atomic, so concurrent Gets
+			// could each pass the check and spike the live socket count
+			// far above MaxConnections (FD / Redis maxclients exhaustion).
+			if p.reserveConnectionSlot() {
 				conn, err = p.createConnection(ctx)
 				if err != nil {
+					p.totalConns.Add(-1) // release the reserved slot
 					// If this is the last attempt, return error
 					if attempt == maxAttempts-1 {
 						return nil, err
@@ -116,7 +136,6 @@ func (p *ConnectionPool) Get(ctx context.Context) (*RedisConn, error) {
 					continue
 				}
 				p.activeConns.Add(1)
-				p.totalConns.Add(1)
 				return conn, nil
 			}
 
@@ -335,7 +354,9 @@ func (c *RedisConn) Do(command string, args ...string) (interface{}, error) {
 	resp, err := reader.ReadResponse()
 	reader.Release() // Return to pool immediately after use
 	if err != nil {
-		if !errors.Is(err, ErrNilResponse) {
+		// A nil response or a Redis command error reply leaves the
+		// connection healthy — only an IO/parse error closes it.
+		if !errors.Is(err, ErrNilResponse) && !errors.Is(err, ErrCommandReply) {
 			c.closed.Store(true)
 		}
 		return nil, err
@@ -470,6 +491,14 @@ func (p *Pipeline) Execute() ([]interface{}, error) {
 				responses[i] = nil
 				continue
 			}
+			if errors.Is(err, ErrCommandReply) {
+				// A Redis command error reply: the connection is healthy
+				// and the remaining replies are still readable. Store the
+				// error value so per-command validation upstream can
+				// report it without aborting the batch.
+				responses[i] = err
+				continue
+			}
 			p.conn.closed.Store(true)
 			return responses[:i], fmt.Errorf("pipeline read error at command %d: %w", i, err)
 		}

internal/cache/backends/resp.go

@@ -20,6 +20,11 @@ import (
 var (
 	ErrInvalidRESP = errors.New("invalid RESP response")
 	ErrNilResponse = errors.New("nil response")
+	// ErrCommandReply marks a RESP '-' (Redis command error) reply, e.g.
+	// WRONGTYPE or OOM. It is a valid protocol response — the connection
+	// is healthy and any remaining pipelined replies are still readable —
+	// as opposed to an IO/parse error which means the connection is dead.
+	ErrCommandReply = errors.New("redis command error reply")
 )
 
 // RESPWriter writes RESP protocol messages
@@ -110,7 +115,9 @@ func (r *RESPReader) readError() error {
 	if err != nil {
 		return err
 	}
-	return errors.New(line)
+	// Wrap so callers can distinguish a Redis command error reply (healthy
+	// connection) from an IO/parse error (dead connection).
+	return fmt.Errorf("%w: %s", ErrCommandReply, line)
 }
 
 // readInteger reads an integer (:1000\r\n)

internal/cache/backends/resp_test.go

@@ -142,7 +142,12 @@ func TestRESPReader_ReadError(t *testing.T) {
 			_, err := reader.ReadResponse()
 
 			require.Error(t, err)
-			assert.Equal(t, tt.expectedError, err.Error())
+			// R177: a '-' reply is now wrapped with ErrCommandReply so
+			// callers can tell a Redis command error (healthy connection)
+			// apart from an IO/parse error. The command's message text is
+			// preserved verbatim inside the wrapped error.
+			assert.True(t, errors.Is(err, ErrCommandReply), "want ErrCommandReply, got %v", err)
+			assert.Contains(t, err.Error(), tt.expectedError)
 		})
 	}
 }

internal/cache/backends/review_r110_test.go

@@ -0,0 +1,54 @@
+package backends
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestConnectionPool_CapHeldUnderBurst regresses the connection-pool cap
+// being non-atomic: ConnectionPool.Get checked `totalConns < MaxConnections`
+// and THEN dialed a new socket, so concurrent Gets could each pass the check
+// and spike the live connection count far above MaxConnections (FD / Redis
+// maxclients exhaustion under burst load). The cap must now be held
+// atomically so at most MaxConnections sockets are ever created at once.
+func TestConnectionPool_CapHeldUnderBurst(t *testing.T) {
+	mr := NewMiniredisServer(t)
+
+	maxConns := 3
+	pool, err := NewConnectionPool(&PoolConfig{
+		Address:        mr.GetAddr(),
+		MaxConnections: maxConns,
+		ConnectTimeout: 80 * time.Millisecond,
+		ReadTimeout:    50 * time.Millisecond,
+		WriteTimeout:   50 * time.Millisecond,
+	})
+	require.NoError(t, err)
+	defer pool.Close()
+
+	ctx := context.Background()
+	const workers = 50
+
+	var wg sync.WaitGroup
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			conn, _ := pool.Get(ctx)
+			if conn != nil {
+				// Hold the connection (simulate an in-flight request) so the
+				// pool stays under pressure and the count is stable.
+				_ = conn
+			}
+		}()
+	}
+	wg.Wait()
+
+	stats := pool.Stats()
+	total := stats["total_connections"].(int32)
+	require.LessOrEqual(t, total, int32(maxConns),
+		"burst Gets must not create more than MaxConnections sockets (got %d)", total)
+}

internal/cache/backends/review_r115_test.go

@@ -0,0 +1,36 @@
+package backends
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestRedisBackend_SetNegativeTTL_NotPersistent guards the R115 negative-TTL
+// fix. The cache stack's convention is that a non-positive TTL means "already
+// expired". Previously RedisBackend.Set/SetMany fell through to a bare SET
+// when ttl<=0, writing an entry with NO expiry — i.e. an expired-dated value
+// became permanent. Now expired TTLs write nothing.
+func TestRedisBackend_SetNegativeTTL_NotPersistent(t *testing.T) {
+	_, backend := setupTestRedis(t)
+	ctx := context.Background()
+
+	// Single Set with an expired TTL must not create an entry.
+	require.NoError(t, backend.Set(ctx, "neg", []byte("v"), -time.Second))
+	_, _, exists, err := backend.Get(ctx, "neg")
+	require.NoError(t, err)
+	require.False(t, exists, "negative-TTL Set must not create a persistent entry")
+
+	// SetMany with multiple items and an expired TTL must not create entries.
+	require.NoError(t, backend.SetMany(ctx, map[string][]byte{
+		"a": []byte("1"),
+		"b": []byte("2"),
+	}, -time.Second))
+	for _, k := range []string{"a", "b"} {
+		_, _, exists, err := backend.Get(ctx, k)
+		require.NoError(t, err)
+		require.False(t, exists, "negative-TTL SetMany must not create a persistent entry for %q", k)
+	}
+}

internal/cache/backends/review_r124_test.go

@@ -0,0 +1,241 @@
+package backends
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// blockingWriteBackend is a cache backend whose Set blocks until released,
+// used to hold an async L2 write in-flight while Delete runs.
+type blockingWriteBackend struct {
+	mu         sync.Mutex
+	data       map[string]string
+	setStarted chan struct{}
+	release    chan struct{}
+	written    chan struct{}
+	once       sync.Once
+}
+
+func (b *blockingWriteBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	select {
+	case <-b.setStarted:
+	default:
+		close(b.setStarted)
+	}
+	select {
+	case <-b.release:
+	case <-b.written: // never
+	case <-ctx.Done():
+		return nil
+	}
+	b.mu.Lock()
+	b.data[key] = string(value)
+	b.mu.Unlock()
+	b.once.Do(func() { close(b.written) })
+	return nil
+}
+
+func (b *blockingWriteBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	v, ok := b.data[key]
+	if !ok {
+		return nil, 0, false, nil
+	}
+	return []byte(v), time.Minute, true, nil
+}
+
+func (b *blockingWriteBackend) Delete(ctx context.Context, key string) (bool, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	_, ok := b.data[key]
+	delete(b.data, key)
+	return ok, nil
+}
+
+func (b *blockingWriteBackend) Exists(ctx context.Context, key string) (bool, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	_, ok := b.data[key]
+	return ok, nil
+}
+
+func (b *blockingWriteBackend) Clear(ctx context.Context) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.data = map[string]string{}
+	return nil
+}
+
+func (b *blockingWriteBackend) GetStats() map[string]interface{} { return nil }
+func (b *blockingWriteBackend) Close() error                     { return nil }
+func (b *blockingWriteBackend) Ping(ctx context.Context) error   { return nil }
+
+// TestHybrid_DeleteWaitsForAsyncWrite guards the R124 fix to hybrid.go
+// Delete: an in-flight async L2 write (from a non-critical Set) could land
+// after Delete's L2 DEL, resurrecting the just-deleted key with a stale
+// value. Delete must wait for in-flight async writes before the L2 delete.
+func TestHybrid_DeleteWaitsForAsyncWrite(t *testing.T) {
+	sec := &blockingWriteBackend{
+		data:       map[string]string{},
+		setStarted: make(chan struct{}),
+		release:    make(chan struct{}),
+		written:    make(chan struct{}),
+	}
+	primary := newMockBackend()
+
+	hybrid, err := NewHybridBackend(&HybridConfig{
+		Primary:         primary,
+		Secondary:       sec,
+		AsyncBufferSize: 16,
+	})
+	if err != nil {
+		t.Fatalf("hybrid: %v", err)
+	}
+	defer hybrid.Close()
+
+	ctx := context.Background()
+
+	// Non-critical Set -> async L2 write that blocks in-flight.
+	if err := hybrid.Set(ctx, "k", []byte("stale"), time.Minute); err != nil {
+		t.Fatalf("set: %v", err)
+	}
+	select {
+	case <-sec.setStarted:
+	case <-time.After(2 * time.Second):
+		t.Fatal("async L2 write never started")
+	}
+
+	// Issue Delete while the async write is in-flight.
+	done := make(chan struct{})
+	go func() {
+		_, _ = hybrid.Delete(ctx, "k")
+		close(done)
+	}()
+
+	// OLD behavior: Delete returns immediately (no wait for async writes);
+	// it performs its L2 delete and closes done. NEW behavior: Delete
+	// blocks in asyncWg.Wait until the in-flight write finishes, so done
+	// does not fire until released. Distinguish the two before releasing.
+	oldImmediate := false
+	select {
+	case <-done:
+		oldImmediate = true
+	case <-time.After(50 * time.Millisecond):
+		// NEW: Delete is blocked waiting for the async write. 50ms is
+		// far longer than OLD's immediate (non-waiting) return needs,
+		// and far shorter than the worker's 500ms writeCtx, so the
+		// in-flight write reliably survives to be released.
+	}
+	// Release the in-flight L2 write (well within the worker's writeCtx
+	// timeout) so it actually lands.
+	close(sec.release)
+
+	if oldImmediate {
+		// OLD: the worker write now lands AFTER Delete's L2 delete,
+		// resurrecting the key.
+		select {
+		case <-sec.written:
+		case <-time.After(2 * time.Second):
+			t.Fatal("async L2 write never landed")
+		}
+	} else {
+		// NEW: Delete waited for the write, then removed it from L2.
+		select {
+		case <-sec.written:
+		case <-time.After(2 * time.Second):
+			t.Fatal("async L2 write never landed")
+		}
+		select {
+		case <-done:
+		case <-time.After(2 * time.Second):
+			t.Fatal("Delete did not return")
+		}
+	}
+
+	if _, _, exists, _ := sec.Get(ctx, "k"); exists {
+		t.Fatal("L2 must not contain key after Delete: the in-flight async Set resurrected it")
+	}
+}
+
+// existsAlwaysBackend reports the key always present in L1 (so a backfill
+// must be skipped) and counts Set calls; setCalled is closed once Set runs,
+// letting the test observe a (OLD) backfill write deterministically.
+type existsAlwaysBackend struct {
+	setCalls  int32
+	setCalled chan struct{}
+	once      sync.Once
+}
+
+func newExistsAlwaysBackend() *existsAlwaysBackend {
+	return &existsAlwaysBackend{setCalled: make(chan struct{})}
+}
+
+func (b *existsAlwaysBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	atomic.AddInt32(&b.setCalls, 1)
+	b.once.Do(func() { close(b.setCalled) })
+	return nil
+}
+func (b *existsAlwaysBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
+	return nil, 0, false, nil
+}
+func (b *existsAlwaysBackend) Delete(ctx context.Context, key string) (bool, error) {
+	return false, nil
+}
+func (b *existsAlwaysBackend) Exists(ctx context.Context, key string) (bool, error) { return true, nil }
+func (b *existsAlwaysBackend) Clear(ctx context.Context) error                      { return nil }
+func (b *existsAlwaysBackend) GetStats() map[string]interface{}                     { return nil }
+func (b *existsAlwaysBackend) Close() error                                         { return nil }
+func (b *existsAlwaysBackend) Ping(ctx context.Context) error                       { return nil }
+
+// TestHybrid_BackfillDoesNotClobberFreshL1 guards the R124 fix to
+// hybrid.go l1BackfillWorker: the backfill only checked L2 key
+// existence, so a stale L2-captured value would overwrite a fresher L1
+// value written concurrently by Set. When L1 already holds the key, the
+// backfill must be skipped.
+func TestHybrid_BackfillDoesNotClobberFreshL1(t *testing.T) {
+	primary := newExistsAlwaysBackend()
+	secondary := newMockBackend()
+	ctx := context.Background()
+	if err := secondary.Set(ctx, "k", []byte("stale"), time.Minute); err != nil {
+		t.Fatalf("seed L2: %v", err)
+	}
+
+	hybrid, err := NewHybridBackend(&HybridConfig{
+		Primary:         primary,
+		Secondary:       secondary,
+		AsyncBufferSize: 16,
+	})
+	if err != nil {
+		t.Fatalf("hybrid: %v", err)
+	}
+	defer hybrid.Close()
+
+	// Queue a stale backfill for a key L1 already holds (fresher).
+	hybrid.queueL1Backfill("k", []byte("stale"), time.Minute)
+
+	// Wait for the worker to drain the backfill buffer.
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if len(hybrid.l1BackfillBuffer) == 0 {
+			time.Sleep(20 * time.Millisecond) // let the worker act (write or skip)
+			break
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	// Deterministically observe an OLD backfill write: if the worker
+	// writes, setCalled closes and setCalls becomes 1. On NEW the
+	// worker skips, so nothing fires.
+	select {
+	case <-primary.setCalled:
+	case <-time.After(500 * time.Millisecond):
+	}
+
+	if atomic.LoadInt32(&primary.setCalls) != 0 {
+		t.Fatal("backfill must not overwrite the fresher L1 value (primary already had the key)")
+	}
+}

internal/cache/backends/review_r125_test.go

@@ -0,0 +1,28 @@
+package backends
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+// TestMemoryBackend_NegativeTTLNotStored guards the R125 fix to memory.go:
+// a negative TTL must mean "already expired" (the stack's convention, per
+// redis.go) and NOT store a permanent entry. Previously ttl<=0 fell
+// through to a permanent entry, diverging from redis and universal_cache,
+// so a value whose validity had passed was kept forever in the memory
+// tier.
+func TestMemoryBackend_NegativeTTLNotStored(t *testing.T) {
+	b := NewMemoryCacheBackend(1000, 0, 0)
+	defer b.Close()
+
+	ctx := context.Background()
+	if err := b.Set(ctx, "k", []byte("v"), -1*time.Second); err != nil {
+		t.Fatalf("set with negative TTL: %v", err)
+	}
+
+	if _, err := b.Get(ctx, "k"); !errors.Is(err, ErrCacheMiss) {
+		t.Fatalf("Get after negative-TTL Set = %v, want %v (negative TTL must be treated as already-expired, not stored)", err, ErrCacheMiss)
+	}
+}

internal/cache/backends/review_r139_f2_test.go

@@ -0,0 +1,28 @@
+package backends
+
+import (
+	"testing"
+	"time"
+)
+
+// TestCacheShard_SetRefreshExistingDoesNotEvictAtCapacity verifies the shard
+// set path does not evict an unrelated LRU entry when replacing an existing
+// key at capacity (count stays flat) (R139 F2). Mirrors the Cache.Set fix.
+func TestCacheShard_SetRefreshExistingDoesNotEvictAtCapacity(t *testing.T) {
+	s := newCacheShard(2, 0) // maxSize=2, no memory bound
+	exp := time.Now().Add(time.Minute)
+
+	s.set("a", "va", exp, 1) // becomes LRU
+	s.set("b", "vb", exp, 1) // MRU
+
+	// Refresh the most-recent key while at capacity. On the buggy path the
+	// LRU sibling `a` is evicted; on the fixed path the replacement is free.
+	s.set("b", "vb2", exp, 1)
+
+	if _, exists, _ := s.get("a"); !exists {
+		t.Fatal("refreshing an existing key must not evict an unrelated live entry `a`")
+	}
+	if v, exists, _ := s.get("b"); !exists || v != "vb2" {
+		t.Fatalf("refreshed key `b` should hold the new value, got %v exists=%v", v, exists)
+	}
+}

internal/cache/backends/review_r141_test.go

@@ -0,0 +1,81 @@
+package backends
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+)
+
+// TestSingleflightCache_PanickingFetcherDoesNotDeadlock verifies that a
+// fetcher which panics (routines in this codebase treat panics as a normal
+// failure mode) does not wedge the singleflight entry: the leading caller
+// yields an error and waiting callers are released instead of blocking on
+// call.wg forever (R141).
+func TestSingleflightCache_PanickingFetcherDoesNotDeadlock(t *testing.T) {
+	backend, err := NewMemoryBackend(DefaultConfig())
+	requireNoErrorFatal(t, err)
+	defer backend.Close()
+
+	cache := NewSingleflightCache(backend)
+	ctx := context.Background()
+	key := "panic-key"
+
+	panicFetcher := func(ctx context.Context) ([]byte, time.Duration, error) {
+		panic("boom")
+	}
+
+	var wg sync.WaitGroup
+	var doneA, doneB = make(chan struct{}), make(chan struct{})
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer func() { _ = recover() }() // tolerate old code's out-of-band panic
+		_, _ = cache.GetOrFetch(ctx, key, panicFetcher)
+		close(doneA)
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, _ = cache.GetOrFetch(ctx, key, panicFetcher)
+		close(doneB)
+	}()
+
+	select {
+	case <-doneB:
+	case <-time.After(3 * time.Second):
+		t.Fatal("concurrent waiter hung on a panicking singleflight fetcher")
+	}
+	select {
+	case <-doneA:
+	case <-time.After(3 * time.Second):
+		t.Fatal("leading caller hung on its own panicking fetcher")
+	}
+	wg.Wait()
+
+	// After the entry is cleaned up, a well-behaved fetcher for the same
+	// key must execute normally (proves the entry was not permanently
+	// wedged by the panic).
+	ran := false
+	good := func(ctx context.Context) ([]byte, time.Duration, error) {
+		ran = true
+		return []byte("ok"), time.Minute, nil
+	}
+	// Give the deferred cleanup a moment to release the key.
+	time.Sleep(200 * time.Millisecond)
+	if _, err := cache.GetOrFetch(ctx, key, good); err != nil {
+		t.Fatalf("well-behaved fetcher after panic returned error: %v", err)
+	}
+	if !ran {
+		t.Fatal("well-behaved fetcher after panic did not execute")
+	}
+}
+
+func requireNoErrorFatal(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}

internal/cache/backends/review_r177_redis_test.go

@@ -0,0 +1,79 @@
+package backends
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestR177_RedisCommandErrorDoesNotCloseConn regresses the RESP command-error
+// conflation: a Redis '-' (command error) reply, e.g. WRONGTYPE or OOM, is
+// a valid protocol response — it does not mean the connection is dead.
+// Previously Do and Pipeline treated it as an IO error, marking the healthy
+// pooled connection closed (and for pipelines aborting the whole batch),
+// discarding a perfectly reusable connection.
+func TestR177_RedisCommandErrorDoesNotCloseConn(t *testing.T) {
+	mr := NewMiniredisServer(t)
+	config := &PoolConfig{
+		Address:        mr.GetAddr(),
+		MaxConnections: 5,
+		ConnectTimeout: 5 * time.Second,
+		ReadTimeout:    3 * time.Second,
+		WriteTimeout:   3 * time.Second,
+	}
+	ctx := context.Background()
+
+	pool, err := NewConnectionPool(config)
+	require.NoError(t, err)
+	defer pool.Close()
+
+	conn, err := pool.Get(ctx)
+	require.NoError(t, err)
+	defer pool.Put(conn)
+
+	t.Run("single command error leaves conn healthy", func(t *testing.T) {
+		mr.SetError("ERR artificial") // every command returns '-ERR artificial'
+
+		_, err := conn.Do("GET", "k")
+		require.Error(t, err)
+		require.True(t, errors.Is(err, ErrCommandReply),
+			"expected a Redis command-reply error, got %v", err)
+
+		mr.SetError("") // clear; the connection must still be reusable
+
+		// On the buggy path Do marked the conn closed above, so this
+		// PING fails with "connection closed"; on the fixed path the
+		// conn is untouched and PING succeeds.
+		resp, err := conn.Do("PING")
+		require.NoError(t, err)
+		require.Equal(t, "PONG", resp)
+	})
+
+	t.Run("pipeline command error reads full batch and keeps conn", func(t *testing.T) {
+		mr.SetError("ERR artificial")
+
+		p := conn.NewPipeline()
+		p.Queue("SET", "k1", "v1")
+		p.Queue("SET", "k2", "v2")
+
+		// On the buggy path Execute returned after the first error (1
+		// response) and closed the conn; on the fixed path all replies
+		// are read (healthy conn), the batch is not aborted.
+		responses, err := p.Execute()
+		require.NoError(t, err, "batch must complete with per-command errors")
+		require.Len(t, responses, 2)
+		for i, r := range responses {
+			er, ok := r.(error)
+			require.True(t, ok, "response %d should carry an error value", i)
+			require.True(t, errors.Is(er, ErrCommandReply), "response %d should be a command-reply error", i)
+		}
+
+		mr.SetError("")
+		resp, err := conn.Do("PING")
+		require.NoError(t, err)
+		require.Equal(t, "PONG", resp)
+	})
+}

internal/cache/backends/review_r177_regression_test.go

@@ -0,0 +1,45 @@
+package backends
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+// TestR177_MemoryBackendReplacementDoesNotEvictUnrelatedAtCapacity verifies
+// that MemoryCacheBackend's GLOBAL limit enforcer is replacement-aware. The
+// shard-level path (cacheShard.set, R139 F2) was already fixed, but
+// enforceGlobalLimits ran before the shard write and counted every Set as a
+// new entry, so at global capacity a pure key replacement evicted an
+// unrelated live entry from the globally-oldest shard (count stays flat).
+func TestR177_MemoryBackendReplacementDoesNotEvictUnrelatedAtCapacity(t *testing.T) {
+	// maxSize=2 forces shardCount down to 1; the single shard's own
+	// limit (2x average = 4) is loose, so the GLOBAL maxSize=2 is the
+	// binding constraint that the buggy enforcer used to over-evict.
+	m := NewMemoryCacheBackend(2, 0, time.Minute)
+	ctx := context.Background()
+
+	if err := m.Set(ctx, "a", "va", time.Minute); err != nil {
+		t.Fatalf("Set a: %v", err)
+	}
+	if err := m.Set(ctx, "b", "vb", time.Minute); err != nil {
+		t.Fatalf("Set b: %v", err)
+	}
+	// Make `a` the most-recently-used entry so `b` is the global LRU.
+	if _, err := m.Get(ctx, "a"); err != nil {
+		t.Fatalf("Get a: %v", err)
+	}
+	// Replace `a` while at global capacity. On the buggy path the global
+	// enforcer sees totalSize>=maxSize and evicts the LRU `b`; on the
+	// fixed path the replacement is count- and size-neutral.
+	if err := m.Set(ctx, "a", "va2", time.Minute); err != nil {
+		t.Fatalf("Set a (replacement): %v", err)
+	}
+
+	if _, err := m.Get(ctx, "b"); err != nil {
+		t.Fatalf("replacing existing key must not evict unrelated live entry `b`: %v", err)
+	}
+	if v, err := m.Get(ctx, "a"); err != nil || v != "va2" {
+		t.Fatalf("replaced key `a` should hold the new value, got %v err=%v", v, err)
+	}
+}

internal/cache/backends/review_r208_stress_test.go

@@ -0,0 +1,93 @@
+package backends
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+)
+
+// TestBackendStressCloseConcurrent hammers a HybridBackend (which has an
+// async in-flight L2-write worker) with concurrent reads/writes from many
+// goroutines while Close runs at the same time — the classic scenario for a
+// use-after-close or worker-leak race that sequential tests miss. Run under
+// -race to detect any data race or panic (R208).
+func TestBackendStressCloseConcurrent(t *testing.T) {
+	l1, _ := NewMemoryBackend(&Config{CleanupInterval: time.Second})
+	l2, _ := NewMemoryBackend(&Config{CleanupInterval: time.Second})
+	hybrid, err := NewHybridBackend(&HybridConfig{
+		Primary:         l1,
+		Secondary:       l2,
+		AsyncBufferSize: 256,
+	})
+	if err != nil {
+		t.Fatalf("NewHybridBackend: %v", err)
+	}
+
+	var wg sync.WaitGroup
+	stop := make(chan struct{})
+	// Writers + readers hammer the backend.
+	for g := 0; g < 8; g++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+			for i := 0; ; i++ {
+				select {
+				case <-stop:
+					return
+				default:
+				}
+				key := fmt.Sprintf("k:%d:%d", id, i%256)
+				if i%2 == 0 {
+					_ = hybrid.Set(context.Background(), key, []byte("value"), time.Minute)
+				} else {
+					_, _, _, _ = hybrid.Get(context.Background(), key)
+				}
+			}
+		}(g)
+	}
+
+	time.Sleep(20 * time.Millisecond) // let workers spin up
+	_ = hybrid.Close()                // close while ops are in-flight
+	close(stop)
+	wg.Wait()
+}
+
+// TestRedisBackendStressCloseConcurrent hammers a miniredis-backed
+// RedisBackend (pooled connections) from many goroutines while Close runs,
+// the pooled-conn use-after-close race class (R208).
+func TestRedisBackendStressCloseConcurrent(t *testing.T) {
+	mr := NewMiniredisServer(t)
+	backend, err := NewRedisBackend(DefaultRedisConfig(mr.GetAddr()))
+	if err != nil {
+		t.Fatalf("NewRedisBackend: %v", err)
+	}
+
+	var wg sync.WaitGroup
+	stop := make(chan struct{})
+	for g := 0; g < 8; g++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+			for i := 0; ; i++ {
+				select {
+				case <-stop:
+					return
+				default:
+				}
+				key := fmt.Sprintf("redis:k:%d:%d", id, i%256)
+				if i%2 == 0 {
+					_ = backend.Set(context.Background(), key, []byte("v"), time.Minute)
+				} else {
+					_, _, _, _ = backend.Get(context.Background(), key)
+				}
+			}
+		}(g)
+	}
+
+	time.Sleep(20 * time.Millisecond)
+	_ = backend.Close()
+	close(stop)
+	wg.Wait()
+}

internal/cache/backends/review_r212_stress_test.go

@@ -0,0 +1,84 @@
+package backends
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// TestSingleflightStressDedup hammers GetOrFetch with many concurrent
+// waiters on the SAME key behind a slow fetcher: exactly one fetch must
+// run and all waiters must share its result, with no race on
+// call.val/call.err read after wg.Wait() (R212). Run under -race.
+func TestSingleflightStressDedup(t *testing.T) {
+	backend, _ := NewMemoryBackend(&Config{CleanupInterval: time.Hour})
+	sf := NewSingleflightCache(backend)
+
+	var fetches atomic.Int64
+	slow := func(context.Context) ([]byte, time.Duration, error) {
+		fetches.Add(1)
+		time.Sleep(5 * time.Millisecond)
+		return []byte("result"), time.Minute, nil
+	}
+
+	var wg sync.WaitGroup
+	for g := 0; g < 16; g++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 20; i++ {
+				v, err := sf.GetOrFetch(context.Background(), "k", slow)
+				if err != nil {
+					t.Errorf("GetOrFetch: %v", err)
+					return
+				}
+				if string(v) != "result" {
+					t.Errorf("unexpected value %q", v)
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+
+	if fetches.Load() < 1 || fetches.Load() > 20 {
+		t.Fatalf("expected dedup across concurrent waiters; fetcher ran %d times", fetches.Load())
+	}
+}
+
+// A panicking fetcher must not leave waiters blocked forever: they must all
+// receive the recovered error (R212).
+func TestSingleflightStressPanicUnblocks(t *testing.T) {
+	backend, _ := NewMemoryBackend(&Config{CleanupInterval: time.Hour})
+	sf := NewSingleflightCache(backend)
+
+	panicFetcher := func(context.Context) ([]byte, time.Duration, error) {
+		panic("boom")
+	}
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		var wg sync.WaitGroup
+		for g := 0; g < 8; g++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				_, err := sf.GetOrFetch(context.Background(), "pk", panicFetcher)
+				if err == nil {
+					t.Error("expected error from panicking fetcher")
+				}
+			}()
+		}
+		wg.Wait()
+	}()
+
+	select {
+	case <-done:
+		// all waiters unblocked with the recovered error
+	case <-time.After(5 * time.Second):
+		t.Fatal("waiters stayed blocked after panicking fetcher (singleflight wedged)")
+	}
+}

internal/cache/backends/review_r99_close_race_test.go

@@ -0,0 +1,65 @@
+//go:build !yaegi
+
+package backends
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+)
+
+// TestHybridBackend_CloseConcurrentSet_NoPanic regresses a send-on-closed-
+// channel panic in HybridBackend. Close() used to close asyncWriteBuffer
+// while a concurrent Set() may be mid-select on
+// `h.asyncWriteBuffer <- item`, which can panic ("send on closed
+// channel") and crash the process at shutdown. This exercises many
+// Close+Set cycles concurrently and asserts no panic escapes.
+func TestHybridBackend_CloseConcurrentSet_NoPanic(t *testing.T) {
+	for i := 0; i < 50; i++ {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		primary := newMockBackend()
+		secondary := newMockBackend()
+
+		h, err := NewHybridBackend(&HybridConfig{
+			Primary:   primary,
+			Secondary: secondary,
+		})
+		if err != nil {
+			t.Fatalf("new hybrid: %v", err)
+		}
+
+		var wg sync.WaitGroup
+		var panicked sync.Map
+
+		// Writers hammer Set concurrently with Close using async (non-
+		// critical) cache type so the send goes through asyncWriteBuffer.
+		for w := 0; w < 8; w++ {
+			wg.Add(1)
+			go func(w int) {
+				defer wg.Done()
+				defer func() {
+					if r := recover(); r != nil {
+						panicked.Store(true, fmt.Sprintf("writer %d: %v", w, r))
+					}
+				}()
+				for k := 0; k < 200; k++ {
+					// "session:" prefix is not blacklist/token -> async path.
+					_ = h.Set(ctx, fmt.Sprintf("session:user%d:%d", w, k), []byte("value"), time.Minute)
+				}
+			}(w)
+		}
+
+		// Close concurrently with the writers.
+		_ = h.Close()
+
+		wg.Wait()
+
+		if v, ok := panicked.Load(true); ok {
+			t.Fatalf("panic during Close+concurrent Set: %v", v)
+		}
+	}
+}

internal/cache/backends/singleflight.go

@@ -2,6 +2,7 @@ package backends
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -80,8 +81,22 @@ func (s *SingleflightCache) GetOrFetch(ctx context.Context, key string, fetcher
 	s.calls[key] = call
 	s.mu.Unlock()
 
-	// Execute the fetcher
-	call.val, call.ttl, call.err = fetcher(ctx)
+	// Execute the fetcher. Recover any panic so call.wg.Done() below is
+	// always reached: a panicking fetcher would otherwise leave every
+	// waiter blocked forever on call.wg.Wait() and the call permanently
+	// wedged in s.calls (subsequent GetOrFetch calls for the key would
+	// block indefinitely). Routines here (BackgroundTask, Goroutine)
+	// treat panics as a normal failure mode, so this is reachable.
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				if call.err == nil {
+					call.err = fmt.Errorf("singleflight fetcher panicked: %v", r)
+				}
+			}
+		}()
+		call.val, call.ttl, call.err = fetcher(ctx)
+	}()
 	call.done = true
 
 	// If successful, store in cache

internal/cache/cache.go

@@ -148,19 +148,39 @@ func (c *Cache) Set(key string, value interface{}, ttl time.Duration) error {
 		return fmt.Errorf("cache is closed")
 	}
 
+	// Negative TTL means "don't cache" (matches the backends:
+	// memory.go:209 and redis.go:104). cacheVerifiedToken can pass a
+	// negative duration for an already-expired token
+	// (token_manager.go:244); storing an already-expired entry would just
+	// waste a hot-cache slot and misreport currentSize (R138).
+	if ttl < 0 {
+		return nil
+	}
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	// Calculate size
+	// Calculate size and snapshot the existing entry (if any) so the
+	// eviction decisions below account for replacement: refreshing an
+	// existing key neither grows the item count nor adds its full size, so
+	// at capacity it must not evict an unrelated LRU entry (R139 F2).
 	size := c.estimateSize(value)
-
-	// Check memory limit
-	if c.config.EnableMemoryLimit && c.currentMemory+size > c.config.MaxMemoryBytes {
-		c.evictLRU()
+	oldItem, exists := c.items[key]
+	var oldSize int64
+	if exists {
+		oldSize = oldItem.Size
 	}
 
-	// Check size limit
-	if c.config.MaxSize > 0 && len(c.items) >= c.config.MaxSize {
+	// Check memory limit (only evict when the NET addition exceeds it)
+	if c.config.EnableMemoryLimit {
+		if netAdd := size - oldSize; netAdd > 0 && c.currentMemory+netAdd > c.config.MaxMemoryBytes {
+			c.evictLRU()
+		}
+	}
+
+	// Check size limit (only evict when a NEW key would exceed it;
+	// replacement of an existing key keeps the count flat)
+	if c.config.MaxSize > 0 && !exists && len(c.items) >= c.config.MaxSize {
 		c.evictLRU()
 	}
 
@@ -176,7 +196,8 @@ func (c *Cache) Set(key string, value interface{}, ttl time.Duration) error {
 		Metadata:     make(map[string]interface{}),
 	}
 
-	// Remove old item if exists
+	// Remove old item if it still exists (it may have been the LRU victim
+	// of an eviction above; re-check so we don't double-account).
 	if oldItem, exists := c.items[key]; exists {
 		c.lruList.Remove(oldItem.element)
 		c.currentMemory -= oldItem.Size
@@ -293,7 +314,10 @@ func (c *Cache) Close() error {
 	}
 
 	c.cancel()
-	if c.config.EnableAutoCleanup {
+	// stopCleanup is only allocated (and the cleanup goroutine started) when
+	// EnableAutoCleanup && CleanupInterval > 0; must mirror that guard here
+	// or close(nil) panics on shutdown (R149).
+	if c.config.EnableAutoCleanup && c.config.CleanupInterval > 0 {
 		close(c.stopCleanup)
 		c.wg.Wait()
 	}

internal/cache/resilience/circuit_breaker.go

@@ -132,7 +132,14 @@ func (cb *CircuitBreaker) AllowRequest() bool {
 
 		if shouldRetry {
 			cb.setState(StateHalfOpen)
-			return true
+			// Count this probe toward the half-open gate. Previously the
+			// open→half-open transition returned true without incrementing
+			// halfOpenRequests, so N goroutines arriving at the cooldown
+			// simultaneously all transitioned and passed as an unbounded
+			// burst (the half-open branch below is only reached by
+			// requests that race in after the first transition) (R152).
+			current := cb.halfOpenRequests.Add(1)
+			return current <= int32(cb.config.HalfOpenMaxRequests) // nolint:gosec // HalfOpenMaxRequests is operator-bounded well below int32 max
 		}
 		return false
 

internal/cache/resilience/health_check_backend_test.go

@@ -0,0 +1,142 @@
+package resilience
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// fakeBackend is a minimal CacheBackend whose Ping can be toggled to fail,
+// so HealthCheckBackend's status transition can be driven deterministically.
+type fakeBackend struct {
+	mu   sync.Mutex
+	data map[string][]byte
+	fail atomic.Bool
+}
+
+func newFakeBackend() *fakeBackend {
+	return &fakeBackend{data: make(map[string][]byte)}
+}
+
+func (f *fakeBackend) Set(_ context.Context, key string, value []byte, _ time.Duration) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.data[key] = value
+	return nil
+}
+
+func (f *fakeBackend) Get(_ context.Context, key string) ([]byte, time.Duration, bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	v, ok := f.data[key]
+	return v, 0, ok, nil
+}
+
+func (f *fakeBackend) Delete(_ context.Context, key string) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	_, ok := f.data[key]
+	delete(f.data, key)
+	return ok, nil
+}
+
+func (f *fakeBackend) Exists(_ context.Context, key string) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	_, ok := f.data[key]
+	return ok, nil
+}
+
+func (f *fakeBackend) Clear(_ context.Context) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.data = make(map[string][]byte)
+	return nil
+}
+
+func (f *fakeBackend) GetStats() map[string]interface{} { return map[string]interface{}{} }
+
+func (f *fakeBackend) Close() error { return nil }
+
+func (f *fakeBackend) Ping(_ context.Context) error {
+	if f.fail.Load() {
+		return errors.New("injected ping failure")
+	}
+	return nil
+}
+
+// TestHealthCheckBackendTransitions covers the previously 0%-covered
+// HealthCheckBackend wrapper: optimistic start, eventual transition to
+// unhealthy under backend failures, and recovery once the backend returns.
+func TestHealthCheckBackendTransitions(t *testing.T) {
+	bt := newFakeBackend()
+	cfg := &HealthCheckConfig{
+		CheckInterval:      5 * time.Millisecond,
+		Timeout:            20 * time.Millisecond,
+		HealthyThreshold:   1,
+		UnhealthyThreshold: 2,
+	}
+	hc := NewHealthCheckBackend(bt, cfg).(*HealthCheckBackend)
+	t.Cleanup(func() { _ = hc.Close() })
+
+	// Optimistic start: healthy before any check runs.
+	if !hc.IsHealthy() {
+		t.Fatal("expected optimistic healthy start")
+	}
+
+	// Delegation: Set/Get pass through the wrapper to the backend.
+	ctx := context.Background()
+	if err := hc.Set(ctx, "k", []byte("v"), time.Minute); err != nil {
+		t.Fatalf("Set through wrapper: %v", err)
+	}
+	if got, _, ok, _ := hc.Get(ctx, "k"); !ok || string(got) != "v" {
+		t.Fatalf("Get through wrapper returned ok=%v value=%q", ok, got)
+	}
+
+	// Fail the backend: the health loop must mark it unhealthy.
+	bt.fail.Store(true)
+	waitFor(t, 3*time.Second, "backend to become unhealthy", func() bool {
+		return !hc.IsHealthy()
+	})
+
+	// Restore the backend: it must recover to healthy.
+	bt.fail.Store(false)
+	waitFor(t, 3*time.Second, "backend to recover", hc.IsHealthy)
+
+	// Stats expose health state.
+	if _, ok := hc.GetStats()["health"]; !ok {
+		t.Fatal("expected health key in GetStats")
+	}
+
+	// Remaining delegation methods route through the wrapper cleanly.
+	if ok, err := hc.Exists(ctx, "k"); err != nil || !ok {
+		t.Fatalf("Exists through wrapper: ok=%v err=%v", ok, err)
+	}
+	if ok, err := hc.Delete(ctx, "k"); err != nil || !ok {
+		t.Fatalf("Delete through wrapper: ok=%v err=%v", ok, err)
+	}
+	if err := hc.Set(ctx, "k", []byte("v"), time.Minute); err != nil {
+		t.Fatalf("Set for Clear: %v", err)
+	}
+	if err := hc.Clear(ctx); err != nil {
+		t.Fatalf("Clear through wrapper: %v", err)
+	}
+	if err := hc.Ping(ctx); err != nil {
+		t.Fatalf("Ping through wrapper: %v", err)
+	}
+}
+
+func waitFor(t *testing.T, d time.Duration, what string, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(d)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for %s", what)
+}

internal/cache/resilience/review_r152_test.go

@@ -0,0 +1,51 @@
+package resilience
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+// TestCircuitBreaker_HalfOpenGateNotExceeded guards the R152 fix to
+// AllowRequest: the open→half-open transition returned true without
+// incrementing halfOpenRequests, so the very probe that opens the
+// half-open window was free — allowing exactly one more request than
+// the configured gate (on a serial refusal loop: gate+1 pass). The
+// probe must be counted so cumulative half-open admissions stay
+// within HalfOpenMaxRequests.
+func TestCircuitBreaker_HalfOpenGateNotExceeded(t *testing.T) {
+	config := &CircuitBreakerConfig{
+		MaxFailures:         2,
+		Timeout:             50 * time.Millisecond,
+		HalfOpenMaxRequests: 2,
+	}
+	cb := NewCircuitBreaker(config)
+	ctx := context.Background()
+
+	// Open the circuit.
+	for i := 0; i < 2; i++ {
+		cb.Execute(ctx, func() error { return errors.New("boom") })
+	}
+	if cb.GetState() != StateOpen {
+		t.Fatal("expected open after failures")
+	}
+
+	time.Sleep(100 * time.Millisecond) // past timeout → cooldown expired
+
+	// Serially offer gate+overflow admission slots. The transition probe
+	// must count toward the gate; un-fixed code allows one extra.
+	allowed := 0
+	for i := 0; i < config.HalfOpenMaxRequests+5; i++ {
+		if !cb.AllowRequest() {
+			break
+		}
+		allowed++
+	}
+	if allowed > config.HalfOpenMaxRequests {
+		t.Fatalf("half-open gate exceeded: %d admitted, max %d", allowed, config.HalfOpenMaxRequests)
+	}
+	if allowed == 0 {
+		t.Fatal("at least one probe must be allowed once the cooldown expires")
+	}
+}

internal/cache/review_r138_test.go

@@ -0,0 +1,38 @@
+package cache
+
+import (
+	"testing"
+	"time"
+)
+
+// TestSet_NegativeTTLIsNoOp verifies Cache.Set with a negative TTL does not
+// store an entry (no slot consumed), matching the memory/redis backends'
+// negative-TTL convention. Previously cacheVerifiedToken fed an
+// already-expired entry (negative duration) into the hot token cache,
+// wasting a slot and misreporting currentSize (R138).
+func TestSet_NegativeTTLIsNoOp(t *testing.T) {
+	c := New(DefaultConfig())
+	defer c.Close()
+
+	// Negative TTL must not consume a cache slot.
+	if err := c.Set("neg", "v", -1*time.Second); err != nil {
+		t.Fatalf("Set with negative TTL returned error: %v", err)
+	}
+	if size := c.Size(); size != 0 {
+		t.Fatalf("negative-TTL Set must not grow cache size; got %d", size)
+	}
+	if _, exists := c.Get("neg"); exists {
+		t.Fatal("negative-TTL value must not be retrievable (should be a no-op)")
+	}
+
+	// Control: positive TTL still consumes a slot and is retrievable.
+	if err := c.Set("pos", "v", time.Minute); err != nil {
+		t.Fatalf("Set with positive TTL returned error: %v", err)
+	}
+	if size := c.Size(); size != 1 {
+		t.Fatalf("positive-TTL Set should grow cache size to 1; got %d", size)
+	}
+	if _, exists := c.Get("pos"); !exists {
+		t.Fatal("positive-TTL value should be retrievable")
+	}
+}

internal/cache/review_r139_f2_test.go

@@ -0,0 +1,42 @@
+package cache
+
+import (
+	"testing"
+	"time"
+)
+
+// TestSet_RefreshExistingDoesNotEvictAtCapacity verifies that replacing an
+// existing key at capacity does not evict an unrelated live entry. The
+// eviction decision must account for the replacement (count stays flat), so
+// refreshing the most-recent key must leave the LRU sibling in place
+// (R139 F2).
+func TestSet_RefreshExistingDoesNotEvictAtCapacity(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.MaxSize = 2
+	cfg.EnableMemoryLimit = false
+	c := New(cfg)
+	defer c.Close()
+
+	if err := c.Set("a", "va", time.Minute); err != nil {
+		t.Fatalf("set a: %v", err)
+	}
+	if err := c.Set("b", "vb", time.Minute); err != nil {
+		t.Fatalf("set b: %v", err)
+	}
+
+	// Refresh the most-recent key while at capacity. On the buggy path
+	// the LRU sibling `a` is evicted; on the fixed path the replacement
+	// is free so both entries survive.
+	if err := c.Set("b", "vb2", time.Minute); err != nil {
+		t.Fatalf("refresh b: %v", err)
+	}
+	if _, ok := c.Get("a"); !ok {
+		t.Fatal("refreshing an existing key must not evict an unrelated live entry `a`")
+	}
+	if got := c.Size(); got != 2 {
+		t.Fatalf("size should remain 2 after an in-place refresh, got %d", got)
+	}
+	if v, ok := c.Get("b"); !ok || v != "vb2" {
+		t.Fatalf("refreshed key `b` should hold the new value, got %v ok=%v", v, ok)
+	}
+}

internal/cache/review_r149_test.go

@@ -0,0 +1,30 @@
+package cache
+
+import (
+	"testing"
+	"time"
+)
+
+// TestCacheClose_NilStopCleanupNoPanic guards the R149 fix in Close():
+// stopCleanup is only allocated when EnableAutoCleanup && CleanupInterval > 0,
+// but Close() previously closed it whenever EnableAutoCleanup was set. A
+// cache built with EnableAutoCleanup=true and CleanupInterval left at its
+// zero value therefore called close(nil) and panicked during shutdown.
+func TestCacheClose_NilStopCleanupNoPanic(t *testing.T) {
+	c := New(Config{EnableAutoCleanup: true}) // CleanupInterval defaults to 0
+	if err := c.Close(); err != nil {
+		t.Fatalf("Close() with CleanupInterval=0 must not panic and return nil, got: %v", err)
+	}
+}
+
+// TestCacheClose_WithCleanupIntervalClosesOnce ensures the normal path
+// (cleanup started) still closes cleanly exactly once.
+func TestCacheClose_WithCleanupIntervalClosesOnce(t *testing.T) {
+	c := New(Config{EnableAutoCleanup: true, CleanupInterval: time.Minute})
+	if err := c.Close(); err != nil {
+		t.Fatalf("Close() with a started cleanup goroutine must succeed, got: %v", err)
+	}
+	if err := c.Close(); err == nil {
+		t.Fatal("second Close() must report already-closed")
+	}
+}