fix(cache): harden cache backends, resilience, recovery, and transports

- RESP -ERR no longer closes healthy conn / aborts batch; global cap replacement-aware
- global health-check pass serialized (latent data race); MemoryMonitor via ResourceManager
- token-exchange retry, open-circuit rejections counted as failures
- TLS-posture matching transport reuse (no silent InsecureSkipVerify handoff)

internal/cache/resilience/circuit_breaker.go

@@ -132,7 +132,14 @@ func (cb *CircuitBreaker) AllowRequest() bool {
 
 		if shouldRetry {
 			cb.setState(StateHalfOpen)
-			return true
+			// Count this probe toward the half-open gate. Previously the
+			// open→half-open transition returned true without incrementing
+			// halfOpenRequests, so N goroutines arriving at the cooldown
+			// simultaneously all transitioned and passed as an unbounded
+			// burst (the half-open branch below is only reached by
+			// requests that race in after the first transition) (R152).
+			current := cb.halfOpenRequests.Add(1)
+			return current <= int32(cb.config.HalfOpenMaxRequests) // nolint:gosec // HalfOpenMaxRequests is operator-bounded well below int32 max
 		}
 		return false
 

internal/cache/resilience/health_check_backend_test.go

@@ -0,0 +1,142 @@
+package resilience
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// fakeBackend is a minimal CacheBackend whose Ping can be toggled to fail,
+// so HealthCheckBackend's status transition can be driven deterministically.
+type fakeBackend struct {
+	mu   sync.Mutex
+	data map[string][]byte
+	fail atomic.Bool
+}
+
+func newFakeBackend() *fakeBackend {
+	return &fakeBackend{data: make(map[string][]byte)}
+}
+
+func (f *fakeBackend) Set(_ context.Context, key string, value []byte, _ time.Duration) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.data[key] = value
+	return nil
+}
+
+func (f *fakeBackend) Get(_ context.Context, key string) ([]byte, time.Duration, bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	v, ok := f.data[key]
+	return v, 0, ok, nil
+}
+
+func (f *fakeBackend) Delete(_ context.Context, key string) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	_, ok := f.data[key]
+	delete(f.data, key)
+	return ok, nil
+}
+
+func (f *fakeBackend) Exists(_ context.Context, key string) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	_, ok := f.data[key]
+	return ok, nil
+}
+
+func (f *fakeBackend) Clear(_ context.Context) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.data = make(map[string][]byte)
+	return nil
+}
+
+func (f *fakeBackend) GetStats() map[string]interface{} { return map[string]interface{}{} }
+
+func (f *fakeBackend) Close() error { return nil }
+
+func (f *fakeBackend) Ping(_ context.Context) error {
+	if f.fail.Load() {
+		return errors.New("injected ping failure")
+	}
+	return nil
+}
+
+// TestHealthCheckBackendTransitions covers the previously 0%-covered
+// HealthCheckBackend wrapper: optimistic start, eventual transition to
+// unhealthy under backend failures, and recovery once the backend returns.
+func TestHealthCheckBackendTransitions(t *testing.T) {
+	bt := newFakeBackend()
+	cfg := &HealthCheckConfig{
+		CheckInterval:      5 * time.Millisecond,
+		Timeout:            20 * time.Millisecond,
+		HealthyThreshold:   1,
+		UnhealthyThreshold: 2,
+	}
+	hc := NewHealthCheckBackend(bt, cfg).(*HealthCheckBackend)
+	t.Cleanup(func() { _ = hc.Close() })
+
+	// Optimistic start: healthy before any check runs.
+	if !hc.IsHealthy() {
+		t.Fatal("expected optimistic healthy start")
+	}
+
+	// Delegation: Set/Get pass through the wrapper to the backend.
+	ctx := context.Background()
+	if err := hc.Set(ctx, "k", []byte("v"), time.Minute); err != nil {
+		t.Fatalf("Set through wrapper: %v", err)
+	}
+	if got, _, ok, _ := hc.Get(ctx, "k"); !ok || string(got) != "v" {
+		t.Fatalf("Get through wrapper returned ok=%v value=%q", ok, got)
+	}
+
+	// Fail the backend: the health loop must mark it unhealthy.
+	bt.fail.Store(true)
+	waitFor(t, 3*time.Second, "backend to become unhealthy", func() bool {
+		return !hc.IsHealthy()
+	})
+
+	// Restore the backend: it must recover to healthy.
+	bt.fail.Store(false)
+	waitFor(t, 3*time.Second, "backend to recover", hc.IsHealthy)
+
+	// Stats expose health state.
+	if _, ok := hc.GetStats()["health"]; !ok {
+		t.Fatal("expected health key in GetStats")
+	}
+
+	// Remaining delegation methods route through the wrapper cleanly.
+	if ok, err := hc.Exists(ctx, "k"); err != nil || !ok {
+		t.Fatalf("Exists through wrapper: ok=%v err=%v", ok, err)
+	}
+	if ok, err := hc.Delete(ctx, "k"); err != nil || !ok {
+		t.Fatalf("Delete through wrapper: ok=%v err=%v", ok, err)
+	}
+	if err := hc.Set(ctx, "k", []byte("v"), time.Minute); err != nil {
+		t.Fatalf("Set for Clear: %v", err)
+	}
+	if err := hc.Clear(ctx); err != nil {
+		t.Fatalf("Clear through wrapper: %v", err)
+	}
+	if err := hc.Ping(ctx); err != nil {
+		t.Fatalf("Ping through wrapper: %v", err)
+	}
+}
+
+func waitFor(t *testing.T, d time.Duration, what string, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(d)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for %s", what)
+}

internal/cache/resilience/review_r152_test.go

@@ -0,0 +1,51 @@
+package resilience
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+// TestCircuitBreaker_HalfOpenGateNotExceeded guards the R152 fix to
+// AllowRequest: the open→half-open transition returned true without
+// incrementing halfOpenRequests, so the very probe that opens the
+// half-open window was free — allowing exactly one more request than
+// the configured gate (on a serial refusal loop: gate+1 pass). The
+// probe must be counted so cumulative half-open admissions stay
+// within HalfOpenMaxRequests.
+func TestCircuitBreaker_HalfOpenGateNotExceeded(t *testing.T) {
+	config := &CircuitBreakerConfig{
+		MaxFailures:         2,
+		Timeout:             50 * time.Millisecond,
+		HalfOpenMaxRequests: 2,
+	}
+	cb := NewCircuitBreaker(config)
+	ctx := context.Background()
+
+	// Open the circuit.
+	for i := 0; i < 2; i++ {
+		cb.Execute(ctx, func() error { return errors.New("boom") })
+	}
+	if cb.GetState() != StateOpen {
+		t.Fatal("expected open after failures")
+	}
+
+	time.Sleep(100 * time.Millisecond) // past timeout → cooldown expired
+
+	// Serially offer gate+overflow admission slots. The transition probe
+	// must count toward the gate; un-fixed code allows one extra.
+	allowed := 0
+	for i := 0; i < config.HalfOpenMaxRequests+5; i++ {
+		if !cb.AllowRequest() {
+			break
+		}
+		allowed++
+	}
+	if allowed > config.HalfOpenMaxRequests {
+		t.Fatalf("half-open gate exceeded: %d admitted, max %d", allowed, config.HalfOpenMaxRequests)
+	}
+	if allowed == 0 {
+		t.Fatal("at least one probe must be allowed once the cooldown expires")
+	}
+}