Fix 6 issues surfaced by adversarial agent team (round 35)

Concurrency/lifecycle (P1, highest-value):
- singleton_resources.go: RegisterBackgroundTask now REPLACES a stopped
  task instead of reusing it. BackgroundTask.Start is sync.Once-guarded,
  so a task stopped on instance close and re-registered (Traefik
  close+recreate for the same provider) was permanently dead: IsTaskRunning
  returned false (started && !stopped) so callers retried a start that
  was a no-op. Metadata refresh, token cleanup, and health checks now
  resume across a recreate cycle.
- error_recovery.go: the shared graceful-degradation health task was
  bound to the FIRST instance's performHealthChecks (later instances'
  checks never ran) and any instance's Close stopped it for all. Added a
  global live-instance registry; the shared task runs every instance's
  checks and is stopped only when the last instance closes (mirrors the
  singleton-token-cleanup gate).

Storage/cache:
- hybrid.go recordL2Error: Store(now) then immediately compared
  time.Since(t)<1s against its own write => ALWAYS fired, so a single
  transient L2 error flipped the whole cache into fallback (full L2
  bypass -> upstream stampede). Now enters fallback only on a PRIOR
  error within the window (true "multiple errors"). Updated two tests
  that had encoded the old single-error-fallback bug.
- redis_pool.go: ConnectionPool.Put checked closed but sent on
  p.connections without holding p.mu; Close closed it under p.mu.
  Window => 'send on closed channel' panic at shutdown. Put now holds
  p.mu (the lock only Close used).

HTTP/transport:
- jwk.go fetchJWKS: added an explicit >1 MiB guard (was silent partial
  read, could yield a truncated keyset and spurious missing-kid).
- http_client_pool.go configKey: now includes every transport-behavior
  field it actually consumes (compression, HTTP2, keep-alives, timeouts,
  buffer sizes, dial) so behaviorally different configs no longer share
  one transport built with the first config's settings.

Each fix has a regression test. Verified: full source-order suite green,
golangci-lint 0 issues, 3x full -race -count=1 -shuffle=on ./... all exit 0.

internal/cache/backends/hybrid.go

@@ -663,18 +663,22 @@ func (h *HybridBackend) healthMonitor() {
 
 // recordL2Error records the timestamp of an L2 error
 func (h *HybridBackend) recordL2Error() {
-	h.lastL2Error.Store(time.Now())
+	now := time.Now()
 
-	// Check if we should enter fallback mode based on recent errors
+	// Enter fallback only if there was a PRIOR error within the last second —
+	// i.e. this is at least the second L2 error in quick succession.
+	// Checking a freshly-written timestamp would always be "recent" and
+	// enter fallback on a single transient L2 error (cache stampede).
 	if !h.fallbackMode.Load() {
-		// Simple heuristic: if we've had an error in the last second, consider L2 unhealthy
 		if lastErr := h.lastL2Error.Load(); lastErr != nil {
-			if t, ok := lastErr.(time.Time); ok && time.Since(t) < time.Second {
+			if t, ok := lastErr.(time.Time); ok && now.Sub(t) < time.Second {
 				h.fallbackMode.Store(true)
 				h.logger.Warnf("Multiple L2 errors detected, entering fallback mode")
 			}
 		}
 	}
+
+	h.lastL2Error.Store(now)
 }
 
 // extractCacheType attempts to determine the cache type from the key

internal/cache/backends/hybrid_test.go

@@ -800,7 +800,8 @@ func TestHybridBackend_Ping_RecoverFromFallback(t *testing.T) {
 
 	ctx := context.Background()
 
-	// First ping fails L2, enters fallback
+	// Repeated L2 failures (two errors within the window) enter fallback.
+	hybrid.Ping(ctx)
 	hybrid.Ping(ctx)
 	assert.True(t, hybrid.fallbackMode.Load())
 
@@ -1184,7 +1185,13 @@ func TestHybridBackend_GetMany_BatchBackendError(t *testing.T) {
 	assert.Len(t, results, 1)
 	assert.Equal(t, []byte("value1"), results["key1"])
 
-	// Batch error should trigger fallback mode
+	// A single batch error is not enough to enter fallback (avoids a cache
+	// stampede on one transient L2 error); a second error within the window is.
+	assert.False(t, hybrid.fallbackMode.Load())
+	_, err = hybrid.GetMany(ctx, []string{"key2"})
+	assert.NoError(t, err)
+
+	// Repeated batch error should trigger fallback mode
 	time.Sleep(50 * time.Millisecond)
 	assert.True(t, hybrid.fallbackMode.Load())
 }
@@ -1488,3 +1495,25 @@ func TestHybridBackend_RecordL2Error_EntersFallbackMode(t *testing.T) {
 	// Should enter fallback mode
 	assert.True(t, hybrid.fallbackMode.Load())
 }
+
+func TestHybridBackend_RecordL2Error_RequiresRepeatedErrors(t *testing.T) {
+	config := &HybridConfig{
+		Primary:   newMockBackend(),
+		Secondary: newMockBackend(),
+	}
+
+	hybrid, err := NewHybridBackend(config)
+	require.NoError(t, err)
+	defer hybrid.Close()
+
+	// A SINGLE transient L2 error must NOT flip the whole cache into fallback
+	// mode (which bypasses L2 for reads and writes -> cache stampede).
+	hybrid.recordL2Error()
+	assert.False(t, hybrid.fallbackMode.Load(),
+		"a single L2 error must not enter fallback mode")
+
+	// A second error within the 1s window (repeated failures) does.
+	hybrid.recordL2Error()
+	assert.True(t, hybrid.fallbackMode.Load(),
+		"repeated L2 errors within the window must enter fallback mode")
+}

internal/cache/backends/redis_pool.go

@@ -158,6 +158,13 @@ func (p *ConnectionPool) Put(conn *RedisConn) {
 	p.puts.Add(1)
 	p.activeConns.Add(-1)
 
+	// Hold p.mu so this is mutually exclusive with Close (which closes
+	// p.connections under the same lock). Without it, a Put that passed
+	// the closed check could send on p.connections just after Close
+	// closed it, panicking "send on closed channel" at shutdown.
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
 	if p.closed.Load() || conn.closed.Load() {
 		_ = conn.Close()
 		p.totalConns.Add(-1)

internal/cache/backends/redis_pool_test.go

@@ -688,3 +688,47 @@ func TestRedisConn_RejectOversizedArgumentBytes(t *testing.T) {
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "arguments too large")
 }
+
+// TestConnectionPool_PutDuringClose_NoPanic hammers Put/Get concurrently with
+// Close. Before the fix, Put checked closed then sent on p.connections
+// without holding p.mu; if Close closed the channel in between, the send
+// panicked ("send on closed channel") at shutdown. Run under -race.
+func TestConnectionPool_PutDuringClose_NoPanic(t *testing.T) {
+	mr := NewMiniredisServer(t)
+
+	config := &PoolConfig{
+		Address:        mr.GetAddr(),
+		MaxConnections: 5,
+		ConnectTimeout: 5 * time.Second,
+		ReadTimeout:    3 * time.Second,
+		WriteTimeout:   3 * time.Second,
+	}
+
+	pool, err := NewConnectionPool(config)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	stop := make(chan struct{})
+	var wg sync.WaitGroup
+	for g := 0; g < 8; g++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case <-stop:
+					return
+				default:
+				}
+				if conn, err := pool.Get(ctx); err == nil {
+					pool.Put(conn) // may race Close
+				}
+			}
+		}()
+	}
+
+	time.Sleep(5 * time.Millisecond)
+	_ = pool.Close()
+	close(stop)
+	wg.Wait()
+}