fix(cache): harden cache backends, resilience, recovery, and transports

- RESP -ERR no longer closes healthy conn / aborts batch; global cap replacement-aware - global health-check pass serialized (latent data race); MemoryMonitor via ResourceManager - token-exchange retry, open-circuit rejections counted as failures - TLS-posture matching transport reuse (no silent InsecureSkipVerify handoff)
2026-10-18 04:47:28 +00:00 · 2026-08-14 16:16:51 +01:00
parent 8640451877
commit 4d23d618f5
49 changed files with 2318 additions and 181 deletions
@@ -0,0 +1,241 @@
+package backends
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// blockingWriteBackend is a cache backend whose Set blocks until released,
+// used to hold an async L2 write in-flight while Delete runs.
+type blockingWriteBackend struct {
+	mu         sync.Mutex
+	data       map[string]string
+	setStarted chan struct{}
+	release    chan struct{}
+	written    chan struct{}
+	once       sync.Once
+}
+
+func (b *blockingWriteBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	select {
+	case <-b.setStarted:
+	default:
+		close(b.setStarted)
+	}
+	select {
+	case <-b.release:
+	case <-b.written: // never
+	case <-ctx.Done():
+		return nil
+	}
+	b.mu.Lock()
+	b.data[key] = string(value)
+	b.mu.Unlock()
+	b.once.Do(func() { close(b.written) })
+	return nil
+}
+
+func (b *blockingWriteBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	v, ok := b.data[key]
+	if !ok {
+		return nil, 0, false, nil
+	}
+	return []byte(v), time.Minute, true, nil
+}
+
+func (b *blockingWriteBackend) Delete(ctx context.Context, key string) (bool, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	_, ok := b.data[key]
+	delete(b.data, key)
+	return ok, nil
+}
+
+func (b *blockingWriteBackend) Exists(ctx context.Context, key string) (bool, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	_, ok := b.data[key]
+	return ok, nil
+}
+
+func (b *blockingWriteBackend) Clear(ctx context.Context) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.data = map[string]string{}
+	return nil
+}
+
+func (b *blockingWriteBackend) GetStats() map[string]interface{} { return nil }
+func (b *blockingWriteBackend) Close() error                     { return nil }
+func (b *blockingWriteBackend) Ping(ctx context.Context) error   { return nil }
+
+// TestHybrid_DeleteWaitsForAsyncWrite guards the R124 fix to hybrid.go
+// Delete: an in-flight async L2 write (from a non-critical Set) could land
+// after Delete's L2 DEL, resurrecting the just-deleted key with a stale
+// value. Delete must wait for in-flight async writes before the L2 delete.
+func TestHybrid_DeleteWaitsForAsyncWrite(t *testing.T) {
+	sec := &blockingWriteBackend{
+		data:       map[string]string{},
+		setStarted: make(chan struct{}),
+		release:    make(chan struct{}),
+		written:    make(chan struct{}),
+	}
+	primary := newMockBackend()
+
+	hybrid, err := NewHybridBackend(&HybridConfig{
+		Primary:         primary,
+		Secondary:       sec,
+		AsyncBufferSize: 16,
+	})
+	if err != nil {
+		t.Fatalf("hybrid: %v", err)
+	}
+	defer hybrid.Close()
+
+	ctx := context.Background()
+
+	// Non-critical Set -> async L2 write that blocks in-flight.
+	if err := hybrid.Set(ctx, "k", []byte("stale"), time.Minute); err != nil {
+		t.Fatalf("set: %v", err)
+	}
+	select {
+	case <-sec.setStarted:
+	case <-time.After(2 * time.Second):
+		t.Fatal("async L2 write never started")
+	}
+
+	// Issue Delete while the async write is in-flight.
+	done := make(chan struct{})
+	go func() {
+		_, _ = hybrid.Delete(ctx, "k")
+		close(done)
+	}()
+
+	// OLD behavior: Delete returns immediately (no wait for async writes);
+	// it performs its L2 delete and closes done. NEW behavior: Delete
+	// blocks in asyncWg.Wait until the in-flight write finishes, so done
+	// does not fire until released. Distinguish the two before releasing.
+	oldImmediate := false
+	select {
+	case <-done:
+		oldImmediate = true
+	case <-time.After(50 * time.Millisecond):
+		// NEW: Delete is blocked waiting for the async write. 50ms is
+		// far longer than OLD's immediate (non-waiting) return needs,
+		// and far shorter than the worker's 500ms writeCtx, so the
+		// in-flight write reliably survives to be released.
+	}
+	// Release the in-flight L2 write (well within the worker's writeCtx
+	// timeout) so it actually lands.
+	close(sec.release)
+
+	if oldImmediate {
+		// OLD: the worker write now lands AFTER Delete's L2 delete,
+		// resurrecting the key.
+		select {
+		case <-sec.written:
+		case <-time.After(2 * time.Second):
+			t.Fatal("async L2 write never landed")
+		}
+	} else {
+		// NEW: Delete waited for the write, then removed it from L2.
+		select {
+		case <-sec.written:
+		case <-time.After(2 * time.Second):
+			t.Fatal("async L2 write never landed")
+		}
+		select {
+		case <-done:
+		case <-time.After(2 * time.Second):
+			t.Fatal("Delete did not return")
+		}
+	}
+
+	if _, _, exists, _ := sec.Get(ctx, "k"); exists {
+		t.Fatal("L2 must not contain key after Delete: the in-flight async Set resurrected it")
+	}
+}
+
+// existsAlwaysBackend reports the key always present in L1 (so a backfill
+// must be skipped) and counts Set calls; setCalled is closed once Set runs,
+// letting the test observe a (OLD) backfill write deterministically.
+type existsAlwaysBackend struct {
+	setCalls  int32
+	setCalled chan struct{}
+	once      sync.Once
+}
+
+func newExistsAlwaysBackend() *existsAlwaysBackend {
+	return &existsAlwaysBackend{setCalled: make(chan struct{})}
+}
+
+func (b *existsAlwaysBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	atomic.AddInt32(&b.setCalls, 1)
+	b.once.Do(func() { close(b.setCalled) })
+	return nil
+}
+func (b *existsAlwaysBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
+	return nil, 0, false, nil
+}
+func (b *existsAlwaysBackend) Delete(ctx context.Context, key string) (bool, error) {
+	return false, nil
+}
+func (b *existsAlwaysBackend) Exists(ctx context.Context, key string) (bool, error) { return true, nil }
+func (b *existsAlwaysBackend) Clear(ctx context.Context) error                      { return nil }
+func (b *existsAlwaysBackend) GetStats() map[string]interface{}                     { return nil }
+func (b *existsAlwaysBackend) Close() error                                         { return nil }
+func (b *existsAlwaysBackend) Ping(ctx context.Context) error                       { return nil }
+
+// TestHybrid_BackfillDoesNotClobberFreshL1 guards the R124 fix to
+// hybrid.go l1BackfillWorker: the backfill only checked L2 key
+// existence, so a stale L2-captured value would overwrite a fresher L1
+// value written concurrently by Set. When L1 already holds the key, the
+// backfill must be skipped.
+func TestHybrid_BackfillDoesNotClobberFreshL1(t *testing.T) {
+	primary := newExistsAlwaysBackend()
+	secondary := newMockBackend()
+	ctx := context.Background()
+	if err := secondary.Set(ctx, "k", []byte("stale"), time.Minute); err != nil {
+		t.Fatalf("seed L2: %v", err)
+	}
+
+	hybrid, err := NewHybridBackend(&HybridConfig{
+		Primary:         primary,
+		Secondary:       secondary,
+		AsyncBufferSize: 16,
+	})
+	if err != nil {
+		t.Fatalf("hybrid: %v", err)
+	}
+	defer hybrid.Close()
+
+	// Queue a stale backfill for a key L1 already holds (fresher).
+	hybrid.queueL1Backfill("k", []byte("stale"), time.Minute)
+
+	// Wait for the worker to drain the backfill buffer.
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if len(hybrid.l1BackfillBuffer) == 0 {
+			time.Sleep(20 * time.Millisecond) // let the worker act (write or skip)
+			break
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	// Deterministically observe an OLD backfill write: if the worker
+	// writes, setCalled closes and setCalls becomes 1. On NEW the
+	// worker skips, so nothing fires.
+	select {
+	case <-primary.setCalled:
+	case <-time.After(500 * time.Millisecond):
+	}
+
+	if atomic.LoadInt32(&primary.setCalls) != 0 {
+		t.Fatal("backfill must not overwrite the fresher L1 value (primary already had the key)")
+	}
+}