traefikoidc/internal/cache/backends/review_r124_test.go

package backends

import (
	"context"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

// blockingWriteBackend is a cache backend whose Set blocks until released,
// used to hold an async L2 write in-flight while Delete runs.
type blockingWriteBackend struct {
	mu         sync.Mutex
	data       map[string]string
	setStarted chan struct{}
	release    chan struct{}
	written    chan struct{}
	once       sync.Once
}

func (b *blockingWriteBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
	select {
	case <-b.setStarted:
	default:
		close(b.setStarted)
	}
	select {
	case <-b.release:
	case <-b.written: // never
	case <-ctx.Done():
		return nil
	}
	b.mu.Lock()
	b.data[key] = string(value)
	b.mu.Unlock()
	b.once.Do(func() { close(b.written) })
	return nil
}

func (b *blockingWriteBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	v, ok := b.data[key]
	if !ok {
		return nil, 0, false, nil
	}
	return []byte(v), time.Minute, true, nil
}

func (b *blockingWriteBackend) Delete(ctx context.Context, key string) (bool, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	_, ok := b.data[key]
	delete(b.data, key)
	return ok, nil
}

func (b *blockingWriteBackend) Exists(ctx context.Context, key string) (bool, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	_, ok := b.data[key]
	return ok, nil
}

func (b *blockingWriteBackend) Clear(ctx context.Context) error {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.data = map[string]string{}
	return nil
}

func (b *blockingWriteBackend) GetStats() map[string]interface{} { return nil }
func (b *blockingWriteBackend) Close() error                     { return nil }
func (b *blockingWriteBackend) Ping(ctx context.Context) error   { return nil }

// TestHybrid_DeleteWaitsForAsyncWrite guards the R124 fix to hybrid.go
// Delete: an in-flight async L2 write (from a non-critical Set) could land
// after Delete's L2 DEL, resurrecting the just-deleted key with a stale
// value. Delete must wait for in-flight async writes before the L2 delete.
func TestHybrid_DeleteWaitsForAsyncWrite(t *testing.T) {
	sec := &blockingWriteBackend{
		data:       map[string]string{},
		setStarted: make(chan struct{}),
		release:    make(chan struct{}),
		written:    make(chan struct{}),
	}
	primary := newMockBackend()

	hybrid, err := NewHybridBackend(&HybridConfig{
		Primary:         primary,
		Secondary:       sec,
		AsyncBufferSize: 16,
	})
	if err != nil {
		t.Fatalf("hybrid: %v", err)
	}
	defer hybrid.Close()

	ctx := context.Background()

	// Non-critical Set -> async L2 write that blocks in-flight.
	if err := hybrid.Set(ctx, "k", []byte("stale"), time.Minute); err != nil {
		t.Fatalf("set: %v", err)
	}
	select {
	case <-sec.setStarted:
	case <-time.After(2 * time.Second):
		t.Fatal("async L2 write never started")
	}

	// Issue Delete while the async write is in-flight.
	done := make(chan struct{})
	go func() {
		_, _ = hybrid.Delete(ctx, "k")
		close(done)
	}()

	// OLD behavior: Delete returns immediately (no wait for async writes);
	// it performs its L2 delete and closes done. NEW behavior: Delete
	// blocks in asyncWg.Wait until the in-flight write finishes, so done
	// does not fire until released. Distinguish the two before releasing.
	oldImmediate := false
	select {
	case <-done:
		oldImmediate = true
	case <-time.After(50 * time.Millisecond):
		// NEW: Delete is blocked waiting for the async write. 50ms is
		// far longer than OLD's immediate (non-waiting) return needs,
		// and far shorter than the worker's 500ms writeCtx, so the
		// in-flight write reliably survives to be released.
	}
	// Release the in-flight L2 write (well within the worker's writeCtx
	// timeout) so it actually lands.
	close(sec.release)

	if oldImmediate {
		// OLD: the worker write now lands AFTER Delete's L2 delete,
		// resurrecting the key.
		select {
		case <-sec.written:
		case <-time.After(2 * time.Second):
			t.Fatal("async L2 write never landed")
		}
	} else {
		// NEW: Delete waited for the write, then removed it from L2.
		select {
		case <-sec.written:
		case <-time.After(2 * time.Second):
			t.Fatal("async L2 write never landed")
		}
		select {
		case <-done:
		case <-time.After(2 * time.Second):
			t.Fatal("Delete did not return")
		}
	}

	if _, _, exists, _ := sec.Get(ctx, "k"); exists {
		t.Fatal("L2 must not contain key after Delete: the in-flight async Set resurrected it")
	}
}

// existsAlwaysBackend reports the key always present in L1 (so a backfill
// must be skipped) and counts Set calls; setCalled is closed once Set runs,
// letting the test observe a (OLD) backfill write deterministically.
type existsAlwaysBackend struct {
	setCalls  int32
	setCalled chan struct{}
	once      sync.Once
}

func newExistsAlwaysBackend() *existsAlwaysBackend {
	return &existsAlwaysBackend{setCalled: make(chan struct{})}
}

func (b *existsAlwaysBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
	atomic.AddInt32(&b.setCalls, 1)
	b.once.Do(func() { close(b.setCalled) })
	return nil
}
func (b *existsAlwaysBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
	return nil, 0, false, nil
}
func (b *existsAlwaysBackend) Delete(ctx context.Context, key string) (bool, error) {
	return false, nil
}
func (b *existsAlwaysBackend) Exists(ctx context.Context, key string) (bool, error) { return true, nil }
func (b *existsAlwaysBackend) Clear(ctx context.Context) error                      { return nil }
func (b *existsAlwaysBackend) GetStats() map[string]interface{}                     { return nil }
func (b *existsAlwaysBackend) Close() error                                         { return nil }
func (b *existsAlwaysBackend) Ping(ctx context.Context) error                       { return nil }

// TestHybrid_BackfillDoesNotClobberFreshL1 guards the R124 fix to
// hybrid.go l1BackfillWorker: the backfill only checked L2 key
// existence, so a stale L2-captured value would overwrite a fresher L1
// value written concurrently by Set. When L1 already holds the key, the
// backfill must be skipped.
func TestHybrid_BackfillDoesNotClobberFreshL1(t *testing.T) {
	primary := newExistsAlwaysBackend()
	secondary := newMockBackend()
	ctx := context.Background()
	if err := secondary.Set(ctx, "k", []byte("stale"), time.Minute); err != nil {
		t.Fatalf("seed L2: %v", err)
	}

	hybrid, err := NewHybridBackend(&HybridConfig{
		Primary:         primary,
		Secondary:       secondary,
		AsyncBufferSize: 16,
	})
	if err != nil {
		t.Fatalf("hybrid: %v", err)
	}
	defer hybrid.Close()

	// Queue a stale backfill for a key L1 already holds (fresher).
	hybrid.queueL1Backfill("k", []byte("stale"), time.Minute)

	// Wait for the worker to drain the backfill buffer.
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if len(hybrid.l1BackfillBuffer) == 0 {
			time.Sleep(20 * time.Millisecond) // let the worker act (write or skip)
			break
		}
		time.Sleep(5 * time.Millisecond)
	}

	// Deterministically observe an OLD backfill write: if the worker
	// writes, setCalled closes and setCalls becomes 1. On NEW the
	// worker skips, so nothing fires.
	select {
	case <-primary.setCalled:
	case <-time.After(500 * time.Millisecond):
	}

	if atomic.LoadInt32(&primary.setCalls) != 0 {
		t.Fatal("backfill must not overwrite the fresher L1 value (primary already had the key)")
	}
}