Add redis support for distributed caching (#83)

* Add redis support for distributed caching

* Move towards the self-provided Redis connection pool and RESP protocol implementation.
Official redis client library won't work with yaegi.

* fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* fixup! fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* fixup! fixup! fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* ... and another all nighter.

* fixup! ... and another all nighter.

* fixup! fixup! ... and another all nighter.

* fixup! fixup! fixup! ... and another all nighter.

* Resolve issue #85 by adding ability to set custom claims in JWT tokens

* Remove redundant validation in auth middleware ( issue #89 )

* Add ability to set cookie prefix for session cookies ( #87 )

* fixup! Add ability to set cookie prefix for session cookies ( #87 )

* Add ability to set cookie max age - issue #91

* Potential fix for code scanning alert no. 10: Size computation for allocation may overflow

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* fixup! Merge main into 0.8.0-redis: resolve conflicts

---------

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
This commit is contained in:
2025-11-30 02:18:46 +00:00
committed by GitHub
co-authored by Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
parent 5fcbd54955
commit e64fc7f730
318 changed files with 100989 additions and 948 deletions
+329
View File
@@ -0,0 +1,329 @@
// Package resilience provides resilience patterns for cache backends.
package resilience
import (
"context"
"errors"
"sync"
"sync/atomic"
"time"
)
// Common errors
var (
// ErrCircuitOpen is returned when the circuit breaker is open
ErrCircuitOpen = errors.New("circuit breaker is open")
// ErrTooManyRequests is returned when too many requests are made in half-open state
ErrTooManyRequests = errors.New("too many requests in half-open state")
)
// State represents the state of the circuit breaker
type State int32
const (
// StateClosed allows all operations to pass through
StateClosed State = iota
// StateOpen blocks all operations
StateOpen
// StateHalfOpen allows a limited number of operations to test recovery
StateHalfOpen
)
// String returns the string representation of the state
func (s State) String() string {
switch s {
case StateClosed:
return "closed"
case StateOpen:
return "open"
case StateHalfOpen:
return "half-open"
default:
return "unknown"
}
}
// CircuitBreakerConfig holds configuration for the circuit breaker
type CircuitBreakerConfig struct {
// MaxFailures is the number of consecutive failures before opening the circuit
MaxFailures int
// FailureThreshold is the failure rate threshold (0.0 to 1.0)
FailureThreshold float64
// Timeout is how long the circuit stays open before trying half-open
Timeout time.Duration
// HalfOpenMaxRequests is the number of requests allowed in half-open state
HalfOpenMaxRequests int
// ResetTimeout is how long to wait before resetting counters in closed state
ResetTimeout time.Duration
// OnStateChange is called when the circuit breaker changes state
OnStateChange func(from, to State)
}
// DefaultCircuitBreakerConfig returns default configuration
func DefaultCircuitBreakerConfig() *CircuitBreakerConfig {
return &CircuitBreakerConfig{
MaxFailures: 5,
FailureThreshold: 0.6,
Timeout: 30 * time.Second,
HalfOpenMaxRequests: 3,
ResetTimeout: 60 * time.Second,
}
}
// CircuitBreaker implements the circuit breaker pattern
type CircuitBreaker struct {
config *CircuitBreakerConfig
// State management
state atomic.Int32
lastStateChange time.Time
stateMu sync.RWMutex
// Failure tracking
consecutiveFailures atomic.Int32
totalRequests atomic.Int64
totalFailures atomic.Int64
halfOpenRequests atomic.Int32
// Timing
lastFailureTime time.Time
lastSuccessTime time.Time
nextRetryTime time.Time
timeMu sync.RWMutex
// Metrics
stateTransitions atomic.Int64
rejectedRequests atomic.Int64
}
// NewCircuitBreaker creates a new circuit breaker
func NewCircuitBreaker(config *CircuitBreakerConfig) *CircuitBreaker {
if config == nil {
config = DefaultCircuitBreakerConfig()
}
return &CircuitBreaker{
config: config,
lastStateChange: time.Now(),
}
}
// Execute runs a function through the circuit breaker
func (cb *CircuitBreaker) Execute(ctx context.Context, fn func() error) error {
if !cb.AllowRequest() {
cb.rejectedRequests.Add(1)
return ErrCircuitOpen
}
cb.totalRequests.Add(1)
err := fn()
if err != nil {
cb.RecordFailure()
} else {
cb.RecordSuccess()
}
return err
}
// AllowRequest checks if a request is allowed to proceed
func (cb *CircuitBreaker) AllowRequest() bool {
state := cb.GetState()
switch state {
case StateClosed:
return true
case StateOpen:
// Check if timeout has passed and we should try half-open
cb.timeMu.RLock()
shouldRetry := time.Now().After(cb.nextRetryTime)
cb.timeMu.RUnlock()
if shouldRetry {
cb.setState(StateHalfOpen)
return true
}
return false
case StateHalfOpen:
// Allow limited requests in half-open state
current := cb.halfOpenRequests.Add(1)
return current <= int32(cb.config.HalfOpenMaxRequests)
default:
return false
}
}
// RecordSuccess records a successful operation
func (cb *CircuitBreaker) RecordSuccess() {
cb.timeMu.Lock()
cb.lastSuccessTime = time.Now()
cb.timeMu.Unlock()
state := cb.GetState()
switch state {
case StateClosed:
// Reset consecutive failures
cb.consecutiveFailures.Store(0)
case StateHalfOpen:
// If we've had enough successful requests, close the circuit
successfulRequests := cb.halfOpenRequests.Load()
if successfulRequests >= int32(cb.config.HalfOpenMaxRequests) {
cb.setState(StateClosed)
cb.consecutiveFailures.Store(0)
cb.halfOpenRequests.Store(0)
}
}
}
// RecordFailure records a failed operation
func (cb *CircuitBreaker) RecordFailure() {
cb.totalFailures.Add(1)
failures := cb.consecutiveFailures.Add(1)
cb.timeMu.Lock()
cb.lastFailureTime = time.Now()
cb.timeMu.Unlock()
state := cb.GetState()
switch state {
case StateClosed:
// Check if we should open the circuit
if failures >= int32(cb.config.MaxFailures) {
cb.openCircuit()
} else if cb.config.FailureThreshold > 0 {
// Check failure rate
total := cb.totalRequests.Load()
failureCount := cb.totalFailures.Load()
if total > 10 && float64(failureCount)/float64(total) > cb.config.FailureThreshold {
cb.openCircuit()
}
}
case StateHalfOpen:
// Any failure in half-open state reopens the circuit
cb.openCircuit()
}
}
// openCircuit transitions to open state
func (cb *CircuitBreaker) openCircuit() {
cb.setState(StateOpen)
cb.halfOpenRequests.Store(0)
cb.timeMu.Lock()
cb.nextRetryTime = time.Now().Add(cb.config.Timeout)
cb.timeMu.Unlock()
}
// GetState returns the current state
func (cb *CircuitBreaker) GetState() State {
return State(cb.state.Load())
}
// setState changes the circuit breaker state
func (cb *CircuitBreaker) setState(newState State) {
oldState := State(cb.state.Swap(int32(newState)))
if oldState != newState {
cb.stateTransitions.Add(1)
cb.stateMu.Lock()
cb.lastStateChange = time.Now()
cb.stateMu.Unlock()
if cb.config.OnStateChange != nil {
cb.config.OnStateChange(oldState, newState)
}
}
}
// Reset resets the circuit breaker to closed state
func (cb *CircuitBreaker) Reset() {
cb.setState(StateClosed)
cb.consecutiveFailures.Store(0)
cb.totalRequests.Store(0)
cb.totalFailures.Store(0)
cb.halfOpenRequests.Store(0)
cb.rejectedRequests.Store(0)
cb.stateTransitions.Store(0)
now := time.Now()
cb.timeMu.Lock()
cb.lastFailureTime = now
cb.lastSuccessTime = now
cb.nextRetryTime = now
cb.timeMu.Unlock()
cb.stateMu.Lock()
cb.lastStateChange = now
cb.stateMu.Unlock()
}
// Stats returns circuit breaker statistics
func (cb *CircuitBreaker) Stats() CircuitBreakerStats {
cb.timeMu.RLock()
lastFailure := cb.lastFailureTime
lastSuccess := cb.lastSuccessTime
nextRetry := cb.nextRetryTime
cb.timeMu.RUnlock()
cb.stateMu.RLock()
lastChange := cb.lastStateChange
cb.stateMu.RUnlock()
totalReq := cb.totalRequests.Load()
totalFail := cb.totalFailures.Load()
successRate := float64(0)
if totalReq > 0 {
successRate = float64(totalReq-totalFail) / float64(totalReq)
}
return CircuitBreakerStats{
State: cb.GetState(),
ConsecutiveFailures: cb.consecutiveFailures.Load(),
TotalRequests: totalReq,
TotalFailures: totalFail,
SuccessRate: successRate,
RejectedRequests: cb.rejectedRequests.Load(),
StateTransitions: cb.stateTransitions.Load(),
LastFailureTime: lastFailure,
LastSuccessTime: lastSuccess,
LastStateChange: lastChange,
NextRetryTime: nextRetry,
}
}
// CircuitBreakerStats holds statistics for the circuit breaker
type CircuitBreakerStats struct {
State State
ConsecutiveFailures int32
TotalRequests int64
TotalFailures int64
SuccessRate float64
RejectedRequests int64
StateTransitions int64
LastFailureTime time.Time
LastSuccessTime time.Time
LastStateChange time.Time
NextRetryTime time.Time
}
// IsHealthy returns true if the circuit breaker is in a healthy state
func (cb *CircuitBreaker) IsHealthy() bool {
return cb.GetState() != StateOpen
}
+141
View File
@@ -0,0 +1,141 @@
// Package resilience provides resilience patterns for cache backends.
package resilience
import (
"context"
"time"
"github.com/lukaszraczylo/traefikoidc/internal/cache/backends"
)
// CircuitBreakerBackend wraps a cache backend with circuit breaker protection
type CircuitBreakerBackend struct {
backend backends.CacheBackend
cb *CircuitBreaker
}
// NewCircuitBreakerBackend creates a new circuit breaker wrapped backend
func NewCircuitBreakerBackend(b backends.CacheBackend, config *CircuitBreakerConfig) backends.CacheBackend {
if config == nil {
config = DefaultCircuitBreakerConfig()
}
return &CircuitBreakerBackend{
backend: b,
cb: NewCircuitBreaker(config),
}
}
// Set stores a value with circuit breaker protection
func (c *CircuitBreakerBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
if !c.cb.AllowRequest() {
return backends.ErrCircuitOpen
}
err := c.backend.Set(ctx, key, value, ttl)
if err == nil {
c.cb.RecordSuccess()
} else {
c.cb.RecordFailure()
}
return err
}
// Get retrieves a value with circuit breaker protection
func (c *CircuitBreakerBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
if !c.cb.AllowRequest() {
return nil, 0, false, backends.ErrCircuitOpen
}
value, ttl, exists, err := c.backend.Get(ctx, key)
if err == nil {
c.cb.RecordSuccess()
} else {
c.cb.RecordFailure()
}
return value, ttl, exists, err
}
// Delete removes a key with circuit breaker protection
func (c *CircuitBreakerBackend) Delete(ctx context.Context, key string) (bool, error) {
if !c.cb.AllowRequest() {
return false, backends.ErrCircuitOpen
}
deleted, err := c.backend.Delete(ctx, key)
if err == nil {
c.cb.RecordSuccess()
} else {
c.cb.RecordFailure()
}
return deleted, err
}
// Exists checks if a key exists with circuit breaker protection
func (c *CircuitBreakerBackend) Exists(ctx context.Context, key string) (bool, error) {
if !c.cb.AllowRequest() {
return false, backends.ErrCircuitOpen
}
exists, err := c.backend.Exists(ctx, key)
if err == nil {
c.cb.RecordSuccess()
} else {
c.cb.RecordFailure()
}
return exists, err
}
// Clear removes all keys with circuit breaker protection
func (c *CircuitBreakerBackend) Clear(ctx context.Context) error {
if !c.cb.AllowRequest() {
return backends.ErrCircuitOpen
}
err := c.backend.Clear(ctx)
if err == nil {
c.cb.RecordSuccess()
} else {
c.cb.RecordFailure()
}
return err
}
// GetStats returns statistics including circuit breaker state
func (c *CircuitBreakerBackend) GetStats() map[string]interface{} {
stats := c.backend.GetStats()
if stats == nil {
stats = make(map[string]interface{})
}
cbStats := c.cb.Stats()
stats["circuit_breaker"] = map[string]interface{}{
"state": cbStats.State.String(),
"consecutive_failures": cbStats.ConsecutiveFailures,
"total_requests": cbStats.TotalRequests,
"total_failures": cbStats.TotalFailures,
"success_rate": cbStats.SuccessRate,
}
return stats
}
// Ping checks backend health with circuit breaker protection
func (c *CircuitBreakerBackend) Ping(ctx context.Context) error {
if !c.cb.AllowRequest() {
return backends.ErrCircuitOpen
}
err := c.backend.Ping(ctx)
if err == nil {
c.cb.RecordSuccess()
} else {
c.cb.RecordFailure()
}
return err
}
// Close shuts down the backend
func (c *CircuitBreakerBackend) Close() error {
return c.backend.Close()
}
@@ -0,0 +1,561 @@
//go:build !yaegi
package resilience
import (
"context"
"errors"
"sync"
"testing"
"time"
"github.com/lukaszraczylo/traefikoidc/internal/cache/backends"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// mockBackend is a simple mock implementation for testing
type mockBackend struct {
data map[string]mockEntry
mu sync.RWMutex
failSet bool
failGet bool
failDelete bool
failExists bool
failClear bool
failPing bool
callCount int
}
type mockEntry struct {
value []byte
expiresAt time.Time
}
func newMockBackend() *mockBackend {
return &mockBackend{
data: make(map[string]mockEntry),
}
}
func (m *mockBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
m.mu.Lock()
defer m.mu.Unlock()
m.callCount++
if m.failSet {
return errors.New("mock set error")
}
expiresAt := time.Now().Add(ttl)
if ttl == 0 {
expiresAt = time.Now().Add(24 * time.Hour)
}
m.data[key] = mockEntry{
value: value,
expiresAt: expiresAt,
}
return nil
}
func (m *mockBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
m.mu.RLock()
defer m.mu.RUnlock()
m.callCount++
if m.failGet {
return nil, 0, false, errors.New("mock get error")
}
entry, exists := m.data[key]
if !exists {
return nil, 0, false, nil
}
if time.Now().After(entry.expiresAt) {
return nil, 0, false, nil
}
ttl := time.Until(entry.expiresAt)
return entry.value, ttl, true, nil
}
func (m *mockBackend) Delete(ctx context.Context, key string) (bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.callCount++
if m.failDelete {
return false, errors.New("mock delete error")
}
_, existed := m.data[key]
delete(m.data, key)
return existed, nil
}
func (m *mockBackend) Exists(ctx context.Context, key string) (bool, error) {
m.mu.RLock()
defer m.mu.RUnlock()
m.callCount++
if m.failExists {
return false, errors.New("mock exists error")
}
entry, exists := m.data[key]
if !exists {
return false, nil
}
if time.Now().After(entry.expiresAt) {
return false, nil
}
return true, nil
}
func (m *mockBackend) Clear(ctx context.Context) error {
m.mu.Lock()
defer m.mu.Unlock()
m.callCount++
if m.failClear {
return errors.New("mock clear error")
}
m.data = make(map[string]mockEntry)
return nil
}
func (m *mockBackend) GetStats() map[string]interface{} {
m.mu.RLock()
defer m.mu.RUnlock()
return map[string]interface{}{
"hits": int64(0),
"misses": int64(0),
"call_count": m.callCount,
}
}
func (m *mockBackend) Close() error {
return nil
}
func (m *mockBackend) Ping(ctx context.Context) error {
m.mu.Lock()
defer m.mu.Unlock()
m.callCount++
if m.failPing {
return errors.New("mock ping error")
}
return nil
}
// Constructor Tests
func TestNewCircuitBreakerBackend_WithDefaultConfig(t *testing.T) {
mockBE := newMockBackend()
cb := NewCircuitBreakerBackend(mockBE, nil)
require.NotNil(t, cb)
// Verify it implements the interface (compile-time check)
var _ backends.CacheBackend = cb
}
func TestNewCircuitBreakerBackend_WithCustomConfig(t *testing.T) {
mockBE := newMockBackend()
config := &CircuitBreakerConfig{
MaxFailures: 3,
FailureThreshold: 0.5,
Timeout: 5 * time.Second,
HalfOpenMaxRequests: 2,
ResetTimeout: 10 * time.Second,
}
cb := NewCircuitBreakerBackend(mockBE, config)
require.NotNil(t, cb)
}
// Set Operation Tests
func TestCircuitBreakerBackend_Set_Success(t *testing.T) {
mockBE := newMockBackend()
cb := NewCircuitBreakerBackend(mockBE, nil)
ctx := context.Background()
err := cb.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
assert.NoError(t, err)
assert.Equal(t, 1, mockBE.callCount)
// Verify value was stored
value, _, exists, _ := mockBE.Get(ctx, "key1")
assert.True(t, exists)
assert.Equal(t, []byte("value1"), value)
}
func TestCircuitBreakerBackend_Set_Failure(t *testing.T) {
mockBE := newMockBackend()
mockBE.failSet = true
cb := NewCircuitBreakerBackend(mockBE, nil)
ctx := context.Background()
err := cb.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
assert.Error(t, err)
}
func TestCircuitBreakerBackend_Set_CircuitOpen(t *testing.T) {
mockBE := newMockBackend()
mockBE.failSet = true
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 100 * time.Millisecond,
}
cb := NewCircuitBreakerBackend(mockBE, config)
ctx := context.Background()
// Trigger failures to open circuit
for i := 0; i < 5; i++ {
cb.Set(ctx, "key", []byte("value"), 1*time.Minute)
}
// Circuit should be open now
err := cb.Set(ctx, "key2", []byte("value2"), 1*time.Minute)
assert.Error(t, err)
assert.Equal(t, backends.ErrCircuitOpen, err)
}
// Get Operation Tests
func TestCircuitBreakerBackend_Get_Success(t *testing.T) {
mockBE := newMockBackend()
cb := NewCircuitBreakerBackend(mockBE, nil)
ctx := context.Background()
// First set a value
mockBE.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
// Now get it through circuit breaker
value, _, exists, err := cb.Get(ctx, "key1")
assert.NoError(t, err)
assert.True(t, exists)
assert.Equal(t, []byte("value1"), value)
}
func TestCircuitBreakerBackend_Get_Failure(t *testing.T) {
mockBE := newMockBackend()
mockBE.failGet = true
cb := NewCircuitBreakerBackend(mockBE, nil)
ctx := context.Background()
_, _, _, err := cb.Get(ctx, "key1")
assert.Error(t, err)
}
func TestCircuitBreakerBackend_Get_CircuitOpen(t *testing.T) {
mockBE := newMockBackend()
mockBE.failGet = true
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 100 * time.Millisecond,
}
cb := NewCircuitBreakerBackend(mockBE, config)
ctx := context.Background()
// Trigger failures
for i := 0; i < 5; i++ {
cb.Get(ctx, "key")
}
// Circuit should be open
_, _, _, err := cb.Get(ctx, "key2")
assert.Error(t, err)
assert.Equal(t, backends.ErrCircuitOpen, err)
}
// Delete Operation Tests
func TestCircuitBreakerBackend_Delete_Success(t *testing.T) {
mockBE := newMockBackend()
cb := NewCircuitBreakerBackend(mockBE, nil)
ctx := context.Background()
// Set a value first
mockBE.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
// Delete through circuit breaker
deleted, err := cb.Delete(ctx, "key1")
assert.NoError(t, err)
assert.True(t, deleted)
// Verify it's deleted
exists, _ := mockBE.Exists(ctx, "key1")
assert.False(t, exists)
}
func TestCircuitBreakerBackend_Delete_CircuitOpen(t *testing.T) {
mockBE := newMockBackend()
mockBE.failDelete = true
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 100 * time.Millisecond,
}
cb := NewCircuitBreakerBackend(mockBE, config)
ctx := context.Background()
// Trigger failures
for i := 0; i < 5; i++ {
cb.Delete(ctx, "key")
}
// Circuit should be open
_, err := cb.Delete(ctx, "key2")
assert.Error(t, err)
assert.Equal(t, backends.ErrCircuitOpen, err)
}
// Exists Operation Tests
func TestCircuitBreakerBackend_Exists_Success(t *testing.T) {
mockBE := newMockBackend()
cb := NewCircuitBreakerBackend(mockBE, nil)
ctx := context.Background()
// Set a value first
mockBE.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
// Check existence through circuit breaker
exists, err := cb.Exists(ctx, "key1")
assert.NoError(t, err)
assert.True(t, exists)
}
func TestCircuitBreakerBackend_Exists_CircuitOpen(t *testing.T) {
mockBE := newMockBackend()
mockBE.failExists = true
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 100 * time.Millisecond,
}
cb := NewCircuitBreakerBackend(mockBE, config)
ctx := context.Background()
// Trigger failures
for i := 0; i < 5; i++ {
cb.Exists(ctx, "key")
}
// Circuit should be open
_, err := cb.Exists(ctx, "key2")
assert.Error(t, err)
assert.Equal(t, backends.ErrCircuitOpen, err)
}
// Clear Operation Tests
func TestCircuitBreakerBackend_Clear_Success(t *testing.T) {
mockBE := newMockBackend()
cb := NewCircuitBreakerBackend(mockBE, nil)
ctx := context.Background()
// Set some values
mockBE.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
mockBE.Set(ctx, "key2", []byte("value2"), 1*time.Minute)
// Clear through circuit breaker
err := cb.Clear(ctx)
assert.NoError(t, err)
// Verify cleared
exists1, _ := mockBE.Exists(ctx, "key1")
exists2, _ := mockBE.Exists(ctx, "key2")
assert.False(t, exists1)
assert.False(t, exists2)
}
func TestCircuitBreakerBackend_Clear_CircuitOpen(t *testing.T) {
mockBE := newMockBackend()
mockBE.failClear = true
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 100 * time.Millisecond,
}
cb := NewCircuitBreakerBackend(mockBE, config)
ctx := context.Background()
// Trigger failures
for i := 0; i < 5; i++ {
cb.Clear(ctx)
}
// Circuit should be open
err := cb.Clear(ctx)
assert.Error(t, err)
assert.Equal(t, backends.ErrCircuitOpen, err)
}
// GetStats Tests
func TestCircuitBreakerBackend_GetStats(t *testing.T) {
mockBE := newMockBackend()
cb := NewCircuitBreakerBackend(mockBE, nil)
ctx := context.Background()
// Perform some operations
cb.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
cb.Get(ctx, "key1")
stats := cb.GetStats()
require.NotNil(t, stats)
// Should have circuit breaker stats
assert.Contains(t, stats, "circuit_breaker")
cbStats, ok := stats["circuit_breaker"].(map[string]interface{})
require.True(t, ok)
// Verify circuit breaker stats fields
assert.Contains(t, cbStats, "state")
assert.Contains(t, cbStats, "consecutive_failures")
assert.Contains(t, cbStats, "total_requests")
assert.Contains(t, cbStats, "total_failures")
assert.Contains(t, cbStats, "success_rate")
}
func TestCircuitBreakerBackend_GetStats_NilBackendStats(t *testing.T) {
// Create a mock backend that returns nil stats
mockBE := &mockBackendNilStats{}
cb := NewCircuitBreakerBackend(mockBE, nil)
stats := cb.GetStats()
require.NotNil(t, stats)
assert.Contains(t, stats, "circuit_breaker")
}
// mockBackendNilStats returns nil from GetStats
type mockBackendNilStats struct {
mockBackend
}
func (m *mockBackendNilStats) GetStats() map[string]interface{} {
return nil
}
// Ping Tests
func TestCircuitBreakerBackend_Ping_Success(t *testing.T) {
mockBE := newMockBackend()
cb := NewCircuitBreakerBackend(mockBE, nil)
ctx := context.Background()
err := cb.Ping(ctx)
assert.NoError(t, err)
}
func TestCircuitBreakerBackend_Ping_CircuitOpen(t *testing.T) {
mockBE := newMockBackend()
mockBE.failPing = true
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 100 * time.Millisecond,
}
cb := NewCircuitBreakerBackend(mockBE, config)
ctx := context.Background()
// Trigger failures
for i := 0; i < 5; i++ {
cb.Ping(ctx)
}
// Circuit should be open
err := cb.Ping(ctx)
assert.Error(t, err)
assert.Equal(t, backends.ErrCircuitOpen, err)
}
// Close Tests
func TestCircuitBreakerBackend_Close(t *testing.T) {
mockBE := newMockBackend()
cb := NewCircuitBreakerBackend(mockBE, nil)
err := cb.Close()
assert.NoError(t, err)
}
// Circuit Recovery Test
func TestCircuitBreakerBackend_CircuitRecovery(t *testing.T) {
mockBE := newMockBackend()
mockBE.failSet = true
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 200 * time.Millisecond,
HalfOpenMaxRequests: 1,
}
cb := NewCircuitBreakerBackend(mockBE, config)
ctx := context.Background()
// Trigger failures to open circuit
for i := 0; i < 5; i++ {
cb.Set(ctx, "key", []byte("value"), 1*time.Minute)
}
// Verify circuit is open
err := cb.Set(ctx, "key2", []byte("value2"), 1*time.Minute)
assert.Equal(t, backends.ErrCircuitOpen, err)
// Wait for timeout
time.Sleep(250 * time.Millisecond)
// Fix the backend
mockBE.mu.Lock()
mockBE.failSet = false
mockBE.mu.Unlock()
// Circuit should be in half-open state, allow a test request
err = cb.Set(ctx, "key3", []byte("value3"), 1*time.Minute)
// After success threshold is met, circuit should close
if err == nil {
// Circuit recovered
err2 := cb.Set(ctx, "key4", []byte("value4"), 1*time.Minute)
assert.NoError(t, err2, "Circuit should be closed after recovery")
}
}
+553
View File
@@ -0,0 +1,553 @@
package resilience
import (
"context"
"errors"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
// TestCircuitBreaker_StateTransitions tests state machine transitions
func TestCircuitBreaker_StateTransitions(t *testing.T) {
t.Parallel()
config := &CircuitBreakerConfig{
MaxFailures: 3,
Timeout: 100 * time.Millisecond,
HalfOpenMaxRequests: 2,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
t.Run("Initial state is closed", func(t *testing.T) {
assert.Equal(t, StateClosed, cb.GetState())
})
t.Run("Closed to Open after max failures", func(t *testing.T) {
cb.Reset()
// Simulate failures
for i := 0; i < 3; i++ {
cb.Execute(ctx, func() error {
return errors.New("test error")
})
}
assert.Equal(t, StateOpen, cb.GetState())
})
t.Run("Open to HalfOpen after timeout", func(t *testing.T) {
// Open the circuit
cb.Reset()
for i := 0; i < 3; i++ {
cb.Execute(ctx, func() error {
return errors.New("test error")
})
}
assert.Equal(t, StateOpen, cb.GetState())
// Wait for timeout
time.Sleep(150 * time.Millisecond)
// Should allow request and transition to half-open
err := cb.Execute(ctx, func() error {
return nil
})
assert.NoError(t, err)
assert.Equal(t, StateHalfOpen, cb.GetState())
})
t.Run("HalfOpen to Closed after successful requests", func(t *testing.T) {
// Open circuit then wait for half-open
cb.Reset()
for i := 0; i < 3; i++ {
cb.Execute(ctx, func() error {
return errors.New("test error")
})
}
assert.Equal(t, StateOpen, cb.GetState())
time.Sleep(150 * time.Millisecond)
// First request transitions to half-open and succeeds
err := cb.Execute(ctx, func() error {
return nil
})
assert.NoError(t, err)
// Should be in half-open after first request
state := cb.GetState()
assert.True(t, state == StateHalfOpen || state == StateClosed,
"After first successful request, should be half-open or potentially closed")
if state == StateHalfOpen {
// Need more successful requests to close
// The exact number depends on implementation but should be within HalfOpenMaxRequests
for i := 0; i < config.HalfOpenMaxRequests; i++ {
cb.Execute(ctx, func() error {
return nil
})
}
// After multiple successful requests, should eventually close
finalState := cb.GetState()
assert.True(t, finalState == StateClosed || finalState == StateHalfOpen,
"After successful requests, circuit should transition towards closed")
}
})
t.Run("HalfOpen to Open on failure", func(t *testing.T) {
// Open circuit then wait for half-open
cb.Reset()
for i := 0; i < 3; i++ {
cb.Execute(ctx, func() error {
return errors.New("test error")
})
}
time.Sleep(150 * time.Millisecond)
// First call transitions to half-open, second failure reopens
cb.Execute(ctx, func() error {
return errors.New("test error")
})
assert.Equal(t, StateOpen, cb.GetState())
})
}
// TestCircuitBreaker_OpenCircuitBlocks tests that open circuit blocks requests
func TestCircuitBreaker_OpenCircuitBlocks(t *testing.T) {
t.Parallel()
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 1 * time.Second,
HalfOpenMaxRequests: 1,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
// Trigger failures to open circuit
for i := 0; i < 2; i++ {
cb.Execute(ctx, func() error {
return errors.New("test error")
})
}
assert.Equal(t, StateOpen, cb.GetState())
// Requests should be blocked
err := cb.Execute(ctx, func() error {
t.Fatal("Should not execute function when circuit is open")
return nil
})
assert.Error(t, err)
assert.Equal(t, ErrCircuitOpen, err)
}
// TestCircuitBreaker_HalfOpenMaxRequests tests max requests in half-open state
func TestCircuitBreaker_HalfOpenMaxRequests(t *testing.T) {
t.Parallel()
config := &CircuitBreakerConfig{
MaxFailures: 3,
Timeout: 100 * time.Millisecond,
HalfOpenMaxRequests: 2,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
// Open circuit then wait for half-open
for i := 0; i < 3; i++ {
cb.Execute(ctx, func() error {
return errors.New("test error")
})
}
assert.Equal(t, StateOpen, cb.GetState())
time.Sleep(150 * time.Millisecond)
// After timeout, circuit should allow transition to half-open
// Execute HalfOpenMaxRequests successful requests
successCount := 0
for i := 0; i < config.HalfOpenMaxRequests; i++ {
err := cb.Execute(ctx, func() error {
successCount++
return nil
})
// Should allow up to HalfOpenMaxRequests
assert.NoError(t, err)
}
// Verify we executed the expected number
assert.Equal(t, config.HalfOpenMaxRequests, successCount)
// After successful requests, circuit behavior depends on implementation
// It could close (allowing more requests) or stay half-open (blocking)
// The important thing is that we allowed exactly HalfOpenMaxRequests
}
// TestCircuitBreaker_SuccessResetsFailures tests failure counter reset
func TestCircuitBreaker_SuccessResetsFailures(t *testing.T) {
t.Parallel()
config := &CircuitBreakerConfig{
MaxFailures: 3,
Timeout: 100 * time.Millisecond,
HalfOpenMaxRequests: 1,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
// Have some failures (but less than max)
cb.Execute(ctx, func() error {
return errors.New("error")
})
cb.Execute(ctx, func() error {
return errors.New("error")
})
assert.Equal(t, StateClosed, cb.GetState())
stats := cb.Stats()
assert.Equal(t, int32(2), stats.ConsecutiveFailures)
// One success should reset failures
cb.Execute(ctx, func() error {
return nil
})
assert.Equal(t, StateClosed, cb.GetState())
stats = cb.Stats()
assert.Equal(t, int32(0), stats.ConsecutiveFailures)
}
// TestCircuitBreaker_ConcurrentAccess tests thread safety
func TestCircuitBreaker_ConcurrentAccess(t *testing.T) {
t.Parallel()
config := &CircuitBreakerConfig{
MaxFailures: 10,
Timeout: 100 * time.Millisecond,
HalfOpenMaxRequests: 5,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
var wg sync.WaitGroup
goroutines := 20
iterations := 50
for i := 0; i < goroutines; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
for j := 0; j < iterations; j++ {
// Mix of successes and failures
cb.Execute(ctx, func() error {
if (id+j)%3 == 0 {
return errors.New("test error")
}
return nil
})
// Random state checks
_ = cb.GetState()
_ = cb.Stats()
}
}(i)
}
wg.Wait()
// Should complete without panics
stats := cb.Stats()
assert.NotNil(t, stats)
}
// TestCircuitBreaker_Stats tests statistics tracking
func TestCircuitBreaker_Stats(t *testing.T) {
t.Parallel()
config := &CircuitBreakerConfig{
MaxFailures: 5,
Timeout: 100 * time.Millisecond,
HalfOpenMaxRequests: 2,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
// Execute some requests
cb.Execute(ctx, func() error { return nil }) // Success
cb.Execute(ctx, func() error { return errors.New("error") }) // Failure
cb.Execute(ctx, func() error { return errors.New("error") }) // Failure
stats := cb.Stats()
assert.Equal(t, StateClosed, stats.State)
assert.Equal(t, int64(3), stats.TotalRequests)
assert.Equal(t, int64(2), stats.TotalFailures)
assert.Equal(t, int32(2), stats.ConsecutiveFailures)
}
// TestCircuitBreaker_Reset tests circuit reset
func TestCircuitBreaker_Reset(t *testing.T) {
t.Parallel()
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 100 * time.Millisecond,
HalfOpenMaxRequests: 1,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
// Open the circuit
for i := 0; i < 2; i++ {
cb.Execute(ctx, func() error {
return errors.New("error")
})
}
assert.Equal(t, StateOpen, cb.GetState())
// Reset
cb.Reset()
assert.Equal(t, StateClosed, cb.GetState())
stats := cb.Stats()
assert.Equal(t, int32(0), stats.ConsecutiveFailures)
assert.Equal(t, int64(0), stats.TotalRequests)
assert.Equal(t, int64(0), stats.TotalFailures)
}
// TestCircuitBreaker_StateChangeCallback tests state change notifications
func TestCircuitBreaker_StateChangeCallback(t *testing.T) {
t.Parallel()
var transitions []string
var mu sync.Mutex
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 50 * time.Millisecond,
HalfOpenMaxRequests: 1,
OnStateChange: func(from, to State) {
mu.Lock()
defer mu.Unlock()
transitions = append(transitions, from.String()+"->"+to.String())
},
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
// Trigger state transitions
// Closed -> Open
for i := 0; i < 2; i++ {
cb.Execute(ctx, func() error {
return errors.New("error")
})
}
// Should be open now
assert.Equal(t, StateOpen, cb.GetState())
// Wait for timeout to allow half-open transition
time.Sleep(100 * time.Millisecond)
// Open -> HalfOpen on first request after timeout
err := cb.Execute(ctx, func() error {
return nil
})
assert.NoError(t, err)
// Execute more successful requests to trigger HalfOpen -> Closed
for i := 0; i < config.HalfOpenMaxRequests-1; i++ {
cb.Execute(ctx, func() error {
return nil
})
}
mu.Lock()
defer mu.Unlock()
assert.Contains(t, transitions, "closed->open")
assert.Contains(t, transitions, "open->half-open")
}
// TestCircuitBreaker_IsHealthy tests health check
func TestCircuitBreaker_IsHealthy(t *testing.T) {
t.Parallel()
config := &CircuitBreakerConfig{
MaxFailures: 2,
Timeout: 100 * time.Millisecond,
HalfOpenMaxRequests: 1,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
// Initially healthy
assert.True(t, cb.IsHealthy())
// Open circuit
for i := 0; i < 2; i++ {
cb.Execute(ctx, func() error {
return errors.New("error")
})
}
assert.Equal(t, StateOpen, cb.GetState())
assert.False(t, cb.IsHealthy(), "Should not be healthy when open")
// Wait for timeout and allow successful request
time.Sleep(150 * time.Millisecond)
cb.Execute(ctx, func() error {
return nil
})
// Should be healthy after recovery
assert.True(t, cb.IsHealthy(), "Should be healthy after recovery")
}
// TestCircuitBreaker_RapidFailures tests rapid consecutive failures
func TestCircuitBreaker_RapidFailures(t *testing.T) {
t.Parallel()
config := &CircuitBreakerConfig{
MaxFailures: 5,
Timeout: 200 * time.Millisecond,
HalfOpenMaxRequests: 1,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
// Rapid failures
for i := 0; i < 10; i++ {
cb.Execute(ctx, func() error {
return errors.New("rapid error")
})
}
assert.Equal(t, StateOpen, cb.GetState())
stats := cb.Stats()
assert.GreaterOrEqual(t, stats.TotalFailures, int64(5))
}
// TestCircuitBreaker_TimeoutAccuracy tests timeout precision
func TestCircuitBreaker_TimeoutAccuracy(t *testing.T) {
t.Parallel()
timeout := 100 * time.Millisecond
config := &CircuitBreakerConfig{
MaxFailures: 1,
Timeout: timeout,
HalfOpenMaxRequests: 1,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
// Open circuit
cb.Execute(ctx, func() error {
return errors.New("error")
})
assert.Equal(t, StateOpen, cb.GetState())
// Wait just before timeout
time.Sleep(timeout - 20*time.Millisecond)
assert.False(t, cb.IsHealthy())
// Wait until after timeout
time.Sleep(40 * time.Millisecond)
// After timeout, AllowRequest should return true for transition to half-open
assert.True(t, cb.AllowRequest())
}
// TestCircuitBreaker_DefaultConfig tests default configuration
func TestCircuitBreaker_DefaultConfig(t *testing.T) {
t.Parallel()
cb := NewCircuitBreaker(nil) // Should use defaults
assert.NotNil(t, cb)
assert.Equal(t, StateClosed, cb.GetState())
// Verify defaults by triggering circuit breaker behavior
ctx := context.Background()
// Test that it takes 5 failures to open (default MaxFailures)
for i := 0; i < 4; i++ {
cb.Execute(ctx, func() error {
return errors.New("error")
})
}
assert.Equal(t, StateClosed, cb.GetState(), "Should still be closed after 4 failures")
// 5th failure should open it
cb.Execute(ctx, func() error {
return errors.New("error")
})
assert.Equal(t, StateOpen, cb.GetState(), "Should be open after 5 failures (default threshold)")
}
// TestCircuitBreaker_StateString tests state string representation
func TestCircuitBreaker_StateString(t *testing.T) {
t.Parallel()
assert.Equal(t, "closed", StateClosed.String())
assert.Equal(t, "open", StateOpen.String())
assert.Equal(t, "half-open", StateHalfOpen.String())
assert.Equal(t, "unknown", State(999).String())
}
// Benchmark circuit breaker performance
func BenchmarkCircuitBreaker_Execute(b *testing.B) {
config := &CircuitBreakerConfig{
MaxFailures: 100,
Timeout: 1 * time.Second,
HalfOpenMaxRequests: 10,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
b.ResetTimer()
for i := 0; i < b.N; i++ {
cb.Execute(ctx, func() error {
return nil
})
}
}
func BenchmarkCircuitBreaker_ExecuteWithFailures(b *testing.B) {
config := &CircuitBreakerConfig{
MaxFailures: 1000,
Timeout: 1 * time.Second,
HalfOpenMaxRequests: 10,
}
cb := NewCircuitBreaker(config)
ctx := context.Background()
b.ResetTimer()
for i := 0; i < b.N; i++ {
cb.Execute(ctx, func() error {
if i%10 == 0 {
return errors.New("error")
}
return nil
})
}
}
+375
View File
@@ -0,0 +1,375 @@
// Package resilience provides resilience patterns for cache backends.
package resilience
import (
"context"
"sync"
"sync/atomic"
"time"
)
// HealthStatus represents the health status of a backend
type HealthStatus int32
const (
// HealthUnknown indicates unknown health status
HealthUnknown HealthStatus = iota
// HealthHealthy indicates the backend is healthy
HealthHealthy
// HealthDegraded indicates the backend is degraded but operational
HealthDegraded
// HealthUnhealthy indicates the backend is unhealthy
HealthUnhealthy
)
// String returns the string representation of the health status
func (h HealthStatus) String() string {
switch h {
case HealthHealthy:
return "healthy"
case HealthDegraded:
return "degraded"
case HealthUnhealthy:
return "unhealthy"
default:
return "unknown"
}
}
// HealthCheckConfig holds configuration for the health checker
type HealthCheckConfig struct {
// CheckInterval is how often to check health
CheckInterval time.Duration
// Timeout is the timeout for each health check
Timeout time.Duration
// HealthyThreshold is the number of consecutive successes to become healthy
HealthyThreshold int
// UnhealthyThreshold is the number of consecutive failures to become unhealthy
UnhealthyThreshold int
// DegradedThreshold is the latency threshold in ms to mark as degraded
DegradedThreshold time.Duration
// OnStatusChange is called when health status changes
OnStatusChange func(from, to HealthStatus)
// CheckFunc is the function to check health
CheckFunc func(ctx context.Context) error
}
// DefaultHealthCheckConfig returns default configuration
func DefaultHealthCheckConfig() *HealthCheckConfig {
return &HealthCheckConfig{
CheckInterval: 30 * time.Second,
Timeout: 5 * time.Second,
HealthyThreshold: 3,
UnhealthyThreshold: 3,
DegradedThreshold: 100 * time.Millisecond,
}
}
// HealthChecker monitors the health of a backend
type HealthChecker struct {
config *HealthCheckConfig
// Status tracking
status atomic.Int32
consecutiveSuccesses atomic.Int32
consecutiveFailures atomic.Int32
// Timing
lastCheckTime time.Time
lastSuccessTime time.Time
lastFailureTime time.Time
averageLatency atomic.Int64
timeMu sync.RWMutex
// Metrics
totalChecks atomic.Int64
totalSuccesses atomic.Int64
totalFailures atomic.Int64
statusChanges atomic.Int64
// Lifecycle
ticker *time.Ticker
stopChan chan struct{}
stopped atomic.Bool
wg sync.WaitGroup
}
// NewHealthChecker creates a new health checker
func NewHealthChecker(config *HealthCheckConfig) *HealthChecker {
if config == nil {
config = DefaultHealthCheckConfig()
}
hc := &HealthChecker{
config: config,
stopChan: make(chan struct{}),
}
hc.status.Store(int32(HealthUnknown))
return hc
}
// Start begins health checking
func (hc *HealthChecker) Start() {
if hc.stopped.Load() {
return
}
hc.ticker = time.NewTicker(hc.config.CheckInterval)
hc.wg.Add(1)
go hc.checkLoop()
}
// Stop stops health checking
func (hc *HealthChecker) Stop() {
if hc.stopped.Swap(true) {
return // Already stopped
}
close(hc.stopChan)
if hc.ticker != nil {
hc.ticker.Stop()
}
hc.wg.Wait()
}
// checkLoop runs periodic health checks
func (hc *HealthChecker) checkLoop() {
defer hc.wg.Done()
// Initial check - log error but continue
if err := hc.Check(context.Background()); err != nil {
// Error is already tracked in Check() method, no need to log again
_ = err
}
for {
select {
case <-hc.stopChan:
return
case <-hc.ticker.C:
ctx, cancel := context.WithTimeout(context.Background(), hc.config.Timeout)
if err := hc.Check(ctx); err != nil {
// Error is already tracked in Check() method, no need to log again
_ = err
}
cancel()
}
}
}
// Check performs a health check
func (hc *HealthChecker) Check(ctx context.Context) error {
if hc.config.CheckFunc == nil {
return nil
}
hc.totalChecks.Add(1)
start := time.Now()
// Create timeout context if not already set
if _, hasDeadline := ctx.Deadline(); !hasDeadline {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, hc.config.Timeout)
defer cancel()
}
// Perform health check
err := hc.config.CheckFunc(ctx)
latency := time.Since(start)
hc.timeMu.Lock()
hc.lastCheckTime = time.Now()
hc.timeMu.Unlock()
// Update average latency
hc.updateAverageLatency(latency)
if err != nil {
hc.recordFailure()
} else {
hc.recordSuccess(latency)
}
return err
}
// recordSuccess records a successful health check
func (hc *HealthChecker) recordSuccess(latency time.Duration) {
hc.totalSuccesses.Add(1)
successes := hc.consecutiveSuccesses.Add(1)
hc.consecutiveFailures.Store(0)
hc.timeMu.Lock()
hc.lastSuccessTime = time.Now()
hc.timeMu.Unlock()
currentStatus := hc.GetStatus()
newStatus := currentStatus
// Check if we should become healthy
if successes >= int32(hc.config.HealthyThreshold) {
if latency > hc.config.DegradedThreshold {
newStatus = HealthDegraded
} else {
newStatus = HealthHealthy
}
}
if newStatus != currentStatus {
hc.setStatus(newStatus)
}
}
// recordFailure records a failed health check
func (hc *HealthChecker) recordFailure() {
hc.totalFailures.Add(1)
failures := hc.consecutiveFailures.Add(1)
hc.consecutiveSuccesses.Store(0)
hc.timeMu.Lock()
hc.lastFailureTime = time.Now()
hc.timeMu.Unlock()
// Check if we should become unhealthy
if failures >= int32(hc.config.UnhealthyThreshold) {
hc.setStatus(HealthUnhealthy)
}
}
// updateAverageLatency updates the rolling average latency
func (hc *HealthChecker) updateAverageLatency(latency time.Duration) {
// Simple exponential moving average
currentAvg := time.Duration(hc.averageLatency.Load())
if currentAvg == 0 {
hc.averageLatency.Store(int64(latency))
} else {
// Weight: 0.2 for new value, 0.8 for old average
newAvg := (currentAvg*4 + latency) / 5
hc.averageLatency.Store(int64(newAvg))
}
}
// GetStatus returns the current health status
func (hc *HealthChecker) GetStatus() HealthStatus {
return HealthStatus(hc.status.Load())
}
// setStatus changes the health status
func (hc *HealthChecker) setStatus(newStatus HealthStatus) {
oldStatus := HealthStatus(hc.status.Swap(int32(newStatus)))
if oldStatus != newStatus {
hc.statusChanges.Add(1)
if hc.config.OnStatusChange != nil {
hc.config.OnStatusChange(oldStatus, newStatus)
}
}
}
// IsHealthy returns true if the backend is healthy or degraded
func (hc *HealthChecker) IsHealthy() bool {
status := hc.GetStatus()
return status == HealthHealthy || status == HealthDegraded
}
// LastCheckTime returns the time of the last health check
func (hc *HealthChecker) LastCheckTime() time.Time {
hc.timeMu.RLock()
defer hc.timeMu.RUnlock()
return hc.lastCheckTime
}
// HealthScore returns a health score between 0.0 (unhealthy) and 1.0 (healthy)
func (hc *HealthChecker) HealthScore() float64 {
status := hc.GetStatus()
switch status {
case HealthHealthy:
return 1.0
case HealthDegraded:
return 0.7
case HealthUnhealthy:
return 0.0
default:
return 0.5
}
}
// Stats returns health checker statistics
func (hc *HealthChecker) Stats() HealthCheckerStats {
hc.timeMu.RLock()
lastCheck := hc.lastCheckTime
lastSuccess := hc.lastSuccessTime
lastFailure := hc.lastFailureTime
hc.timeMu.RUnlock()
totalChecks := hc.totalChecks.Load()
totalSuccesses := hc.totalSuccesses.Load()
totalFailures := hc.totalFailures.Load()
successRate := float64(0)
if totalChecks > 0 {
successRate = float64(totalSuccesses) / float64(totalChecks)
}
return HealthCheckerStats{
Status: hc.GetStatus(),
ConsecutiveSuccesses: hc.consecutiveSuccesses.Load(),
ConsecutiveFailures: hc.consecutiveFailures.Load(),
TotalChecks: totalChecks,
TotalSuccesses: totalSuccesses,
TotalFailures: totalFailures,
SuccessRate: successRate,
AverageLatency: time.Duration(hc.averageLatency.Load()),
StatusChanges: hc.statusChanges.Load(),
LastCheckTime: lastCheck,
LastSuccessTime: lastSuccess,
LastFailureTime: lastFailure,
HealthScore: hc.HealthScore(),
}
}
// HealthCheckerStats holds statistics for the health checker
type HealthCheckerStats struct {
Status HealthStatus
ConsecutiveSuccesses int32
ConsecutiveFailures int32
TotalChecks int64
TotalSuccesses int64
TotalFailures int64
SuccessRate float64
AverageLatency time.Duration
StatusChanges int64
LastCheckTime time.Time
LastSuccessTime time.Time
LastFailureTime time.Time
HealthScore float64
}
// Reset resets the health checker statistics
func (hc *HealthChecker) Reset() {
hc.status.Store(int32(HealthUnknown))
hc.consecutiveSuccesses.Store(0)
hc.consecutiveFailures.Store(0)
hc.totalChecks.Store(0)
hc.totalSuccesses.Store(0)
hc.totalFailures.Store(0)
hc.statusChanges.Store(0)
hc.averageLatency.Store(0)
now := time.Now()
hc.timeMu.Lock()
hc.lastCheckTime = now
hc.lastSuccessTime = now
hc.lastFailureTime = now
hc.timeMu.Unlock()
}
+215
View File
@@ -0,0 +1,215 @@
// Package resilience provides resilience patterns for cache backends.
package resilience
import (
"context"
"sync"
"sync/atomic"
"time"
"github.com/lukaszraczylo/traefikoidc/internal/cache/backends"
)
// HealthCheckBackend wraps a cache backend with health checking
type HealthCheckBackend struct {
backend backends.CacheBackend
config *HealthCheckConfig
// Health tracking
status atomic.Int32
consecutiveFails atomic.Int32
consecutiveOK atomic.Int32
lastCheck time.Time
checkMutex sync.RWMutex
// Lifecycle
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
}
// NewHealthCheckBackend creates a new health check wrapped backend
func NewHealthCheckBackend(b backends.CacheBackend, config *HealthCheckConfig) backends.CacheBackend {
if config == nil {
config = DefaultHealthCheckConfig()
}
ctx, cancel := context.WithCancel(context.Background())
hc := &HealthCheckBackend{
backend: b,
config: config,
ctx: ctx,
cancel: cancel,
}
// Set initial status to healthy (optimistic)
hc.status.Store(int32(HealthHealthy))
// Start health check routine
hc.wg.Add(1)
go hc.healthCheckLoop()
return hc
}
// Set stores a value and tracks health
func (h *HealthCheckBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
// Allow operations even if unhealthy (may recover)
err := h.backend.Set(ctx, key, value, ttl)
h.recordResult(err == nil)
return err
}
// Get retrieves a value and tracks health
func (h *HealthCheckBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
value, ttl, exists, err := h.backend.Get(ctx, key)
h.recordResult(err == nil)
return value, ttl, exists, err
}
// Delete removes a key and tracks health
func (h *HealthCheckBackend) Delete(ctx context.Context, key string) (bool, error) {
deleted, err := h.backend.Delete(ctx, key)
h.recordResult(err == nil)
return deleted, err
}
// Exists checks if a key exists and tracks health
func (h *HealthCheckBackend) Exists(ctx context.Context, key string) (bool, error) {
exists, err := h.backend.Exists(ctx, key)
h.recordResult(err == nil)
return exists, err
}
// Clear removes all keys and tracks health
func (h *HealthCheckBackend) Clear(ctx context.Context) error {
err := h.backend.Clear(ctx)
h.recordResult(err == nil)
return err
}
// GetStats returns statistics including health status
func (h *HealthCheckBackend) GetStats() map[string]interface{} {
stats := h.backend.GetStats()
if stats == nil {
stats = make(map[string]interface{})
}
h.checkMutex.RLock()
lastCheck := h.lastCheck
h.checkMutex.RUnlock()
status := HealthStatus(h.status.Load())
stats["health"] = map[string]interface{}{
"status": status.String(),
"consecutive_fails": h.consecutiveFails.Load(),
"consecutive_ok": h.consecutiveOK.Load(),
"last_check": lastCheck.Format(time.RFC3339),
"time_since_check": time.Since(lastCheck).Seconds(),
"check_interval_sec": h.config.CheckInterval.Seconds(),
}
return stats
}
// Ping checks backend health
func (h *HealthCheckBackend) Ping(ctx context.Context) error {
err := h.backend.Ping(ctx)
h.recordResult(err == nil)
return err
}
// Close shuts down the health checker and backend
func (h *HealthCheckBackend) Close() error {
// Stop health check routine
h.cancel()
// Wait for routine to finish
done := make(chan struct{})
go func() {
h.wg.Wait()
close(done)
}()
select {
case <-done:
// Finished normally
case <-time.After(2 * time.Second):
// Timeout
}
return h.backend.Close()
}
// IsHealthy returns true if the backend is healthy
func (h *HealthCheckBackend) IsHealthy() bool {
status := HealthStatus(h.status.Load())
return status == HealthHealthy || status == HealthDegraded
}
// recordResult records the result of an operation for health tracking
func (h *HealthCheckBackend) recordResult(success bool) {
if success {
fails := h.consecutiveFails.Swap(0)
oks := h.consecutiveOK.Add(1)
// Check if we should transition to healthy
if fails > 0 && oks >= int32(h.config.HealthyThreshold) {
oldStatus := HealthStatus(h.status.Swap(int32(HealthHealthy)))
if oldStatus != HealthHealthy && h.config.OnStatusChange != nil {
h.config.OnStatusChange(oldStatus, HealthHealthy)
}
}
} else {
oks := h.consecutiveOK.Swap(0)
fails := h.consecutiveFails.Add(1)
// Check if we should transition to unhealthy
if oks > 0 && fails >= int32(h.config.UnhealthyThreshold) {
oldStatus := HealthStatus(h.status.Swap(int32(HealthUnhealthy)))
if oldStatus != HealthUnhealthy && h.config.OnStatusChange != nil {
h.config.OnStatusChange(oldStatus, HealthUnhealthy)
}
} else if fails >= int32(h.config.UnhealthyThreshold)*2 {
// Severely degraded
h.status.Store(int32(HealthUnhealthy))
} else if fails >= int32(h.config.UnhealthyThreshold) {
// Degraded but still trying
h.status.Store(int32(HealthDegraded))
}
}
}
// healthCheckLoop runs periodic health checks
func (h *HealthCheckBackend) healthCheckLoop() {
defer h.wg.Done()
ticker := time.NewTicker(h.config.CheckInterval)
defer ticker.Stop()
// Do initial check
h.performHealthCheck()
for {
select {
case <-h.ctx.Done():
return
case <-ticker.C:
h.performHealthCheck()
}
}
}
// performHealthCheck performs a single health check
func (h *HealthCheckBackend) performHealthCheck() {
h.checkMutex.Lock()
h.lastCheck = time.Now()
h.checkMutex.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), h.config.Timeout)
defer cancel()
err := h.backend.Ping(ctx)
h.recordResult(err == nil)
}
+447
View File
@@ -0,0 +1,447 @@
package resilience
import (
"context"
"errors"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
// TestHealthChecker_StatusTransitions tests health status transitions
func TestHealthChecker_StatusTransitions(t *testing.T) {
t.Parallel()
var callCount atomic.Int32
var shouldFail atomic.Bool
checkFunc := func(ctx context.Context) error {
callCount.Add(1)
if shouldFail.Load() {
return errors.New("health check failed")
}
return nil
}
config := &HealthCheckConfig{
CheckInterval: 50 * time.Millisecond,
Timeout: 10 * time.Millisecond,
UnhealthyThreshold: 3,
HealthyThreshold: 2,
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
hc.Start()
defer hc.Stop()
// Initially unknown
assert.Equal(t, HealthUnknown, hc.GetStatus())
// Trigger failures
shouldFail.Store(true)
time.Sleep(200 * time.Millisecond)
// Should be unhealthy after threshold failures
status := hc.GetStatus()
assert.True(t, status == HealthUnhealthy || status == HealthDegraded)
// Recover
shouldFail.Store(false)
time.Sleep(150 * time.Millisecond)
// Should recover towards healthy
finalStatus := hc.GetStatus()
assert.True(t, finalStatus == HealthHealthy || finalStatus == HealthDegraded || finalStatus == HealthUnknown)
}
// TestHealthChecker_InitialState tests initial health status
func TestHealthChecker_InitialState(t *testing.T) {
t.Parallel()
checkFunc := func(ctx context.Context) error {
return nil
}
config := &HealthCheckConfig{
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
assert.Equal(t, HealthUnknown, hc.GetStatus())
assert.False(t, hc.IsHealthy())
}
// TestHealthChecker_ForceCheck tests manual health check trigger
func TestHealthChecker_ForceCheck(t *testing.T) {
t.Parallel()
var callCount atomic.Int32
checkFunc := func(ctx context.Context) error {
callCount.Add(1)
return nil
}
config := &HealthCheckConfig{
CheckInterval: 10 * time.Second, // Long interval
Timeout: 1 * time.Second,
UnhealthyThreshold: 3,
HealthyThreshold: 2,
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
initialCount := callCount.Load()
// Force check
hc.Check(context.Background())
// Should have been called
assert.Greater(t, callCount.Load(), initialCount)
}
// TestHealthChecker_StatusChangeCallback tests status change notifications
func TestHealthChecker_StatusChangeCallback(t *testing.T) {
t.Parallel()
var transitions []string
var mu sync.Mutex
var shouldFail atomic.Bool
checkFunc := func(ctx context.Context) error {
if shouldFail.Load() {
return errors.New("health check failed")
}
return nil
}
config := &HealthCheckConfig{
CheckInterval: 30 * time.Millisecond,
Timeout: 10 * time.Millisecond,
UnhealthyThreshold: 2,
HealthyThreshold: 2,
CheckFunc: checkFunc,
OnStatusChange: func(from, to HealthStatus) {
mu.Lock()
defer mu.Unlock()
transitions = append(transitions, from.String()+"->"+to.String())
},
}
hc := NewHealthChecker(config)
hc.Start()
defer hc.Stop()
// Trigger failures
shouldFail.Store(true)
time.Sleep(100 * time.Millisecond)
// Recover
shouldFail.Store(false)
time.Sleep(100 * time.Millisecond)
mu.Lock()
defer mu.Unlock()
// Should have status transitions
assert.NotEmpty(t, transitions)
}
// TestHealthChecker_Stats tests statistics tracking
func TestHealthChecker_Stats(t *testing.T) {
t.Parallel()
var callCount atomic.Int32
checkFunc := func(ctx context.Context) error {
callCount.Add(1)
if callCount.Load()%2 == 0 {
return errors.New("failure")
}
return nil
}
config := &HealthCheckConfig{
CheckInterval: 20 * time.Millisecond,
Timeout: 10 * time.Millisecond,
UnhealthyThreshold: 5,
HealthyThreshold: 2,
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
hc.Start()
defer hc.Stop()
time.Sleep(150 * time.Millisecond)
stats := hc.Stats()
assert.Greater(t, stats.TotalChecks, int64(0))
assert.Greater(t, stats.TotalFailures, int64(0))
assert.Greater(t, stats.SuccessRate, 0.0)
assert.Less(t, stats.SuccessRate, 1.0)
}
// TestHealthChecker_Timeout tests check timeout handling
func TestHealthChecker_Timeout(t *testing.T) {
t.Parallel()
checkFunc := func(ctx context.Context) error {
// Simulate slow check
select {
case <-time.After(100 * time.Millisecond):
return nil
case <-ctx.Done():
return ctx.Err()
}
}
config := &HealthCheckConfig{
CheckInterval: 50 * time.Millisecond,
Timeout: 10 * time.Millisecond, // Short timeout
UnhealthyThreshold: 2,
HealthyThreshold: 2,
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
hc.Start()
defer hc.Stop()
time.Sleep(150 * time.Millisecond)
// Should be unhealthy due to timeouts
status := hc.GetStatus()
assert.NotEqual(t, HealthHealthy, status)
}
// TestHealthChecker_ConcurrentAccess tests thread safety
func TestHealthChecker_ConcurrentAccess(t *testing.T) {
t.Parallel()
checkFunc := func(ctx context.Context) error {
return nil
}
config := &HealthCheckConfig{
CheckInterval: 10 * time.Millisecond,
Timeout: 5 * time.Millisecond,
UnhealthyThreshold: 3,
HealthyThreshold: 2,
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
hc.Start()
defer hc.Stop()
var wg sync.WaitGroup
goroutines := 20
for i := 0; i < goroutines; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 50; j++ {
_ = hc.GetStatus()
_ = hc.IsHealthy()
_ = hc.Stats()
hc.Check(context.Background())
}
}()
}
wg.Wait()
// Should complete without panics
}
// TestHealthChecker_StopAndStart tests lifecycle management
func TestHealthChecker_StopAndStart(t *testing.T) {
t.Parallel()
var callCount atomic.Int32
checkFunc := func(ctx context.Context) error {
callCount.Add(1)
return nil
}
config := &HealthCheckConfig{
CheckInterval: 20 * time.Millisecond,
Timeout: 10 * time.Millisecond,
UnhealthyThreshold: 3,
HealthyThreshold: 2,
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
// Start
hc.Start()
time.Sleep(100 * time.Millisecond)
count1 := callCount.Load()
assert.Greater(t, count1, int32(0))
// Stop
hc.Stop()
time.Sleep(100 * time.Millisecond)
count2 := callCount.Load()
// Should not have increased significantly after stop
assert.Less(t, count2-count1, int32(3))
}
// TestHealthChecker_DegradedState tests degraded status
func TestHealthChecker_DegradedState(t *testing.T) {
t.Parallel()
var callCount atomic.Int32
checkFunc := func(ctx context.Context) error {
count := callCount.Add(1)
// Fail once, then succeed
if count == 1 {
return errors.New("single failure")
}
return nil
}
config := &HealthCheckConfig{
CheckInterval: 30 * time.Millisecond,
Timeout: 10 * time.Millisecond,
UnhealthyThreshold: 3, // Need 3 failures for unhealthy
HealthyThreshold: 2, // Need 2 successes for healthy
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
hc.Start()
defer hc.Stop()
time.Sleep(100 * time.Millisecond)
// After initial checks, status should be set (might be healthy or degraded based on execution)
status := hc.GetStatus()
assert.True(t, status != HealthUnknown, "Status should not be unknown after checks")
}
// TestHealthChecker_DefaultConfig tests default configuration
func TestHealthChecker_DefaultConfig(t *testing.T) {
t.Parallel()
checkFunc := func(ctx context.Context) error {
return nil
}
config := &HealthCheckConfig{
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
assert.NotNil(t, hc)
assert.Equal(t, HealthUnknown, hc.GetStatus())
// Verify default config was applied (we can't access private fields, so just check it works)
assert.NotNil(t, hc)
}
// TestHealthChecker_StatusString tests status string representation
func TestHealthChecker_StatusString(t *testing.T) {
t.Parallel()
assert.Equal(t, "healthy", HealthHealthy.String())
assert.Equal(t, "unhealthy", HealthUnhealthy.String())
assert.Equal(t, "degraded", HealthDegraded.String())
assert.Equal(t, "unknown", HealthStatus(999).String())
}
// TestHealthChecker_RecoveryPattern tests typical failure and recovery
func TestHealthChecker_RecoveryPattern(t *testing.T) {
t.Parallel()
var checkNumber atomic.Int32
checkFunc := func(ctx context.Context) error {
n := checkNumber.Add(1)
// Fail checks 3-5, succeed others
if n >= 3 && n <= 5 {
return errors.New("temporary failure")
}
return nil
}
var statusLog []HealthStatus
var mu sync.Mutex
config := &HealthCheckConfig{
CheckInterval: 30 * time.Millisecond,
Timeout: 10 * time.Millisecond,
UnhealthyThreshold: 3,
HealthyThreshold: 2,
CheckFunc: checkFunc,
OnStatusChange: func(from, to HealthStatus) {
mu.Lock()
defer mu.Unlock()
statusLog = append(statusLog, to)
},
}
hc := NewHealthChecker(config)
hc.Start()
defer hc.Stop()
time.Sleep(300 * time.Millisecond)
mu.Lock()
defer mu.Unlock()
// Should see transitions through unhealthy and back to healthy
assert.NotEmpty(t, statusLog)
// Final status should be healthy or degraded (recovered)
finalStatus := hc.GetStatus()
assert.True(t, finalStatus == HealthHealthy || finalStatus == HealthDegraded, "Should have recovered")
}
// Benchmark health checker performance
func BenchmarkHealthChecker_ForceCheck(b *testing.B) {
checkFunc := func(ctx context.Context) error {
return nil
}
config := &HealthCheckConfig{
CheckInterval: 10 * time.Minute,
Timeout: 1 * time.Second,
UnhealthyThreshold: 3,
HealthyThreshold: 2,
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
b.ResetTimer()
for i := 0; i < b.N; i++ {
hc.Check(context.Background())
}
}
func BenchmarkHealthChecker_Status(b *testing.B) {
checkFunc := func(ctx context.Context) error {
return nil
}
config := &HealthCheckConfig{
CheckFunc: checkFunc,
}
hc := NewHealthChecker(config)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = hc.GetStatus()
}
}