mirror of
https://github.com/lukaszraczylo/traefikoidc.git
synced 2026-10-14 04:07:46 +00:00
Add redis support for distributed caching (#83)
* Add redis support for distributed caching * Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! fixup! fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * ... and another all nighter. * fixup! ... and another all nighter. * fixup! fixup! ... and another all nighter. * fixup! fixup! fixup! ... and another all nighter. * Resolve issue #85 by adding ability to set custom claims in JWT tokens * Remove redundant validation in auth middleware ( issue #89 ) * Add ability to set cookie prefix for session cookies ( #87 ) * fixup! Add ability to set cookie prefix for session cookies ( #87 ) * Add ability to set cookie max age - issue #91 * Potential fix for code scanning alert no. 10: Size computation for allocation may overflow Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> * fixup! Merge main into 0.8.0-redis: resolve conflicts --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
parent
5fcbd54955
commit
e64fc7f730
+329
@@ -0,0 +1,329 @@
|
||||
// Package resilience provides resilience patterns for cache backends.
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Common errors
|
||||
var (
|
||||
// ErrCircuitOpen is returned when the circuit breaker is open
|
||||
ErrCircuitOpen = errors.New("circuit breaker is open")
|
||||
|
||||
// ErrTooManyRequests is returned when too many requests are made in half-open state
|
||||
ErrTooManyRequests = errors.New("too many requests in half-open state")
|
||||
)
|
||||
|
||||
// State represents the state of the circuit breaker
|
||||
type State int32
|
||||
|
||||
const (
|
||||
// StateClosed allows all operations to pass through
|
||||
StateClosed State = iota
|
||||
|
||||
// StateOpen blocks all operations
|
||||
StateOpen
|
||||
|
||||
// StateHalfOpen allows a limited number of operations to test recovery
|
||||
StateHalfOpen
|
||||
)
|
||||
|
||||
// String returns the string representation of the state
|
||||
func (s State) String() string {
|
||||
switch s {
|
||||
case StateClosed:
|
||||
return "closed"
|
||||
case StateOpen:
|
||||
return "open"
|
||||
case StateHalfOpen:
|
||||
return "half-open"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// CircuitBreakerConfig holds configuration for the circuit breaker
|
||||
type CircuitBreakerConfig struct {
|
||||
// MaxFailures is the number of consecutive failures before opening the circuit
|
||||
MaxFailures int
|
||||
|
||||
// FailureThreshold is the failure rate threshold (0.0 to 1.0)
|
||||
FailureThreshold float64
|
||||
|
||||
// Timeout is how long the circuit stays open before trying half-open
|
||||
Timeout time.Duration
|
||||
|
||||
// HalfOpenMaxRequests is the number of requests allowed in half-open state
|
||||
HalfOpenMaxRequests int
|
||||
|
||||
// ResetTimeout is how long to wait before resetting counters in closed state
|
||||
ResetTimeout time.Duration
|
||||
|
||||
// OnStateChange is called when the circuit breaker changes state
|
||||
OnStateChange func(from, to State)
|
||||
}
|
||||
|
||||
// DefaultCircuitBreakerConfig returns default configuration
|
||||
func DefaultCircuitBreakerConfig() *CircuitBreakerConfig {
|
||||
return &CircuitBreakerConfig{
|
||||
MaxFailures: 5,
|
||||
FailureThreshold: 0.6,
|
||||
Timeout: 30 * time.Second,
|
||||
HalfOpenMaxRequests: 3,
|
||||
ResetTimeout: 60 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// CircuitBreaker implements the circuit breaker pattern
|
||||
type CircuitBreaker struct {
|
||||
config *CircuitBreakerConfig
|
||||
|
||||
// State management
|
||||
state atomic.Int32
|
||||
lastStateChange time.Time
|
||||
stateMu sync.RWMutex
|
||||
|
||||
// Failure tracking
|
||||
consecutiveFailures atomic.Int32
|
||||
totalRequests atomic.Int64
|
||||
totalFailures atomic.Int64
|
||||
halfOpenRequests atomic.Int32
|
||||
|
||||
// Timing
|
||||
lastFailureTime time.Time
|
||||
lastSuccessTime time.Time
|
||||
nextRetryTime time.Time
|
||||
timeMu sync.RWMutex
|
||||
|
||||
// Metrics
|
||||
stateTransitions atomic.Int64
|
||||
rejectedRequests atomic.Int64
|
||||
}
|
||||
|
||||
// NewCircuitBreaker creates a new circuit breaker
|
||||
func NewCircuitBreaker(config *CircuitBreakerConfig) *CircuitBreaker {
|
||||
if config == nil {
|
||||
config = DefaultCircuitBreakerConfig()
|
||||
}
|
||||
|
||||
return &CircuitBreaker{
|
||||
config: config,
|
||||
lastStateChange: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// Execute runs a function through the circuit breaker
|
||||
func (cb *CircuitBreaker) Execute(ctx context.Context, fn func() error) error {
|
||||
if !cb.AllowRequest() {
|
||||
cb.rejectedRequests.Add(1)
|
||||
return ErrCircuitOpen
|
||||
}
|
||||
|
||||
cb.totalRequests.Add(1)
|
||||
|
||||
err := fn()
|
||||
if err != nil {
|
||||
cb.RecordFailure()
|
||||
} else {
|
||||
cb.RecordSuccess()
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// AllowRequest checks if a request is allowed to proceed
|
||||
func (cb *CircuitBreaker) AllowRequest() bool {
|
||||
state := cb.GetState()
|
||||
|
||||
switch state {
|
||||
case StateClosed:
|
||||
return true
|
||||
|
||||
case StateOpen:
|
||||
// Check if timeout has passed and we should try half-open
|
||||
cb.timeMu.RLock()
|
||||
shouldRetry := time.Now().After(cb.nextRetryTime)
|
||||
cb.timeMu.RUnlock()
|
||||
|
||||
if shouldRetry {
|
||||
cb.setState(StateHalfOpen)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
case StateHalfOpen:
|
||||
// Allow limited requests in half-open state
|
||||
current := cb.halfOpenRequests.Add(1)
|
||||
return current <= int32(cb.config.HalfOpenMaxRequests)
|
||||
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// RecordSuccess records a successful operation
|
||||
func (cb *CircuitBreaker) RecordSuccess() {
|
||||
cb.timeMu.Lock()
|
||||
cb.lastSuccessTime = time.Now()
|
||||
cb.timeMu.Unlock()
|
||||
|
||||
state := cb.GetState()
|
||||
|
||||
switch state {
|
||||
case StateClosed:
|
||||
// Reset consecutive failures
|
||||
cb.consecutiveFailures.Store(0)
|
||||
|
||||
case StateHalfOpen:
|
||||
// If we've had enough successful requests, close the circuit
|
||||
successfulRequests := cb.halfOpenRequests.Load()
|
||||
if successfulRequests >= int32(cb.config.HalfOpenMaxRequests) {
|
||||
cb.setState(StateClosed)
|
||||
cb.consecutiveFailures.Store(0)
|
||||
cb.halfOpenRequests.Store(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RecordFailure records a failed operation
|
||||
func (cb *CircuitBreaker) RecordFailure() {
|
||||
cb.totalFailures.Add(1)
|
||||
failures := cb.consecutiveFailures.Add(1)
|
||||
|
||||
cb.timeMu.Lock()
|
||||
cb.lastFailureTime = time.Now()
|
||||
cb.timeMu.Unlock()
|
||||
|
||||
state := cb.GetState()
|
||||
|
||||
switch state {
|
||||
case StateClosed:
|
||||
// Check if we should open the circuit
|
||||
if failures >= int32(cb.config.MaxFailures) {
|
||||
cb.openCircuit()
|
||||
} else if cb.config.FailureThreshold > 0 {
|
||||
// Check failure rate
|
||||
total := cb.totalRequests.Load()
|
||||
failureCount := cb.totalFailures.Load()
|
||||
if total > 10 && float64(failureCount)/float64(total) > cb.config.FailureThreshold {
|
||||
cb.openCircuit()
|
||||
}
|
||||
}
|
||||
|
||||
case StateHalfOpen:
|
||||
// Any failure in half-open state reopens the circuit
|
||||
cb.openCircuit()
|
||||
}
|
||||
}
|
||||
|
||||
// openCircuit transitions to open state
|
||||
func (cb *CircuitBreaker) openCircuit() {
|
||||
cb.setState(StateOpen)
|
||||
cb.halfOpenRequests.Store(0)
|
||||
|
||||
cb.timeMu.Lock()
|
||||
cb.nextRetryTime = time.Now().Add(cb.config.Timeout)
|
||||
cb.timeMu.Unlock()
|
||||
}
|
||||
|
||||
// GetState returns the current state
|
||||
func (cb *CircuitBreaker) GetState() State {
|
||||
return State(cb.state.Load())
|
||||
}
|
||||
|
||||
// setState changes the circuit breaker state
|
||||
func (cb *CircuitBreaker) setState(newState State) {
|
||||
oldState := State(cb.state.Swap(int32(newState)))
|
||||
|
||||
if oldState != newState {
|
||||
cb.stateTransitions.Add(1)
|
||||
|
||||
cb.stateMu.Lock()
|
||||
cb.lastStateChange = time.Now()
|
||||
cb.stateMu.Unlock()
|
||||
|
||||
if cb.config.OnStateChange != nil {
|
||||
cb.config.OnStateChange(oldState, newState)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reset resets the circuit breaker to closed state
|
||||
func (cb *CircuitBreaker) Reset() {
|
||||
cb.setState(StateClosed)
|
||||
cb.consecutiveFailures.Store(0)
|
||||
cb.totalRequests.Store(0)
|
||||
cb.totalFailures.Store(0)
|
||||
cb.halfOpenRequests.Store(0)
|
||||
cb.rejectedRequests.Store(0)
|
||||
cb.stateTransitions.Store(0)
|
||||
|
||||
now := time.Now()
|
||||
cb.timeMu.Lock()
|
||||
cb.lastFailureTime = now
|
||||
cb.lastSuccessTime = now
|
||||
cb.nextRetryTime = now
|
||||
cb.timeMu.Unlock()
|
||||
|
||||
cb.stateMu.Lock()
|
||||
cb.lastStateChange = now
|
||||
cb.stateMu.Unlock()
|
||||
}
|
||||
|
||||
// Stats returns circuit breaker statistics
|
||||
func (cb *CircuitBreaker) Stats() CircuitBreakerStats {
|
||||
cb.timeMu.RLock()
|
||||
lastFailure := cb.lastFailureTime
|
||||
lastSuccess := cb.lastSuccessTime
|
||||
nextRetry := cb.nextRetryTime
|
||||
cb.timeMu.RUnlock()
|
||||
|
||||
cb.stateMu.RLock()
|
||||
lastChange := cb.lastStateChange
|
||||
cb.stateMu.RUnlock()
|
||||
|
||||
totalReq := cb.totalRequests.Load()
|
||||
totalFail := cb.totalFailures.Load()
|
||||
successRate := float64(0)
|
||||
if totalReq > 0 {
|
||||
successRate = float64(totalReq-totalFail) / float64(totalReq)
|
||||
}
|
||||
|
||||
return CircuitBreakerStats{
|
||||
State: cb.GetState(),
|
||||
ConsecutiveFailures: cb.consecutiveFailures.Load(),
|
||||
TotalRequests: totalReq,
|
||||
TotalFailures: totalFail,
|
||||
SuccessRate: successRate,
|
||||
RejectedRequests: cb.rejectedRequests.Load(),
|
||||
StateTransitions: cb.stateTransitions.Load(),
|
||||
LastFailureTime: lastFailure,
|
||||
LastSuccessTime: lastSuccess,
|
||||
LastStateChange: lastChange,
|
||||
NextRetryTime: nextRetry,
|
||||
}
|
||||
}
|
||||
|
||||
// CircuitBreakerStats holds statistics for the circuit breaker
|
||||
type CircuitBreakerStats struct {
|
||||
State State
|
||||
ConsecutiveFailures int32
|
||||
TotalRequests int64
|
||||
TotalFailures int64
|
||||
SuccessRate float64
|
||||
RejectedRequests int64
|
||||
StateTransitions int64
|
||||
LastFailureTime time.Time
|
||||
LastSuccessTime time.Time
|
||||
LastStateChange time.Time
|
||||
NextRetryTime time.Time
|
||||
}
|
||||
|
||||
// IsHealthy returns true if the circuit breaker is in a healthy state
|
||||
func (cb *CircuitBreaker) IsHealthy() bool {
|
||||
return cb.GetState() != StateOpen
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// Package resilience provides resilience patterns for cache backends.
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/traefikoidc/internal/cache/backends"
|
||||
)
|
||||
|
||||
// CircuitBreakerBackend wraps a cache backend with circuit breaker protection
|
||||
type CircuitBreakerBackend struct {
|
||||
backend backends.CacheBackend
|
||||
cb *CircuitBreaker
|
||||
}
|
||||
|
||||
// NewCircuitBreakerBackend creates a new circuit breaker wrapped backend
|
||||
func NewCircuitBreakerBackend(b backends.CacheBackend, config *CircuitBreakerConfig) backends.CacheBackend {
|
||||
if config == nil {
|
||||
config = DefaultCircuitBreakerConfig()
|
||||
}
|
||||
|
||||
return &CircuitBreakerBackend{
|
||||
backend: b,
|
||||
cb: NewCircuitBreaker(config),
|
||||
}
|
||||
}
|
||||
|
||||
// Set stores a value with circuit breaker protection
|
||||
func (c *CircuitBreakerBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
|
||||
if !c.cb.AllowRequest() {
|
||||
return backends.ErrCircuitOpen
|
||||
}
|
||||
|
||||
err := c.backend.Set(ctx, key, value, ttl)
|
||||
if err == nil {
|
||||
c.cb.RecordSuccess()
|
||||
} else {
|
||||
c.cb.RecordFailure()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Get retrieves a value with circuit breaker protection
|
||||
func (c *CircuitBreakerBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
|
||||
if !c.cb.AllowRequest() {
|
||||
return nil, 0, false, backends.ErrCircuitOpen
|
||||
}
|
||||
|
||||
value, ttl, exists, err := c.backend.Get(ctx, key)
|
||||
if err == nil {
|
||||
c.cb.RecordSuccess()
|
||||
} else {
|
||||
c.cb.RecordFailure()
|
||||
}
|
||||
return value, ttl, exists, err
|
||||
}
|
||||
|
||||
// Delete removes a key with circuit breaker protection
|
||||
func (c *CircuitBreakerBackend) Delete(ctx context.Context, key string) (bool, error) {
|
||||
if !c.cb.AllowRequest() {
|
||||
return false, backends.ErrCircuitOpen
|
||||
}
|
||||
|
||||
deleted, err := c.backend.Delete(ctx, key)
|
||||
if err == nil {
|
||||
c.cb.RecordSuccess()
|
||||
} else {
|
||||
c.cb.RecordFailure()
|
||||
}
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
// Exists checks if a key exists with circuit breaker protection
|
||||
func (c *CircuitBreakerBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
if !c.cb.AllowRequest() {
|
||||
return false, backends.ErrCircuitOpen
|
||||
}
|
||||
|
||||
exists, err := c.backend.Exists(ctx, key)
|
||||
if err == nil {
|
||||
c.cb.RecordSuccess()
|
||||
} else {
|
||||
c.cb.RecordFailure()
|
||||
}
|
||||
return exists, err
|
||||
}
|
||||
|
||||
// Clear removes all keys with circuit breaker protection
|
||||
func (c *CircuitBreakerBackend) Clear(ctx context.Context) error {
|
||||
if !c.cb.AllowRequest() {
|
||||
return backends.ErrCircuitOpen
|
||||
}
|
||||
|
||||
err := c.backend.Clear(ctx)
|
||||
if err == nil {
|
||||
c.cb.RecordSuccess()
|
||||
} else {
|
||||
c.cb.RecordFailure()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// GetStats returns statistics including circuit breaker state
|
||||
func (c *CircuitBreakerBackend) GetStats() map[string]interface{} {
|
||||
stats := c.backend.GetStats()
|
||||
if stats == nil {
|
||||
stats = make(map[string]interface{})
|
||||
}
|
||||
|
||||
cbStats := c.cb.Stats()
|
||||
stats["circuit_breaker"] = map[string]interface{}{
|
||||
"state": cbStats.State.String(),
|
||||
"consecutive_failures": cbStats.ConsecutiveFailures,
|
||||
"total_requests": cbStats.TotalRequests,
|
||||
"total_failures": cbStats.TotalFailures,
|
||||
"success_rate": cbStats.SuccessRate,
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// Ping checks backend health with circuit breaker protection
|
||||
func (c *CircuitBreakerBackend) Ping(ctx context.Context) error {
|
||||
if !c.cb.AllowRequest() {
|
||||
return backends.ErrCircuitOpen
|
||||
}
|
||||
|
||||
err := c.backend.Ping(ctx)
|
||||
if err == nil {
|
||||
c.cb.RecordSuccess()
|
||||
} else {
|
||||
c.cb.RecordFailure()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Close shuts down the backend
|
||||
func (c *CircuitBreakerBackend) Close() error {
|
||||
return c.backend.Close()
|
||||
}
|
||||
@@ -0,0 +1,561 @@
|
||||
//go:build !yaegi
|
||||
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/traefikoidc/internal/cache/backends"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// mockBackend is a simple mock implementation for testing
|
||||
type mockBackend struct {
|
||||
data map[string]mockEntry
|
||||
mu sync.RWMutex
|
||||
failSet bool
|
||||
failGet bool
|
||||
failDelete bool
|
||||
failExists bool
|
||||
failClear bool
|
||||
failPing bool
|
||||
callCount int
|
||||
}
|
||||
|
||||
type mockEntry struct {
|
||||
value []byte
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
func newMockBackend() *mockBackend {
|
||||
return &mockBackend{
|
||||
data: make(map[string]mockEntry),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.callCount++
|
||||
|
||||
if m.failSet {
|
||||
return errors.New("mock set error")
|
||||
}
|
||||
|
||||
expiresAt := time.Now().Add(ttl)
|
||||
if ttl == 0 {
|
||||
expiresAt = time.Now().Add(24 * time.Hour)
|
||||
}
|
||||
|
||||
m.data[key] = mockEntry{
|
||||
value: value,
|
||||
expiresAt: expiresAt,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
m.callCount++
|
||||
|
||||
if m.failGet {
|
||||
return nil, 0, false, errors.New("mock get error")
|
||||
}
|
||||
|
||||
entry, exists := m.data[key]
|
||||
if !exists {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
|
||||
if time.Now().After(entry.expiresAt) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
|
||||
ttl := time.Until(entry.expiresAt)
|
||||
return entry.value, ttl, true, nil
|
||||
}
|
||||
|
||||
func (m *mockBackend) Delete(ctx context.Context, key string) (bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.callCount++
|
||||
|
||||
if m.failDelete {
|
||||
return false, errors.New("mock delete error")
|
||||
}
|
||||
|
||||
_, existed := m.data[key]
|
||||
delete(m.data, key)
|
||||
return existed, nil
|
||||
}
|
||||
|
||||
func (m *mockBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
m.callCount++
|
||||
|
||||
if m.failExists {
|
||||
return false, errors.New("mock exists error")
|
||||
}
|
||||
|
||||
entry, exists := m.data[key]
|
||||
if !exists {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if time.Now().After(entry.expiresAt) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *mockBackend) Clear(ctx context.Context) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.callCount++
|
||||
|
||||
if m.failClear {
|
||||
return errors.New("mock clear error")
|
||||
}
|
||||
|
||||
m.data = make(map[string]mockEntry)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBackend) GetStats() map[string]interface{} {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
return map[string]interface{}{
|
||||
"hits": int64(0),
|
||||
"misses": int64(0),
|
||||
"call_count": m.callCount,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockBackend) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBackend) Ping(ctx context.Context) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.callCount++
|
||||
|
||||
if m.failPing {
|
||||
return errors.New("mock ping error")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Constructor Tests
|
||||
|
||||
func TestNewCircuitBreakerBackend_WithDefaultConfig(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
require.NotNil(t, cb)
|
||||
|
||||
// Verify it implements the interface (compile-time check)
|
||||
var _ backends.CacheBackend = cb
|
||||
}
|
||||
|
||||
func TestNewCircuitBreakerBackend_WithCustomConfig(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 3,
|
||||
FailureThreshold: 0.5,
|
||||
Timeout: 5 * time.Second,
|
||||
HalfOpenMaxRequests: 2,
|
||||
ResetTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
cb := NewCircuitBreakerBackend(mockBE, config)
|
||||
require.NotNil(t, cb)
|
||||
}
|
||||
|
||||
// Set Operation Tests
|
||||
|
||||
func TestCircuitBreakerBackend_Set_Success(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
err := cb.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, mockBE.callCount)
|
||||
|
||||
// Verify value was stored
|
||||
value, _, exists, _ := mockBE.Get(ctx, "key1")
|
||||
assert.True(t, exists)
|
||||
assert.Equal(t, []byte("value1"), value)
|
||||
}
|
||||
|
||||
func TestCircuitBreakerBackend_Set_Failure(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
mockBE.failSet = true
|
||||
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
err := cb.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestCircuitBreakerBackend_Set_CircuitOpen(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
mockBE.failSet = true
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
}
|
||||
cb := NewCircuitBreakerBackend(mockBE, config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Trigger failures to open circuit
|
||||
for i := 0; i < 5; i++ {
|
||||
cb.Set(ctx, "key", []byte("value"), 1*time.Minute)
|
||||
}
|
||||
|
||||
// Circuit should be open now
|
||||
err := cb.Set(ctx, "key2", []byte("value2"), 1*time.Minute)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, backends.ErrCircuitOpen, err)
|
||||
}
|
||||
|
||||
// Get Operation Tests
|
||||
|
||||
func TestCircuitBreakerBackend_Get_Success(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// First set a value
|
||||
mockBE.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
|
||||
|
||||
// Now get it through circuit breaker
|
||||
value, _, exists, err := cb.Get(ctx, "key1")
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, exists)
|
||||
assert.Equal(t, []byte("value1"), value)
|
||||
}
|
||||
|
||||
func TestCircuitBreakerBackend_Get_Failure(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
mockBE.failGet = true
|
||||
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
_, _, _, err := cb.Get(ctx, "key1")
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestCircuitBreakerBackend_Get_CircuitOpen(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
mockBE.failGet = true
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
}
|
||||
cb := NewCircuitBreakerBackend(mockBE, config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Trigger failures
|
||||
for i := 0; i < 5; i++ {
|
||||
cb.Get(ctx, "key")
|
||||
}
|
||||
|
||||
// Circuit should be open
|
||||
_, _, _, err := cb.Get(ctx, "key2")
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, backends.ErrCircuitOpen, err)
|
||||
}
|
||||
|
||||
// Delete Operation Tests
|
||||
|
||||
func TestCircuitBreakerBackend_Delete_Success(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Set a value first
|
||||
mockBE.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
|
||||
|
||||
// Delete through circuit breaker
|
||||
deleted, err := cb.Delete(ctx, "key1")
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, deleted)
|
||||
|
||||
// Verify it's deleted
|
||||
exists, _ := mockBE.Exists(ctx, "key1")
|
||||
assert.False(t, exists)
|
||||
}
|
||||
|
||||
func TestCircuitBreakerBackend_Delete_CircuitOpen(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
mockBE.failDelete = true
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
}
|
||||
cb := NewCircuitBreakerBackend(mockBE, config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Trigger failures
|
||||
for i := 0; i < 5; i++ {
|
||||
cb.Delete(ctx, "key")
|
||||
}
|
||||
|
||||
// Circuit should be open
|
||||
_, err := cb.Delete(ctx, "key2")
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, backends.ErrCircuitOpen, err)
|
||||
}
|
||||
|
||||
// Exists Operation Tests
|
||||
|
||||
func TestCircuitBreakerBackend_Exists_Success(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Set a value first
|
||||
mockBE.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
|
||||
|
||||
// Check existence through circuit breaker
|
||||
exists, err := cb.Exists(ctx, "key1")
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, exists)
|
||||
}
|
||||
|
||||
func TestCircuitBreakerBackend_Exists_CircuitOpen(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
mockBE.failExists = true
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
}
|
||||
cb := NewCircuitBreakerBackend(mockBE, config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Trigger failures
|
||||
for i := 0; i < 5; i++ {
|
||||
cb.Exists(ctx, "key")
|
||||
}
|
||||
|
||||
// Circuit should be open
|
||||
_, err := cb.Exists(ctx, "key2")
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, backends.ErrCircuitOpen, err)
|
||||
}
|
||||
|
||||
// Clear Operation Tests
|
||||
|
||||
func TestCircuitBreakerBackend_Clear_Success(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Set some values
|
||||
mockBE.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
|
||||
mockBE.Set(ctx, "key2", []byte("value2"), 1*time.Minute)
|
||||
|
||||
// Clear through circuit breaker
|
||||
err := cb.Clear(ctx)
|
||||
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify cleared
|
||||
exists1, _ := mockBE.Exists(ctx, "key1")
|
||||
exists2, _ := mockBE.Exists(ctx, "key2")
|
||||
assert.False(t, exists1)
|
||||
assert.False(t, exists2)
|
||||
}
|
||||
|
||||
func TestCircuitBreakerBackend_Clear_CircuitOpen(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
mockBE.failClear = true
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
}
|
||||
cb := NewCircuitBreakerBackend(mockBE, config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Trigger failures
|
||||
for i := 0; i < 5; i++ {
|
||||
cb.Clear(ctx)
|
||||
}
|
||||
|
||||
// Circuit should be open
|
||||
err := cb.Clear(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, backends.ErrCircuitOpen, err)
|
||||
}
|
||||
|
||||
// GetStats Tests
|
||||
|
||||
func TestCircuitBreakerBackend_GetStats(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Perform some operations
|
||||
cb.Set(ctx, "key1", []byte("value1"), 1*time.Minute)
|
||||
cb.Get(ctx, "key1")
|
||||
|
||||
stats := cb.GetStats()
|
||||
|
||||
require.NotNil(t, stats)
|
||||
|
||||
// Should have circuit breaker stats
|
||||
assert.Contains(t, stats, "circuit_breaker")
|
||||
|
||||
cbStats, ok := stats["circuit_breaker"].(map[string]interface{})
|
||||
require.True(t, ok)
|
||||
|
||||
// Verify circuit breaker stats fields
|
||||
assert.Contains(t, cbStats, "state")
|
||||
assert.Contains(t, cbStats, "consecutive_failures")
|
||||
assert.Contains(t, cbStats, "total_requests")
|
||||
assert.Contains(t, cbStats, "total_failures")
|
||||
assert.Contains(t, cbStats, "success_rate")
|
||||
}
|
||||
|
||||
func TestCircuitBreakerBackend_GetStats_NilBackendStats(t *testing.T) {
|
||||
// Create a mock backend that returns nil stats
|
||||
mockBE := &mockBackendNilStats{}
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
stats := cb.GetStats()
|
||||
|
||||
require.NotNil(t, stats)
|
||||
assert.Contains(t, stats, "circuit_breaker")
|
||||
}
|
||||
|
||||
// mockBackendNilStats returns nil from GetStats
|
||||
type mockBackendNilStats struct {
|
||||
mockBackend
|
||||
}
|
||||
|
||||
func (m *mockBackendNilStats) GetStats() map[string]interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ping Tests
|
||||
|
||||
func TestCircuitBreakerBackend_Ping_Success(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
err := cb.Ping(ctx)
|
||||
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestCircuitBreakerBackend_Ping_CircuitOpen(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
mockBE.failPing = true
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
}
|
||||
cb := NewCircuitBreakerBackend(mockBE, config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Trigger failures
|
||||
for i := 0; i < 5; i++ {
|
||||
cb.Ping(ctx)
|
||||
}
|
||||
|
||||
// Circuit should be open
|
||||
err := cb.Ping(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, backends.ErrCircuitOpen, err)
|
||||
}
|
||||
|
||||
// Close Tests
|
||||
|
||||
func TestCircuitBreakerBackend_Close(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
cb := NewCircuitBreakerBackend(mockBE, nil)
|
||||
|
||||
err := cb.Close()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// Circuit Recovery Test
|
||||
|
||||
func TestCircuitBreakerBackend_CircuitRecovery(t *testing.T) {
|
||||
mockBE := newMockBackend()
|
||||
mockBE.failSet = true
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 200 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 1,
|
||||
}
|
||||
cb := NewCircuitBreakerBackend(mockBE, config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Trigger failures to open circuit
|
||||
for i := 0; i < 5; i++ {
|
||||
cb.Set(ctx, "key", []byte("value"), 1*time.Minute)
|
||||
}
|
||||
|
||||
// Verify circuit is open
|
||||
err := cb.Set(ctx, "key2", []byte("value2"), 1*time.Minute)
|
||||
assert.Equal(t, backends.ErrCircuitOpen, err)
|
||||
|
||||
// Wait for timeout
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
|
||||
// Fix the backend
|
||||
mockBE.mu.Lock()
|
||||
mockBE.failSet = false
|
||||
mockBE.mu.Unlock()
|
||||
|
||||
// Circuit should be in half-open state, allow a test request
|
||||
err = cb.Set(ctx, "key3", []byte("value3"), 1*time.Minute)
|
||||
|
||||
// After success threshold is met, circuit should close
|
||||
if err == nil {
|
||||
// Circuit recovered
|
||||
err2 := cb.Set(ctx, "key4", []byte("value4"), 1*time.Minute)
|
||||
assert.NoError(t, err2, "Circuit should be closed after recovery")
|
||||
}
|
||||
}
|
||||
+553
@@ -0,0 +1,553 @@
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestCircuitBreaker_StateTransitions tests state machine transitions
|
||||
func TestCircuitBreaker_StateTransitions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 3,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 2,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("Initial state is closed", func(t *testing.T) {
|
||||
assert.Equal(t, StateClosed, cb.GetState())
|
||||
})
|
||||
|
||||
t.Run("Closed to Open after max failures", func(t *testing.T) {
|
||||
cb.Reset()
|
||||
|
||||
// Simulate failures
|
||||
for i := 0; i < 3; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("test error")
|
||||
})
|
||||
}
|
||||
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
})
|
||||
|
||||
t.Run("Open to HalfOpen after timeout", func(t *testing.T) {
|
||||
// Open the circuit
|
||||
cb.Reset()
|
||||
for i := 0; i < 3; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("test error")
|
||||
})
|
||||
}
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
|
||||
// Wait for timeout
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
// Should allow request and transition to half-open
|
||||
err := cb.Execute(ctx, func() error {
|
||||
return nil
|
||||
})
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, StateHalfOpen, cb.GetState())
|
||||
})
|
||||
|
||||
t.Run("HalfOpen to Closed after successful requests", func(t *testing.T) {
|
||||
// Open circuit then wait for half-open
|
||||
cb.Reset()
|
||||
for i := 0; i < 3; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("test error")
|
||||
})
|
||||
}
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
// First request transitions to half-open and succeeds
|
||||
err := cb.Execute(ctx, func() error {
|
||||
return nil
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
// Should be in half-open after first request
|
||||
state := cb.GetState()
|
||||
assert.True(t, state == StateHalfOpen || state == StateClosed,
|
||||
"After first successful request, should be half-open or potentially closed")
|
||||
|
||||
if state == StateHalfOpen {
|
||||
// Need more successful requests to close
|
||||
// The exact number depends on implementation but should be within HalfOpenMaxRequests
|
||||
for i := 0; i < config.HalfOpenMaxRequests; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
// After multiple successful requests, should eventually close
|
||||
finalState := cb.GetState()
|
||||
assert.True(t, finalState == StateClosed || finalState == StateHalfOpen,
|
||||
"After successful requests, circuit should transition towards closed")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("HalfOpen to Open on failure", func(t *testing.T) {
|
||||
// Open circuit then wait for half-open
|
||||
cb.Reset()
|
||||
for i := 0; i < 3; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("test error")
|
||||
})
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
// First call transitions to half-open, second failure reopens
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("test error")
|
||||
})
|
||||
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
})
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_OpenCircuitBlocks tests that open circuit blocks requests
|
||||
func TestCircuitBreaker_OpenCircuitBlocks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 1 * time.Second,
|
||||
HalfOpenMaxRequests: 1,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Trigger failures to open circuit
|
||||
for i := 0; i < 2; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("test error")
|
||||
})
|
||||
}
|
||||
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
|
||||
// Requests should be blocked
|
||||
err := cb.Execute(ctx, func() error {
|
||||
t.Fatal("Should not execute function when circuit is open")
|
||||
return nil
|
||||
})
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, ErrCircuitOpen, err)
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_HalfOpenMaxRequests tests max requests in half-open state
|
||||
func TestCircuitBreaker_HalfOpenMaxRequests(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 3,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 2,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Open circuit then wait for half-open
|
||||
for i := 0; i < 3; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("test error")
|
||||
})
|
||||
}
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
// After timeout, circuit should allow transition to half-open
|
||||
// Execute HalfOpenMaxRequests successful requests
|
||||
successCount := 0
|
||||
for i := 0; i < config.HalfOpenMaxRequests; i++ {
|
||||
err := cb.Execute(ctx, func() error {
|
||||
successCount++
|
||||
return nil
|
||||
})
|
||||
// Should allow up to HalfOpenMaxRequests
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// Verify we executed the expected number
|
||||
assert.Equal(t, config.HalfOpenMaxRequests, successCount)
|
||||
|
||||
// After successful requests, circuit behavior depends on implementation
|
||||
// It could close (allowing more requests) or stay half-open (blocking)
|
||||
// The important thing is that we allowed exactly HalfOpenMaxRequests
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_SuccessResetsFailures tests failure counter reset
|
||||
func TestCircuitBreaker_SuccessResetsFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 3,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 1,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Have some failures (but less than max)
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("error")
|
||||
})
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("error")
|
||||
})
|
||||
|
||||
assert.Equal(t, StateClosed, cb.GetState())
|
||||
stats := cb.Stats()
|
||||
assert.Equal(t, int32(2), stats.ConsecutiveFailures)
|
||||
|
||||
// One success should reset failures
|
||||
cb.Execute(ctx, func() error {
|
||||
return nil
|
||||
})
|
||||
|
||||
assert.Equal(t, StateClosed, cb.GetState())
|
||||
stats = cb.Stats()
|
||||
assert.Equal(t, int32(0), stats.ConsecutiveFailures)
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_ConcurrentAccess tests thread safety
|
||||
func TestCircuitBreaker_ConcurrentAccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 10,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 5,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
var wg sync.WaitGroup
|
||||
goroutines := 20
|
||||
iterations := 50
|
||||
|
||||
for i := 0; i < goroutines; i++ {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < iterations; j++ {
|
||||
// Mix of successes and failures
|
||||
cb.Execute(ctx, func() error {
|
||||
if (id+j)%3 == 0 {
|
||||
return errors.New("test error")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Random state checks
|
||||
_ = cb.GetState()
|
||||
_ = cb.Stats()
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Should complete without panics
|
||||
stats := cb.Stats()
|
||||
assert.NotNil(t, stats)
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_Stats tests statistics tracking
|
||||
func TestCircuitBreaker_Stats(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 5,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 2,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Execute some requests
|
||||
cb.Execute(ctx, func() error { return nil }) // Success
|
||||
cb.Execute(ctx, func() error { return errors.New("error") }) // Failure
|
||||
cb.Execute(ctx, func() error { return errors.New("error") }) // Failure
|
||||
|
||||
stats := cb.Stats()
|
||||
|
||||
assert.Equal(t, StateClosed, stats.State)
|
||||
assert.Equal(t, int64(3), stats.TotalRequests)
|
||||
assert.Equal(t, int64(2), stats.TotalFailures)
|
||||
assert.Equal(t, int32(2), stats.ConsecutiveFailures)
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_Reset tests circuit reset
|
||||
func TestCircuitBreaker_Reset(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 1,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Open the circuit
|
||||
for i := 0; i < 2; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("error")
|
||||
})
|
||||
}
|
||||
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
|
||||
// Reset
|
||||
cb.Reset()
|
||||
|
||||
assert.Equal(t, StateClosed, cb.GetState())
|
||||
stats := cb.Stats()
|
||||
assert.Equal(t, int32(0), stats.ConsecutiveFailures)
|
||||
assert.Equal(t, int64(0), stats.TotalRequests)
|
||||
assert.Equal(t, int64(0), stats.TotalFailures)
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_StateChangeCallback tests state change notifications
|
||||
func TestCircuitBreaker_StateChangeCallback(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var transitions []string
|
||||
var mu sync.Mutex
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 50 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 1,
|
||||
OnStateChange: func(from, to State) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
transitions = append(transitions, from.String()+"->"+to.String())
|
||||
},
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Trigger state transitions
|
||||
// Closed -> Open
|
||||
for i := 0; i < 2; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("error")
|
||||
})
|
||||
}
|
||||
|
||||
// Should be open now
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
|
||||
// Wait for timeout to allow half-open transition
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Open -> HalfOpen on first request after timeout
|
||||
err := cb.Execute(ctx, func() error {
|
||||
return nil
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Execute more successful requests to trigger HalfOpen -> Closed
|
||||
for i := 0; i < config.HalfOpenMaxRequests-1; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
assert.Contains(t, transitions, "closed->open")
|
||||
assert.Contains(t, transitions, "open->half-open")
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_IsHealthy tests health check
|
||||
func TestCircuitBreaker_IsHealthy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 2,
|
||||
Timeout: 100 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 1,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Initially healthy
|
||||
assert.True(t, cb.IsHealthy())
|
||||
|
||||
// Open circuit
|
||||
for i := 0; i < 2; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("error")
|
||||
})
|
||||
}
|
||||
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
assert.False(t, cb.IsHealthy(), "Should not be healthy when open")
|
||||
|
||||
// Wait for timeout and allow successful request
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
cb.Execute(ctx, func() error {
|
||||
return nil
|
||||
})
|
||||
|
||||
// Should be healthy after recovery
|
||||
assert.True(t, cb.IsHealthy(), "Should be healthy after recovery")
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_RapidFailures tests rapid consecutive failures
|
||||
func TestCircuitBreaker_RapidFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 5,
|
||||
Timeout: 200 * time.Millisecond,
|
||||
HalfOpenMaxRequests: 1,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Rapid failures
|
||||
for i := 0; i < 10; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("rapid error")
|
||||
})
|
||||
}
|
||||
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
|
||||
stats := cb.Stats()
|
||||
assert.GreaterOrEqual(t, stats.TotalFailures, int64(5))
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_TimeoutAccuracy tests timeout precision
|
||||
func TestCircuitBreaker_TimeoutAccuracy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
timeout := 100 * time.Millisecond
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 1,
|
||||
Timeout: timeout,
|
||||
HalfOpenMaxRequests: 1,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Open circuit
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("error")
|
||||
})
|
||||
|
||||
assert.Equal(t, StateOpen, cb.GetState())
|
||||
|
||||
// Wait just before timeout
|
||||
time.Sleep(timeout - 20*time.Millisecond)
|
||||
assert.False(t, cb.IsHealthy())
|
||||
|
||||
// Wait until after timeout
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
// After timeout, AllowRequest should return true for transition to half-open
|
||||
assert.True(t, cb.AllowRequest())
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_DefaultConfig tests default configuration
|
||||
func TestCircuitBreaker_DefaultConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cb := NewCircuitBreaker(nil) // Should use defaults
|
||||
|
||||
assert.NotNil(t, cb)
|
||||
assert.Equal(t, StateClosed, cb.GetState())
|
||||
|
||||
// Verify defaults by triggering circuit breaker behavior
|
||||
ctx := context.Background()
|
||||
|
||||
// Test that it takes 5 failures to open (default MaxFailures)
|
||||
for i := 0; i < 4; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("error")
|
||||
})
|
||||
}
|
||||
assert.Equal(t, StateClosed, cb.GetState(), "Should still be closed after 4 failures")
|
||||
|
||||
// 5th failure should open it
|
||||
cb.Execute(ctx, func() error {
|
||||
return errors.New("error")
|
||||
})
|
||||
assert.Equal(t, StateOpen, cb.GetState(), "Should be open after 5 failures (default threshold)")
|
||||
}
|
||||
|
||||
// TestCircuitBreaker_StateString tests state string representation
|
||||
func TestCircuitBreaker_StateString(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(t, "closed", StateClosed.String())
|
||||
assert.Equal(t, "open", StateOpen.String())
|
||||
assert.Equal(t, "half-open", StateHalfOpen.String())
|
||||
assert.Equal(t, "unknown", State(999).String())
|
||||
}
|
||||
|
||||
// Benchmark circuit breaker performance
|
||||
func BenchmarkCircuitBreaker_Execute(b *testing.B) {
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 100,
|
||||
Timeout: 1 * time.Second,
|
||||
HalfOpenMaxRequests: 10,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
ctx := context.Background()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkCircuitBreaker_ExecuteWithFailures(b *testing.B) {
|
||||
config := &CircuitBreakerConfig{
|
||||
MaxFailures: 1000,
|
||||
Timeout: 1 * time.Second,
|
||||
HalfOpenMaxRequests: 10,
|
||||
}
|
||||
cb := NewCircuitBreaker(config)
|
||||
ctx := context.Background()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
cb.Execute(ctx, func() error {
|
||||
if i%10 == 0 {
|
||||
return errors.New("error")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
+375
@@ -0,0 +1,375 @@
|
||||
// Package resilience provides resilience patterns for cache backends.
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// HealthStatus represents the health status of a backend
|
||||
type HealthStatus int32
|
||||
|
||||
const (
|
||||
// HealthUnknown indicates unknown health status
|
||||
HealthUnknown HealthStatus = iota
|
||||
|
||||
// HealthHealthy indicates the backend is healthy
|
||||
HealthHealthy
|
||||
|
||||
// HealthDegraded indicates the backend is degraded but operational
|
||||
HealthDegraded
|
||||
|
||||
// HealthUnhealthy indicates the backend is unhealthy
|
||||
HealthUnhealthy
|
||||
)
|
||||
|
||||
// String returns the string representation of the health status
|
||||
func (h HealthStatus) String() string {
|
||||
switch h {
|
||||
case HealthHealthy:
|
||||
return "healthy"
|
||||
case HealthDegraded:
|
||||
return "degraded"
|
||||
case HealthUnhealthy:
|
||||
return "unhealthy"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// HealthCheckConfig holds configuration for the health checker
|
||||
type HealthCheckConfig struct {
|
||||
// CheckInterval is how often to check health
|
||||
CheckInterval time.Duration
|
||||
|
||||
// Timeout is the timeout for each health check
|
||||
Timeout time.Duration
|
||||
|
||||
// HealthyThreshold is the number of consecutive successes to become healthy
|
||||
HealthyThreshold int
|
||||
|
||||
// UnhealthyThreshold is the number of consecutive failures to become unhealthy
|
||||
UnhealthyThreshold int
|
||||
|
||||
// DegradedThreshold is the latency threshold in ms to mark as degraded
|
||||
DegradedThreshold time.Duration
|
||||
|
||||
// OnStatusChange is called when health status changes
|
||||
OnStatusChange func(from, to HealthStatus)
|
||||
|
||||
// CheckFunc is the function to check health
|
||||
CheckFunc func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// DefaultHealthCheckConfig returns default configuration
|
||||
func DefaultHealthCheckConfig() *HealthCheckConfig {
|
||||
return &HealthCheckConfig{
|
||||
CheckInterval: 30 * time.Second,
|
||||
Timeout: 5 * time.Second,
|
||||
HealthyThreshold: 3,
|
||||
UnhealthyThreshold: 3,
|
||||
DegradedThreshold: 100 * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
// HealthChecker monitors the health of a backend
|
||||
type HealthChecker struct {
|
||||
config *HealthCheckConfig
|
||||
|
||||
// Status tracking
|
||||
status atomic.Int32
|
||||
consecutiveSuccesses atomic.Int32
|
||||
consecutiveFailures atomic.Int32
|
||||
|
||||
// Timing
|
||||
lastCheckTime time.Time
|
||||
lastSuccessTime time.Time
|
||||
lastFailureTime time.Time
|
||||
averageLatency atomic.Int64
|
||||
timeMu sync.RWMutex
|
||||
|
||||
// Metrics
|
||||
totalChecks atomic.Int64
|
||||
totalSuccesses atomic.Int64
|
||||
totalFailures atomic.Int64
|
||||
statusChanges atomic.Int64
|
||||
|
||||
// Lifecycle
|
||||
ticker *time.Ticker
|
||||
stopChan chan struct{}
|
||||
stopped atomic.Bool
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewHealthChecker creates a new health checker
|
||||
func NewHealthChecker(config *HealthCheckConfig) *HealthChecker {
|
||||
if config == nil {
|
||||
config = DefaultHealthCheckConfig()
|
||||
}
|
||||
|
||||
hc := &HealthChecker{
|
||||
config: config,
|
||||
stopChan: make(chan struct{}),
|
||||
}
|
||||
hc.status.Store(int32(HealthUnknown))
|
||||
|
||||
return hc
|
||||
}
|
||||
|
||||
// Start begins health checking
|
||||
func (hc *HealthChecker) Start() {
|
||||
if hc.stopped.Load() {
|
||||
return
|
||||
}
|
||||
|
||||
hc.ticker = time.NewTicker(hc.config.CheckInterval)
|
||||
hc.wg.Add(1)
|
||||
go hc.checkLoop()
|
||||
}
|
||||
|
||||
// Stop stops health checking
|
||||
func (hc *HealthChecker) Stop() {
|
||||
if hc.stopped.Swap(true) {
|
||||
return // Already stopped
|
||||
}
|
||||
|
||||
close(hc.stopChan)
|
||||
if hc.ticker != nil {
|
||||
hc.ticker.Stop()
|
||||
}
|
||||
hc.wg.Wait()
|
||||
}
|
||||
|
||||
// checkLoop runs periodic health checks
|
||||
func (hc *HealthChecker) checkLoop() {
|
||||
defer hc.wg.Done()
|
||||
|
||||
// Initial check - log error but continue
|
||||
if err := hc.Check(context.Background()); err != nil {
|
||||
// Error is already tracked in Check() method, no need to log again
|
||||
_ = err
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-hc.stopChan:
|
||||
return
|
||||
case <-hc.ticker.C:
|
||||
ctx, cancel := context.WithTimeout(context.Background(), hc.config.Timeout)
|
||||
if err := hc.Check(ctx); err != nil {
|
||||
// Error is already tracked in Check() method, no need to log again
|
||||
_ = err
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check performs a health check
|
||||
func (hc *HealthChecker) Check(ctx context.Context) error {
|
||||
if hc.config.CheckFunc == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
hc.totalChecks.Add(1)
|
||||
start := time.Now()
|
||||
|
||||
// Create timeout context if not already set
|
||||
if _, hasDeadline := ctx.Deadline(); !hasDeadline {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, hc.config.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
// Perform health check
|
||||
err := hc.config.CheckFunc(ctx)
|
||||
latency := time.Since(start)
|
||||
|
||||
hc.timeMu.Lock()
|
||||
hc.lastCheckTime = time.Now()
|
||||
hc.timeMu.Unlock()
|
||||
|
||||
// Update average latency
|
||||
hc.updateAverageLatency(latency)
|
||||
|
||||
if err != nil {
|
||||
hc.recordFailure()
|
||||
} else {
|
||||
hc.recordSuccess(latency)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// recordSuccess records a successful health check
|
||||
func (hc *HealthChecker) recordSuccess(latency time.Duration) {
|
||||
hc.totalSuccesses.Add(1)
|
||||
successes := hc.consecutiveSuccesses.Add(1)
|
||||
hc.consecutiveFailures.Store(0)
|
||||
|
||||
hc.timeMu.Lock()
|
||||
hc.lastSuccessTime = time.Now()
|
||||
hc.timeMu.Unlock()
|
||||
|
||||
currentStatus := hc.GetStatus()
|
||||
newStatus := currentStatus
|
||||
|
||||
// Check if we should become healthy
|
||||
if successes >= int32(hc.config.HealthyThreshold) {
|
||||
if latency > hc.config.DegradedThreshold {
|
||||
newStatus = HealthDegraded
|
||||
} else {
|
||||
newStatus = HealthHealthy
|
||||
}
|
||||
}
|
||||
|
||||
if newStatus != currentStatus {
|
||||
hc.setStatus(newStatus)
|
||||
}
|
||||
}
|
||||
|
||||
// recordFailure records a failed health check
|
||||
func (hc *HealthChecker) recordFailure() {
|
||||
hc.totalFailures.Add(1)
|
||||
failures := hc.consecutiveFailures.Add(1)
|
||||
hc.consecutiveSuccesses.Store(0)
|
||||
|
||||
hc.timeMu.Lock()
|
||||
hc.lastFailureTime = time.Now()
|
||||
hc.timeMu.Unlock()
|
||||
|
||||
// Check if we should become unhealthy
|
||||
if failures >= int32(hc.config.UnhealthyThreshold) {
|
||||
hc.setStatus(HealthUnhealthy)
|
||||
}
|
||||
}
|
||||
|
||||
// updateAverageLatency updates the rolling average latency
|
||||
func (hc *HealthChecker) updateAverageLatency(latency time.Duration) {
|
||||
// Simple exponential moving average
|
||||
currentAvg := time.Duration(hc.averageLatency.Load())
|
||||
if currentAvg == 0 {
|
||||
hc.averageLatency.Store(int64(latency))
|
||||
} else {
|
||||
// Weight: 0.2 for new value, 0.8 for old average
|
||||
newAvg := (currentAvg*4 + latency) / 5
|
||||
hc.averageLatency.Store(int64(newAvg))
|
||||
}
|
||||
}
|
||||
|
||||
// GetStatus returns the current health status
|
||||
func (hc *HealthChecker) GetStatus() HealthStatus {
|
||||
return HealthStatus(hc.status.Load())
|
||||
}
|
||||
|
||||
// setStatus changes the health status
|
||||
func (hc *HealthChecker) setStatus(newStatus HealthStatus) {
|
||||
oldStatus := HealthStatus(hc.status.Swap(int32(newStatus)))
|
||||
|
||||
if oldStatus != newStatus {
|
||||
hc.statusChanges.Add(1)
|
||||
if hc.config.OnStatusChange != nil {
|
||||
hc.config.OnStatusChange(oldStatus, newStatus)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IsHealthy returns true if the backend is healthy or degraded
|
||||
func (hc *HealthChecker) IsHealthy() bool {
|
||||
status := hc.GetStatus()
|
||||
return status == HealthHealthy || status == HealthDegraded
|
||||
}
|
||||
|
||||
// LastCheckTime returns the time of the last health check
|
||||
func (hc *HealthChecker) LastCheckTime() time.Time {
|
||||
hc.timeMu.RLock()
|
||||
defer hc.timeMu.RUnlock()
|
||||
return hc.lastCheckTime
|
||||
}
|
||||
|
||||
// HealthScore returns a health score between 0.0 (unhealthy) and 1.0 (healthy)
|
||||
func (hc *HealthChecker) HealthScore() float64 {
|
||||
status := hc.GetStatus()
|
||||
switch status {
|
||||
case HealthHealthy:
|
||||
return 1.0
|
||||
case HealthDegraded:
|
||||
return 0.7
|
||||
case HealthUnhealthy:
|
||||
return 0.0
|
||||
default:
|
||||
return 0.5
|
||||
}
|
||||
}
|
||||
|
||||
// Stats returns health checker statistics
|
||||
func (hc *HealthChecker) Stats() HealthCheckerStats {
|
||||
hc.timeMu.RLock()
|
||||
lastCheck := hc.lastCheckTime
|
||||
lastSuccess := hc.lastSuccessTime
|
||||
lastFailure := hc.lastFailureTime
|
||||
hc.timeMu.RUnlock()
|
||||
|
||||
totalChecks := hc.totalChecks.Load()
|
||||
totalSuccesses := hc.totalSuccesses.Load()
|
||||
totalFailures := hc.totalFailures.Load()
|
||||
|
||||
successRate := float64(0)
|
||||
if totalChecks > 0 {
|
||||
successRate = float64(totalSuccesses) / float64(totalChecks)
|
||||
}
|
||||
|
||||
return HealthCheckerStats{
|
||||
Status: hc.GetStatus(),
|
||||
ConsecutiveSuccesses: hc.consecutiveSuccesses.Load(),
|
||||
ConsecutiveFailures: hc.consecutiveFailures.Load(),
|
||||
TotalChecks: totalChecks,
|
||||
TotalSuccesses: totalSuccesses,
|
||||
TotalFailures: totalFailures,
|
||||
SuccessRate: successRate,
|
||||
AverageLatency: time.Duration(hc.averageLatency.Load()),
|
||||
StatusChanges: hc.statusChanges.Load(),
|
||||
LastCheckTime: lastCheck,
|
||||
LastSuccessTime: lastSuccess,
|
||||
LastFailureTime: lastFailure,
|
||||
HealthScore: hc.HealthScore(),
|
||||
}
|
||||
}
|
||||
|
||||
// HealthCheckerStats holds statistics for the health checker
|
||||
type HealthCheckerStats struct {
|
||||
Status HealthStatus
|
||||
ConsecutiveSuccesses int32
|
||||
ConsecutiveFailures int32
|
||||
TotalChecks int64
|
||||
TotalSuccesses int64
|
||||
TotalFailures int64
|
||||
SuccessRate float64
|
||||
AverageLatency time.Duration
|
||||
StatusChanges int64
|
||||
LastCheckTime time.Time
|
||||
LastSuccessTime time.Time
|
||||
LastFailureTime time.Time
|
||||
HealthScore float64
|
||||
}
|
||||
|
||||
// Reset resets the health checker statistics
|
||||
func (hc *HealthChecker) Reset() {
|
||||
hc.status.Store(int32(HealthUnknown))
|
||||
hc.consecutiveSuccesses.Store(0)
|
||||
hc.consecutiveFailures.Store(0)
|
||||
hc.totalChecks.Store(0)
|
||||
hc.totalSuccesses.Store(0)
|
||||
hc.totalFailures.Store(0)
|
||||
hc.statusChanges.Store(0)
|
||||
hc.averageLatency.Store(0)
|
||||
|
||||
now := time.Now()
|
||||
hc.timeMu.Lock()
|
||||
hc.lastCheckTime = now
|
||||
hc.lastSuccessTime = now
|
||||
hc.lastFailureTime = now
|
||||
hc.timeMu.Unlock()
|
||||
}
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
// Package resilience provides resilience patterns for cache backends.
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/traefikoidc/internal/cache/backends"
|
||||
)
|
||||
|
||||
// HealthCheckBackend wraps a cache backend with health checking
|
||||
type HealthCheckBackend struct {
|
||||
backend backends.CacheBackend
|
||||
config *HealthCheckConfig
|
||||
|
||||
// Health tracking
|
||||
status atomic.Int32
|
||||
consecutiveFails atomic.Int32
|
||||
consecutiveOK atomic.Int32
|
||||
lastCheck time.Time
|
||||
checkMutex sync.RWMutex
|
||||
|
||||
// Lifecycle
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewHealthCheckBackend creates a new health check wrapped backend
|
||||
func NewHealthCheckBackend(b backends.CacheBackend, config *HealthCheckConfig) backends.CacheBackend {
|
||||
if config == nil {
|
||||
config = DefaultHealthCheckConfig()
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
hc := &HealthCheckBackend{
|
||||
backend: b,
|
||||
config: config,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
|
||||
// Set initial status to healthy (optimistic)
|
||||
hc.status.Store(int32(HealthHealthy))
|
||||
|
||||
// Start health check routine
|
||||
hc.wg.Add(1)
|
||||
go hc.healthCheckLoop()
|
||||
|
||||
return hc
|
||||
}
|
||||
|
||||
// Set stores a value and tracks health
|
||||
func (h *HealthCheckBackend) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
|
||||
// Allow operations even if unhealthy (may recover)
|
||||
err := h.backend.Set(ctx, key, value, ttl)
|
||||
h.recordResult(err == nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// Get retrieves a value and tracks health
|
||||
func (h *HealthCheckBackend) Get(ctx context.Context, key string) ([]byte, time.Duration, bool, error) {
|
||||
value, ttl, exists, err := h.backend.Get(ctx, key)
|
||||
h.recordResult(err == nil)
|
||||
return value, ttl, exists, err
|
||||
}
|
||||
|
||||
// Delete removes a key and tracks health
|
||||
func (h *HealthCheckBackend) Delete(ctx context.Context, key string) (bool, error) {
|
||||
deleted, err := h.backend.Delete(ctx, key)
|
||||
h.recordResult(err == nil)
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
// Exists checks if a key exists and tracks health
|
||||
func (h *HealthCheckBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
exists, err := h.backend.Exists(ctx, key)
|
||||
h.recordResult(err == nil)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
// Clear removes all keys and tracks health
|
||||
func (h *HealthCheckBackend) Clear(ctx context.Context) error {
|
||||
err := h.backend.Clear(ctx)
|
||||
h.recordResult(err == nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetStats returns statistics including health status
|
||||
func (h *HealthCheckBackend) GetStats() map[string]interface{} {
|
||||
stats := h.backend.GetStats()
|
||||
if stats == nil {
|
||||
stats = make(map[string]interface{})
|
||||
}
|
||||
|
||||
h.checkMutex.RLock()
|
||||
lastCheck := h.lastCheck
|
||||
h.checkMutex.RUnlock()
|
||||
|
||||
status := HealthStatus(h.status.Load())
|
||||
stats["health"] = map[string]interface{}{
|
||||
"status": status.String(),
|
||||
"consecutive_fails": h.consecutiveFails.Load(),
|
||||
"consecutive_ok": h.consecutiveOK.Load(),
|
||||
"last_check": lastCheck.Format(time.RFC3339),
|
||||
"time_since_check": time.Since(lastCheck).Seconds(),
|
||||
"check_interval_sec": h.config.CheckInterval.Seconds(),
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// Ping checks backend health
|
||||
func (h *HealthCheckBackend) Ping(ctx context.Context) error {
|
||||
err := h.backend.Ping(ctx)
|
||||
h.recordResult(err == nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// Close shuts down the health checker and backend
|
||||
func (h *HealthCheckBackend) Close() error {
|
||||
// Stop health check routine
|
||||
h.cancel()
|
||||
|
||||
// Wait for routine to finish
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
h.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Finished normally
|
||||
case <-time.After(2 * time.Second):
|
||||
// Timeout
|
||||
}
|
||||
|
||||
return h.backend.Close()
|
||||
}
|
||||
|
||||
// IsHealthy returns true if the backend is healthy
|
||||
func (h *HealthCheckBackend) IsHealthy() bool {
|
||||
status := HealthStatus(h.status.Load())
|
||||
return status == HealthHealthy || status == HealthDegraded
|
||||
}
|
||||
|
||||
// recordResult records the result of an operation for health tracking
|
||||
func (h *HealthCheckBackend) recordResult(success bool) {
|
||||
if success {
|
||||
fails := h.consecutiveFails.Swap(0)
|
||||
oks := h.consecutiveOK.Add(1)
|
||||
|
||||
// Check if we should transition to healthy
|
||||
if fails > 0 && oks >= int32(h.config.HealthyThreshold) {
|
||||
oldStatus := HealthStatus(h.status.Swap(int32(HealthHealthy)))
|
||||
if oldStatus != HealthHealthy && h.config.OnStatusChange != nil {
|
||||
h.config.OnStatusChange(oldStatus, HealthHealthy)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
oks := h.consecutiveOK.Swap(0)
|
||||
fails := h.consecutiveFails.Add(1)
|
||||
|
||||
// Check if we should transition to unhealthy
|
||||
if oks > 0 && fails >= int32(h.config.UnhealthyThreshold) {
|
||||
oldStatus := HealthStatus(h.status.Swap(int32(HealthUnhealthy)))
|
||||
if oldStatus != HealthUnhealthy && h.config.OnStatusChange != nil {
|
||||
h.config.OnStatusChange(oldStatus, HealthUnhealthy)
|
||||
}
|
||||
} else if fails >= int32(h.config.UnhealthyThreshold)*2 {
|
||||
// Severely degraded
|
||||
h.status.Store(int32(HealthUnhealthy))
|
||||
} else if fails >= int32(h.config.UnhealthyThreshold) {
|
||||
// Degraded but still trying
|
||||
h.status.Store(int32(HealthDegraded))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// healthCheckLoop runs periodic health checks
|
||||
func (h *HealthCheckBackend) healthCheckLoop() {
|
||||
defer h.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(h.config.CheckInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Do initial check
|
||||
h.performHealthCheck()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-h.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
h.performHealthCheck()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// performHealthCheck performs a single health check
|
||||
func (h *HealthCheckBackend) performHealthCheck() {
|
||||
h.checkMutex.Lock()
|
||||
h.lastCheck = time.Now()
|
||||
h.checkMutex.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), h.config.Timeout)
|
||||
defer cancel()
|
||||
|
||||
err := h.backend.Ping(ctx)
|
||||
h.recordResult(err == nil)
|
||||
}
|
||||
+447
@@ -0,0 +1,447 @@
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestHealthChecker_StatusTransitions tests health status transitions
|
||||
func TestHealthChecker_StatusTransitions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var callCount atomic.Int32
|
||||
var shouldFail atomic.Bool
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
callCount.Add(1)
|
||||
if shouldFail.Load() {
|
||||
return errors.New("health check failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 50 * time.Millisecond,
|
||||
Timeout: 10 * time.Millisecond,
|
||||
UnhealthyThreshold: 3,
|
||||
HealthyThreshold: 2,
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
hc.Start()
|
||||
defer hc.Stop()
|
||||
|
||||
// Initially unknown
|
||||
assert.Equal(t, HealthUnknown, hc.GetStatus())
|
||||
|
||||
// Trigger failures
|
||||
shouldFail.Store(true)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
// Should be unhealthy after threshold failures
|
||||
status := hc.GetStatus()
|
||||
assert.True(t, status == HealthUnhealthy || status == HealthDegraded)
|
||||
|
||||
// Recover
|
||||
shouldFail.Store(false)
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
// Should recover towards healthy
|
||||
finalStatus := hc.GetStatus()
|
||||
assert.True(t, finalStatus == HealthHealthy || finalStatus == HealthDegraded || finalStatus == HealthUnknown)
|
||||
}
|
||||
|
||||
// TestHealthChecker_InitialState tests initial health status
|
||||
func TestHealthChecker_InitialState(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
hc := NewHealthChecker(config)
|
||||
assert.Equal(t, HealthUnknown, hc.GetStatus())
|
||||
assert.False(t, hc.IsHealthy())
|
||||
}
|
||||
|
||||
// TestHealthChecker_ForceCheck tests manual health check trigger
|
||||
func TestHealthChecker_ForceCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var callCount atomic.Int32
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
callCount.Add(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 10 * time.Second, // Long interval
|
||||
Timeout: 1 * time.Second,
|
||||
UnhealthyThreshold: 3,
|
||||
HealthyThreshold: 2,
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
|
||||
initialCount := callCount.Load()
|
||||
|
||||
// Force check
|
||||
hc.Check(context.Background())
|
||||
|
||||
// Should have been called
|
||||
assert.Greater(t, callCount.Load(), initialCount)
|
||||
}
|
||||
|
||||
// TestHealthChecker_StatusChangeCallback tests status change notifications
|
||||
func TestHealthChecker_StatusChangeCallback(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var transitions []string
|
||||
var mu sync.Mutex
|
||||
var shouldFail atomic.Bool
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
if shouldFail.Load() {
|
||||
return errors.New("health check failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 30 * time.Millisecond,
|
||||
Timeout: 10 * time.Millisecond,
|
||||
UnhealthyThreshold: 2,
|
||||
HealthyThreshold: 2,
|
||||
CheckFunc: checkFunc,
|
||||
OnStatusChange: func(from, to HealthStatus) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
transitions = append(transitions, from.String()+"->"+to.String())
|
||||
},
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
hc.Start()
|
||||
defer hc.Stop()
|
||||
|
||||
// Trigger failures
|
||||
shouldFail.Store(true)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Recover
|
||||
shouldFail.Store(false)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
// Should have status transitions
|
||||
assert.NotEmpty(t, transitions)
|
||||
}
|
||||
|
||||
// TestHealthChecker_Stats tests statistics tracking
|
||||
func TestHealthChecker_Stats(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var callCount atomic.Int32
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
callCount.Add(1)
|
||||
if callCount.Load()%2 == 0 {
|
||||
return errors.New("failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 20 * time.Millisecond,
|
||||
Timeout: 10 * time.Millisecond,
|
||||
UnhealthyThreshold: 5,
|
||||
HealthyThreshold: 2,
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
hc.Start()
|
||||
defer hc.Stop()
|
||||
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
stats := hc.Stats()
|
||||
|
||||
assert.Greater(t, stats.TotalChecks, int64(0))
|
||||
assert.Greater(t, stats.TotalFailures, int64(0))
|
||||
assert.Greater(t, stats.SuccessRate, 0.0)
|
||||
assert.Less(t, stats.SuccessRate, 1.0)
|
||||
}
|
||||
|
||||
// TestHealthChecker_Timeout tests check timeout handling
|
||||
func TestHealthChecker_Timeout(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
// Simulate slow check
|
||||
select {
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 50 * time.Millisecond,
|
||||
Timeout: 10 * time.Millisecond, // Short timeout
|
||||
UnhealthyThreshold: 2,
|
||||
HealthyThreshold: 2,
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
hc.Start()
|
||||
defer hc.Stop()
|
||||
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
// Should be unhealthy due to timeouts
|
||||
status := hc.GetStatus()
|
||||
assert.NotEqual(t, HealthHealthy, status)
|
||||
}
|
||||
|
||||
// TestHealthChecker_ConcurrentAccess tests thread safety
|
||||
func TestHealthChecker_ConcurrentAccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 10 * time.Millisecond,
|
||||
Timeout: 5 * time.Millisecond,
|
||||
UnhealthyThreshold: 3,
|
||||
HealthyThreshold: 2,
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
hc.Start()
|
||||
defer hc.Stop()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
goroutines := 20
|
||||
|
||||
for i := 0; i < goroutines; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 50; j++ {
|
||||
_ = hc.GetStatus()
|
||||
_ = hc.IsHealthy()
|
||||
_ = hc.Stats()
|
||||
hc.Check(context.Background())
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
// Should complete without panics
|
||||
}
|
||||
|
||||
// TestHealthChecker_StopAndStart tests lifecycle management
|
||||
func TestHealthChecker_StopAndStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var callCount atomic.Int32
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
callCount.Add(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 20 * time.Millisecond,
|
||||
Timeout: 10 * time.Millisecond,
|
||||
UnhealthyThreshold: 3,
|
||||
HealthyThreshold: 2,
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
|
||||
// Start
|
||||
hc.Start()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
count1 := callCount.Load()
|
||||
assert.Greater(t, count1, int32(0))
|
||||
|
||||
// Stop
|
||||
hc.Stop()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
count2 := callCount.Load()
|
||||
|
||||
// Should not have increased significantly after stop
|
||||
assert.Less(t, count2-count1, int32(3))
|
||||
}
|
||||
|
||||
// TestHealthChecker_DegradedState tests degraded status
|
||||
func TestHealthChecker_DegradedState(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var callCount atomic.Int32
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
count := callCount.Add(1)
|
||||
// Fail once, then succeed
|
||||
if count == 1 {
|
||||
return errors.New("single failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 30 * time.Millisecond,
|
||||
Timeout: 10 * time.Millisecond,
|
||||
UnhealthyThreshold: 3, // Need 3 failures for unhealthy
|
||||
HealthyThreshold: 2, // Need 2 successes for healthy
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
hc.Start()
|
||||
defer hc.Stop()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// After initial checks, status should be set (might be healthy or degraded based on execution)
|
||||
status := hc.GetStatus()
|
||||
assert.True(t, status != HealthUnknown, "Status should not be unknown after checks")
|
||||
}
|
||||
|
||||
// TestHealthChecker_DefaultConfig tests default configuration
|
||||
func TestHealthChecker_DefaultConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
hc := NewHealthChecker(config)
|
||||
|
||||
assert.NotNil(t, hc)
|
||||
assert.Equal(t, HealthUnknown, hc.GetStatus())
|
||||
|
||||
// Verify default config was applied (we can't access private fields, so just check it works)
|
||||
assert.NotNil(t, hc)
|
||||
}
|
||||
|
||||
// TestHealthChecker_StatusString tests status string representation
|
||||
func TestHealthChecker_StatusString(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(t, "healthy", HealthHealthy.String())
|
||||
assert.Equal(t, "unhealthy", HealthUnhealthy.String())
|
||||
assert.Equal(t, "degraded", HealthDegraded.String())
|
||||
assert.Equal(t, "unknown", HealthStatus(999).String())
|
||||
}
|
||||
|
||||
// TestHealthChecker_RecoveryPattern tests typical failure and recovery
|
||||
func TestHealthChecker_RecoveryPattern(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var checkNumber atomic.Int32
|
||||
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
n := checkNumber.Add(1)
|
||||
// Fail checks 3-5, succeed others
|
||||
if n >= 3 && n <= 5 {
|
||||
return errors.New("temporary failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var statusLog []HealthStatus
|
||||
var mu sync.Mutex
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 30 * time.Millisecond,
|
||||
Timeout: 10 * time.Millisecond,
|
||||
UnhealthyThreshold: 3,
|
||||
HealthyThreshold: 2,
|
||||
CheckFunc: checkFunc,
|
||||
OnStatusChange: func(from, to HealthStatus) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
statusLog = append(statusLog, to)
|
||||
},
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
hc.Start()
|
||||
defer hc.Stop()
|
||||
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
// Should see transitions through unhealthy and back to healthy
|
||||
assert.NotEmpty(t, statusLog)
|
||||
|
||||
// Final status should be healthy or degraded (recovered)
|
||||
finalStatus := hc.GetStatus()
|
||||
assert.True(t, finalStatus == HealthHealthy || finalStatus == HealthDegraded, "Should have recovered")
|
||||
}
|
||||
|
||||
// Benchmark health checker performance
|
||||
func BenchmarkHealthChecker_ForceCheck(b *testing.B) {
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckInterval: 10 * time.Minute,
|
||||
Timeout: 1 * time.Second,
|
||||
UnhealthyThreshold: 3,
|
||||
HealthyThreshold: 2,
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
|
||||
hc := NewHealthChecker(config)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
hc.Check(context.Background())
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkHealthChecker_Status(b *testing.B) {
|
||||
checkFunc := func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &HealthCheckConfig{
|
||||
CheckFunc: checkFunc,
|
||||
}
|
||||
hc := NewHealthChecker(config)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = hc.GetStatus()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user