Add redis support for distributed caching (#83)

* Add redis support for distributed caching

* Move towards the self-provided Redis connection pool and RESP protocol implementation.
Official redis client library won't work with yaegi.

* fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* fixup! fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* fixup! fixup! fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi.

* ... and another all nighter.

* fixup! ... and another all nighter.

* fixup! fixup! ... and another all nighter.

* fixup! fixup! fixup! ... and another all nighter.

* Resolve issue #85 by adding ability to set custom claims in JWT tokens

* Remove redundant validation in auth middleware ( issue #89 )

* Add ability to set cookie prefix for session cookies ( #87 )

* fixup! Add ability to set cookie prefix for session cookies ( #87 )

* Add ability to set cookie max age - issue #91

* Potential fix for code scanning alert no. 10: Size computation for allocation may overflow

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* fixup! Merge main into 0.8.0-redis: resolve conflicts

---------

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

internal/recovery/base.go

@@ -0,0 +1,307 @@
+// Package recovery provides error recovery and resilience mechanisms for OIDC authentication.
+package recovery
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// ErrorRecoveryMechanism defines the interface for error recovery strategies.
+// It provides a common contract for implementing various resilience patterns
+// such as circuit breakers, retry mechanisms, and fallback strategies.
+type ErrorRecoveryMechanism interface {
+	// ExecuteWithContext runs a function with error recovery using the provided context
+	ExecuteWithContext(ctx context.Context, fn func() error) error
+	// Reset resets the recovery mechanism state
+	Reset()
+	// IsAvailable checks if the mechanism is currently available for use
+	IsAvailable() bool
+	// GetMetrics returns metrics about the recovery mechanism's performance
+	GetMetrics() map[string]interface{}
+}
+
+// Logger defines the logging interface
+type Logger interface {
+	Logf(format string, args ...interface{})
+	ErrorLogf(format string, args ...interface{})
+	DebugLogf(format string, args ...interface{})
+}
+
+// BaseRecoveryMechanism provides common functionality and metrics tracking
+// for all recovery mechanism implementations. It handles request counting,
+// success/failure tracking, and timestamp management in a thread-safe manner.
+type BaseRecoveryMechanism struct {
+	// name identifies the recovery mechanism instance
+	name string
+	// logger provides structured logging capabilities
+	logger Logger
+
+	// Metrics tracked with atomic operations for thread safety
+	totalRequests  int64
+	successCount   int64
+	failureCount   int64
+	lastSuccessStr string
+	lastFailureStr string
+
+	// mutexes for thread-safe timestamp updates
+	successMutex sync.RWMutex
+	failureMutex sync.RWMutex
+}
+
+// NewBaseRecoveryMechanism creates a new base recovery mechanism with the given name and logger.
+// This serves as the foundation for specific recovery mechanism implementations.
+// Parameters:
+//   - name: Identifier for this recovery mechanism instance
+//   - logger: Logger instance for outputting diagnostic information
+//
+// Returns:
+//   - A new BaseRecoveryMechanism instance with initialized metrics
+func NewBaseRecoveryMechanism(name string, logger Logger) *BaseRecoveryMechanism {
+	return &BaseRecoveryMechanism{
+		name:           name,
+		logger:         logger,
+		totalRequests:  0,
+		successCount:   0,
+		failureCount:   0,
+		lastSuccessStr: "never",
+		lastFailureStr: "never",
+	}
+}
+
+// RecordRequest increments the total request counter.
+// This method is thread-safe using atomic operations.
+func (b *BaseRecoveryMechanism) RecordRequest() {
+	atomic.AddInt64(&b.totalRequests, 1)
+}
+
+// RecordSuccess increments the success counter and updates the last success timestamp.
+// This method is thread-safe using atomic operations for counters
+// and mutex protection for timestamp updates.
+func (b *BaseRecoveryMechanism) RecordSuccess() {
+	atomic.AddInt64(&b.successCount, 1)
+	b.successMutex.Lock()
+	b.lastSuccessStr = time.Now().Format(time.RFC3339)
+	b.successMutex.Unlock()
+}
+
+// RecordFailure increments the failure counter and updates the last failure timestamp.
+// This method is thread-safe using atomic operations for counters
+// and mutex protection for timestamp updates.
+func (b *BaseRecoveryMechanism) RecordFailure() {
+	atomic.AddInt64(&b.failureCount, 1)
+	b.failureMutex.Lock()
+	b.lastFailureStr = time.Now().Format(time.RFC3339)
+	b.failureMutex.Unlock()
+}
+
+// GetBaseMetrics returns comprehensive metrics about the recovery mechanism.
+// Includes request counts, success/failure rates, timing information,
+// and calculated percentages. All access is thread-safe.
+func (b *BaseRecoveryMechanism) GetBaseMetrics() map[string]interface{} {
+	total := atomic.LoadInt64(&b.totalRequests)
+	success := atomic.LoadInt64(&b.successCount)
+	failure := atomic.LoadInt64(&b.failureCount)
+
+	b.successMutex.RLock()
+	lastSuccess := b.lastSuccessStr
+	b.successMutex.RUnlock()
+
+	b.failureMutex.RLock()
+	lastFailure := b.lastFailureStr
+	b.failureMutex.RUnlock()
+
+	metrics := map[string]interface{}{
+		"name":          b.name,
+		"totalRequests": total,
+		"successCount":  success,
+		"failureCount":  failure,
+		"lastSuccess":   lastSuccess,
+		"lastFailure":   lastFailure,
+	}
+
+	// Calculate success and failure rates
+	if total > 0 {
+		successRate := float64(success) / float64(total) * 100
+		failureRate := float64(failure) / float64(total) * 100
+		metrics["successRate"] = fmt.Sprintf("%.2f%%", successRate)
+		metrics["failureRate"] = fmt.Sprintf("%.2f%%", failureRate)
+	} else {
+		metrics["successRate"] = "0.00%"
+		metrics["failureRate"] = "0.00%"
+	}
+
+	return metrics
+}
+
+// LogInfo logs an informational message with the mechanism name as prefix.
+// Provides consistent logging format across all recovery mechanisms.
+func (b *BaseRecoveryMechanism) LogInfo(format string, args ...interface{}) {
+	if b.logger != nil {
+		b.logger.Logf("[%s] %s", b.name, fmt.Sprintf(format, args...))
+	}
+}
+
+// LogError logs an error message with the mechanism name as prefix.
+// Used for reporting failures and error conditions in recovery mechanisms.
+func (b *BaseRecoveryMechanism) LogError(format string, args ...interface{}) {
+	if b.logger != nil {
+		b.logger.ErrorLogf("[%s] %s", b.name, fmt.Sprintf(format, args...))
+	}
+}
+
+// LogDebug logs a debug message with the mechanism name as prefix.
+// Useful for detailed troubleshooting of recovery mechanism behavior.
+func (b *BaseRecoveryMechanism) LogDebug(format string, args ...interface{}) {
+	if b.logger != nil {
+		b.logger.DebugLogf("[%s] %s", b.name, fmt.Sprintf(format, args...))
+	}
+}
+
+// ErrorType represents different categories of errors
+type ErrorType int
+
+const (
+	// ErrorTypeUnknown represents an unknown error type
+	ErrorTypeUnknown ErrorType = iota
+	// ErrorTypeNetwork represents network-related errors
+	ErrorTypeNetwork
+	// ErrorTypeTimeout represents timeout errors
+	ErrorTypeTimeout
+	// ErrorTypeAuthentication represents authentication errors
+	ErrorTypeAuthentication
+	// ErrorTypeRateLimit represents rate limiting errors
+	ErrorTypeRateLimit
+	// ErrorTypeServerError represents server errors (5xx)
+	ErrorTypeServerError
+	// ErrorTypeClientError represents client errors (4xx)
+	ErrorTypeClientError
+)
+
+// HTTPError represents an HTTP error with status code and message
+type HTTPError struct {
+	StatusCode int
+	Message    string
+	Body       []byte
+	Headers    map[string]string
+}
+
+// Error implements the error interface
+func (e *HTTPError) Error() string {
+	return fmt.Sprintf("HTTP %d: %s", e.StatusCode, e.Message)
+}
+
+// IsRetryable checks if the HTTP error is retryable
+func (e *HTTPError) IsRetryable() bool {
+	// Retry on 5xx errors and specific 4xx errors
+	return e.StatusCode >= 500 || e.StatusCode == 429 || e.StatusCode == 408
+}
+
+// OIDCError represents an OIDC-specific error
+type OIDCError struct {
+	Code        string
+	Description string
+	URI         string
+	State       string
+}
+
+// Error implements the error interface
+func (e *OIDCError) Error() string {
+	if e.Description != "" {
+		return fmt.Sprintf("OIDC error %s: %s", e.Code, e.Description)
+	}
+	return fmt.Sprintf("OIDC error: %s", e.Code)
+}
+
+// IsRetryable checks if the OIDC error is retryable
+func (e *OIDCError) IsRetryable() bool {
+	// Some OIDC errors are retryable
+	switch e.Code {
+	case "temporarily_unavailable", "server_error":
+		return true
+	default:
+		return false
+	}
+}
+
+// FallbackMechanism provides a simple fallback recovery strategy
+type FallbackMechanism struct {
+	*BaseRecoveryMechanism
+	fallbackFunc func() error
+}
+
+// NewFallbackMechanism creates a new fallback mechanism
+func NewFallbackMechanism(name string, logger Logger, fallbackFunc func() error) *FallbackMechanism {
+	return &FallbackMechanism{
+		BaseRecoveryMechanism: NewBaseRecoveryMechanism(name, logger),
+		fallbackFunc:          fallbackFunc,
+	}
+}
+
+// ExecuteWithContext executes the primary function and falls back on error
+func (f *FallbackMechanism) ExecuteWithContext(ctx context.Context, fn func() error) error {
+	f.RecordRequest()
+
+	// Check context first
+	select {
+	case <-ctx.Done():
+		f.RecordFailure()
+		return ctx.Err()
+	default:
+	}
+
+	// Try primary function
+	if err := fn(); err != nil {
+		f.LogInfo("Primary function failed: %v, trying fallback", err)
+
+		// Try fallback
+		if f.fallbackFunc != nil {
+			if fallbackErr := f.fallbackFunc(); fallbackErr == nil {
+				f.RecordSuccess()
+				return nil
+			} else {
+				f.LogError("Fallback also failed: %v", fallbackErr)
+				f.RecordFailure()
+				return fmt.Errorf("both primary and fallback failed: primary=%v, fallback=%v", err, fallbackErr)
+			}
+		}
+
+		f.RecordFailure()
+		return err
+	}
+
+	f.RecordSuccess()
+	return nil
+}
+
+// Reset resets the fallback mechanism state
+func (f *FallbackMechanism) Reset() {
+	// Reset metrics
+	atomic.StoreInt64(&f.totalRequests, 0)
+	atomic.StoreInt64(&f.successCount, 0)
+	atomic.StoreInt64(&f.failureCount, 0)
+
+	f.successMutex.Lock()
+	f.lastSuccessStr = "never"
+	f.successMutex.Unlock()
+
+	f.failureMutex.Lock()
+	f.lastFailureStr = "never"
+	f.failureMutex.Unlock()
+}
+
+// IsAvailable checks if the fallback mechanism is available
+func (f *FallbackMechanism) IsAvailable() bool {
+	// Fallback is always available
+	return true
+}
+
+// GetMetrics returns metrics about the fallback mechanism
+func (f *FallbackMechanism) GetMetrics() map[string]interface{} {
+	metrics := f.GetBaseMetrics()
+	metrics["type"] = "fallback"
+	metrics["hasFallback"] = f.fallbackFunc != nil
+	return metrics
+}

internal/recovery/circuit_breaker.go

@@ -0,0 +1,336 @@
+// Package recovery provides error recovery and resilience mechanisms for OIDC authentication.
+package recovery
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// CircuitBreakerState represents the current state of the circuit breaker
+type CircuitBreakerState int
+
+const (
+	// CircuitBreakerClosed allows all requests to pass through
+	CircuitBreakerClosed CircuitBreakerState = iota
+	// CircuitBreakerOpen blocks all requests
+	CircuitBreakerOpen
+	// CircuitBreakerHalfOpen allows limited requests for testing
+	CircuitBreakerHalfOpen
+)
+
+// String returns the string representation of the circuit breaker state
+func (s CircuitBreakerState) String() string {
+	switch s {
+	case CircuitBreakerClosed:
+		return "closed"
+	case CircuitBreakerOpen:
+		return "open"
+	case CircuitBreakerHalfOpen:
+		return "half-open"
+	default:
+		return "unknown"
+	}
+}
+
+// CircuitBreakerConfig defines configuration for the circuit breaker
+type CircuitBreakerConfig struct {
+	// FailureThreshold is the number of failures before opening the circuit
+	FailureThreshold int
+	// SuccessThreshold is the number of successes in half-open state before closing
+	SuccessThreshold int
+	// Timeout is the duration to wait before transitioning from open to half-open
+	Timeout time.Duration
+	// MaxRequests is the maximum number of requests allowed in half-open state
+	MaxRequests int
+}
+
+// DefaultCircuitBreakerConfig returns sensible default configuration
+func DefaultCircuitBreakerConfig() CircuitBreakerConfig {
+	return CircuitBreakerConfig{
+		FailureThreshold: 5,
+		SuccessThreshold: 2,
+		Timeout:          30 * time.Second,
+		MaxRequests:      3,
+	}
+}
+
+// CircuitBreaker implements the circuit breaker pattern for fault tolerance.
+// It prevents cascading failures by temporarily blocking requests to a failing service.
+type CircuitBreaker struct {
+	*BaseRecoveryMechanism
+	config CircuitBreakerConfig
+
+	// State management
+	state           int32 // atomic: CircuitBreakerState
+	lastStateChange time.Time
+	stateMutex      sync.RWMutex
+
+	// Failure tracking
+	consecutiveFailures  int32 // atomic
+	consecutiveSuccesses int32 // atomic
+
+	// Half-open state management
+	halfOpenRequests int32 // atomic
+}
+
+// NewCircuitBreaker creates a new circuit breaker with the given configuration
+func NewCircuitBreaker(config CircuitBreakerConfig, logger Logger) *CircuitBreaker {
+	return &CircuitBreaker{
+		BaseRecoveryMechanism: NewBaseRecoveryMechanism("CircuitBreaker", logger),
+		config:                config,
+		state:                 int32(CircuitBreakerClosed),
+		lastStateChange:       time.Now(),
+		consecutiveFailures:   0,
+		consecutiveSuccesses:  0,
+		halfOpenRequests:      0,
+	}
+}
+
+// ExecuteWithContext executes a function with circuit breaker protection
+func (cb *CircuitBreaker) ExecuteWithContext(ctx context.Context, fn func() error) error {
+	cb.RecordRequest()
+
+	// Check if request is allowed
+	if !cb.allowRequest() {
+		cb.RecordFailure()
+		return fmt.Errorf("circuit breaker is open")
+	}
+
+	// Execute the function
+	err := fn()
+
+	if err != nil {
+		cb.recordFailure()
+		return err
+	}
+
+	cb.recordSuccess()
+	return nil
+}
+
+// Execute executes a function with circuit breaker protection (legacy method)
+func (cb *CircuitBreaker) Execute(fn func() error) error {
+	return cb.ExecuteWithContext(context.Background(), fn)
+}
+
+// allowRequest determines if a request should be allowed based on the circuit state
+func (cb *CircuitBreaker) allowRequest() bool {
+	state := CircuitBreakerState(atomic.LoadInt32(&cb.state))
+
+	switch state {
+	case CircuitBreakerClosed:
+		return true
+
+	case CircuitBreakerOpen:
+		// Check if timeout has elapsed
+		cb.stateMutex.RLock()
+		lastChange := cb.lastStateChange
+		cb.stateMutex.RUnlock()
+
+		if time.Since(lastChange) > cb.config.Timeout {
+			// Transition to half-open
+			cb.transitionToHalfOpen()
+			return cb.allowHalfOpenRequest()
+		}
+		return false
+
+	case CircuitBreakerHalfOpen:
+		return cb.allowHalfOpenRequest()
+
+	default:
+		return false
+	}
+}
+
+// allowHalfOpenRequest checks if a request is allowed in half-open state
+func (cb *CircuitBreaker) allowHalfOpenRequest() bool {
+	current := atomic.AddInt32(&cb.halfOpenRequests, 1)
+	if current <= int32(cb.config.MaxRequests) {
+		return true
+	}
+	atomic.AddInt32(&cb.halfOpenRequests, -1)
+	return false
+}
+
+// recordFailure records a failure and potentially opens the circuit
+func (cb *CircuitBreaker) recordFailure() {
+	cb.RecordFailure()
+
+	failures := atomic.AddInt32(&cb.consecutiveFailures, 1)
+	atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
+
+	state := CircuitBreakerState(atomic.LoadInt32(&cb.state))
+
+	if state == CircuitBreakerClosed && failures >= int32(cb.config.FailureThreshold) {
+		cb.transitionToOpen()
+	} else if state == CircuitBreakerHalfOpen {
+		cb.transitionToOpen()
+	}
+}
+
+// recordSuccess records a success and potentially closes the circuit
+func (cb *CircuitBreaker) recordSuccess() {
+	cb.RecordSuccess()
+
+	successes := atomic.AddInt32(&cb.consecutiveSuccesses, 1)
+	atomic.StoreInt32(&cb.consecutiveFailures, 0)
+
+	state := CircuitBreakerState(atomic.LoadInt32(&cb.state))
+
+	if state == CircuitBreakerHalfOpen && successes >= int32(cb.config.SuccessThreshold) {
+		cb.transitionToClosed()
+	}
+}
+
+// transitionToClosed transitions the circuit to closed state
+func (cb *CircuitBreaker) transitionToClosed() {
+	if atomic.CompareAndSwapInt32(&cb.state, int32(CircuitBreakerHalfOpen), int32(CircuitBreakerClosed)) {
+		cb.stateMutex.Lock()
+		cb.lastStateChange = time.Now()
+		cb.stateMutex.Unlock()
+
+		atomic.StoreInt32(&cb.consecutiveFailures, 0)
+		atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
+		atomic.StoreInt32(&cb.halfOpenRequests, 0)
+
+		cb.LogInfo("Circuit breaker closed")
+	}
+}
+
+// transitionToOpen transitions the circuit to open state
+func (cb *CircuitBreaker) transitionToOpen() {
+	oldState := atomic.SwapInt32(&cb.state, int32(CircuitBreakerOpen))
+	if oldState != int32(CircuitBreakerOpen) {
+		cb.stateMutex.Lock()
+		cb.lastStateChange = time.Now()
+		cb.stateMutex.Unlock()
+
+		atomic.StoreInt32(&cb.consecutiveFailures, 0)
+		atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
+		atomic.StoreInt32(&cb.halfOpenRequests, 0)
+
+		cb.LogError("Circuit breaker opened due to failures")
+	}
+}
+
+// transitionToHalfOpen transitions the circuit to half-open state
+func (cb *CircuitBreaker) transitionToHalfOpen() {
+	if atomic.CompareAndSwapInt32(&cb.state, int32(CircuitBreakerOpen), int32(CircuitBreakerHalfOpen)) {
+		cb.stateMutex.Lock()
+		cb.lastStateChange = time.Now()
+		cb.stateMutex.Unlock()
+
+		atomic.StoreInt32(&cb.consecutiveFailures, 0)
+		atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
+		atomic.StoreInt32(&cb.halfOpenRequests, 0)
+
+		cb.LogInfo("Circuit breaker half-open, testing recovery")
+	}
+}
+
+// GetState returns the current state of the circuit breaker
+func (cb *CircuitBreaker) GetState() CircuitBreakerState {
+	return CircuitBreakerState(atomic.LoadInt32(&cb.state))
+}
+
+// Reset resets the circuit breaker to closed state
+func (cb *CircuitBreaker) Reset() {
+	atomic.StoreInt32(&cb.state, int32(CircuitBreakerClosed))
+
+	cb.stateMutex.Lock()
+	cb.lastStateChange = time.Now()
+	cb.stateMutex.Unlock()
+
+	atomic.StoreInt32(&cb.consecutiveFailures, 0)
+	atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
+	atomic.StoreInt32(&cb.halfOpenRequests, 0)
+
+	// Reset base metrics
+	atomic.StoreInt64(&cb.totalRequests, 0)
+	atomic.StoreInt64(&cb.successCount, 0)
+	atomic.StoreInt64(&cb.failureCount, 0)
+
+	cb.LogInfo("Circuit breaker reset to closed state")
+}
+
+// IsAvailable returns true if the circuit breaker is not fully open
+func (cb *CircuitBreaker) IsAvailable() bool {
+	state := cb.GetState()
+	return state != CircuitBreakerOpen || time.Since(cb.getLastStateChange()) > cb.config.Timeout
+}
+
+// getLastStateChange returns the last state change time safely
+func (cb *CircuitBreaker) getLastStateChange() time.Time {
+	cb.stateMutex.RLock()
+	defer cb.stateMutex.RUnlock()
+	return cb.lastStateChange
+}
+
+// GetMetrics returns comprehensive metrics about the circuit breaker
+func (cb *CircuitBreaker) GetMetrics() map[string]interface{} {
+	metrics := cb.GetBaseMetrics()
+
+	state := cb.GetState()
+	metrics["state"] = state.String()
+	metrics["consecutiveFailures"] = atomic.LoadInt32(&cb.consecutiveFailures)
+	metrics["consecutiveSuccesses"] = atomic.LoadInt32(&cb.consecutiveSuccesses)
+	metrics["halfOpenRequests"] = atomic.LoadInt32(&cb.halfOpenRequests)
+
+	cb.stateMutex.RLock()
+	metrics["lastStateChange"] = cb.lastStateChange.Format(time.RFC3339)
+	metrics["timeSinceLastChange"] = time.Since(cb.lastStateChange).String()
+	cb.stateMutex.RUnlock()
+
+	// Configuration
+	metrics["config"] = map[string]interface{}{
+		"failureThreshold": cb.config.FailureThreshold,
+		"successThreshold": cb.config.SuccessThreshold,
+		"timeout":          cb.config.Timeout.String(),
+		"maxRequests":      cb.config.MaxRequests,
+	}
+
+	// Health indicator
+	switch state {
+	case CircuitBreakerClosed:
+		metrics["health"] = "healthy"
+	case CircuitBreakerHalfOpen:
+		metrics["health"] = "recovering"
+	case CircuitBreakerOpen:
+		if time.Since(cb.getLastStateChange()) > cb.config.Timeout {
+			metrics["health"] = "ready-to-recover"
+		} else {
+			metrics["health"] = "unhealthy"
+		}
+	}
+
+	return metrics
+}
+
+// ForceOpen forces the circuit breaker to open state
+func (cb *CircuitBreaker) ForceOpen() {
+	atomic.StoreInt32(&cb.state, int32(CircuitBreakerOpen))
+
+	cb.stateMutex.Lock()
+	cb.lastStateChange = time.Now()
+	cb.stateMutex.Unlock()
+
+	cb.LogInfo("Circuit breaker forced open")
+}
+
+// ForceClosed forces the circuit breaker to closed state
+func (cb *CircuitBreaker) ForceClosed() {
+	atomic.StoreInt32(&cb.state, int32(CircuitBreakerClosed))
+
+	cb.stateMutex.Lock()
+	cb.lastStateChange = time.Now()
+	cb.stateMutex.Unlock()
+
+	atomic.StoreInt32(&cb.consecutiveFailures, 0)
+	atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
+	atomic.StoreInt32(&cb.halfOpenRequests, 0)
+
+	cb.LogInfo("Circuit breaker forced closed")
+}

internal/recovery/metrics.go

@@ -0,0 +1,391 @@
+// Package recovery provides error recovery and resilience mechanisms for OIDC authentication.
+package recovery
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"math/rand"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// RetryConfig defines configuration for the retry executor
+type RetryConfig struct {
+	// MaxAttempts is the maximum number of retry attempts
+	MaxAttempts int
+	// InitialDelay is the initial delay between retries
+	InitialDelay time.Duration
+	// MaxDelay is the maximum delay between retries
+	MaxDelay time.Duration
+	// Multiplier is the backoff multiplier
+	Multiplier float64
+	// RandomizationFactor adds jitter to delays (0.0 to 1.0)
+	RandomizationFactor float64
+	// RetryableErrors defines which errors should trigger a retry
+	RetryableErrors []string
+	// RetryableStatusCodes defines which HTTP status codes should trigger a retry
+	RetryableStatusCodes []int
+}
+
+// DefaultRetryConfig returns sensible default retry configuration
+func DefaultRetryConfig() RetryConfig {
+	return RetryConfig{
+		MaxAttempts:          3,
+		InitialDelay:         100 * time.Millisecond,
+		MaxDelay:             30 * time.Second,
+		Multiplier:           2.0,
+		RandomizationFactor:  0.1,
+		RetryableErrors:      []string{"connection refused", "timeout", "EOF"},
+		RetryableStatusCodes: []int{408, 429, 500, 502, 503, 504},
+	}
+}
+
+// RetryExecutor implements retry logic with exponential backoff
+type RetryExecutor struct {
+	*BaseRecoveryMechanism
+	config RetryConfig
+
+	// Metrics
+	totalRetries   int64
+	maxRetriesHit  int64
+	lastRetryTime  time.Time
+	retryTimeMutex sync.RWMutex
+}
+
+// NewRetryExecutor creates a new retry executor with the given configuration
+func NewRetryExecutor(config RetryConfig, logger Logger) *RetryExecutor {
+	if config.MaxAttempts < 1 {
+		config.MaxAttempts = 1
+	}
+	if config.Multiplier < 1.0 {
+		config.Multiplier = 1.0
+	}
+	return &RetryExecutor{
+		BaseRecoveryMechanism: NewBaseRecoveryMechanism("RetryExecutor", logger),
+		config:                config,
+		totalRetries:          0,
+		maxRetriesHit:         0,
+	}
+}
+
+// ExecuteWithContext executes a function with retry logic
+func (re *RetryExecutor) ExecuteWithContext(ctx context.Context, fn func() error) error {
+	re.RecordRequest()
+
+	var lastErr error
+	for attempt := 1; attempt <= re.config.MaxAttempts; attempt++ {
+		// Check context before attempting
+		select {
+		case <-ctx.Done():
+			re.RecordFailure()
+			return ctx.Err()
+		default:
+		}
+
+		// Execute the function
+		lastErr = fn()
+
+		if lastErr == nil {
+			re.RecordSuccess()
+			if attempt > 1 {
+				re.LogInfo("Succeeded after %d attempts", attempt)
+			}
+			return nil
+		}
+
+		// Check if error is retryable
+		if !re.isRetryableError(lastErr) {
+			re.LogDebug("Error is not retryable: %v", lastErr)
+			re.RecordFailure()
+			return lastErr
+		}
+
+		// Don't retry if this was the last attempt
+		if attempt >= re.config.MaxAttempts {
+			atomic.AddInt64(&re.maxRetriesHit, 1)
+			re.LogError("Max retries (%d) exhausted", re.config.MaxAttempts)
+			break
+		}
+
+		// Calculate and apply delay
+		delay := re.calculateDelay(attempt)
+		re.LogInfo("Attempt %d failed: %v, retrying in %v", attempt, lastErr, delay)
+
+		atomic.AddInt64(&re.totalRetries, 1)
+		re.retryTimeMutex.Lock()
+		re.lastRetryTime = time.Now()
+		re.retryTimeMutex.Unlock()
+
+		select {
+		case <-time.After(delay):
+			// Continue to next attempt
+		case <-ctx.Done():
+			re.RecordFailure()
+			return fmt.Errorf("retry cancelled: %w", ctx.Err())
+		}
+	}
+
+	re.RecordFailure()
+	return fmt.Errorf("all retry attempts failed: %w", lastErr)
+}
+
+// Execute executes a function with retry logic (legacy method)
+func (re *RetryExecutor) Execute(ctx context.Context, fn func() error) error {
+	return re.ExecuteWithContext(ctx, fn)
+}
+
+// isRetryableError determines if an error should trigger a retry
+func (re *RetryExecutor) isRetryableError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := strings.ToLower(err.Error())
+
+	// Check for retryable error patterns
+	for _, pattern := range re.config.RetryableErrors {
+		if strings.Contains(errStr, strings.ToLower(pattern)) {
+			return true
+		}
+	}
+
+	// Check for HTTP errors
+	if httpErr, ok := err.(*HTTPError); ok {
+		for _, code := range re.config.RetryableStatusCodes {
+			if httpErr.StatusCode == code {
+				return true
+			}
+		}
+		// Also retry on any 5xx error
+		if httpErr.StatusCode >= 500 && httpErr.StatusCode < 600 {
+			return true
+		}
+	}
+
+	// Check for OIDC errors
+	if oidcErr, ok := err.(*OIDCError); ok {
+		return oidcErr.IsRetryable()
+	}
+
+	// Check for context errors (don't retry these)
+	if err == context.Canceled || err == context.DeadlineExceeded {
+		return false
+	}
+
+	// Default: don't retry unknown errors
+	return false
+}
+
+// calculateDelay calculates the delay before the next retry attempt
+func (re *RetryExecutor) calculateDelay(attempt int) time.Duration {
+	// Exponential backoff
+	delay := float64(re.config.InitialDelay) * math.Pow(re.config.Multiplier, float64(attempt-1))
+
+	// Cap at max delay
+	if delay > float64(re.config.MaxDelay) {
+		delay = float64(re.config.MaxDelay)
+	}
+
+	// Add jitter
+	if re.config.RandomizationFactor > 0 {
+		jitter := delay * re.config.RandomizationFactor
+		minDelay := delay - jitter
+		maxDelay := delay + jitter
+		delay = minDelay + rand.Float64()*(maxDelay-minDelay)
+	}
+
+	return time.Duration(delay)
+}
+
+// Reset resets the retry executor state
+func (re *RetryExecutor) Reset() {
+	atomic.StoreInt64(&re.totalRetries, 0)
+	atomic.StoreInt64(&re.maxRetriesHit, 0)
+	atomic.StoreInt64(&re.totalRequests, 0)
+	atomic.StoreInt64(&re.successCount, 0)
+	atomic.StoreInt64(&re.failureCount, 0)
+
+	re.retryTimeMutex.Lock()
+	re.lastRetryTime = time.Time{}
+	re.retryTimeMutex.Unlock()
+}
+
+// IsAvailable always returns true for retry executor
+func (re *RetryExecutor) IsAvailable() bool {
+	return true
+}
+
+// GetMetrics returns comprehensive metrics about the retry executor
+func (re *RetryExecutor) GetMetrics() map[string]interface{} {
+	metrics := re.GetBaseMetrics()
+
+	metrics["totalRetries"] = atomic.LoadInt64(&re.totalRetries)
+	metrics["maxRetriesHit"] = atomic.LoadInt64(&re.maxRetriesHit)
+
+	re.retryTimeMutex.RLock()
+	if !re.lastRetryTime.IsZero() {
+		metrics["lastRetryTime"] = re.lastRetryTime.Format(time.RFC3339)
+		metrics["timeSinceLastRetry"] = time.Since(re.lastRetryTime).String()
+	} else {
+		metrics["lastRetryTime"] = "never"
+	}
+	re.retryTimeMutex.RUnlock()
+
+	// Configuration
+	metrics["config"] = map[string]interface{}{
+		"maxAttempts":         re.config.MaxAttempts,
+		"initialDelay":        re.config.InitialDelay.String(),
+		"maxDelay":            re.config.MaxDelay.String(),
+		"multiplier":          re.config.Multiplier,
+		"randomizationFactor": re.config.RandomizationFactor,
+	}
+
+	// Calculate average retries per request
+	totalRequests := atomic.LoadInt64(&re.totalRequests)
+	if totalRequests > 0 {
+		avgRetries := float64(atomic.LoadInt64(&re.totalRetries)) / float64(totalRequests)
+		metrics["averageRetriesPerRequest"] = fmt.Sprintf("%.2f", avgRetries)
+	}
+
+	return metrics
+}
+
+// RecoveryMetrics aggregates metrics from multiple recovery mechanisms
+type RecoveryMetrics struct {
+	mechanisms map[string]ErrorRecoveryMechanism
+	mu         sync.RWMutex
+}
+
+// NewRecoveryMetrics creates a new recovery metrics aggregator
+func NewRecoveryMetrics() *RecoveryMetrics {
+	return &RecoveryMetrics{
+		mechanisms: make(map[string]ErrorRecoveryMechanism),
+	}
+}
+
+// RegisterMechanism registers a recovery mechanism for metrics collection
+func (rm *RecoveryMetrics) RegisterMechanism(name string, mechanism ErrorRecoveryMechanism) {
+	rm.mu.Lock()
+	defer rm.mu.Unlock()
+	rm.mechanisms[name] = mechanism
+}
+
+// UnregisterMechanism removes a recovery mechanism from metrics collection
+func (rm *RecoveryMetrics) UnregisterMechanism(name string) {
+	rm.mu.Lock()
+	defer rm.mu.Unlock()
+	delete(rm.mechanisms, name)
+}
+
+// GetAllMetrics returns aggregated metrics from all registered mechanisms
+func (rm *RecoveryMetrics) GetAllMetrics() map[string]interface{} {
+	rm.mu.RLock()
+	defer rm.mu.RUnlock()
+
+	allMetrics := make(map[string]interface{})
+	for name, mechanism := range rm.mechanisms {
+		allMetrics[name] = mechanism.GetMetrics()
+	}
+
+	// Add summary statistics
+	totalRequests := int64(0)
+	totalSuccesses := int64(0)
+	totalFailures := int64(0)
+
+	for _, mechanism := range rm.mechanisms {
+		metrics := mechanism.GetMetrics()
+		if requests, ok := metrics["totalRequests"].(int64); ok {
+			totalRequests += requests
+		}
+		if successes, ok := metrics["successCount"].(int64); ok {
+			totalSuccesses += successes
+		}
+		if failures, ok := metrics["failureCount"].(int64); ok {
+			totalFailures += failures
+		}
+	}
+
+	allMetrics["summary"] = map[string]interface{}{
+		"totalMechanisms": len(rm.mechanisms),
+		"totalRequests":   totalRequests,
+		"totalSuccesses":  totalSuccesses,
+		"totalFailures":   totalFailures,
+	}
+
+	if totalRequests > 0 {
+		successRate := float64(totalSuccesses) / float64(totalRequests) * 100
+		allMetrics["summary"].(map[string]interface{})["overallSuccessRate"] = fmt.Sprintf("%.2f%%", successRate)
+	}
+
+	return allMetrics
+}
+
+// GetMechanismMetrics returns metrics for a specific mechanism
+func (rm *RecoveryMetrics) GetMechanismMetrics(name string) (map[string]interface{}, bool) {
+	rm.mu.RLock()
+	defer rm.mu.RUnlock()
+
+	if mechanism, exists := rm.mechanisms[name]; exists {
+		return mechanism.GetMetrics(), true
+	}
+	return nil, false
+}
+
+// HealthCheck performs a health check on all registered mechanisms
+func (rm *RecoveryMetrics) HealthCheck() map[string]interface{} {
+	rm.mu.RLock()
+	defer rm.mu.RUnlock()
+
+	health := make(map[string]interface{})
+	healthyCount := 0
+	unhealthyCount := 0
+
+	for name, mechanism := range rm.mechanisms {
+		if mechanism.IsAvailable() {
+			health[name] = "healthy"
+			healthyCount++
+		} else {
+			health[name] = "unhealthy"
+			unhealthyCount++
+		}
+	}
+
+	overallHealth := "healthy"
+	if unhealthyCount > 0 {
+		if healthyCount > 0 {
+			overallHealth = "degraded"
+		} else {
+			overallHealth = "unhealthy"
+		}
+	}
+
+	return map[string]interface{}{
+		"status":     overallHealth,
+		"mechanisms": health,
+		"healthy":    healthyCount,
+		"unhealthy":  unhealthyCount,
+		"timestamp":  time.Now().Format(time.RFC3339),
+	}
+}
+
+// HTTPMetricsHandler creates an HTTP handler for serving recovery metrics
+func (rm *RecoveryMetrics) HTTPMetricsHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		metrics := rm.GetAllMetrics()
+		health := rm.HealthCheck()
+
+		response := map[string]interface{}{
+			"metrics": metrics,
+			"health":  health,
+		}
+
+		// Would normally use json.Marshal here, but keeping it simple for the module
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintf(w, "%v", response)
+	}
+}

internal/recovery/recovery_boost_test.go

@@ -0,0 +1,524 @@
+//go:build !yaegi
+
+package recovery
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+// LogDebug Tests
+func TestBaseRecoveryMechanism_LogDebug(t *testing.T) {
+	logger := &mockLogger{}
+	base := NewBaseRecoveryMechanism("test-debug", logger)
+
+	// Call LogDebug
+	base.LogDebug("test message: %s", "value")
+
+	// Verify debug log was called
+	if len(logger.debugLog) != 1 {
+		t.Errorf("Expected 1 debug log entry, got %d", len(logger.debugLog))
+	}
+}
+
+func TestBaseRecoveryMechanism_LogDebug_NilLogger(t *testing.T) {
+	base := NewBaseRecoveryMechanism("test", nil)
+
+	// Should not panic with nil logger
+	base.LogDebug("this should not crash")
+}
+
+// HTTPError Tests
+func TestHTTPError_Error(t *testing.T) {
+	err := &HTTPError{
+		StatusCode: 404,
+		Message:    "Not Found",
+	}
+
+	expected := "HTTP 404: Not Found"
+	if err.Error() != expected {
+		t.Errorf("Expected '%s', got '%s'", expected, err.Error())
+	}
+}
+
+func TestHTTPError_IsRetryable(t *testing.T) {
+	tests := []struct {
+		name       string
+		statusCode int
+		retryable  bool
+	}{
+		{"500 Internal Server Error", 500, true},
+		{"502 Bad Gateway", 502, true},
+		{"503 Service Unavailable", 503, true},
+		{"504 Gateway Timeout", 504, true},
+		{"429 Too Many Requests", 429, true},
+		{"408 Request Timeout", 408, true},
+		{"400 Bad Request", 400, false},
+		{"401 Unauthorized", 401, false},
+		{"403 Forbidden", 403, false},
+		{"404 Not Found", 404, false},
+		{"200 OK", 200, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := &HTTPError{
+				StatusCode: tt.statusCode,
+				Message:    tt.name,
+			}
+
+			if err.IsRetryable() != tt.retryable {
+				t.Errorf("StatusCode %d: expected retryable=%v, got %v",
+					tt.statusCode, tt.retryable, err.IsRetryable())
+			}
+		})
+	}
+}
+
+// OIDCError Tests
+func TestOIDCError_Error_WithDescription(t *testing.T) {
+	err := &OIDCError{
+		Code:        "invalid_request",
+		Description: "Missing required parameter",
+	}
+
+	expected := "OIDC error invalid_request: Missing required parameter"
+	if err.Error() != expected {
+		t.Errorf("Expected '%s', got '%s'", expected, err.Error())
+	}
+}
+
+func TestOIDCError_Error_WithoutDescription(t *testing.T) {
+	err := &OIDCError{
+		Code: "server_error",
+	}
+
+	expected := "OIDC error: server_error"
+	if err.Error() != expected {
+		t.Errorf("Expected '%s', got '%s'", expected, err.Error())
+	}
+}
+
+func TestOIDCError_IsRetryable(t *testing.T) {
+	tests := []struct {
+		code      string
+		retryable bool
+	}{
+		{"temporarily_unavailable", true},
+		{"server_error", true},
+		{"invalid_request", false},
+		{"invalid_client", false},
+		{"invalid_grant", false},
+		{"unauthorized_client", false},
+		{"unsupported_grant_type", false},
+		{"access_denied", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.code, func(t *testing.T) {
+			err := &OIDCError{
+				Code: tt.code,
+			}
+
+			if err.IsRetryable() != tt.retryable {
+				t.Errorf("Code '%s': expected retryable=%v, got %v",
+					tt.code, tt.retryable, err.IsRetryable())
+			}
+		})
+	}
+}
+
+// FallbackMechanism Tests
+func TestNewFallbackMechanism(t *testing.T) {
+	logger := &mockLogger{}
+	fallbackFunc := func() error { return nil }
+
+	fm := NewFallbackMechanism("test-fallback", logger, fallbackFunc)
+
+	if fm == nil {
+		t.Fatal("Expected NewFallbackMechanism to return non-nil")
+	}
+
+	if fm.name != "test-fallback" {
+		t.Errorf("Expected name 'test-fallback', got '%s'", fm.name)
+	}
+
+	if fm.fallbackFunc == nil {
+		t.Error("Expected fallbackFunc to be set")
+	}
+}
+
+func TestFallbackMechanism_ExecuteWithContext_PrimarySuccess(t *testing.T) {
+	logger := &mockLogger{}
+	fallbackCalled := false
+	fallbackFunc := func() error {
+		fallbackCalled = true
+		return nil
+	}
+
+	fm := NewFallbackMechanism("test", logger, fallbackFunc)
+
+	// Primary function succeeds
+	err := fm.ExecuteWithContext(context.Background(), func() error {
+		return nil
+	})
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+
+	if fallbackCalled {
+		t.Error("Expected fallback to not be called when primary succeeds")
+	}
+
+	if fm.successCount != 1 {
+		t.Errorf("Expected successCount=1, got %d", fm.successCount)
+	}
+}
+
+func TestFallbackMechanism_ExecuteWithContext_FallbackSuccess(t *testing.T) {
+	logger := &mockLogger{}
+	fallbackCalled := false
+	fallbackFunc := func() error {
+		fallbackCalled = true
+		return nil
+	}
+
+	fm := NewFallbackMechanism("test", logger, fallbackFunc)
+
+	// Primary fails, fallback succeeds
+	err := fm.ExecuteWithContext(context.Background(), func() error {
+		return errors.New("primary failed")
+	})
+
+	if err != nil {
+		t.Errorf("Expected no error (fallback succeeded), got %v", err)
+	}
+
+	if !fallbackCalled {
+		t.Error("Expected fallback to be called")
+	}
+
+	if fm.successCount != 1 {
+		t.Errorf("Expected successCount=1, got %d", fm.successCount)
+	}
+}
+
+func TestFallbackMechanism_ExecuteWithContext_BothFail(t *testing.T) {
+	logger := &mockLogger{}
+	fallbackFunc := func() error {
+		return errors.New("fallback failed")
+	}
+
+	fm := NewFallbackMechanism("test", logger, fallbackFunc)
+
+	// Both primary and fallback fail
+	err := fm.ExecuteWithContext(context.Background(), func() error {
+		return errors.New("primary failed")
+	})
+
+	if err == nil {
+		t.Error("Expected error when both primary and fallback fail")
+	}
+
+	if fm.failureCount != 1 {
+		t.Errorf("Expected failureCount=1, got %d", fm.failureCount)
+	}
+}
+
+func TestFallbackMechanism_ExecuteWithContext_NoFallback(t *testing.T) {
+	logger := &mockLogger{}
+	fm := NewFallbackMechanism("test", logger, nil) // No fallback function
+
+	// Primary fails, no fallback
+	primaryErr := errors.New("primary failed")
+	err := fm.ExecuteWithContext(context.Background(), func() error {
+		return primaryErr
+	})
+
+	if err != primaryErr {
+		t.Errorf("Expected primary error %v, got %v", primaryErr, err)
+	}
+
+	if fm.failureCount != 1 {
+		t.Errorf("Expected failureCount=1, got %d", fm.failureCount)
+	}
+}
+
+func TestFallbackMechanism_ExecuteWithContext_ContextCanceled(t *testing.T) {
+	logger := &mockLogger{}
+	fallbackFunc := func() error { return nil }
+	fm := NewFallbackMechanism("test", logger, fallbackFunc)
+
+	// Context already canceled
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	err := fm.ExecuteWithContext(ctx, func() error {
+		t.Error("Function should not be called when context is canceled")
+		return nil
+	})
+
+	if err != context.Canceled {
+		t.Errorf("Expected context.Canceled error, got %v", err)
+	}
+
+	if fm.failureCount != 1 {
+		t.Errorf("Expected failureCount=1, got %d", fm.failureCount)
+	}
+}
+
+func TestFallbackMechanism_Reset(t *testing.T) {
+	logger := &mockLogger{}
+	fm := NewFallbackMechanism("test", logger, nil)
+
+	// Record some metrics
+	fm.RecordRequest()
+	fm.RecordSuccess()
+	fm.RecordFailure()
+
+	if fm.totalRequests == 0 {
+		t.Error("Expected some requests before reset")
+	}
+
+	// Reset
+	fm.Reset()
+
+	if fm.totalRequests != 0 {
+		t.Errorf("Expected totalRequests=0 after reset, got %d", fm.totalRequests)
+	}
+
+	if fm.successCount != 0 {
+		t.Errorf("Expected successCount=0 after reset, got %d", fm.successCount)
+	}
+
+	if fm.failureCount != 0 {
+		t.Errorf("Expected failureCount=0 after reset, got %d", fm.failureCount)
+	}
+
+	if fm.lastSuccessStr != "never" {
+		t.Errorf("Expected lastSuccessStr='never' after reset, got '%s'", fm.lastSuccessStr)
+	}
+
+	if fm.lastFailureStr != "never" {
+		t.Errorf("Expected lastFailureStr='never' after reset, got '%s'", fm.lastFailureStr)
+	}
+}
+
+func TestFallbackMechanism_IsAvailable(t *testing.T) {
+	logger := &mockLogger{}
+	fm := NewFallbackMechanism("test", logger, nil)
+
+	// Fallback mechanism is always available
+	if !fm.IsAvailable() {
+		t.Error("Expected IsAvailable to return true")
+	}
+}
+
+func TestFallbackMechanism_GetMetrics(t *testing.T) {
+	logger := &mockLogger{}
+	fallbackFunc := func() error { return nil }
+	fm := NewFallbackMechanism("test-metrics", logger, fallbackFunc)
+
+	fm.RecordRequest()
+	fm.RecordSuccess()
+
+	metrics := fm.GetMetrics()
+
+	if metrics == nil {
+		t.Fatal("Expected GetMetrics to return non-nil")
+	}
+
+	if metrics["type"] != "fallback" {
+		t.Errorf("Expected type='fallback', got %v", metrics["type"])
+	}
+
+	if metrics["name"] != "test-metrics" {
+		t.Errorf("Expected name='test-metrics', got %v", metrics["name"])
+	}
+
+	if metrics["hasFallback"] != true {
+		t.Error("Expected hasFallback=true")
+	}
+
+	if metrics["totalRequests"].(int64) != 1 {
+		t.Errorf("Expected totalRequests=1, got %v", metrics["totalRequests"])
+	}
+}
+
+func TestFallbackMechanism_GetMetrics_NoFallback(t *testing.T) {
+	logger := &mockLogger{}
+	fm := NewFallbackMechanism("test", logger, nil)
+
+	metrics := fm.GetMetrics()
+
+	if metrics["hasFallback"] != false {
+		t.Error("Expected hasFallback=false when no fallback function")
+	}
+}
+
+// ============================================================================
+// CIRCUIT BREAKER ADDITIONAL TESTS
+// ============================================================================
+
+// TestCircuitBreaker_Execute tests the legacy Execute method
+func TestCircuitBreaker_Execute(t *testing.T) {
+	logger := &mockLogger{}
+	config := DefaultCircuitBreakerConfig()
+	cb := NewCircuitBreaker(config, logger)
+
+	// Test successful execution via Execute (legacy method)
+	called := false
+	err := cb.Execute(func() error {
+		called = true
+		return nil
+	})
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+
+	if !called {
+		t.Error("Expected function to be called")
+	}
+
+	// Test error propagation via Execute
+	expectedErr := errors.New("test error")
+	err = cb.Execute(func() error {
+		return expectedErr
+	})
+
+	if err != expectedErr {
+		t.Errorf("Expected error %v, got %v", expectedErr, err)
+	}
+}
+
+// TestCircuitBreaker_ForceOpen tests forcing circuit breaker to open state
+func TestCircuitBreaker_ForceOpen(t *testing.T) {
+	logger := &mockLogger{}
+	config := DefaultCircuitBreakerConfig()
+	cb := NewCircuitBreaker(config, logger)
+
+	// Initially circuit should be closed
+	if cb.GetState() != CircuitBreakerClosed {
+		t.Errorf("Expected initial state Closed, got %v", cb.GetState())
+	}
+
+	// Force open
+	cb.ForceOpen()
+
+	// Verify state is now open
+	if cb.GetState() != CircuitBreakerOpen {
+		t.Errorf("Expected state Open after ForceOpen, got %v", cb.GetState())
+	}
+
+	// Verify circuit blocks requests
+	err := cb.ExecuteWithContext(context.Background(), func() error {
+		t.Error("Function should not be called when circuit is forced open")
+		return nil
+	})
+
+	if err == nil {
+		t.Error("Expected error when circuit is forced open")
+	}
+
+	// Verify logger was called
+	if len(logger.logs) == 0 {
+		t.Error("Expected info log when forcing circuit open")
+	}
+}
+
+// TestCircuitBreaker_ForceClosed tests forcing circuit breaker to closed state
+func TestCircuitBreaker_ForceClosed(t *testing.T) {
+	logger := &mockLogger{}
+	config := DefaultCircuitBreakerConfig()
+	config.FailureThreshold = 1
+	cb := NewCircuitBreaker(config, logger)
+
+	// Trigger failures to open circuit
+	cb.ExecuteWithContext(context.Background(), func() error {
+		return errors.New("failure")
+	})
+	cb.ExecuteWithContext(context.Background(), func() error {
+		return errors.New("failure")
+	})
+	cb.ExecuteWithContext(context.Background(), func() error {
+		return errors.New("failure")
+	})
+
+	// Circuit should be open after failures
+	if cb.GetState() != CircuitBreakerOpen {
+		t.Errorf("Expected state Open after failures, got %v", cb.GetState())
+	}
+
+	// Force closed
+	cb.ForceClosed()
+
+	// Verify state is now closed
+	if cb.GetState() != CircuitBreakerClosed {
+		t.Errorf("Expected state Closed after ForceClosed, got %v", cb.GetState())
+	}
+
+	// Verify circuit allows requests
+	called := false
+	err := cb.ExecuteWithContext(context.Background(), func() error {
+		called = true
+		return nil
+	})
+
+	if err != nil {
+		t.Errorf("Expected no error after forcing closed, got %v", err)
+	}
+
+	if !called {
+		t.Error("Expected function to be called after forcing closed")
+	}
+
+	// Verify counters are reset
+	metrics := cb.GetMetrics()
+	consecutiveFailures, ok := metrics["consecutiveFailures"].(int32)
+	if !ok || consecutiveFailures != 0 {
+		t.Errorf("Expected consecutiveFailures=0 after ForceClosed, got %v", consecutiveFailures)
+	}
+
+	// Verify logger was called
+	if len(logger.logs) == 0 {
+		t.Error("Expected info log when forcing circuit closed")
+	}
+}
+
+// TestCircuitBreaker_ForceOpen_AllowsRecovery tests that forced open can transition to half-open
+func TestCircuitBreaker_ForceOpen_AllowsRecovery(t *testing.T) {
+	logger := &mockLogger{}
+	config := DefaultCircuitBreakerConfig()
+	config.Timeout = 50 * time.Millisecond // Very short timeout for testing
+	cb := NewCircuitBreaker(config, logger)
+
+	// Force open
+	cb.ForceOpen()
+
+	// Wait for timeout to allow transition to half-open
+	time.Sleep(100 * time.Millisecond)
+
+	// Circuit should allow a test request in half-open state
+	called := false
+	err := cb.ExecuteWithContext(context.Background(), func() error {
+		called = true
+		return nil
+	})
+
+	// After successful execution, circuit should close
+	if err != nil {
+		t.Logf("Note: Circuit may still be in transition, error: %v", err)
+	}
+
+	if called {
+		// If called, verify circuit recovered
+		state := cb.GetState()
+		if state != CircuitBreakerClosed && state != CircuitBreakerHalfOpen {
+			t.Errorf("Expected Closed or HalfOpen after successful recovery, got %v", state)
+		}
+	}
+}

internal/recovery/recovery_test.go

@@ -0,0 +1,547 @@
+//go:build !yaegi
+
+package recovery
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// Mock logger for testing
+type mockLogger struct {
+	mu       sync.Mutex
+	logs     []string
+	errLogs  []string
+	debugLog []string
+}
+
+func (m *mockLogger) Logf(format string, args ...interface{}) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.logs = append(m.logs, format)
+}
+
+func (m *mockLogger) ErrorLogf(format string, args ...interface{}) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.errLogs = append(m.errLogs, format)
+}
+
+func (m *mockLogger) DebugLogf(format string, args ...interface{}) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.debugLog = append(m.debugLog, format)
+}
+
+// BaseRecoveryMechanism tests
+func TestNewBaseRecoveryMechanism(t *testing.T) {
+	logger := &mockLogger{}
+	base := NewBaseRecoveryMechanism("test-recovery", logger)
+
+	if base == nil {
+		t.Fatal("Expected NewBaseRecoveryMechanism to return non-nil")
+	}
+
+	if base.name != "test-recovery" {
+		t.Errorf("Expected name 'test-recovery', got '%s'", base.name)
+	}
+
+	if base.totalRequests != 0 {
+		t.Error("Expected totalRequests to be 0")
+	}
+
+	if base.successCount != 0 {
+		t.Error("Expected successCount to be 0")
+	}
+
+	if base.failureCount != 0 {
+		t.Error("Expected failureCount to be 0")
+	}
+
+	if base.lastSuccessStr != "never" {
+		t.Error("Expected lastSuccessStr to be 'never'")
+	}
+
+	if base.lastFailureStr != "never" {
+		t.Error("Expected lastFailureStr to be 'never'")
+	}
+}
+
+func TestBaseRecoveryMechanism_RecordRequest(t *testing.T) {
+	base := NewBaseRecoveryMechanism("test", &mockLogger{})
+
+	base.RecordRequest()
+	if atomic.LoadInt64(&base.totalRequests) != 1 {
+		t.Error("Expected totalRequests to be 1")
+	}
+
+	base.RecordRequest()
+	base.RecordRequest()
+	if atomic.LoadInt64(&base.totalRequests) != 3 {
+		t.Error("Expected totalRequests to be 3")
+	}
+}
+
+func TestBaseRecoveryMechanism_RecordSuccess(t *testing.T) {
+	base := NewBaseRecoveryMechanism("test", &mockLogger{})
+
+	base.RecordSuccess()
+	if atomic.LoadInt64(&base.successCount) != 1 {
+		t.Error("Expected successCount to be 1")
+	}
+
+	base.successMutex.RLock()
+	lastSuccess := base.lastSuccessStr
+	base.successMutex.RUnlock()
+
+	if lastSuccess == "never" {
+		t.Error("Expected lastSuccessStr to be updated")
+	}
+}
+
+func TestBaseRecoveryMechanism_RecordFailure(t *testing.T) {
+	base := NewBaseRecoveryMechanism("test", &mockLogger{})
+
+	base.RecordFailure()
+	if atomic.LoadInt64(&base.failureCount) != 1 {
+		t.Error("Expected failureCount to be 1")
+	}
+
+	base.failureMutex.RLock()
+	lastFailure := base.lastFailureStr
+	base.failureMutex.RUnlock()
+
+	if lastFailure == "never" {
+		t.Error("Expected lastFailureStr to be updated")
+	}
+}
+
+func TestBaseRecoveryMechanism_GetBaseMetrics(t *testing.T) {
+	base := NewBaseRecoveryMechanism("test", &mockLogger{})
+
+	base.RecordRequest()
+	base.RecordRequest()
+	base.RecordSuccess()
+	base.RecordFailure()
+
+	metrics := base.GetBaseMetrics()
+
+	if metrics["totalRequests"].(int64) != 2 {
+		t.Error("Expected totalRequests to be 2")
+	}
+
+	if metrics["successCount"].(int64) != 1 {
+		t.Error("Expected successCount to be 1")
+	}
+
+	if metrics["failureCount"].(int64) != 1 {
+		t.Error("Expected failureCount to be 1")
+	}
+
+	if metrics["successRate"].(string) != "50.00%" {
+		t.Errorf("Expected successRate to be '50.00%%', got %v", metrics["successRate"])
+	}
+
+	if metrics["name"].(string) != "test" {
+		t.Error("Expected name to be 'test'")
+	}
+}
+
+func TestBaseRecoveryMechanism_ConcurrentAccess(t *testing.T) {
+	base := NewBaseRecoveryMechanism("test", &mockLogger{})
+
+	var wg sync.WaitGroup
+	iterations := 100
+
+	// Concurrent requests
+	for i := 0; i < iterations; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			base.RecordRequest()
+		}()
+	}
+
+	// Concurrent successes
+	for i := 0; i < iterations; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			base.RecordSuccess()
+		}()
+	}
+
+	// Concurrent failures
+	for i := 0; i < iterations; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			base.RecordFailure()
+		}()
+	}
+
+	wg.Wait()
+
+	if atomic.LoadInt64(&base.totalRequests) != int64(iterations) {
+		t.Errorf("Expected %d total requests, got %d", iterations, base.totalRequests)
+	}
+
+	if atomic.LoadInt64(&base.successCount) != int64(iterations) {
+		t.Errorf("Expected %d successes, got %d", iterations, base.successCount)
+	}
+
+	if atomic.LoadInt64(&base.failureCount) != int64(iterations) {
+		t.Errorf("Expected %d failures, got %d", iterations, base.failureCount)
+	}
+}
+
+// CircuitBreakerState tests
+func TestCircuitBreakerState_String(t *testing.T) {
+	tests := []struct {
+		state    CircuitBreakerState
+		expected string
+	}{
+		{CircuitBreakerClosed, "closed"},
+		{CircuitBreakerOpen, "open"},
+		{CircuitBreakerHalfOpen, "half-open"},
+		{CircuitBreakerState(99), "unknown"},
+	}
+
+	for _, tt := range tests {
+		if tt.state.String() != tt.expected {
+			t.Errorf("Expected state %d to be '%s', got '%s'", tt.state, tt.expected, tt.state.String())
+		}
+	}
+}
+
+// CircuitBreakerConfig tests
+func TestDefaultCircuitBreakerConfig(t *testing.T) {
+	config := DefaultCircuitBreakerConfig()
+
+	if config.FailureThreshold != 5 {
+		t.Errorf("Expected FailureThreshold 5, got %d", config.FailureThreshold)
+	}
+
+	if config.SuccessThreshold != 2 {
+		t.Errorf("Expected SuccessThreshold 2, got %d", config.SuccessThreshold)
+	}
+
+	if config.Timeout != 30*time.Second {
+		t.Errorf("Expected Timeout 30s, got %v", config.Timeout)
+	}
+
+	if config.MaxRequests != 3 {
+		t.Errorf("Expected MaxRequests 3, got %d", config.MaxRequests)
+	}
+}
+
+// CircuitBreaker tests
+func TestNewCircuitBreaker(t *testing.T) {
+	config := DefaultCircuitBreakerConfig()
+	logger := &mockLogger{}
+	cb := NewCircuitBreaker(config, logger)
+
+	if cb == nil {
+		t.Fatal("Expected NewCircuitBreaker to return non-nil")
+	}
+
+	if cb.GetState() != CircuitBreakerClosed {
+		t.Error("Expected initial state to be Closed")
+	}
+
+	if cb.config.FailureThreshold != 5 {
+		t.Error("Expected config to be set")
+	}
+}
+
+func TestCircuitBreaker_InitiallyClosed(t *testing.T) {
+	config := DefaultCircuitBreakerConfig()
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	if !cb.IsAvailable() {
+		t.Error("Expected circuit breaker to be available initially")
+	}
+
+	if cb.GetState() != CircuitBreakerClosed {
+		t.Error("Expected state to be Closed")
+	}
+}
+
+func TestCircuitBreaker_ExecuteWithContext_Success(t *testing.T) {
+	config := DefaultCircuitBreakerConfig()
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	callCount := 0
+	err := cb.ExecuteWithContext(context.Background(), func() error {
+		callCount++
+		return nil
+	})
+
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+
+	if callCount != 1 {
+		t.Error("Expected function to be called once")
+	}
+
+	if atomic.LoadInt64(&cb.successCount) != 1 {
+		t.Error("Expected success count to be 1")
+	}
+}
+
+func TestCircuitBreaker_ExecuteWithContext_Failure(t *testing.T) {
+	config := DefaultCircuitBreakerConfig()
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	testErr := errors.New("test error")
+	err := cb.ExecuteWithContext(context.Background(), func() error {
+		return testErr
+	})
+
+	if err != testErr {
+		t.Errorf("Expected error %v, got %v", testErr, err)
+	}
+
+	if atomic.LoadInt64(&cb.failureCount) != 1 {
+		t.Error("Expected failure count to be 1")
+	}
+}
+
+func TestCircuitBreaker_OpensAfterThresholdFailures(t *testing.T) {
+	config := CircuitBreakerConfig{
+		FailureThreshold: 3,
+		SuccessThreshold: 2,
+		Timeout:          100 * time.Millisecond,
+		MaxRequests:      2,
+	}
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	testErr := errors.New("test error")
+
+	// Cause failures to reach threshold
+	for i := 0; i < 3; i++ {
+		_ = cb.ExecuteWithContext(context.Background(), func() error {
+			return testErr
+		})
+	}
+
+	// Circuit should now be open
+	if cb.GetState() != CircuitBreakerOpen {
+		t.Errorf("Expected state to be Open after %d failures, got %s", config.FailureThreshold, cb.GetState())
+	}
+
+	if cb.IsAvailable() {
+		t.Error("Expected circuit breaker to be unavailable when open")
+	}
+
+	// Subsequent requests should be blocked
+	err := cb.ExecuteWithContext(context.Background(), func() error {
+		t.Error("Function should not be called when circuit is open")
+		return nil
+	})
+
+	if err == nil {
+		t.Error("Expected error when circuit is open")
+	}
+
+	if err.Error() != "circuit breaker is open" {
+		t.Errorf("Expected 'circuit breaker is open' error, got: %v", err)
+	}
+}
+
+func TestCircuitBreaker_TransitionsToHalfOpen(t *testing.T) {
+	config := CircuitBreakerConfig{
+		FailureThreshold: 2,
+		SuccessThreshold: 1,
+		Timeout:          50 * time.Millisecond,
+		MaxRequests:      2,
+	}
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	// Open the circuit
+	for i := 0; i < 2; i++ {
+		_ = cb.ExecuteWithContext(context.Background(), func() error {
+			return errors.New("fail")
+		})
+	}
+
+	if cb.GetState() != CircuitBreakerOpen {
+		t.Error("Expected circuit to be open")
+	}
+
+	// Wait for timeout
+	time.Sleep(60 * time.Millisecond)
+
+	// Next request should transition to half-open
+	err := cb.ExecuteWithContext(context.Background(), func() error {
+		return nil
+	})
+
+	if err != nil {
+		t.Errorf("Expected no error in half-open state, got %v", err)
+	}
+
+	// State should be closed after successful request in half-open
+	if cb.GetState() != CircuitBreakerClosed {
+		t.Errorf("Expected state to be Closed after success in half-open, got %s", cb.GetState())
+	}
+}
+
+func TestCircuitBreaker_Reset(t *testing.T) {
+	config := DefaultCircuitBreakerConfig()
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	// Record some metrics
+	cb.RecordRequest()
+	cb.RecordSuccess()
+	cb.RecordFailure()
+
+	// Reset
+	cb.Reset()
+
+	if atomic.LoadInt64(&cb.totalRequests) != 0 {
+		t.Error("Expected totalRequests to be 0 after reset")
+	}
+
+	if atomic.LoadInt32(&cb.consecutiveFailures) != 0 {
+		t.Error("Expected consecutiveFailures to be 0 after reset")
+	}
+
+	if cb.GetState() != CircuitBreakerClosed {
+		t.Error("Expected state to be Closed after reset")
+	}
+}
+
+func TestCircuitBreaker_GetMetrics(t *testing.T) {
+	config := DefaultCircuitBreakerConfig()
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	cb.RecordRequest()
+	cb.RecordSuccess()
+
+	metrics := cb.GetMetrics()
+
+	if metrics == nil {
+		t.Fatal("Expected metrics to be non-nil")
+	}
+
+	if metrics["state"] != "closed" {
+		t.Errorf("Expected state 'closed', got %v", metrics["state"])
+	}
+
+	if metrics["totalRequests"].(int64) != 1 {
+		t.Errorf("Expected totalRequests 1, got %v", metrics["totalRequests"])
+	}
+
+	if metrics["successCount"].(int64) != 1 {
+		t.Error("Expected successCount to be 1")
+	}
+
+	if _, ok := metrics["config"]; !ok {
+		t.Error("Expected config in metrics")
+	}
+}
+
+func TestCircuitBreaker_ConcurrentExecute(t *testing.T) {
+	config := CircuitBreakerConfig{
+		FailureThreshold: 10,
+		SuccessThreshold: 2,
+		Timeout:          100 * time.Millisecond,
+		MaxRequests:      5,
+	}
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	var wg sync.WaitGroup
+	successCount := atomic.Int32{}
+	iterations := 50
+
+	for i := 0; i < iterations; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			err := cb.ExecuteWithContext(context.Background(), func() error {
+				time.Sleep(time.Millisecond)
+				if idx%2 == 0 {
+					return nil
+				}
+				return errors.New("error")
+			})
+			if err == nil {
+				successCount.Add(1)
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
+	// Should have processed requests without panicking
+	if atomic.LoadInt64(&cb.totalRequests) < int64(iterations) {
+		t.Logf("Processed %d requests out of %d (some may have been blocked)", cb.totalRequests, iterations)
+	}
+}
+
+func TestCircuitBreaker_ContextCancellation(t *testing.T) {
+	config := DefaultCircuitBreakerConfig()
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Execute with valid context
+	err := cb.ExecuteWithContext(ctx, func() error {
+		// Cancel during execution
+		cancel()
+		// Circuit breaker doesn't check context during execution by design
+		// It's the responsibility of the function to check context
+		return nil
+	})
+
+	// Should complete successfully - circuit breaker passes context but doesn't enforce it
+	if err != nil {
+		t.Errorf("Expected no error, got %v", err)
+	}
+}
+
+func TestCircuitBreaker_HalfOpenMaxRequests(t *testing.T) {
+	config := CircuitBreakerConfig{
+		FailureThreshold: 2,
+		SuccessThreshold: 1,
+		Timeout:          50 * time.Millisecond,
+		MaxRequests:      1,
+	}
+	cb := NewCircuitBreaker(config, &mockLogger{})
+
+	// Open the circuit
+	for i := 0; i < 2; i++ {
+		_ = cb.ExecuteWithContext(context.Background(), func() error {
+			return errors.New("fail")
+		})
+	}
+
+	// Wait for timeout to transition to half-open
+	time.Sleep(60 * time.Millisecond)
+
+	// First request should be allowed
+	allowed := cb.allowRequest()
+	if !allowed {
+		t.Error("Expected first request to be allowed in half-open state")
+	}
+
+	// Manually transition to half-open if not already
+	cb.stateMutex.Lock()
+	atomic.StoreInt32(&cb.state, int32(CircuitBreakerHalfOpen))
+	cb.stateMutex.Unlock()
+
+	// Increment half-open requests to max
+	atomic.StoreInt32(&cb.halfOpenRequests, int32(config.MaxRequests))
+
+	// Next request should be blocked
+	allowed = cb.allowRequest()
+	if allowed {
+		t.Error("Expected request to be blocked when max half-open requests reached")
+	}
+}