mirror of
https://github.com/lukaszraczylo/traefikoidc.git
synced 2026-10-15 04:18:03 +00:00
... and another all nighter.
This commit is contained in:
@@ -0,0 +1,307 @@
|
||||
// Package recovery provides error recovery and resilience mechanisms for OIDC authentication.
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrorRecoveryMechanism defines the interface for error recovery strategies.
|
||||
// It provides a common contract for implementing various resilience patterns
|
||||
// such as circuit breakers, retry mechanisms, and fallback strategies.
|
||||
type ErrorRecoveryMechanism interface {
|
||||
// ExecuteWithContext runs a function with error recovery using the provided context
|
||||
ExecuteWithContext(ctx context.Context, fn func() error) error
|
||||
// Reset resets the recovery mechanism state
|
||||
Reset()
|
||||
// IsAvailable checks if the mechanism is currently available for use
|
||||
IsAvailable() bool
|
||||
// GetMetrics returns metrics about the recovery mechanism's performance
|
||||
GetMetrics() map[string]interface{}
|
||||
}
|
||||
|
||||
// Logger defines the logging interface
|
||||
type Logger interface {
|
||||
Logf(format string, args ...interface{})
|
||||
ErrorLogf(format string, args ...interface{})
|
||||
DebugLogf(format string, args ...interface{})
|
||||
}
|
||||
|
||||
// BaseRecoveryMechanism provides common functionality and metrics tracking
|
||||
// for all recovery mechanism implementations. It handles request counting,
|
||||
// success/failure tracking, and timestamp management in a thread-safe manner.
|
||||
type BaseRecoveryMechanism struct {
|
||||
// name identifies the recovery mechanism instance
|
||||
name string
|
||||
// logger provides structured logging capabilities
|
||||
logger Logger
|
||||
|
||||
// Metrics tracked with atomic operations for thread safety
|
||||
totalRequests int64
|
||||
successCount int64
|
||||
failureCount int64
|
||||
lastSuccessStr string
|
||||
lastFailureStr string
|
||||
|
||||
// mutexes for thread-safe timestamp updates
|
||||
successMutex sync.RWMutex
|
||||
failureMutex sync.RWMutex
|
||||
}
|
||||
|
||||
// NewBaseRecoveryMechanism creates a new base recovery mechanism with the given name and logger.
|
||||
// This serves as the foundation for specific recovery mechanism implementations.
|
||||
// Parameters:
|
||||
// - name: Identifier for this recovery mechanism instance
|
||||
// - logger: Logger instance for outputting diagnostic information
|
||||
//
|
||||
// Returns:
|
||||
// - A new BaseRecoveryMechanism instance with initialized metrics
|
||||
func NewBaseRecoveryMechanism(name string, logger Logger) *BaseRecoveryMechanism {
|
||||
return &BaseRecoveryMechanism{
|
||||
name: name,
|
||||
logger: logger,
|
||||
totalRequests: 0,
|
||||
successCount: 0,
|
||||
failureCount: 0,
|
||||
lastSuccessStr: "never",
|
||||
lastFailureStr: "never",
|
||||
}
|
||||
}
|
||||
|
||||
// RecordRequest increments the total request counter.
|
||||
// This method is thread-safe using atomic operations.
|
||||
func (b *BaseRecoveryMechanism) RecordRequest() {
|
||||
atomic.AddInt64(&b.totalRequests, 1)
|
||||
}
|
||||
|
||||
// RecordSuccess increments the success counter and updates the last success timestamp.
|
||||
// This method is thread-safe using atomic operations for counters
|
||||
// and mutex protection for timestamp updates.
|
||||
func (b *BaseRecoveryMechanism) RecordSuccess() {
|
||||
atomic.AddInt64(&b.successCount, 1)
|
||||
b.successMutex.Lock()
|
||||
b.lastSuccessStr = time.Now().Format(time.RFC3339)
|
||||
b.successMutex.Unlock()
|
||||
}
|
||||
|
||||
// RecordFailure increments the failure counter and updates the last failure timestamp.
|
||||
// This method is thread-safe using atomic operations for counters
|
||||
// and mutex protection for timestamp updates.
|
||||
func (b *BaseRecoveryMechanism) RecordFailure() {
|
||||
atomic.AddInt64(&b.failureCount, 1)
|
||||
b.failureMutex.Lock()
|
||||
b.lastFailureStr = time.Now().Format(time.RFC3339)
|
||||
b.failureMutex.Unlock()
|
||||
}
|
||||
|
||||
// GetBaseMetrics returns comprehensive metrics about the recovery mechanism.
|
||||
// Includes request counts, success/failure rates, timing information,
|
||||
// and calculated percentages. All access is thread-safe.
|
||||
func (b *BaseRecoveryMechanism) GetBaseMetrics() map[string]interface{} {
|
||||
total := atomic.LoadInt64(&b.totalRequests)
|
||||
success := atomic.LoadInt64(&b.successCount)
|
||||
failure := atomic.LoadInt64(&b.failureCount)
|
||||
|
||||
b.successMutex.RLock()
|
||||
lastSuccess := b.lastSuccessStr
|
||||
b.successMutex.RUnlock()
|
||||
|
||||
b.failureMutex.RLock()
|
||||
lastFailure := b.lastFailureStr
|
||||
b.failureMutex.RUnlock()
|
||||
|
||||
metrics := map[string]interface{}{
|
||||
"name": b.name,
|
||||
"totalRequests": total,
|
||||
"successCount": success,
|
||||
"failureCount": failure,
|
||||
"lastSuccess": lastSuccess,
|
||||
"lastFailure": lastFailure,
|
||||
}
|
||||
|
||||
// Calculate success and failure rates
|
||||
if total > 0 {
|
||||
successRate := float64(success) / float64(total) * 100
|
||||
failureRate := float64(failure) / float64(total) * 100
|
||||
metrics["successRate"] = fmt.Sprintf("%.2f%%", successRate)
|
||||
metrics["failureRate"] = fmt.Sprintf("%.2f%%", failureRate)
|
||||
} else {
|
||||
metrics["successRate"] = "0.00%"
|
||||
metrics["failureRate"] = "0.00%"
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
// LogInfo logs an informational message with the mechanism name as prefix.
|
||||
// Provides consistent logging format across all recovery mechanisms.
|
||||
func (b *BaseRecoveryMechanism) LogInfo(format string, args ...interface{}) {
|
||||
if b.logger != nil {
|
||||
b.logger.Logf("[%s] %s", b.name, fmt.Sprintf(format, args...))
|
||||
}
|
||||
}
|
||||
|
||||
// LogError logs an error message with the mechanism name as prefix.
|
||||
// Used for reporting failures and error conditions in recovery mechanisms.
|
||||
func (b *BaseRecoveryMechanism) LogError(format string, args ...interface{}) {
|
||||
if b.logger != nil {
|
||||
b.logger.ErrorLogf("[%s] %s", b.name, fmt.Sprintf(format, args...))
|
||||
}
|
||||
}
|
||||
|
||||
// LogDebug logs a debug message with the mechanism name as prefix.
|
||||
// Useful for detailed troubleshooting of recovery mechanism behavior.
|
||||
func (b *BaseRecoveryMechanism) LogDebug(format string, args ...interface{}) {
|
||||
if b.logger != nil {
|
||||
b.logger.DebugLogf("[%s] %s", b.name, fmt.Sprintf(format, args...))
|
||||
}
|
||||
}
|
||||
|
||||
// ErrorType represents different categories of errors
|
||||
type ErrorType int
|
||||
|
||||
const (
|
||||
// ErrorTypeUnknown represents an unknown error type
|
||||
ErrorTypeUnknown ErrorType = iota
|
||||
// ErrorTypeNetwork represents network-related errors
|
||||
ErrorTypeNetwork
|
||||
// ErrorTypeTimeout represents timeout errors
|
||||
ErrorTypeTimeout
|
||||
// ErrorTypeAuthentication represents authentication errors
|
||||
ErrorTypeAuthentication
|
||||
// ErrorTypeRateLimit represents rate limiting errors
|
||||
ErrorTypeRateLimit
|
||||
// ErrorTypeServerError represents server errors (5xx)
|
||||
ErrorTypeServerError
|
||||
// ErrorTypeClientError represents client errors (4xx)
|
||||
ErrorTypeClientError
|
||||
)
|
||||
|
||||
// HTTPError represents an HTTP error with status code and message
|
||||
type HTTPError struct {
|
||||
StatusCode int
|
||||
Message string
|
||||
Body []byte
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
// Error implements the error interface
|
||||
func (e *HTTPError) Error() string {
|
||||
return fmt.Sprintf("HTTP %d: %s", e.StatusCode, e.Message)
|
||||
}
|
||||
|
||||
// IsRetryable checks if the HTTP error is retryable
|
||||
func (e *HTTPError) IsRetryable() bool {
|
||||
// Retry on 5xx errors and specific 4xx errors
|
||||
return e.StatusCode >= 500 || e.StatusCode == 429 || e.StatusCode == 408
|
||||
}
|
||||
|
||||
// OIDCError represents an OIDC-specific error
|
||||
type OIDCError struct {
|
||||
Code string
|
||||
Description string
|
||||
URI string
|
||||
State string
|
||||
}
|
||||
|
||||
// Error implements the error interface
|
||||
func (e *OIDCError) Error() string {
|
||||
if e.Description != "" {
|
||||
return fmt.Sprintf("OIDC error %s: %s", e.Code, e.Description)
|
||||
}
|
||||
return fmt.Sprintf("OIDC error: %s", e.Code)
|
||||
}
|
||||
|
||||
// IsRetryable checks if the OIDC error is retryable
|
||||
func (e *OIDCError) IsRetryable() bool {
|
||||
// Some OIDC errors are retryable
|
||||
switch e.Code {
|
||||
case "temporarily_unavailable", "server_error":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// FallbackMechanism provides a simple fallback recovery strategy
|
||||
type FallbackMechanism struct {
|
||||
*BaseRecoveryMechanism
|
||||
fallbackFunc func() error
|
||||
}
|
||||
|
||||
// NewFallbackMechanism creates a new fallback mechanism
|
||||
func NewFallbackMechanism(name string, logger Logger, fallbackFunc func() error) *FallbackMechanism {
|
||||
return &FallbackMechanism{
|
||||
BaseRecoveryMechanism: NewBaseRecoveryMechanism(name, logger),
|
||||
fallbackFunc: fallbackFunc,
|
||||
}
|
||||
}
|
||||
|
||||
// ExecuteWithContext executes the primary function and falls back on error
|
||||
func (f *FallbackMechanism) ExecuteWithContext(ctx context.Context, fn func() error) error {
|
||||
f.RecordRequest()
|
||||
|
||||
// Check context first
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
f.RecordFailure()
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Try primary function
|
||||
if err := fn(); err != nil {
|
||||
f.LogInfo("Primary function failed: %v, trying fallback", err)
|
||||
|
||||
// Try fallback
|
||||
if f.fallbackFunc != nil {
|
||||
if fallbackErr := f.fallbackFunc(); fallbackErr == nil {
|
||||
f.RecordSuccess()
|
||||
return nil
|
||||
} else {
|
||||
f.LogError("Fallback also failed: %v", fallbackErr)
|
||||
f.RecordFailure()
|
||||
return fmt.Errorf("both primary and fallback failed: primary=%v, fallback=%v", err, fallbackErr)
|
||||
}
|
||||
}
|
||||
|
||||
f.RecordFailure()
|
||||
return err
|
||||
}
|
||||
|
||||
f.RecordSuccess()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reset resets the fallback mechanism state
|
||||
func (f *FallbackMechanism) Reset() {
|
||||
// Reset metrics
|
||||
atomic.StoreInt64(&f.totalRequests, 0)
|
||||
atomic.StoreInt64(&f.successCount, 0)
|
||||
atomic.StoreInt64(&f.failureCount, 0)
|
||||
|
||||
f.successMutex.Lock()
|
||||
f.lastSuccessStr = "never"
|
||||
f.successMutex.Unlock()
|
||||
|
||||
f.failureMutex.Lock()
|
||||
f.lastFailureStr = "never"
|
||||
f.failureMutex.Unlock()
|
||||
}
|
||||
|
||||
// IsAvailable checks if the fallback mechanism is available
|
||||
func (f *FallbackMechanism) IsAvailable() bool {
|
||||
// Fallback is always available
|
||||
return true
|
||||
}
|
||||
|
||||
// GetMetrics returns metrics about the fallback mechanism
|
||||
func (f *FallbackMechanism) GetMetrics() map[string]interface{} {
|
||||
metrics := f.GetBaseMetrics()
|
||||
metrics["type"] = "fallback"
|
||||
metrics["hasFallback"] = f.fallbackFunc != nil
|
||||
return metrics
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
// Package recovery provides error recovery and resilience mechanisms for OIDC authentication.
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CircuitBreakerState represents the current state of the circuit breaker
|
||||
type CircuitBreakerState int
|
||||
|
||||
const (
|
||||
// CircuitBreakerClosed allows all requests to pass through
|
||||
CircuitBreakerClosed CircuitBreakerState = iota
|
||||
// CircuitBreakerOpen blocks all requests
|
||||
CircuitBreakerOpen
|
||||
// CircuitBreakerHalfOpen allows limited requests for testing
|
||||
CircuitBreakerHalfOpen
|
||||
)
|
||||
|
||||
// String returns the string representation of the circuit breaker state
|
||||
func (s CircuitBreakerState) String() string {
|
||||
switch s {
|
||||
case CircuitBreakerClosed:
|
||||
return "closed"
|
||||
case CircuitBreakerOpen:
|
||||
return "open"
|
||||
case CircuitBreakerHalfOpen:
|
||||
return "half-open"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// CircuitBreakerConfig defines configuration for the circuit breaker
|
||||
type CircuitBreakerConfig struct {
|
||||
// FailureThreshold is the number of failures before opening the circuit
|
||||
FailureThreshold int
|
||||
// SuccessThreshold is the number of successes in half-open state before closing
|
||||
SuccessThreshold int
|
||||
// Timeout is the duration to wait before transitioning from open to half-open
|
||||
Timeout time.Duration
|
||||
// MaxRequests is the maximum number of requests allowed in half-open state
|
||||
MaxRequests int
|
||||
}
|
||||
|
||||
// DefaultCircuitBreakerConfig returns sensible default configuration
|
||||
func DefaultCircuitBreakerConfig() CircuitBreakerConfig {
|
||||
return CircuitBreakerConfig{
|
||||
FailureThreshold: 5,
|
||||
SuccessThreshold: 2,
|
||||
Timeout: 30 * time.Second,
|
||||
MaxRequests: 3,
|
||||
}
|
||||
}
|
||||
|
||||
// CircuitBreaker implements the circuit breaker pattern for fault tolerance.
|
||||
// It prevents cascading failures by temporarily blocking requests to a failing service.
|
||||
type CircuitBreaker struct {
|
||||
*BaseRecoveryMechanism
|
||||
config CircuitBreakerConfig
|
||||
|
||||
// State management
|
||||
state int32 // atomic: CircuitBreakerState
|
||||
lastStateChange time.Time
|
||||
stateMutex sync.RWMutex
|
||||
|
||||
// Failure tracking
|
||||
consecutiveFailures int32 // atomic
|
||||
consecutiveSuccesses int32 // atomic
|
||||
|
||||
// Half-open state management
|
||||
halfOpenRequests int32 // atomic
|
||||
}
|
||||
|
||||
// NewCircuitBreaker creates a new circuit breaker with the given configuration
|
||||
func NewCircuitBreaker(config CircuitBreakerConfig, logger Logger) *CircuitBreaker {
|
||||
return &CircuitBreaker{
|
||||
BaseRecoveryMechanism: NewBaseRecoveryMechanism("CircuitBreaker", logger),
|
||||
config: config,
|
||||
state: int32(CircuitBreakerClosed),
|
||||
lastStateChange: time.Now(),
|
||||
consecutiveFailures: 0,
|
||||
consecutiveSuccesses: 0,
|
||||
halfOpenRequests: 0,
|
||||
}
|
||||
}
|
||||
|
||||
// ExecuteWithContext executes a function with circuit breaker protection
|
||||
func (cb *CircuitBreaker) ExecuteWithContext(ctx context.Context, fn func() error) error {
|
||||
cb.RecordRequest()
|
||||
|
||||
// Check if request is allowed
|
||||
if !cb.allowRequest() {
|
||||
cb.RecordFailure()
|
||||
return fmt.Errorf("circuit breaker is open")
|
||||
}
|
||||
|
||||
// Execute the function
|
||||
err := fn()
|
||||
|
||||
if err != nil {
|
||||
cb.recordFailure()
|
||||
return err
|
||||
}
|
||||
|
||||
cb.recordSuccess()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Execute executes a function with circuit breaker protection (legacy method)
|
||||
func (cb *CircuitBreaker) Execute(fn func() error) error {
|
||||
return cb.ExecuteWithContext(context.Background(), fn)
|
||||
}
|
||||
|
||||
// allowRequest determines if a request should be allowed based on the circuit state
|
||||
func (cb *CircuitBreaker) allowRequest() bool {
|
||||
state := CircuitBreakerState(atomic.LoadInt32(&cb.state))
|
||||
|
||||
switch state {
|
||||
case CircuitBreakerClosed:
|
||||
return true
|
||||
|
||||
case CircuitBreakerOpen:
|
||||
// Check if timeout has elapsed
|
||||
cb.stateMutex.RLock()
|
||||
lastChange := cb.lastStateChange
|
||||
cb.stateMutex.RUnlock()
|
||||
|
||||
if time.Since(lastChange) > cb.config.Timeout {
|
||||
// Transition to half-open
|
||||
cb.transitionToHalfOpen()
|
||||
return cb.allowHalfOpenRequest()
|
||||
}
|
||||
return false
|
||||
|
||||
case CircuitBreakerHalfOpen:
|
||||
return cb.allowHalfOpenRequest()
|
||||
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// allowHalfOpenRequest checks if a request is allowed in half-open state
|
||||
func (cb *CircuitBreaker) allowHalfOpenRequest() bool {
|
||||
current := atomic.AddInt32(&cb.halfOpenRequests, 1)
|
||||
if current <= int32(cb.config.MaxRequests) {
|
||||
return true
|
||||
}
|
||||
atomic.AddInt32(&cb.halfOpenRequests, -1)
|
||||
return false
|
||||
}
|
||||
|
||||
// recordFailure records a failure and potentially opens the circuit
|
||||
func (cb *CircuitBreaker) recordFailure() {
|
||||
cb.RecordFailure()
|
||||
|
||||
failures := atomic.AddInt32(&cb.consecutiveFailures, 1)
|
||||
atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
|
||||
|
||||
state := CircuitBreakerState(atomic.LoadInt32(&cb.state))
|
||||
|
||||
if state == CircuitBreakerClosed && failures >= int32(cb.config.FailureThreshold) {
|
||||
cb.transitionToOpen()
|
||||
} else if state == CircuitBreakerHalfOpen {
|
||||
cb.transitionToOpen()
|
||||
}
|
||||
}
|
||||
|
||||
// recordSuccess records a success and potentially closes the circuit
|
||||
func (cb *CircuitBreaker) recordSuccess() {
|
||||
cb.RecordSuccess()
|
||||
|
||||
successes := atomic.AddInt32(&cb.consecutiveSuccesses, 1)
|
||||
atomic.StoreInt32(&cb.consecutiveFailures, 0)
|
||||
|
||||
state := CircuitBreakerState(atomic.LoadInt32(&cb.state))
|
||||
|
||||
if state == CircuitBreakerHalfOpen && successes >= int32(cb.config.SuccessThreshold) {
|
||||
cb.transitionToClosed()
|
||||
}
|
||||
}
|
||||
|
||||
// transitionToClosed transitions the circuit to closed state
|
||||
func (cb *CircuitBreaker) transitionToClosed() {
|
||||
if atomic.CompareAndSwapInt32(&cb.state, int32(CircuitBreakerHalfOpen), int32(CircuitBreakerClosed)) {
|
||||
cb.stateMutex.Lock()
|
||||
cb.lastStateChange = time.Now()
|
||||
cb.stateMutex.Unlock()
|
||||
|
||||
atomic.StoreInt32(&cb.consecutiveFailures, 0)
|
||||
atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
|
||||
atomic.StoreInt32(&cb.halfOpenRequests, 0)
|
||||
|
||||
cb.LogInfo("Circuit breaker closed")
|
||||
}
|
||||
}
|
||||
|
||||
// transitionToOpen transitions the circuit to open state
|
||||
func (cb *CircuitBreaker) transitionToOpen() {
|
||||
oldState := atomic.SwapInt32(&cb.state, int32(CircuitBreakerOpen))
|
||||
if oldState != int32(CircuitBreakerOpen) {
|
||||
cb.stateMutex.Lock()
|
||||
cb.lastStateChange = time.Now()
|
||||
cb.stateMutex.Unlock()
|
||||
|
||||
atomic.StoreInt32(&cb.consecutiveFailures, 0)
|
||||
atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
|
||||
atomic.StoreInt32(&cb.halfOpenRequests, 0)
|
||||
|
||||
cb.LogError("Circuit breaker opened due to failures")
|
||||
}
|
||||
}
|
||||
|
||||
// transitionToHalfOpen transitions the circuit to half-open state
|
||||
func (cb *CircuitBreaker) transitionToHalfOpen() {
|
||||
if atomic.CompareAndSwapInt32(&cb.state, int32(CircuitBreakerOpen), int32(CircuitBreakerHalfOpen)) {
|
||||
cb.stateMutex.Lock()
|
||||
cb.lastStateChange = time.Now()
|
||||
cb.stateMutex.Unlock()
|
||||
|
||||
atomic.StoreInt32(&cb.consecutiveFailures, 0)
|
||||
atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
|
||||
atomic.StoreInt32(&cb.halfOpenRequests, 0)
|
||||
|
||||
cb.LogInfo("Circuit breaker half-open, testing recovery")
|
||||
}
|
||||
}
|
||||
|
||||
// GetState returns the current state of the circuit breaker
|
||||
func (cb *CircuitBreaker) GetState() CircuitBreakerState {
|
||||
return CircuitBreakerState(atomic.LoadInt32(&cb.state))
|
||||
}
|
||||
|
||||
// Reset resets the circuit breaker to closed state
|
||||
func (cb *CircuitBreaker) Reset() {
|
||||
atomic.StoreInt32(&cb.state, int32(CircuitBreakerClosed))
|
||||
|
||||
cb.stateMutex.Lock()
|
||||
cb.lastStateChange = time.Now()
|
||||
cb.stateMutex.Unlock()
|
||||
|
||||
atomic.StoreInt32(&cb.consecutiveFailures, 0)
|
||||
atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
|
||||
atomic.StoreInt32(&cb.halfOpenRequests, 0)
|
||||
|
||||
// Reset base metrics
|
||||
atomic.StoreInt64(&cb.totalRequests, 0)
|
||||
atomic.StoreInt64(&cb.successCount, 0)
|
||||
atomic.StoreInt64(&cb.failureCount, 0)
|
||||
|
||||
cb.LogInfo("Circuit breaker reset to closed state")
|
||||
}
|
||||
|
||||
// IsAvailable returns true if the circuit breaker is not fully open
|
||||
func (cb *CircuitBreaker) IsAvailable() bool {
|
||||
state := cb.GetState()
|
||||
return state != CircuitBreakerOpen || time.Since(cb.getLastStateChange()) > cb.config.Timeout
|
||||
}
|
||||
|
||||
// getLastStateChange returns the last state change time safely
|
||||
func (cb *CircuitBreaker) getLastStateChange() time.Time {
|
||||
cb.stateMutex.RLock()
|
||||
defer cb.stateMutex.RUnlock()
|
||||
return cb.lastStateChange
|
||||
}
|
||||
|
||||
// GetMetrics returns comprehensive metrics about the circuit breaker
|
||||
func (cb *CircuitBreaker) GetMetrics() map[string]interface{} {
|
||||
metrics := cb.GetBaseMetrics()
|
||||
|
||||
state := cb.GetState()
|
||||
metrics["state"] = state.String()
|
||||
metrics["consecutiveFailures"] = atomic.LoadInt32(&cb.consecutiveFailures)
|
||||
metrics["consecutiveSuccesses"] = atomic.LoadInt32(&cb.consecutiveSuccesses)
|
||||
metrics["halfOpenRequests"] = atomic.LoadInt32(&cb.halfOpenRequests)
|
||||
|
||||
cb.stateMutex.RLock()
|
||||
metrics["lastStateChange"] = cb.lastStateChange.Format(time.RFC3339)
|
||||
metrics["timeSinceLastChange"] = time.Since(cb.lastStateChange).String()
|
||||
cb.stateMutex.RUnlock()
|
||||
|
||||
// Configuration
|
||||
metrics["config"] = map[string]interface{}{
|
||||
"failureThreshold": cb.config.FailureThreshold,
|
||||
"successThreshold": cb.config.SuccessThreshold,
|
||||
"timeout": cb.config.Timeout.String(),
|
||||
"maxRequests": cb.config.MaxRequests,
|
||||
}
|
||||
|
||||
// Health indicator
|
||||
switch state {
|
||||
case CircuitBreakerClosed:
|
||||
metrics["health"] = "healthy"
|
||||
case CircuitBreakerHalfOpen:
|
||||
metrics["health"] = "recovering"
|
||||
case CircuitBreakerOpen:
|
||||
if time.Since(cb.getLastStateChange()) > cb.config.Timeout {
|
||||
metrics["health"] = "ready-to-recover"
|
||||
} else {
|
||||
metrics["health"] = "unhealthy"
|
||||
}
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
// ForceOpen forces the circuit breaker to open state
|
||||
func (cb *CircuitBreaker) ForceOpen() {
|
||||
atomic.StoreInt32(&cb.state, int32(CircuitBreakerOpen))
|
||||
|
||||
cb.stateMutex.Lock()
|
||||
cb.lastStateChange = time.Now()
|
||||
cb.stateMutex.Unlock()
|
||||
|
||||
cb.LogInfo("Circuit breaker forced open")
|
||||
}
|
||||
|
||||
// ForceClosed forces the circuit breaker to closed state
|
||||
func (cb *CircuitBreaker) ForceClosed() {
|
||||
atomic.StoreInt32(&cb.state, int32(CircuitBreakerClosed))
|
||||
|
||||
cb.stateMutex.Lock()
|
||||
cb.lastStateChange = time.Now()
|
||||
cb.stateMutex.Unlock()
|
||||
|
||||
atomic.StoreInt32(&cb.consecutiveFailures, 0)
|
||||
atomic.StoreInt32(&cb.consecutiveSuccesses, 0)
|
||||
atomic.StoreInt32(&cb.halfOpenRequests, 0)
|
||||
|
||||
cb.LogInfo("Circuit breaker forced closed")
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
// Package recovery provides error recovery and resilience mechanisms for OIDC authentication.
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RetryConfig defines configuration for the retry executor
|
||||
type RetryConfig struct {
|
||||
// MaxAttempts is the maximum number of retry attempts
|
||||
MaxAttempts int
|
||||
// InitialDelay is the initial delay between retries
|
||||
InitialDelay time.Duration
|
||||
// MaxDelay is the maximum delay between retries
|
||||
MaxDelay time.Duration
|
||||
// Multiplier is the backoff multiplier
|
||||
Multiplier float64
|
||||
// RandomizationFactor adds jitter to delays (0.0 to 1.0)
|
||||
RandomizationFactor float64
|
||||
// RetryableErrors defines which errors should trigger a retry
|
||||
RetryableErrors []string
|
||||
// RetryableStatusCodes defines which HTTP status codes should trigger a retry
|
||||
RetryableStatusCodes []int
|
||||
}
|
||||
|
||||
// DefaultRetryConfig returns sensible default retry configuration
|
||||
func DefaultRetryConfig() RetryConfig {
|
||||
return RetryConfig{
|
||||
MaxAttempts: 3,
|
||||
InitialDelay: 100 * time.Millisecond,
|
||||
MaxDelay: 30 * time.Second,
|
||||
Multiplier: 2.0,
|
||||
RandomizationFactor: 0.1,
|
||||
RetryableErrors: []string{"connection refused", "timeout", "EOF"},
|
||||
RetryableStatusCodes: []int{408, 429, 500, 502, 503, 504},
|
||||
}
|
||||
}
|
||||
|
||||
// RetryExecutor implements retry logic with exponential backoff
|
||||
type RetryExecutor struct {
|
||||
*BaseRecoveryMechanism
|
||||
config RetryConfig
|
||||
|
||||
// Metrics
|
||||
totalRetries int64
|
||||
maxRetriesHit int64
|
||||
lastRetryTime time.Time
|
||||
retryTimeMutex sync.RWMutex
|
||||
}
|
||||
|
||||
// NewRetryExecutor creates a new retry executor with the given configuration
|
||||
func NewRetryExecutor(config RetryConfig, logger Logger) *RetryExecutor {
|
||||
if config.MaxAttempts < 1 {
|
||||
config.MaxAttempts = 1
|
||||
}
|
||||
if config.Multiplier < 1.0 {
|
||||
config.Multiplier = 1.0
|
||||
}
|
||||
return &RetryExecutor{
|
||||
BaseRecoveryMechanism: NewBaseRecoveryMechanism("RetryExecutor", logger),
|
||||
config: config,
|
||||
totalRetries: 0,
|
||||
maxRetriesHit: 0,
|
||||
}
|
||||
}
|
||||
|
||||
// ExecuteWithContext executes a function with retry logic
|
||||
func (re *RetryExecutor) ExecuteWithContext(ctx context.Context, fn func() error) error {
|
||||
re.RecordRequest()
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= re.config.MaxAttempts; attempt++ {
|
||||
// Check context before attempting
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
re.RecordFailure()
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Execute the function
|
||||
lastErr = fn()
|
||||
|
||||
if lastErr == nil {
|
||||
re.RecordSuccess()
|
||||
if attempt > 1 {
|
||||
re.LogInfo("Succeeded after %d attempts", attempt)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if error is retryable
|
||||
if !re.isRetryableError(lastErr) {
|
||||
re.LogDebug("Error is not retryable: %v", lastErr)
|
||||
re.RecordFailure()
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// Don't retry if this was the last attempt
|
||||
if attempt >= re.config.MaxAttempts {
|
||||
atomic.AddInt64(&re.maxRetriesHit, 1)
|
||||
re.LogError("Max retries (%d) exhausted", re.config.MaxAttempts)
|
||||
break
|
||||
}
|
||||
|
||||
// Calculate and apply delay
|
||||
delay := re.calculateDelay(attempt)
|
||||
re.LogInfo("Attempt %d failed: %v, retrying in %v", attempt, lastErr, delay)
|
||||
|
||||
atomic.AddInt64(&re.totalRetries, 1)
|
||||
re.retryTimeMutex.Lock()
|
||||
re.lastRetryTime = time.Now()
|
||||
re.retryTimeMutex.Unlock()
|
||||
|
||||
select {
|
||||
case <-time.After(delay):
|
||||
// Continue to next attempt
|
||||
case <-ctx.Done():
|
||||
re.RecordFailure()
|
||||
return fmt.Errorf("retry cancelled: %w", ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
re.RecordFailure()
|
||||
return fmt.Errorf("all retry attempts failed: %w", lastErr)
|
||||
}
|
||||
|
||||
// Execute executes a function with retry logic (legacy method)
|
||||
func (re *RetryExecutor) Execute(ctx context.Context, fn func() error) error {
|
||||
return re.ExecuteWithContext(ctx, fn)
|
||||
}
|
||||
|
||||
// isRetryableError determines if an error should trigger a retry
|
||||
func (re *RetryExecutor) isRetryableError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
errStr := strings.ToLower(err.Error())
|
||||
|
||||
// Check for retryable error patterns
|
||||
for _, pattern := range re.config.RetryableErrors {
|
||||
if strings.Contains(errStr, strings.ToLower(pattern)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Check for HTTP errors
|
||||
if httpErr, ok := err.(*HTTPError); ok {
|
||||
for _, code := range re.config.RetryableStatusCodes {
|
||||
if httpErr.StatusCode == code {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Also retry on any 5xx error
|
||||
if httpErr.StatusCode >= 500 && httpErr.StatusCode < 600 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Check for OIDC errors
|
||||
if oidcErr, ok := err.(*OIDCError); ok {
|
||||
return oidcErr.IsRetryable()
|
||||
}
|
||||
|
||||
// Check for context errors (don't retry these)
|
||||
if err == context.Canceled || err == context.DeadlineExceeded {
|
||||
return false
|
||||
}
|
||||
|
||||
// Default: don't retry unknown errors
|
||||
return false
|
||||
}
|
||||
|
||||
// calculateDelay calculates the delay before the next retry attempt
|
||||
func (re *RetryExecutor) calculateDelay(attempt int) time.Duration {
|
||||
// Exponential backoff
|
||||
delay := float64(re.config.InitialDelay) * math.Pow(re.config.Multiplier, float64(attempt-1))
|
||||
|
||||
// Cap at max delay
|
||||
if delay > float64(re.config.MaxDelay) {
|
||||
delay = float64(re.config.MaxDelay)
|
||||
}
|
||||
|
||||
// Add jitter
|
||||
if re.config.RandomizationFactor > 0 {
|
||||
jitter := delay * re.config.RandomizationFactor
|
||||
minDelay := delay - jitter
|
||||
maxDelay := delay + jitter
|
||||
delay = minDelay + rand.Float64()*(maxDelay-minDelay)
|
||||
}
|
||||
|
||||
return time.Duration(delay)
|
||||
}
|
||||
|
||||
// Reset resets the retry executor state
|
||||
func (re *RetryExecutor) Reset() {
|
||||
atomic.StoreInt64(&re.totalRetries, 0)
|
||||
atomic.StoreInt64(&re.maxRetriesHit, 0)
|
||||
atomic.StoreInt64(&re.totalRequests, 0)
|
||||
atomic.StoreInt64(&re.successCount, 0)
|
||||
atomic.StoreInt64(&re.failureCount, 0)
|
||||
|
||||
re.retryTimeMutex.Lock()
|
||||
re.lastRetryTime = time.Time{}
|
||||
re.retryTimeMutex.Unlock()
|
||||
}
|
||||
|
||||
// IsAvailable always returns true for retry executor
|
||||
func (re *RetryExecutor) IsAvailable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// GetMetrics returns comprehensive metrics about the retry executor
|
||||
func (re *RetryExecutor) GetMetrics() map[string]interface{} {
|
||||
metrics := re.GetBaseMetrics()
|
||||
|
||||
metrics["totalRetries"] = atomic.LoadInt64(&re.totalRetries)
|
||||
metrics["maxRetriesHit"] = atomic.LoadInt64(&re.maxRetriesHit)
|
||||
|
||||
re.retryTimeMutex.RLock()
|
||||
if !re.lastRetryTime.IsZero() {
|
||||
metrics["lastRetryTime"] = re.lastRetryTime.Format(time.RFC3339)
|
||||
metrics["timeSinceLastRetry"] = time.Since(re.lastRetryTime).String()
|
||||
} else {
|
||||
metrics["lastRetryTime"] = "never"
|
||||
}
|
||||
re.retryTimeMutex.RUnlock()
|
||||
|
||||
// Configuration
|
||||
metrics["config"] = map[string]interface{}{
|
||||
"maxAttempts": re.config.MaxAttempts,
|
||||
"initialDelay": re.config.InitialDelay.String(),
|
||||
"maxDelay": re.config.MaxDelay.String(),
|
||||
"multiplier": re.config.Multiplier,
|
||||
"randomizationFactor": re.config.RandomizationFactor,
|
||||
}
|
||||
|
||||
// Calculate average retries per request
|
||||
totalRequests := atomic.LoadInt64(&re.totalRequests)
|
||||
if totalRequests > 0 {
|
||||
avgRetries := float64(atomic.LoadInt64(&re.totalRetries)) / float64(totalRequests)
|
||||
metrics["averageRetriesPerRequest"] = fmt.Sprintf("%.2f", avgRetries)
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
// RecoveryMetrics aggregates metrics from multiple recovery mechanisms
|
||||
type RecoveryMetrics struct {
|
||||
mechanisms map[string]ErrorRecoveryMechanism
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewRecoveryMetrics creates a new recovery metrics aggregator
|
||||
func NewRecoveryMetrics() *RecoveryMetrics {
|
||||
return &RecoveryMetrics{
|
||||
mechanisms: make(map[string]ErrorRecoveryMechanism),
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterMechanism registers a recovery mechanism for metrics collection
|
||||
func (rm *RecoveryMetrics) RegisterMechanism(name string, mechanism ErrorRecoveryMechanism) {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
rm.mechanisms[name] = mechanism
|
||||
}
|
||||
|
||||
// UnregisterMechanism removes a recovery mechanism from metrics collection
|
||||
func (rm *RecoveryMetrics) UnregisterMechanism(name string) {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
delete(rm.mechanisms, name)
|
||||
}
|
||||
|
||||
// GetAllMetrics returns aggregated metrics from all registered mechanisms
|
||||
func (rm *RecoveryMetrics) GetAllMetrics() map[string]interface{} {
|
||||
rm.mu.RLock()
|
||||
defer rm.mu.RUnlock()
|
||||
|
||||
allMetrics := make(map[string]interface{})
|
||||
for name, mechanism := range rm.mechanisms {
|
||||
allMetrics[name] = mechanism.GetMetrics()
|
||||
}
|
||||
|
||||
// Add summary statistics
|
||||
totalRequests := int64(0)
|
||||
totalSuccesses := int64(0)
|
||||
totalFailures := int64(0)
|
||||
|
||||
for _, mechanism := range rm.mechanisms {
|
||||
metrics := mechanism.GetMetrics()
|
||||
if requests, ok := metrics["totalRequests"].(int64); ok {
|
||||
totalRequests += requests
|
||||
}
|
||||
if successes, ok := metrics["successCount"].(int64); ok {
|
||||
totalSuccesses += successes
|
||||
}
|
||||
if failures, ok := metrics["failureCount"].(int64); ok {
|
||||
totalFailures += failures
|
||||
}
|
||||
}
|
||||
|
||||
allMetrics["summary"] = map[string]interface{}{
|
||||
"totalMechanisms": len(rm.mechanisms),
|
||||
"totalRequests": totalRequests,
|
||||
"totalSuccesses": totalSuccesses,
|
||||
"totalFailures": totalFailures,
|
||||
}
|
||||
|
||||
if totalRequests > 0 {
|
||||
successRate := float64(totalSuccesses) / float64(totalRequests) * 100
|
||||
allMetrics["summary"].(map[string]interface{})["overallSuccessRate"] = fmt.Sprintf("%.2f%%", successRate)
|
||||
}
|
||||
|
||||
return allMetrics
|
||||
}
|
||||
|
||||
// GetMechanismMetrics returns metrics for a specific mechanism
|
||||
func (rm *RecoveryMetrics) GetMechanismMetrics(name string) (map[string]interface{}, bool) {
|
||||
rm.mu.RLock()
|
||||
defer rm.mu.RUnlock()
|
||||
|
||||
if mechanism, exists := rm.mechanisms[name]; exists {
|
||||
return mechanism.GetMetrics(), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// HealthCheck performs a health check on all registered mechanisms
|
||||
func (rm *RecoveryMetrics) HealthCheck() map[string]interface{} {
|
||||
rm.mu.RLock()
|
||||
defer rm.mu.RUnlock()
|
||||
|
||||
health := make(map[string]interface{})
|
||||
healthyCount := 0
|
||||
unhealthyCount := 0
|
||||
|
||||
for name, mechanism := range rm.mechanisms {
|
||||
if mechanism.IsAvailable() {
|
||||
health[name] = "healthy"
|
||||
healthyCount++
|
||||
} else {
|
||||
health[name] = "unhealthy"
|
||||
unhealthyCount++
|
||||
}
|
||||
}
|
||||
|
||||
overallHealth := "healthy"
|
||||
if unhealthyCount > 0 {
|
||||
if healthyCount > 0 {
|
||||
overallHealth = "degraded"
|
||||
} else {
|
||||
overallHealth = "unhealthy"
|
||||
}
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"status": overallHealth,
|
||||
"mechanisms": health,
|
||||
"healthy": healthyCount,
|
||||
"unhealthy": unhealthyCount,
|
||||
"timestamp": time.Now().Format(time.RFC3339),
|
||||
}
|
||||
}
|
||||
|
||||
// HTTPMetricsHandler creates an HTTP handler for serving recovery metrics
|
||||
func (rm *RecoveryMetrics) HTTPMetricsHandler() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
metrics := rm.GetAllMetrics()
|
||||
health := rm.HealthCheck()
|
||||
|
||||
response := map[string]interface{}{
|
||||
"metrics": metrics,
|
||||
"health": health,
|
||||
}
|
||||
|
||||
// Would normally use json.Marshal here, but keeping it simple for the module
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, "%v", response)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user