traefikoidc/internal/recovery/metrics.go

// Package recovery provides error recovery and resilience mechanisms for OIDC authentication.
package recovery

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"math/rand"
	"net/http"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// RetryConfig defines configuration for the retry executor
type RetryConfig struct {
	RetryableErrors      []string
	RetryableStatusCodes []int
	MaxAttempts          int
	InitialDelay         time.Duration
	MaxDelay             time.Duration
	Multiplier           float64
	RandomizationFactor  float64
}

// DefaultRetryConfig returns sensible default retry configuration
func DefaultRetryConfig() RetryConfig {
	return RetryConfig{
		MaxAttempts:          3,
		InitialDelay:         100 * time.Millisecond,
		MaxDelay:             30 * time.Second,
		Multiplier:           2.0,
		RandomizationFactor:  0.1,
		RetryableErrors:      []string{"connection refused", "timeout", "EOF"},
		RetryableStatusCodes: []int{408, 429, 500, 502, 503, 504},
	}
}

// RetryExecutor implements retry logic with exponential backoff
type RetryExecutor struct {
	lastRetryTime time.Time
	*BaseRecoveryMechanism
	config         RetryConfig
	totalRetries   int64
	maxRetriesHit  int64
	retryTimeMutex sync.RWMutex
}

// NewRetryExecutor creates a new retry executor with the given configuration
func NewRetryExecutor(config RetryConfig, logger Logger) *RetryExecutor {
	if config.MaxAttempts < 1 {
		config.MaxAttempts = 1
	}
	if config.Multiplier < 1.0 {
		config.Multiplier = 1.0
	}
	return &RetryExecutor{
		BaseRecoveryMechanism: NewBaseRecoveryMechanism("RetryExecutor", logger),
		config:                config,
		totalRetries:          0,
		maxRetriesHit:         0,
	}
}

// ExecuteWithContext executes a function with retry logic
func (re *RetryExecutor) ExecuteWithContext(ctx context.Context, fn func() error) error {
	re.RecordRequest()

	var lastErr error
	for attempt := 1; attempt <= re.config.MaxAttempts; attempt++ {
		// Check context before attempting
		select {
		case <-ctx.Done():
			re.RecordFailure()
			return ctx.Err()
		default:
		}

		// Execute the function
		lastErr = fn()

		if lastErr == nil {
			re.RecordSuccess()
			if attempt > 1 {
				re.LogInfo("Succeeded after %d attempts", attempt)
			}
			return nil
		}

		// Check if error is retryable
		if !re.isRetryableError(lastErr) {
			re.LogDebug("Error is not retryable: %v", lastErr)
			re.RecordFailure()
			return lastErr
		}

		// Don't retry if this was the last attempt
		if attempt >= re.config.MaxAttempts {
			atomic.AddInt64(&re.maxRetriesHit, 1)
			re.LogError("Max retries (%d) exhausted", re.config.MaxAttempts)
			break
		}

		// Calculate and apply delay
		delay := re.calculateDelay(attempt)
		re.LogInfo("Attempt %d failed: %v, retrying in %v", attempt, lastErr, delay)

		atomic.AddInt64(&re.totalRetries, 1)
		re.retryTimeMutex.Lock()
		re.lastRetryTime = time.Now()
		re.retryTimeMutex.Unlock()

		select {
		case <-time.After(delay):
			// Continue to next attempt
		case <-ctx.Done():
			re.RecordFailure()
			return fmt.Errorf("retry canceled: %w", ctx.Err())
		}
	}

	re.RecordFailure()
	return fmt.Errorf("all retry attempts failed: %w", lastErr)
}

// Execute executes a function with retry logic (legacy method)
func (re *RetryExecutor) Execute(ctx context.Context, fn func() error) error {
	return re.ExecuteWithContext(ctx, fn)
}

// isRetryableError determines if an error should trigger a retry
func (re *RetryExecutor) isRetryableError(err error) bool {
	if err == nil {
		return false
	}

	errStr := strings.ToLower(err.Error())

	// Check for retryable error patterns
	for _, pattern := range re.config.RetryableErrors {
		if strings.Contains(errStr, strings.ToLower(pattern)) {
			return true
		}
	}

	// Check for HTTP errors. Use errors.As (not a direct type assertion) so an
	// error that wraps an *HTTPError — e.g. fmt.Errorf("token exchange failed:
	// %w", httpErr) — is still recognized as retryable: previously a wrapped
	// retryable error was misclassified as terminal and bailed after one
	// attempt, defeating retry despite the production wrapping pattern.
	var httpErr *HTTPError
	if errors.As(err, &httpErr) {
		for _, code := range re.config.RetryableStatusCodes {
			if httpErr.StatusCode == code {
				return true
			}
		}
		// Also retry on any 5xx error
		if httpErr.StatusCode >= 500 && httpErr.StatusCode < 600 {
			return true
		}
	}

	// Check for OIDC errors (unwrapped or wrapped).
	var oidcErr *OIDCError
	if errors.As(err, &oidcErr) {
		return oidcErr.IsRetryable()
	}

	// Check for context errors (don't retry these)
	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
		return false
	}

	// Default: don't retry unknown errors
	return false
}

// calculateDelay calculates the delay before the next retry attempt
func (re *RetryExecutor) calculateDelay(attempt int) time.Duration {
	// Exponential backoff
	delay := float64(re.config.InitialDelay) * math.Pow(re.config.Multiplier, float64(attempt-1))

	// Cap at max delay
	if delay > float64(re.config.MaxDelay) {
		delay = float64(re.config.MaxDelay)
	}

	// Add jitter
	// #nosec G404 -- math/rand is acceptable for jitter timing, not security-sensitive
	if re.config.RandomizationFactor > 0 {
		jitter := delay * re.config.RandomizationFactor
		minDelay := delay - jitter
		// A RandomizationFactor > 1 makes minDelay negative, which turns
		// time.After(negative) into an immediate (zero) backoff and defeats
		// the retry spacing (R150). Clamp to zero so backoff stays bounded.
		if minDelay < 0 {
			minDelay = 0
		}
		maxDelay := delay + jitter
		delay = minDelay + rand.Float64()*(maxDelay-minDelay)
	}

	return time.Duration(delay)
}

// Reset resets the retry executor state
func (re *RetryExecutor) Reset() {
	atomic.StoreInt64(&re.totalRetries, 0)
	atomic.StoreInt64(&re.maxRetriesHit, 0)
	atomic.StoreInt64(&re.totalRequests, 0)
	atomic.StoreInt64(&re.successCount, 0)
	atomic.StoreInt64(&re.failureCount, 0)

	re.retryTimeMutex.Lock()
	re.lastRetryTime = time.Time{}
	re.retryTimeMutex.Unlock()
}

// IsAvailable always returns true for retry executor
func (re *RetryExecutor) IsAvailable() bool {
	return true
}

// GetMetrics returns comprehensive metrics about the retry executor
func (re *RetryExecutor) GetMetrics() map[string]interface{} {
	metrics := re.GetBaseMetrics()

	metrics["totalRetries"] = atomic.LoadInt64(&re.totalRetries)
	metrics["maxRetriesHit"] = atomic.LoadInt64(&re.maxRetriesHit)

	re.retryTimeMutex.RLock()
	if !re.lastRetryTime.IsZero() {
		metrics["lastRetryTime"] = re.lastRetryTime.Format(time.RFC3339)
		metrics["timeSinceLastRetry"] = time.Since(re.lastRetryTime).String()
	} else {
		metrics["lastRetryTime"] = "never"
	}
	re.retryTimeMutex.RUnlock()

	// Configuration
	metrics["config"] = map[string]interface{}{
		"maxAttempts":         re.config.MaxAttempts,
		"initialDelay":        re.config.InitialDelay.String(),
		"maxDelay":            re.config.MaxDelay.String(),
		"multiplier":          re.config.Multiplier,
		"randomizationFactor": re.config.RandomizationFactor,
	}

	// Calculate average retries per request
	totalRequests := atomic.LoadInt64(&re.totalRequests)
	if totalRequests > 0 {
		avgRetries := float64(atomic.LoadInt64(&re.totalRetries)) / float64(totalRequests)
		metrics["averageRetriesPerRequest"] = fmt.Sprintf("%.2f", avgRetries)
	}

	return metrics
}

// RecoveryMetrics aggregates metrics from multiple recovery mechanisms
type RecoveryMetrics struct {
	mechanisms map[string]ErrorRecoveryMechanism
	mu         sync.RWMutex
}

// NewRecoveryMetrics creates a new recovery metrics aggregator
func NewRecoveryMetrics() *RecoveryMetrics {
	return &RecoveryMetrics{
		mechanisms: make(map[string]ErrorRecoveryMechanism),
	}
}

// RegisterMechanism registers a recovery mechanism for metrics collection
func (rm *RecoveryMetrics) RegisterMechanism(name string, mechanism ErrorRecoveryMechanism) {
	rm.mu.Lock()
	defer rm.mu.Unlock()
	rm.mechanisms[name] = mechanism
}

// UnregisterMechanism removes a recovery mechanism from metrics collection
func (rm *RecoveryMetrics) UnregisterMechanism(name string) {
	rm.mu.Lock()
	defer rm.mu.Unlock()
	delete(rm.mechanisms, name)
}

// GetAllMetrics returns aggregated metrics from all registered mechanisms
func (rm *RecoveryMetrics) GetAllMetrics() map[string]interface{} {
	rm.mu.RLock()
	defer rm.mu.RUnlock()

	allMetrics := make(map[string]interface{})
	for name, mechanism := range rm.mechanisms {
		allMetrics[name] = mechanism.GetMetrics()
	}

	// Add summary statistics
	totalRequests := int64(0)
	totalSuccesses := int64(0)
	totalFailures := int64(0)

	for _, mechanism := range rm.mechanisms {
		metrics := mechanism.GetMetrics()
		if requests, ok := metrics["totalRequests"].(int64); ok {
			totalRequests += requests
		}
		if successes, ok := metrics["successCount"].(int64); ok {
			totalSuccesses += successes
		}
		if failures, ok := metrics["failureCount"].(int64); ok {
			totalFailures += failures
		}
	}

	summary := map[string]interface{}{
		"totalMechanisms": len(rm.mechanisms),
		"totalRequests":   totalRequests,
		"totalSuccesses":  totalSuccesses,
		"totalFailures":   totalFailures,
	}

	if totalRequests > 0 {
		successRate := float64(totalSuccesses) / float64(totalRequests) * 100
		summary["overallSuccessRate"] = fmt.Sprintf("%.2f%%", successRate)
	}
	allMetrics["summary"] = summary

	return allMetrics
}

// GetMechanismMetrics returns metrics for a specific mechanism
func (rm *RecoveryMetrics) GetMechanismMetrics(name string) (map[string]interface{}, bool) {
	rm.mu.RLock()
	defer rm.mu.RUnlock()

	if mechanism, exists := rm.mechanisms[name]; exists {
		return mechanism.GetMetrics(), true
	}
	return nil, false
}

// HealthCheck performs a health check on all registered mechanisms
func (rm *RecoveryMetrics) HealthCheck() map[string]interface{} {
	rm.mu.RLock()
	defer rm.mu.RUnlock()

	health := make(map[string]interface{})
	healthyCount := 0
	unhealthyCount := 0

	for name, mechanism := range rm.mechanisms {
		if mechanism.IsAvailable() {
			health[name] = "healthy"
			healthyCount++
		} else {
			health[name] = "unhealthy"
			unhealthyCount++
		}
	}

	overallHealth := "healthy"
	if unhealthyCount > 0 {
		if healthyCount > 0 {
			overallHealth = "degraded"
		} else {
			overallHealth = "unhealthy"
		}
	}

	return map[string]interface{}{
		"status":     overallHealth,
		"mechanisms": health,
		"healthy":    healthyCount,
		"unhealthy":  unhealthyCount,
		"timestamp":  time.Now().Format(time.RFC3339),
	}
}

// HTTPMetricsHandler creates an HTTP handler for serving recovery metrics
func (rm *RecoveryMetrics) HTTPMetricsHandler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		metrics := rm.GetAllMetrics()
		health := rm.HealthCheck()

		response := map[string]interface{}{
			"metrics": metrics,
			"health":  health,
		}

		body, err := json.Marshal(response)
		if err != nil {
			http.Error(w, "failed to marshal metrics", http.StatusInternalServerError)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write(body)
	}
}