package traefikoidc import ( "maps" "runtime" "sync" "sync/atomic" "time" ) // PerformanceMetrics tracks various performance-related metrics type PerformanceMetrics struct { startTime time.Time logger *Logger verificationTimes []time.Duration refreshTimes []time.Duration validationTimes []time.Duration memoryUsage int64 heapSize int64 successfulVerifications int64 successfulValidations int64 successfulRefreshes int64 failedVerifications int64 failedValidations int64 failedRefreshes int64 avgVerificationTime time.Duration avgValidationTime time.Duration avgRefreshTime time.Duration cacheHits int64 goroutineCount int64 memoryPressure int64 gcPauseTime int64 tokenRefreshes int64 heapInUse int64 verificationErrors int64 validationErrors int64 refreshErrors int64 rateLimitedRequests int64 activeSessions int64 sessionCreations int64 sessionDeletions int64 cacheMisses int64 tokenValidations int64 tokenVerifications int64 cacheSize int64 cacheEvictions int64 timingMutex sync.RWMutex } // NewPerformanceMetrics creates a new performance metrics tracker func NewPerformanceMetrics(logger *Logger) *PerformanceMetrics { pm := &PerformanceMetrics{ startTime: time.Now(), verificationTimes: make([]time.Duration, 0, 1000), // Keep last 1000 measurements validationTimes: make([]time.Duration, 0, 1000), refreshTimes: make([]time.Duration, 0, 1000), logger: logger, } // Start background metrics collection go pm.startMetricsCollection() return pm } // RecordCacheHit records a cache hit func (pm *PerformanceMetrics) RecordCacheHit() { atomic.AddInt64(&pm.cacheHits, 1) } // RecordCacheMiss records a cache miss func (pm *PerformanceMetrics) RecordCacheMiss() { atomic.AddInt64(&pm.cacheMisses, 1) } // RecordCacheEviction records a cache eviction func (pm *PerformanceMetrics) RecordCacheEviction() { atomic.AddInt64(&pm.cacheEvictions, 1) } // UpdateCacheSize updates the current cache size func (pm *PerformanceMetrics) UpdateCacheSize(size int64) { atomic.StoreInt64(&pm.cacheSize, size) } // RecordTokenVerification records a token verification operation func (pm *PerformanceMetrics) RecordTokenVerification(duration time.Duration, success bool) { atomic.AddInt64(&pm.tokenVerifications, 1) if success { atomic.AddInt64(&pm.successfulVerifications, 1) pm.addVerificationTime(duration) } else { atomic.AddInt64(&pm.failedVerifications, 1) atomic.AddInt64(&pm.verificationErrors, 1) } } // RecordTokenValidation records a token validation operation func (pm *PerformanceMetrics) RecordTokenValidation(duration time.Duration, success bool) { atomic.AddInt64(&pm.tokenValidations, 1) if success { atomic.AddInt64(&pm.successfulValidations, 1) pm.addValidationTime(duration) } else { atomic.AddInt64(&pm.failedValidations, 1) atomic.AddInt64(&pm.validationErrors, 1) } } // RecordTokenRefresh records a token refresh operation func (pm *PerformanceMetrics) RecordTokenRefresh(duration time.Duration, success bool) { atomic.AddInt64(&pm.tokenRefreshes, 1) if success { atomic.AddInt64(&pm.successfulRefreshes, 1) pm.addRefreshTime(duration) } else { atomic.AddInt64(&pm.failedRefreshes, 1) atomic.AddInt64(&pm.refreshErrors, 1) } } // RecordRateLimitedRequest records a rate-limited request func (pm *PerformanceMetrics) RecordRateLimitedRequest() { atomic.AddInt64(&pm.rateLimitedRequests, 1) } // RecordSessionCreation records a session creation func (pm *PerformanceMetrics) RecordSessionCreation() { atomic.AddInt64(&pm.sessionCreations, 1) atomic.AddInt64(&pm.activeSessions, 1) } // RecordSessionDeletion records a session deletion func (pm *PerformanceMetrics) RecordSessionDeletion() { atomic.AddInt64(&pm.sessionDeletions, 1) atomic.AddInt64(&pm.activeSessions, -1) } // addVerificationTime adds a verification time measurement func (pm *PerformanceMetrics) addVerificationTime(duration time.Duration) { pm.timingMutex.Lock() defer pm.timingMutex.Unlock() pm.verificationTimes = append(pm.verificationTimes, duration) if len(pm.verificationTimes) > 1000 { pm.verificationTimes = pm.verificationTimes[1:] } pm.updateAverageVerificationTime() } // addValidationTime adds a validation time measurement func (pm *PerformanceMetrics) addValidationTime(duration time.Duration) { pm.timingMutex.Lock() defer pm.timingMutex.Unlock() pm.validationTimes = append(pm.validationTimes, duration) if len(pm.validationTimes) > 1000 { pm.validationTimes = pm.validationTimes[1:] } pm.updateAverageValidationTime() } // addRefreshTime adds a refresh time measurement func (pm *PerformanceMetrics) addRefreshTime(duration time.Duration) { pm.timingMutex.Lock() defer pm.timingMutex.Unlock() pm.refreshTimes = append(pm.refreshTimes, duration) if len(pm.refreshTimes) > 1000 { pm.refreshTimes = pm.refreshTimes[1:] } pm.updateAverageRefreshTime() } // updateAverageVerificationTime calculates the average verification time func (pm *PerformanceMetrics) updateAverageVerificationTime() { if len(pm.verificationTimes) == 0 { pm.avgVerificationTime = 0 return } var total time.Duration for _, t := range pm.verificationTimes { total += t } pm.avgVerificationTime = total / time.Duration(len(pm.verificationTimes)) } // updateAverageValidationTime calculates the average validation time func (pm *PerformanceMetrics) updateAverageValidationTime() { if len(pm.validationTimes) == 0 { pm.avgValidationTime = 0 return } var total time.Duration for _, t := range pm.validationTimes { total += t } pm.avgValidationTime = total / time.Duration(len(pm.validationTimes)) } // updateAverageRefreshTime calculates the average refresh time func (pm *PerformanceMetrics) updateAverageRefreshTime() { if len(pm.refreshTimes) == 0 { pm.avgRefreshTime = 0 return } var total time.Duration for _, t := range pm.refreshTimes { total += t } pm.avgRefreshTime = total / time.Duration(len(pm.refreshTimes)) } // startMetricsCollection starts background collection of system metrics func (pm *PerformanceMetrics) startMetricsCollection() { ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() for range ticker.C { pm.collectSystemMetrics() } } // collectSystemMetrics collects system-level metrics func (pm *PerformanceMetrics) collectSystemMetrics() { // Memory statistics var m runtime.MemStats runtime.ReadMemStats(&m) atomic.StoreInt64(&pm.memoryUsage, int64(m.Alloc)) atomic.StoreInt64(&pm.heapSize, int64(m.HeapSys)) atomic.StoreInt64(&pm.heapInUse, int64(m.HeapInuse)) atomic.StoreInt64(&pm.gcPauseTime, int64(m.PauseNs[(m.NumGC+255)%256])) // Calculate memory pressure (0-100 scale) // Based on heap utilization and GC frequency heapUtilization := float64(m.HeapInuse) / float64(m.HeapSys) gcFrequency := float64(m.NumGC) / time.Since(pm.startTime).Minutes() // Memory pressure calculation pressure := int64(heapUtilization * 50) // 0-50 based on heap utilization if gcFrequency > 10 { // High GC frequency indicates pressure pressure += int64((gcFrequency - 10) * 2) // Add up to 50 more } if pressure > 100 { pressure = 100 } atomic.StoreInt64(&pm.memoryPressure, pressure) // Goroutine count atomic.StoreInt64(&pm.goroutineCount, int64(runtime.NumGoroutine())) // Log memory pressure warnings if pressure > 80 { pm.logger.Errorf("High memory pressure detected: %d%% (heap utilization: %.1f%%, GC frequency: %.1f/min)", pressure, heapUtilization*100, gcFrequency) } else if pressure > 60 { pm.logger.Infof("Moderate memory pressure: %d%% (heap utilization: %.1f%%, GC frequency: %.1f/min)", pressure, heapUtilization*100, gcFrequency) } } // GetMetrics returns all current performance metrics func (pm *PerformanceMetrics) GetMetrics() map[string]any { pm.timingMutex.RLock() defer pm.timingMutex.RUnlock() // Calculate cache hit ratio hits := atomic.LoadInt64(&pm.cacheHits) misses := atomic.LoadInt64(&pm.cacheMisses) var hitRatio float64 if hits+misses > 0 { hitRatio = float64(hits) / float64(hits+misses) } // Calculate error rates verifications := atomic.LoadInt64(&pm.tokenVerifications) validations := atomic.LoadInt64(&pm.tokenValidations) refreshes := atomic.LoadInt64(&pm.tokenRefreshes) var verificationErrorRate, validationErrorRate, refreshErrorRate float64 if verifications > 0 { verificationErrorRate = float64(atomic.LoadInt64(&pm.verificationErrors)) / float64(verifications) } if validations > 0 { validationErrorRate = float64(atomic.LoadInt64(&pm.validationErrors)) / float64(validations) } if refreshes > 0 { refreshErrorRate = float64(atomic.LoadInt64(&pm.refreshErrors)) / float64(refreshes) } return map[string]any{ // Cache metrics "cache_hits": hits, "cache_misses": misses, "cache_hit_ratio": hitRatio, "cache_evictions": atomic.LoadInt64(&pm.cacheEvictions), "cache_size": atomic.LoadInt64(&pm.cacheSize), // Token operation metrics "token_verifications": verifications, "token_validations": validations, "token_refreshes": refreshes, "verification_error_rate": verificationErrorRate, "validation_error_rate": validationErrorRate, "refresh_error_rate": refreshErrorRate, // Success/failure metrics "successful_verifications": atomic.LoadInt64(&pm.successfulVerifications), "successful_validations": atomic.LoadInt64(&pm.successfulValidations), "successful_refreshes": atomic.LoadInt64(&pm.successfulRefreshes), "failed_verifications": atomic.LoadInt64(&pm.failedVerifications), "failed_validations": atomic.LoadInt64(&pm.failedValidations), "failed_refreshes": atomic.LoadInt64(&pm.failedRefreshes), // Timing metrics "avg_verification_time_ms": pm.avgVerificationTime.Milliseconds(), "avg_validation_time_ms": pm.avgValidationTime.Milliseconds(), "avg_refresh_time_ms": pm.avgRefreshTime.Milliseconds(), // Resource metrics "memory_usage_bytes": atomic.LoadInt64(&pm.memoryUsage), "memory_pressure": atomic.LoadInt64(&pm.memoryPressure), "heap_size_bytes": atomic.LoadInt64(&pm.heapSize), "heap_inuse_bytes": atomic.LoadInt64(&pm.heapInUse), "gc_pause_time_ns": atomic.LoadInt64(&pm.gcPauseTime), "goroutine_count": atomic.LoadInt64(&pm.goroutineCount), // Rate limiting metrics "rate_limited_requests": atomic.LoadInt64(&pm.rateLimitedRequests), // Session metrics "active_sessions": atomic.LoadInt64(&pm.activeSessions), "sessions_created": atomic.LoadInt64(&pm.sessionCreations), "sessions_deleted": atomic.LoadInt64(&pm.sessionDeletions), "session_creations": atomic.LoadInt64(&pm.sessionCreations), "session_deletions": atomic.LoadInt64(&pm.sessionDeletions), // Uptime "uptime_seconds": time.Since(pm.startTime).Seconds(), } } // GetDetailedTimingMetrics returns detailed timing statistics func (pm *PerformanceMetrics) GetDetailedTimingMetrics() map[string]any { pm.timingMutex.RLock() defer pm.timingMutex.RUnlock() return map[string]any{ "verification_stats": pm.calculateTimingStats(pm.verificationTimes), "verification_timing": pm.calculateTimingStats(pm.verificationTimes), "validation_stats": pm.calculateTimingStats(pm.validationTimes), "validation_timing": pm.calculateTimingStats(pm.validationTimes), "refresh_stats": pm.calculateTimingStats(pm.refreshTimes), "refresh_timing": pm.calculateTimingStats(pm.refreshTimes), } } // calculateTimingStats calculates statistical metrics for timing data func (pm *PerformanceMetrics) calculateTimingStats(times []time.Duration) map[string]any { if len(times) == 0 { return map[string]any{ "count": 0, "min_ms": float64(0), "max_ms": float64(0), "avg_ms": float64(0), "average_ms": float64(0), "median_ms": float64(0), "p95_ms": float64(0), "p99_ms": float64(0), } } // Sort times for percentile calculations sortedTimes := make([]time.Duration, len(times)) copy(sortedTimes, times) // Simple bubble sort for small arrays for i := range sortedTimes { for j := i + 1; j < len(sortedTimes); j++ { if sortedTimes[i] > sortedTimes[j] { sortedTimes[i], sortedTimes[j] = sortedTimes[j], sortedTimes[i] } } } // Calculate statistics min := sortedTimes[0] max := sortedTimes[len(sortedTimes)-1] var total time.Duration for _, t := range sortedTimes { total += t } avg := total / time.Duration(len(sortedTimes)) median := sortedTimes[len(sortedTimes)/2] p95 := sortedTimes[int(float64(len(sortedTimes))*0.95)] p99 := sortedTimes[int(float64(len(sortedTimes))*0.99)] return map[string]any{ "count": len(sortedTimes), "min_ms": float64(min.Nanoseconds()) / 1e6, "max_ms": float64(max.Nanoseconds()) / 1e6, "avg_ms": float64(avg.Nanoseconds()) / 1e6, "average_ms": float64(avg.Nanoseconds()) / 1e6, "median_ms": float64(median.Nanoseconds()) / 1e6, "p95_ms": float64(p95.Nanoseconds()) / 1e6, "p99_ms": float64(p99.Nanoseconds()) / 1e6, } } // ResourceMonitor tracks resource usage and limits type ResourceMonitor struct { cacheSizes map[string]int64 alertThresholds map[string]float64 perfMetrics *PerformanceMetrics logger *Logger alerts []ResourceAlert maxMemoryBytes int64 maxCacheSize int64 maxSessions int64 cacheMutex sync.RWMutex alertsMutex sync.RWMutex } // ResourceAlert represents a resource usage alert type ResourceAlert struct { Timestamp time.Time `json:"timestamp"` Type string `json:"type"` Message string `json:"message"` Severity string `json:"severity"` Threshold float64 `json:"threshold"` CurrentValue float64 `json:"current_value"` } // NewResourceMonitor creates a new resource monitor func NewResourceMonitor(perfMetrics *PerformanceMetrics, logger *Logger) *ResourceMonitor { rm := &ResourceMonitor{ maxMemoryBytes: 100 * 1024 * 1024, // 100MB default maxCacheSize: 10000, // 10k items default maxSessions: 1000, // 1k sessions default cacheSizes: make(map[string]int64), alertThresholds: map[string]float64{ "memory_usage": 0.8, // 80% "memory_pressure": 0.7, // 70% "cache_usage": 0.9, // 90% "session_usage": 0.85, // 85% "error_rate": 0.1, // 10% }, alerts: make([]ResourceAlert, 0), perfMetrics: perfMetrics, logger: logger, } // Start monitoring routine go rm.startMonitoring() return rm } // SetMemoryLimit sets the maximum memory usage limit func (rm *ResourceMonitor) SetMemoryLimit(bytes int64) { rm.maxMemoryBytes = bytes } // SetCacheLimit sets the maximum cache size limit func (rm *ResourceMonitor) SetCacheLimit(size int64) { rm.maxCacheSize = size } // SetSessionLimit sets the maximum session count limit func (rm *ResourceMonitor) SetSessionLimit(count int64) { rm.maxSessions = count } // UpdateCacheSize updates the size of a specific cache func (rm *ResourceMonitor) UpdateCacheSize(cacheName string, size int64) { rm.cacheMutex.Lock() defer rm.cacheMutex.Unlock() rm.cacheSizes[cacheName] = size } // GetCacheSizes returns current cache sizes func (rm *ResourceMonitor) GetCacheSizes() map[string]int64 { rm.cacheMutex.RLock() defer rm.cacheMutex.RUnlock() sizes := make(map[string]int64) maps.Copy(sizes, rm.cacheSizes) return sizes } // startMonitoring starts the background monitoring routine func (rm *ResourceMonitor) startMonitoring() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for range ticker.C { rm.checkResourceUsage() } } // checkResourceUsage checks current resource usage against limits func (rm *ResourceMonitor) checkResourceUsage() { metrics := rm.perfMetrics.GetMetrics() // Check memory usage if memUsage, ok := metrics["memory_usage_bytes"].(int64); ok { memUsageRatio := float64(memUsage) / float64(rm.maxMemoryBytes) if memUsageRatio > rm.alertThresholds["memory_usage"] { rm.addAlert(ResourceAlert{ Type: "memory_usage", Message: "Memory usage exceeds threshold", Threshold: rm.alertThresholds["memory_usage"], CurrentValue: memUsageRatio, Timestamp: time.Now(), Severity: rm.getSeverity(memUsageRatio, rm.alertThresholds["memory_usage"]), }) } } // Check memory pressure if memPressure, ok := metrics["memory_pressure"].(int64); ok { pressureRatio := float64(memPressure) / 100.0 // Convert to 0-1 scale if pressureRatio > rm.alertThresholds["memory_pressure"] { rm.addAlert(ResourceAlert{ Type: "memory_pressure", Message: "Memory pressure exceeds threshold", Threshold: rm.alertThresholds["memory_pressure"], CurrentValue: pressureRatio, Timestamp: time.Now(), Severity: rm.getSeverity(pressureRatio, rm.alertThresholds["memory_pressure"]), }) } } // Check cache usage if cacheSize, ok := metrics["cache_size"].(int64); ok { cacheUsageRatio := float64(cacheSize) / float64(rm.maxCacheSize) if cacheUsageRatio > rm.alertThresholds["cache_usage"] { rm.addAlert(ResourceAlert{ Type: "cache_usage", Message: "Cache usage exceeds threshold", Threshold: rm.alertThresholds["cache_usage"], CurrentValue: cacheUsageRatio, Timestamp: time.Now(), Severity: rm.getSeverity(cacheUsageRatio, rm.alertThresholds["cache_usage"]), }) } } // Check session usage if activeSessions, ok := metrics["active_sessions"].(int64); ok { sessionUsageRatio := float64(activeSessions) / float64(rm.maxSessions) if sessionUsageRatio > rm.alertThresholds["session_usage"] { rm.addAlert(ResourceAlert{ Type: "session_usage", Message: "Active session count exceeds threshold", Threshold: rm.alertThresholds["session_usage"], CurrentValue: sessionUsageRatio, Timestamp: time.Now(), Severity: rm.getSeverity(sessionUsageRatio, rm.alertThresholds["session_usage"]), }) } } // Check error rates if errorRate, ok := metrics["verification_error_rate"].(float64); ok { if errorRate > rm.alertThresholds["error_rate"] { rm.addAlert(ResourceAlert{ Type: "verification_error_rate", Message: "Token verification error rate exceeds threshold", Threshold: rm.alertThresholds["error_rate"], CurrentValue: errorRate, Timestamp: time.Now(), Severity: rm.getSeverity(errorRate, rm.alertThresholds["error_rate"]), }) } } } // getSeverity determines the severity level based on how much the threshold is exceeded func (rm *ResourceMonitor) getSeverity(currentValue, threshold float64) string { ratio := currentValue / threshold if ratio >= 1.5 { return "critical" } else if ratio >= 1.2 { return "high" } else if ratio >= 1.0 { return "medium" } return "low" } // addAlert adds a new resource alert func (rm *ResourceMonitor) addAlert(alert ResourceAlert) { rm.alertsMutex.Lock() defer rm.alertsMutex.Unlock() // Add alert rm.alerts = append(rm.alerts, alert) // Keep only last 100 alerts if len(rm.alerts) > 100 { rm.alerts = rm.alerts[1:] } // Log the alert rm.logger.Errorf("Resource Alert [%s/%s]: %s (%.2f%% > %.2f%%)", alert.Type, alert.Severity, alert.Message, alert.CurrentValue*100, alert.Threshold*100) } // GetAlerts returns current resource alerts func (rm *ResourceMonitor) GetAlerts() []ResourceAlert { rm.alertsMutex.RLock() defer rm.alertsMutex.RUnlock() alerts := make([]ResourceAlert, len(rm.alerts)) copy(alerts, rm.alerts) return alerts } // GetResourceStatus returns current resource status func (rm *ResourceMonitor) GetResourceStatus() map[string]any { metrics := rm.perfMetrics.GetMetrics() cacheSizes := rm.GetCacheSizes() status := map[string]any{ "limits": map[string]any{ "max_memory_bytes": rm.maxMemoryBytes, "max_cache_size": rm.maxCacheSize, "max_sessions": rm.maxSessions, }, "thresholds": rm.alertThresholds, "current": metrics, "cache_sizes": cacheSizes, // Add expected keys for tests "memory_limit": uint64(rm.maxMemoryBytes), "cache_limit": int(rm.maxCacheSize), "session_limit": int(rm.maxSessions), } // Calculate usage ratios if memUsage, ok := metrics["memory_usage_bytes"].(int64); ok { status["memory_usage_ratio"] = float64(memUsage) / float64(rm.maxMemoryBytes) } if memPressure, ok := metrics["memory_pressure"].(int64); ok { status["memory_pressure_ratio"] = float64(memPressure) / 100.0 } if cacheSize, ok := metrics["cache_size"].(int64); ok { status["cache_usage_ratio"] = float64(cacheSize) / float64(rm.maxCacheSize) } if activeSessions, ok := metrics["active_sessions"].(int64); ok { status["session_usage_ratio"] = float64(activeSessions) / float64(rm.maxSessions) } // Calculate total cache size across all caches var totalCacheSize int64 for _, size := range cacheSizes { totalCacheSize += size } status["total_cache_size"] = totalCacheSize return status }