fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Further pursue of perfection.

2026-10-18 04:47:28 +00:00 · 2025-09-06 03:12:59 +01:00
parent f96940921b
commit 05c316002d
15 changed files with 4647 additions and 99 deletions
@@ -0,0 +1,375 @@
+package traefikoidc
+
+import (
+	"context"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// MemoryStats holds comprehensive memory statistics
+type MemoryStats struct {
+	// Go runtime memory stats
+	HeapAllocBytes    uint64    // bytes allocated and still in use
+	HeapSysBytes      uint64    // bytes obtained from system
+	HeapIdleBytes     uint64    // bytes in idle (unused) spans
+	HeapInuseBytes    uint64    // bytes in in-use spans
+	HeapReleasedBytes uint64    // bytes released to the OS
+	HeapObjects       uint64    // total number of allocated objects
+	StackInuseBytes   uint64    // bytes in stack spans
+	StackSysBytes     uint64    // bytes obtained from system for stack
+	GCSysBytes        uint64    // bytes used for garbage collection system metadata
+	NumGoroutines     int       // number of goroutines that currently exist
+	LastGCTime        time.Time // time of last garbage collection
+
+	// Application-specific memory tracking
+	SessionCount    int   // current number of sessions
+	TaskCount       int   // current number of background tasks
+	CacheSize       int64 // estimated cache memory usage
+	ConnectionPools int   // number of HTTP connection pools
+
+	// Memory pressure indicators
+	MemoryPressure MemoryPressureLevel // overall memory pressure level
+	GCFrequency    float64             // garbage collections per minute
+
+	Timestamp time.Time
+}
+
+// MemoryPressureLevel indicates the current memory pressure
+type MemoryPressureLevel int
+
+const (
+	MemoryPressureNone MemoryPressureLevel = iota
+	MemoryPressureLow
+	MemoryPressureModerate
+	MemoryPressureHigh
+	MemoryPressureCritical
+)
+
+func (mpl MemoryPressureLevel) String() string {
+	switch mpl {
+	case MemoryPressureNone:
+		return "None"
+	case MemoryPressureLow:
+		return "Low"
+	case MemoryPressureModerate:
+		return "Moderate"
+	case MemoryPressureHigh:
+		return "High"
+	case MemoryPressureCritical:
+		return "Critical"
+	default:
+		return "Unknown"
+	}
+}
+
+// MemoryMonitor provides comprehensive memory monitoring and alerting
+type MemoryMonitor struct {
+	logger          *Logger
+	mu              sync.RWMutex
+	lastStats       *MemoryStats
+	lastGCCount     uint32
+	lastGCTime      time.Time
+	startTime       time.Time
+	alertThresholds MemoryAlertThresholds
+
+	// Memory leak detection
+	baselineHeap     uint64
+	heapGrowthRate   float64 // bytes per second
+	suspiciousGrowth bool
+
+	// Goroutine tracking
+	baselineGoroutines int
+	maxGoroutines      int64
+	goroutineLeakAlert bool
+}
+
+// MemoryAlertThresholds defines when to trigger memory alerts
+type MemoryAlertThresholds struct {
+	HeapSizeMB          uint64  // Alert when heap exceeds this size in MB
+	HeapGrowthRateMB    float64 // Alert when heap grows faster than this MB/sec
+	GoroutineCount      int     // Alert when goroutine count exceeds this
+	GoroutineGrowthRate float64 // Alert when goroutines grow faster than this per minute
+	GCFrequency         float64 // Alert when GC frequency exceeds this per minute
+}
+
+// DefaultMemoryAlertThresholds returns sensible default alert thresholds
+func DefaultMemoryAlertThresholds() MemoryAlertThresholds {
+	return MemoryAlertThresholds{
+		HeapSizeMB:          256,  // 256MB heap size
+		HeapGrowthRateMB:    10.0, // 10MB/sec heap growth
+		GoroutineCount:      1000, // 1000 goroutines
+		GoroutineGrowthRate: 10.0, // 10 goroutines/minute growth
+		GCFrequency:         30.0, // 30 GCs/minute
+	}
+}
+
+// NewMemoryMonitor creates a new memory monitor
+func NewMemoryMonitor(logger *Logger, thresholds MemoryAlertThresholds) *MemoryMonitor {
+	if logger == nil {
+		logger = GetSingletonNoOpLogger()
+	}
+
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+
+	return &MemoryMonitor{
+		logger:             logger,
+		startTime:          time.Now(),
+		alertThresholds:    thresholds,
+		baselineHeap:       memStats.HeapAlloc,
+		baselineGoroutines: runtime.NumGoroutine(),
+		lastGCTime:         time.Unix(0, int64(memStats.LastGC)),
+		lastGCCount:        memStats.NumGC,
+	}
+}
+
+// GetCurrentStats collects current memory statistics
+func (mm *MemoryMonitor) GetCurrentStats() *MemoryStats {
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+
+	now := time.Now()
+
+	// Calculate GC frequency
+	gcFrequency := 0.0
+	if mm.lastStats != nil {
+		timeDiff := now.Sub(mm.lastStats.Timestamp).Minutes()
+		if timeDiff > 0 {
+			gcDiff := float64(memStats.NumGC - mm.lastGCCount)
+			gcFrequency = gcDiff / timeDiff
+		}
+	}
+
+	mm.lastGCCount = memStats.NumGC
+
+	stats := &MemoryStats{
+		HeapAllocBytes:    memStats.HeapAlloc,
+		HeapSysBytes:      memStats.HeapSys,
+		HeapIdleBytes:     memStats.HeapIdle,
+		HeapInuseBytes:    memStats.HeapInuse,
+		HeapReleasedBytes: memStats.HeapReleased,
+		HeapObjects:       memStats.HeapObjects,
+		StackInuseBytes:   memStats.StackInuse,
+		StackSysBytes:     memStats.StackSys,
+		GCSysBytes:        memStats.GCSys,
+		NumGoroutines:     runtime.NumGoroutine(),
+		LastGCTime:        time.Unix(0, int64(memStats.LastGC)),
+		GCFrequency:       gcFrequency,
+		Timestamp:         now,
+	}
+
+	// Get application-specific stats
+	mm.collectApplicationStats(stats)
+
+	// Calculate memory pressure
+	stats.MemoryPressure = mm.calculateMemoryPressure(stats)
+
+	// Update goroutine tracking
+	mm.updateGoroutineTracking(stats)
+
+	// Update heap growth tracking
+	mm.updateHeapGrowthTracking(stats)
+
+	mm.mu.Lock()
+	mm.lastStats = stats
+	mm.mu.Unlock()
+
+	return stats
+}
+
+// collectApplicationStats gathers application-specific memory stats
+func (mm *MemoryMonitor) collectApplicationStats(stats *MemoryStats) {
+	// Get session count from ChunkManager if available
+	// This is a placeholder - real implementation would access actual managers
+	stats.SessionCount = 0 // Would be populated from actual session manager
+
+	// Get background task count from TaskRegistry
+	registry := GetGlobalTaskRegistry()
+	stats.TaskCount = registry.GetTaskCount()
+
+	// Estimate cache size
+	stats.CacheSize = 0 // Would be populated from actual cache implementations
+
+	// Count HTTP connection pools
+	stats.ConnectionPools = 1 // Would be counted from actual HTTP clients
+}
+
+// calculateMemoryPressure determines the current memory pressure level
+func (mm *MemoryMonitor) calculateMemoryPressure(stats *MemoryStats) MemoryPressureLevel {
+	heapMB := float64(stats.HeapAllocBytes) / (1024 * 1024)
+
+	// Critical: Heap > 512MB or very frequent GC
+	if heapMB > 512 || stats.GCFrequency > 60 {
+		return MemoryPressureCritical
+	}
+
+	// High: Heap > 256MB or frequent GC
+	if heapMB > 256 || stats.GCFrequency > 30 {
+		return MemoryPressureHigh
+	}
+
+	// Moderate: Heap > 128MB or elevated GC
+	if heapMB > 128 || stats.GCFrequency > 15 {
+		return MemoryPressureModerate
+	}
+
+	// Low: Heap > 64MB or some GC activity
+	if heapMB > 64 || stats.GCFrequency > 5 {
+		return MemoryPressureLow
+	}
+
+	return MemoryPressureNone
+}
+
+// updateGoroutineTracking monitors goroutine counts for leaks
+func (mm *MemoryMonitor) updateGoroutineTracking(stats *MemoryStats) {
+	currentCount := int64(stats.NumGoroutines)
+
+	// Update max goroutines
+	if currentCount > atomic.LoadInt64(&mm.maxGoroutines) {
+		atomic.StoreInt64(&mm.maxGoroutines, currentCount)
+	}
+
+	// Check for potential goroutine leak
+	if stats.NumGoroutines > mm.baselineGoroutines+int(mm.alertThresholds.GoroutineCount) {
+		if !mm.goroutineLeakAlert {
+			mm.goroutineLeakAlert = true
+			mm.logger.Error("Potential goroutine leak detected: %d goroutines (baseline: %d)",
+				stats.NumGoroutines, mm.baselineGoroutines)
+		}
+	} else {
+		mm.goroutineLeakAlert = false
+	}
+}
+
+// updateHeapGrowthTracking monitors heap growth rate
+func (mm *MemoryMonitor) updateHeapGrowthTracking(stats *MemoryStats) {
+	if mm.lastStats != nil {
+		timeDiff := stats.Timestamp.Sub(mm.lastStats.Timestamp).Seconds()
+		if timeDiff > 0 {
+			heapDiff := float64(stats.HeapAllocBytes) - float64(mm.lastStats.HeapAllocBytes)
+			mm.heapGrowthRate = heapDiff / timeDiff // bytes per second
+
+			growthRateMB := mm.heapGrowthRate / (1024 * 1024)
+			if growthRateMB > mm.alertThresholds.HeapGrowthRateMB {
+				if !mm.suspiciousGrowth {
+					mm.suspiciousGrowth = true
+					mm.logger.Error("Suspicious heap growth rate: %.2f MB/sec", growthRateMB)
+				}
+			} else {
+				mm.suspiciousGrowth = false
+			}
+		}
+	}
+}
+
+// LogMemoryStats logs comprehensive memory statistics
+func (mm *MemoryMonitor) LogMemoryStats(stats *MemoryStats) {
+	heapMB := float64(stats.HeapAllocBytes) / (1024 * 1024)
+	sysMB := float64(stats.HeapSysBytes) / (1024 * 1024)
+
+	mm.logger.Info("Memory Stats - Heap: %.1fMB/%.1fMB, Goroutines: %d, Pressure: %s, GC: %.1f/min",
+		heapMB, sysMB, stats.NumGoroutines, stats.MemoryPressure.String(), stats.GCFrequency)
+
+	// Log additional details at debug level
+	mm.logger.Debug("Memory Details - Sessions: %d, Tasks: %d, Cache: %dB, Pools: %d",
+		stats.SessionCount, stats.TaskCount, stats.CacheSize, stats.ConnectionPools)
+}
+
+// StartMonitoring starts continuous memory monitoring
+func (mm *MemoryMonitor) StartMonitoring(ctx context.Context, interval time.Duration) {
+	if interval <= 0 {
+		interval = 30 * time.Second
+	}
+
+	registry := GetGlobalTaskRegistry()
+
+	task, err := registry.CreateSingletonTask(
+		"memory-monitor",
+		interval,
+		func() {
+			stats := mm.GetCurrentStats()
+			mm.LogMemoryStats(stats)
+			mm.checkAlerts(stats)
+		},
+		mm.logger,
+		nil,
+	)
+
+	if err != nil {
+		mm.logger.Errorf("Failed to create memory monitoring task: %v", err)
+		return
+	}
+
+	task.Start()
+	mm.logger.Info("Started memory monitoring with %v interval", interval)
+}
+
+// checkAlerts checks for memory-related alerts
+func (mm *MemoryMonitor) checkAlerts(stats *MemoryStats) {
+	heapMB := float64(stats.HeapAllocBytes) / (1024 * 1024)
+
+	// Heap size alert
+	if heapMB > float64(mm.alertThresholds.HeapSizeMB) {
+		mm.logger.Error("Memory Alert: Heap size %.1fMB exceeds threshold %dMB",
+			heapMB, mm.alertThresholds.HeapSizeMB)
+	}
+
+	// GC frequency alert
+	if stats.GCFrequency > mm.alertThresholds.GCFrequency {
+		mm.logger.Error("Memory Alert: GC frequency %.1f/min exceeds threshold %.1f/min",
+			stats.GCFrequency, mm.alertThresholds.GCFrequency)
+	}
+
+	// Critical memory pressure
+	if stats.MemoryPressure >= MemoryPressureHigh {
+		mm.logger.Error("Memory Alert: %s memory pressure detected", stats.MemoryPressure.String())
+	}
+}
+
+// TriggerGC forces garbage collection and logs the impact
+func (mm *MemoryMonitor) TriggerGC() {
+	before := mm.GetCurrentStats()
+
+	runtime.GC()
+	runtime.GC() // Run twice to ensure full collection
+
+	after := mm.GetCurrentStats()
+
+	freedBytes := int64(before.HeapAllocBytes) - int64(after.HeapAllocBytes)
+	freedMB := float64(freedBytes) / (1024 * 1024)
+
+	mm.logger.Info("Manual GC completed - Freed: %.1fMB, Before: %.1fMB, After: %.1fMB",
+		freedMB,
+		float64(before.HeapAllocBytes)/(1024*1024),
+		float64(after.HeapAllocBytes)/(1024*1024))
+}
+
+// GetMemoryPressure returns the current memory pressure level
+func (mm *MemoryMonitor) GetMemoryPressure() MemoryPressureLevel {
+	mm.mu.RLock()
+	defer mm.mu.RUnlock()
+
+	if mm.lastStats != nil {
+		return mm.lastStats.MemoryPressure
+	}
+	return MemoryPressureNone
+}
+
+// Global memory monitor instance
+var (
+	globalMemoryMonitor     *MemoryMonitor
+	globalMemoryMonitorOnce sync.Once
+)
+
+// GetGlobalMemoryMonitor returns the singleton memory monitor
+func GetGlobalMemoryMonitor() *MemoryMonitor {
+	globalMemoryMonitorOnce.Do(func() {
+		logger := GetSingletonNoOpLogger()
+		thresholds := DefaultMemoryAlertThresholds()
+		globalMemoryMonitor = NewMemoryMonitor(logger, thresholds)
+	})
+	return globalMemoryMonitor
+}