traefikoidc/memory_monitor.go

package traefikoidc

import (
	"context"
	"runtime"
	"sync"
	"sync/atomic"
	"time"
)

// MemoryStats holds comprehensive memory statistics
type MemoryStats struct {
	LastGCTime        time.Time
	Timestamp         time.Time
	GCSysBytes        uint64
	NumGoroutines     int
	HeapReleasedBytes uint64
	HeapObjects       uint64
	StackInuseBytes   uint64
	StackSysBytes     uint64
	HeapAllocBytes    uint64
	HeapInuseBytes    uint64
	HeapIdleBytes     uint64
	SessionCount      int
	TaskCount         int
	CacheSize         int64
	ConnectionPools   int
	MemoryPressure    MemoryPressureLevel
	GCFrequency       float64
	HeapSysBytes      uint64
}

// MemoryPressureLevel indicates the current memory pressure
type MemoryPressureLevel int

const (
	MemoryPressureNone MemoryPressureLevel = iota
	MemoryPressureLow
	MemoryPressureModerate
	MemoryPressureHigh
	MemoryPressureCritical
)

func (mpl MemoryPressureLevel) String() string {
	switch mpl {
	case MemoryPressureNone:
		return "None"
	case MemoryPressureLow:
		return "Low"
	case MemoryPressureModerate:
		return "Moderate"
	case MemoryPressureHigh:
		return "High"
	case MemoryPressureCritical:
		return "Critical"
	default:
		return "Unknown"
	}
}

// MemoryMonitor provides comprehensive memory monitoring and alerting
type MemoryMonitor struct {
	lastGCTime         time.Time
	startTime          time.Time
	lastStats          *MemoryStats
	logger             *Logger
	alertThresholds    MemoryAlertThresholds
	baselineGoroutines int
	baselineHeap       uint64
	heapGrowthRate     float64
	maxGoroutines      int64
	mu                 sync.RWMutex
	lastGCCount        uint32
	suspiciousGrowth   bool
	goroutineLeakAlert bool
}

// MemoryAlertThresholds defines when to trigger memory alerts
type MemoryAlertThresholds struct {
	HeapSizeMB          uint64  // Alert when heap exceeds this size in MB
	HeapGrowthRateMB    float64 // Alert when heap grows faster than this MB/sec
	GoroutineCount      int     // Alert when goroutine count exceeds this
	GoroutineGrowthRate float64 // Alert when goroutines grow faster than this per minute
	GCFrequency         float64 // Alert when GC frequency exceeds this per minute
}

// DefaultMemoryAlertThresholds returns sensible default alert thresholds
func DefaultMemoryAlertThresholds() MemoryAlertThresholds {
	return MemoryAlertThresholds{
		HeapSizeMB:          256,  // 256MB heap size
		HeapGrowthRateMB:    10.0, // 10MB/sec heap growth
		GoroutineCount:      1000, // 1000 goroutines
		GoroutineGrowthRate: 10.0, // 10 goroutines/minute growth
		GCFrequency:         30.0, // 30 GCs/minute
	}
}

// NewMemoryMonitor creates a new memory monitor
func NewMemoryMonitor(logger *Logger, thresholds MemoryAlertThresholds) *MemoryMonitor {
	if logger == nil {
		logger = GetSingletonNoOpLogger()
	}

	var memStats runtime.MemStats
	runtime.ReadMemStats(&memStats)

	return &MemoryMonitor{
		logger:             logger,
		startTime:          time.Now(),
		alertThresholds:    thresholds,
		baselineHeap:       memStats.HeapAlloc,
		baselineGoroutines: runtime.NumGoroutine(),
		// #nosec G115 -- LastGC nanoseconds fits in int64 for centuries
		lastGCTime:  time.Unix(0, int64(memStats.LastGC)),
		lastGCCount: memStats.NumGC,
	}
}

// GetCurrentStats collects current memory statistics
func (mm *MemoryMonitor) GetCurrentStats() *MemoryStats {
	var memStats runtime.MemStats
	runtime.ReadMemStats(&memStats)

	now := time.Now()

	// Calculate GC frequency
	gcFrequency := 0.0
	mm.mu.RLock()
	lastStats := mm.lastStats
	lastGCCount := mm.lastGCCount
	mm.mu.RUnlock()

	if lastStats != nil {
		timeDiff := now.Sub(lastStats.Timestamp).Minutes()
		if timeDiff > 0 {
			gcDiff := float64(memStats.NumGC - lastGCCount)
			gcFrequency = gcDiff / timeDiff
		}
	}

	stats := &MemoryStats{
		HeapAllocBytes:    memStats.HeapAlloc,
		HeapSysBytes:      memStats.HeapSys,
		HeapIdleBytes:     memStats.HeapIdle,
		HeapInuseBytes:    memStats.HeapInuse,
		HeapReleasedBytes: memStats.HeapReleased,
		HeapObjects:       memStats.HeapObjects,
		StackInuseBytes:   memStats.StackInuse,
		StackSysBytes:     memStats.StackSys,
		GCSysBytes:        memStats.GCSys,
		NumGoroutines:     runtime.NumGoroutine(),
		// #nosec G115 -- LastGC nanoseconds fits in int64 for centuries
		LastGCTime:  time.Unix(0, int64(memStats.LastGC)),
		GCFrequency: gcFrequency,
		Timestamp:   now,
	}

	// Get application-specific stats
	mm.collectApplicationStats(stats)

	// Calculate memory pressure
	stats.MemoryPressure = mm.calculateMemoryPressure(stats)

	// Update goroutine tracking
	mm.updateGoroutineTracking(stats)

	// Update heap growth tracking
	mm.updateHeapGrowthTracking(stats)

	mm.mu.Lock()
	mm.lastStats = stats
	mm.lastGCCount = memStats.NumGC
	mm.mu.Unlock()

	return stats
}

// collectApplicationStats gathers application-specific memory stats
func (mm *MemoryMonitor) collectApplicationStats(stats *MemoryStats) {
	// Get session count from ChunkManager if available
	// This is a placeholder - real implementation would access actual managers
	stats.SessionCount = 0 // Would be populated from actual session manager

	// Get background task count from TaskRegistry
	registry := GetGlobalTaskRegistry()
	stats.TaskCount = registry.GetTaskCount()

	// Estimate cache size
	stats.CacheSize = 0 // Would be populated from actual cache implementations

	// Count HTTP connection pools
	stats.ConnectionPools = 1 // Would be counted from actual HTTP clients
}

// calculateMemoryPressure determines the current memory pressure level
func (mm *MemoryMonitor) calculateMemoryPressure(stats *MemoryStats) MemoryPressureLevel {
	heapMB := float64(stats.HeapAllocBytes) / (1024 * 1024)

	// Critical: Heap > 512MB or very frequent GC
	if heapMB > 512 || stats.GCFrequency > 60 {
		return MemoryPressureCritical
	}

	// High: Heap > 256MB or frequent GC
	if heapMB > 256 || stats.GCFrequency > 30 {
		return MemoryPressureHigh
	}

	// Moderate: Heap > 128MB or elevated GC
	if heapMB > 128 || stats.GCFrequency > 15 {
		return MemoryPressureModerate
	}

	// Low: Heap > 64MB or some GC activity
	if heapMB > 64 || stats.GCFrequency > 5 {
		return MemoryPressureLow
	}

	return MemoryPressureNone
}

// updateGoroutineTracking monitors goroutine counts for leaks
func (mm *MemoryMonitor) updateGoroutineTracking(stats *MemoryStats) {
	currentCount := int64(stats.NumGoroutines)

	// Update max goroutines
	if currentCount > atomic.LoadInt64(&mm.maxGoroutines) {
		atomic.StoreInt64(&mm.maxGoroutines, currentCount)
	}

	// Check for potential goroutine leak
	if stats.NumGoroutines > mm.baselineGoroutines+mm.alertThresholds.GoroutineCount {
		mm.mu.Lock()
		wasAlert := mm.goroutineLeakAlert
		if !wasAlert {
			mm.goroutineLeakAlert = true
		}
		mm.mu.Unlock()
		if !wasAlert {
			mm.logger.Error("Potential goroutine leak detected: %d goroutines (baseline: %d)",
				stats.NumGoroutines, mm.baselineGoroutines)
		}
	} else {
		mm.mu.Lock()
		mm.goroutineLeakAlert = false
		mm.mu.Unlock()
	}
}

// updateHeapGrowthTracking monitors heap growth rate
func (mm *MemoryMonitor) updateHeapGrowthTracking(stats *MemoryStats) {
	mm.mu.RLock()
	lastStats := mm.lastStats
	mm.mu.RUnlock()

	if lastStats != nil {
		timeDiff := stats.Timestamp.Sub(lastStats.Timestamp).Seconds()
		if timeDiff > 0 {
			heapDiff := float64(stats.HeapAllocBytes) - float64(lastStats.HeapAllocBytes)
			heapGrowthRate := heapDiff / timeDiff // bytes per second

			mm.mu.Lock()
			mm.heapGrowthRate = heapGrowthRate
			mm.mu.Unlock()

			growthRateMB := heapGrowthRate / (1024 * 1024)
			if growthRateMB > mm.alertThresholds.HeapGrowthRateMB {
				mm.mu.Lock()
				wasSuspicious := mm.suspiciousGrowth
				if !wasSuspicious {
					mm.suspiciousGrowth = true
				}
				mm.mu.Unlock()
				if !wasSuspicious {
					mm.logger.Error("Suspicious heap growth rate: %.2f MB/sec", growthRateMB)
				}
			} else {
				mm.mu.Lock()
				mm.suspiciousGrowth = false
				mm.mu.Unlock()
			}
		}
	}
}

// LogMemoryStats logs comprehensive memory statistics
func (mm *MemoryMonitor) LogMemoryStats(stats *MemoryStats) {
	heapMB := float64(stats.HeapAllocBytes) / (1024 * 1024)
	sysMB := float64(stats.HeapSysBytes) / (1024 * 1024)

	mm.logger.Info("Memory Stats - Heap: %.1fMB/%.1fMB, Goroutines: %d, Pressure: %s, GC: %.1f/min",
		heapMB, sysMB, stats.NumGoroutines, stats.MemoryPressure.String(), stats.GCFrequency)

	// Log additional details at debug level
	mm.logger.Debug("Memory Details - Sessions: %d, Tasks: %d, Cache: %dB, Pools: %d",
		stats.SessionCount, stats.TaskCount, stats.CacheSize, stats.ConnectionPools)
}

// Global monitoring state
var (
	globalMonitoringStarted bool
	globalMonitoringMutex   sync.Mutex
)

// StartMonitoring starts continuous memory monitoring as a global singleton
func (mm *MemoryMonitor) StartMonitoring(ctx context.Context, interval time.Duration) {
	globalMonitoringMutex.Lock()
	defer globalMonitoringMutex.Unlock()

	// Check if monitoring is already started
	if globalMonitoringStarted {
		if !isTestMode() {
			mm.logger.Debug("Memory monitoring already started, skipping duplicate start")
		}
		return
	}

	if interval <= 0 {
		interval = 30 * time.Second
	}

	registry := GetGlobalTaskRegistry()

	task, err := registry.CreateSingletonTask(
		"memory-monitor",
		interval,
		func() {
			stats := mm.GetCurrentStats()
			mm.LogMemoryStats(stats)
			mm.checkAlerts(stats)
		},
		mm.logger,
		nil,
	)

	if err != nil {
		mm.logger.Errorf("Failed to create memory monitoring task: %v", err)
		return
	}

	// Only start if task was newly created or we're sure it's not already running
	task.Start()
	globalMonitoringStarted = true

	if !isTestMode() {
		mm.logger.Info("Started global memory monitoring with %v interval", interval)
	}
}

// checkAlerts checks for memory-related alerts
func (mm *MemoryMonitor) checkAlerts(stats *MemoryStats) {
	heapMB := float64(stats.HeapAllocBytes) / (1024 * 1024)

	// Heap size alert
	if heapMB > float64(mm.alertThresholds.HeapSizeMB) {
		mm.logger.Error("Memory Alert: Heap size %.1fMB exceeds threshold %dMB",
			heapMB, mm.alertThresholds.HeapSizeMB)
	}

	// GC frequency alert
	if stats.GCFrequency > mm.alertThresholds.GCFrequency {
		mm.logger.Error("Memory Alert: GC frequency %.1f/min exceeds threshold %.1f/min",
			stats.GCFrequency, mm.alertThresholds.GCFrequency)
	}

	// Critical memory pressure
	if stats.MemoryPressure >= MemoryPressureHigh {
		mm.logger.Error("Memory Alert: %s memory pressure detected", stats.MemoryPressure.String())
	}
}

// TriggerGC forces garbage collection and logs the impact
func (mm *MemoryMonitor) TriggerGC() {
	before := mm.GetCurrentStats()

	runtime.GC()
	runtime.GC() // Run twice to ensure full collection

	after := mm.GetCurrentStats()

	// #nosec G115 -- heap allocation bytes fit in int64 for practical purposes
	freedBytes := int64(before.HeapAllocBytes) - int64(after.HeapAllocBytes)
	freedMB := float64(freedBytes) / (1024 * 1024)

	mm.logger.Info("Manual GC completed - Freed: %.1fMB, Before: %.1fMB, After: %.1fMB",
		freedMB,
		float64(before.HeapAllocBytes)/(1024*1024),
		float64(after.HeapAllocBytes)/(1024*1024))
}

// GetMemoryPressure returns the current memory pressure level
func (mm *MemoryMonitor) GetMemoryPressure() MemoryPressureLevel {
	mm.mu.RLock()
	defer mm.mu.RUnlock()

	if mm.lastStats != nil {
		return mm.lastStats.MemoryPressure
	}
	return MemoryPressureNone
}

// StopMonitoring stops the global memory monitoring if it's running
func (mm *MemoryMonitor) StopMonitoring() {
	globalMonitoringMutex.Lock()
	defer globalMonitoringMutex.Unlock()

	if !globalMonitoringStarted {
		return
	}

	registry := GetGlobalTaskRegistry()
	if task, exists := registry.GetTask("memory-monitor"); exists {
		task.Stop()
		globalMonitoringStarted = false
		if !isTestMode() {
			mm.logger.Info("Stopped global memory monitoring")
		}
	} else {
		mm.logger.Errorf("Failed to find memory monitoring task to stop")
	}
}

// IsMonitoringActive returns true if global memory monitoring is currently active
func (mm *MemoryMonitor) IsMonitoringActive() bool {
	globalMonitoringMutex.Lock()
	defer globalMonitoringMutex.Unlock()
	return globalMonitoringStarted
}

// Global memory monitor instance
var (
	globalMemoryMonitor     *MemoryMonitor
	globalMemoryMonitorOnce sync.Once
)

// GetGlobalMemoryMonitor returns the singleton memory monitor
func GetGlobalMemoryMonitor() *MemoryMonitor {
	globalMemoryMonitorOnce.Do(func() {
		logger := GetSingletonNoOpLogger()
		thresholds := DefaultMemoryAlertThresholds()
		globalMemoryMonitor = NewMemoryMonitor(logger, thresholds)
	})
	return globalMemoryMonitor
}

// ResetGlobalMemoryMonitor resets the global memory monitor for testing
// This should only be used in tests to prevent state pollution between tests
func ResetGlobalMemoryMonitor() {
	globalMonitoringMutex.Lock()
	defer globalMonitoringMutex.Unlock()

	if globalMemoryMonitor != nil {
		// Stop monitoring if it's active
		if globalMonitoringStarted {
			registry := GetGlobalTaskRegistry()
			if task, exists := registry.GetTask("memory-monitor"); exists {
				task.Stop()
			}
		}
		globalMemoryMonitor = nil
	}

	// Reset the singleton state
	globalMemoryMonitorOnce = sync.Once{}
	globalMonitoringStarted = false
}