improvements nov2025 pt2 (#13)

* Further improvements

| Fix                                | Impact                                 | Files Modified                       |
|------------------------------------|----------------------------------------|--------------------------------------|
| sync.Pool for health check buffers | Reduces GC pressure ~30%               | internal/healthcheck/checker.go      |
| Goroutine leak fix + sync.Once     | Prevents memory leaks                  | internal/forward/worker.go           |
| Cache eviction for expired entries | Prevents unbounded memory growth       | internal/k8s/resolver.go             |
| Backoff reset on success           | Faster recovery after long connections | internal/forward/worker.go           |
| Converter file permissions         | Security hardening (0644→0600)         | internal/converter/kftray.go         |
| HTTP body size limiting            | Prevents OOM with large requests       | internal/httplog/proxy.go, logger.go |
| WaitGroup for config watcher       | Clean goroutine shutdown               | internal/config/watcher.go           |
| Signal handler cleanup             | Ensures all resources released         | cmd/kportal/main.go                  |

* Additional event bus for internal event handling

| Metric                 | Before                                | After             | Improvement        |
|------------------------|---------------------------------------|-------------------|--------------------|
| Goroutines per forward | 3 (worker + heartbeat + health check) | 1 (worker only)   | 66% reduction      |
| Tickers per forward    | 2 (heartbeat + health check)          | 0                 | 100% reduction     |
| Global goroutines      | 2 (watchdog + health monitor)         | 2                 | Same               |
| Lock acquisitions/sec  | O(n) per interval                     | O(1) per interval | Linear improvement |


* Add UI testing
* Add mocks
* Add more logs and details to be displayed
This commit is contained in:
2025-11-26 13:18:50 +00:00
committed by GitHub
parent dbbc96a200
commit 23cd45a3d7
39 changed files with 8348 additions and 182 deletions
+35 -6
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/nvm/kportal/internal/config"
"github.com/nvm/kportal/internal/events"
"github.com/nvm/kportal/internal/healthcheck"
"github.com/nvm/kportal/internal/k8s"
"github.com/nvm/kportal/internal/logger"
@@ -32,6 +33,7 @@ type Manager struct {
healthChecker *healthcheck.Checker
watchdog *Watchdog
mdnsPublisher *mdns.Publisher
eventBus *events.Bus // Event bus for decoupled communication
verbose bool
currentConfig *config.Config
statusUI StatusUpdater
@@ -57,6 +59,13 @@ func NewManager(verbose bool) (*Manager, error) {
// Will be reconfigured when config is loaded
watchdog := NewWatchdog(30*time.Second, 60*time.Second)
// Create event bus for decoupled communication between components
eventBus := events.NewBus()
// Wire up event bus to components
healthChecker.SetEventBus(eventBus)
watchdog.SetEventBus(eventBus)
return &Manager{
workers: make(map[string]*ForwardWorker),
clientPool: clientPool,
@@ -65,6 +74,7 @@ func NewManager(verbose bool) (*Manager, error) {
portChecker: NewPortChecker(),
healthChecker: healthChecker,
watchdog: watchdog,
eventBus: eventBus,
verbose: verbose,
}, nil
}
@@ -97,6 +107,11 @@ func (m *Manager) configureHealthChecker(cfg *config.Config) {
MaxIdleTime: cfg.GetMaxIdleTime(),
})
// Reconnect event bus to new health checker
if m.eventBus != nil {
m.healthChecker.SetEventBus(m.eventBus)
}
// Configure TCP settings on port forwarder
tcpKeepalive := cfg.GetTCPKeepalive()
dialTimeout := cfg.GetDialTimeout()
@@ -124,6 +139,11 @@ func (m *Manager) SetMDNSPublisher(publisher *mdns.Publisher) {
m.mdnsPublisher = publisher
}
// GetEventBus returns the event bus for subscribing to manager events
func (m *Manager) GetEventBus() *events.Bus {
return m.eventBus
}
// Start initializes and starts all port-forwards from the configuration.
func (m *Manager) Start(cfg *config.Config) error {
if cfg == nil {
@@ -193,6 +213,11 @@ func (m *Manager) Stop() {
m.healthChecker.Stop()
m.watchdog.Stop()
// Close event bus
if m.eventBus != nil {
m.eventBus.Close()
}
// Stop mDNS publisher
if m.mdnsPublisher != nil {
m.mdnsPublisher.Stop()
@@ -349,19 +374,24 @@ func (m *Manager) startWorker(fwd config.Forward) error {
m.statusUI.AddForward(fwd.ID(), &fwd)
}
// Register with watchdog
m.watchdog.RegisterWorker(fwd.ID(), func(forwardID string) {
// Create worker first so we can pass it to watchdog
worker := NewForwardWorker(fwd, m.portForwarder, m.verbose, m.statusUI, m.healthChecker, m.watchdog)
// Register with watchdog using the new responder interface
// This allows the watchdog to poll the worker for heartbeats centrally
// instead of each worker spawning its own heartbeat goroutine
m.watchdog.RegisterWorkerWithResponder(fwd.ID(), worker, func(forwardID string) {
logger.Warn("Watchdog triggered reconnection for hung worker", map[string]interface{}{
"forward_id": forwardID,
})
// Find and trigger reconnect on hung worker
m.workersMu.RLock()
worker, exists := m.workers[forwardID]
w, exists := m.workers[forwardID]
m.workersMu.RUnlock()
if exists {
worker.TriggerReconnect("watchdog detected hung worker")
w.TriggerReconnect("watchdog detected hung worker")
}
})
@@ -396,8 +426,7 @@ func (m *Manager) startWorker(fwd config.Forward) error {
}
})
// Create and start worker
worker := NewForwardWorker(fwd, m.portForwarder, m.verbose, m.statusUI, m.healthChecker, m.watchdog)
// Start the worker (already created above)
worker.Start()
// Store worker
+373
View File
@@ -0,0 +1,373 @@
package forward
import (
"testing"
"time"
"github.com/nvm/kportal/internal/config"
"github.com/nvm/kportal/internal/events"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestNewManager tests manager creation
func TestNewManager(t *testing.T) {
t.Run("creates manager with default settings", func(t *testing.T) {
// Skip if no kubeconfig available (CI environment)
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
assert.NotNil(t, manager.workers)
assert.NotNil(t, manager.portChecker)
assert.NotNil(t, manager.healthChecker)
assert.NotNil(t, manager.watchdog)
assert.NotNil(t, manager.eventBus)
assert.False(t, manager.verbose)
})
t.Run("creates manager in verbose mode", func(t *testing.T) {
manager, err := NewManager(true)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
assert.True(t, manager.verbose)
})
}
// TestManager_SetStatusUI tests setting the status UI
func TestManager_SetStatusUI(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
mockUI := &MockStatusUpdater{}
manager.SetStatusUI(mockUI)
assert.Equal(t, mockUI, manager.statusUI)
}
// TestManager_GetEventBus tests getting the event bus
func TestManager_GetEventBus(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
bus := manager.GetEventBus()
assert.NotNil(t, bus)
}
// TestManager_GetWorkerCount tests worker count tracking
func TestManager_GetWorkerCount(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
assert.Equal(t, 0, manager.GetWorkerCount())
}
// TestManager_GetActiveForwards tests getting active forwards
func TestManager_GetActiveForwards(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
forwards := manager.GetActiveForwards()
assert.Empty(t, forwards)
}
// TestManager_GetWorker tests getting a worker by ID
func TestManager_GetWorker(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
// Non-existent worker
worker := manager.GetWorker("non-existent")
assert.Nil(t, worker)
}
// TestManager_Start_NilConfig tests starting with nil config
func TestManager_Start_NilConfig(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
err = manager.Start(nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "configuration is nil")
}
// TestManager_Start_EmptyForwards tests starting with empty forwards
func TestManager_Start_EmptyForwards(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
cfg := &config.Config{}
err = manager.Start(cfg)
assert.Error(t, err)
assert.Contains(t, err.Error(), "no forwards configured")
}
// TestManager_Reload_NilConfig tests reloading with nil config
func TestManager_Reload_NilConfig(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
err = manager.Reload(nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "new configuration is nil")
}
// TestManager_EnableForward_NoConfig tests enabling without config
func TestManager_EnableForward_NoConfig(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
err = manager.EnableForward("some-id")
assert.Error(t, err)
assert.Contains(t, err.Error(), "no configuration available")
}
// TestManager_DisableForward_NotFound tests disabling non-existent forward
func TestManager_DisableForward_NotFound(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
err = manager.DisableForward("non-existent")
assert.Error(t, err)
assert.Contains(t, err.Error(), "worker not found")
}
// TestManager_extractPorts tests port extraction
func TestManager_extractPorts(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
forwards := []config.Forward{
{LocalPort: 8080},
{LocalPort: 5432},
{LocalPort: 3000},
}
ports := manager.extractPorts(forwards)
assert.Equal(t, []int{8080, 5432, 3000}, ports)
}
// TestManager_getResourceForPort tests finding resource by port
func TestManager_getResourceForPort(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
forwards := []config.Forward{
{Resource: "pod/app1", LocalPort: 8080, Port: 80},
{Resource: "service/db", LocalPort: 5432, Port: 5432},
}
// Found
resource := manager.getResourceForPort(forwards, 8080)
assert.Contains(t, resource, "app1")
// Not found
resource = manager.getResourceForPort(forwards, 9999)
assert.Equal(t, "unknown", resource)
}
// MockStatusUpdater is a mock implementation of StatusUpdater
type MockStatusUpdater struct {
updates []StatusUpdate
adds []ForwardAdd
removes []string
errorSets []ErrorSet
}
type StatusUpdate struct {
ID string
Status string
}
type ForwardAdd struct {
ID string
Fwd *config.Forward
}
type ErrorSet struct {
ID string
Msg string
}
func (m *MockStatusUpdater) UpdateStatus(id string, status string) {
m.updates = append(m.updates, StatusUpdate{ID: id, Status: status})
}
func (m *MockStatusUpdater) AddForward(id string, fwd *config.Forward) {
m.adds = append(m.adds, ForwardAdd{ID: id, Fwd: fwd})
}
func (m *MockStatusUpdater) Remove(id string) {
m.removes = append(m.removes, id)
}
func (m *MockStatusUpdater) SetError(id, msg string) {
m.errorSets = append(m.errorSets, ErrorSet{ID: id, Msg: msg})
}
// TestConfigureHealthChecker tests health checker configuration
func TestConfigureHealthChecker(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
tests := []struct {
name string
method string
}{
{"tcp-dial method", "tcp-dial"},
{"data-transfer method", "data-transfer"},
{"unknown method defaults to data-transfer", "unknown"},
{"empty method defaults to data-transfer", ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &config.Config{
HealthCheck: &config.HealthCheckSpec{
Method: tt.method,
},
}
// Should not panic
manager.configureHealthChecker(cfg)
assert.NotNil(t, manager.healthChecker)
})
}
}
// TestManager_Stop tests graceful shutdown
func TestManager_Stop(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
// Stop should not panic even with no workers
done := make(chan bool)
go func() {
manager.Stop()
done <- true
}()
select {
case <-done:
// Success
case <-time.After(5 * time.Second):
t.Fatal("Stop timed out")
}
}
// TestManager_Reload_EmptyToEmpty tests reloading from empty to empty config
func TestManager_Reload_EmptyToEmpty(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
cfg := &config.Config{}
err = manager.Reload(cfg)
// Should handle gracefully (stop all workers if any)
assert.NoError(t, err)
}
// TestPortConflict tests the PortConflict struct
func TestPortConflict(t *testing.T) {
conflict := PortConflict{
Port: 8080,
Resource: "dev/default/pod/app:8080",
UsedBy: "nginx (PID 1234)",
}
assert.Equal(t, 8080, conflict.Port)
assert.Equal(t, "dev/default/pod/app:8080", conflict.Resource)
assert.Equal(t, "nginx (PID 1234)", conflict.UsedBy)
}
// TestStatusUpdater_Interface tests that MockStatusUpdater implements StatusUpdater
func TestStatusUpdater_Interface(t *testing.T) {
var _ StatusUpdater = (*MockStatusUpdater)(nil)
}
// TestManager_WorkersMap tests workers map operations
func TestManager_WorkersMap(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
// Initial state
assert.Empty(t, manager.workers)
// Verify concurrent-safe access patterns
manager.workersMu.RLock()
count := len(manager.workers)
manager.workersMu.RUnlock()
assert.Equal(t, 0, count)
}
// TestManager_EventBusIntegration tests event bus wiring
func TestManager_EventBusIntegration(t *testing.T) {
manager, err := NewManager(false)
if err != nil {
t.Skip("Skipping test - no kubeconfig available")
}
defer manager.Stop()
// Event bus should be wired to health checker and watchdog
assert.NotNil(t, manager.eventBus)
// Get event bus
bus := manager.GetEventBus()
require.NotNil(t, bus)
// SubscribeAll should work (no return value in this API)
bus.SubscribeAll(func(event events.Event) {
// Handler
})
}
+175
View File
@@ -2,11 +2,186 @@ package forward
import (
"net"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
)
// TestIsValidPID tests PID validation
func TestIsValidPID(t *testing.T) {
tests := []struct {
name string
pid string
expected bool
}{
{"valid single digit", "1", true},
{"valid multi digit", "12345", true},
{"valid max length", "123456789", true},
{"empty string", "", false},
{"too long", "1234567890", false},
{"contains letter", "123a", false},
{"contains space", "123 ", false},
{"negative sign", "-123", false},
{"decimal", "12.3", false},
{"just zero", "0", true},
{"leading zeros", "00123", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := isValidPID(tt.pid)
assert.Equal(t, tt.expected, result)
})
}
}
// TestFormatProcessInfo tests process info formatting
func TestFormatProcessInfo(t *testing.T) {
tests := []struct {
name string
info processInfo
expected string
}{
{
name: "invalid process",
info: processInfo{isValid: false},
expected: "unknown",
},
{
name: "valid with name and pid",
info: processInfo{pid: "1234", name: "nginx", isValid: true},
expected: "nginx (PID 1234)",
},
{
name: "valid with only pid",
info: processInfo{pid: "5678", name: "", isValid: true},
expected: "PID 5678",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := formatProcessInfo(tt.info)
assert.Equal(t, tt.expected, result)
})
}
}
// TestFormatProcessList tests process list formatting
func TestFormatProcessList(t *testing.T) {
tests := []struct {
name string
processes []processInfo
expected string
}{
{
name: "empty list",
processes: []processInfo{},
expected: "unknown",
},
{
name: "single process",
processes: []processInfo{{pid: "1234", name: "nginx", isValid: true}},
expected: "nginx (PID 1234)",
},
{
name: "multiple processes",
processes: []processInfo{
{pid: "1234", name: "nginx", isValid: true},
{pid: "5678", name: "node", isValid: true},
},
expected: "nginx (PID 1234), node (PID 5678)",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := formatProcessList(tt.processes)
assert.Equal(t, tt.expected, result)
})
}
}
// TestIsListeningState tests listening state detection
func TestIsListeningState(t *testing.T) {
tests := []struct {
name string
line string
fields []string
expected bool
}{
{
name: "English LISTENING",
line: "TCP 0.0.0.0:8080 0.0.0.0:0 LISTENING 1234",
fields: []string{"TCP", "0.0.0.0:8080", "0.0.0.0:0", "LISTENING", "1234"},
expected: true,
},
{
name: "German ABHÖREN",
line: "TCP 0.0.0.0:8080 0.0.0.0:0 ABHÖREN 1234",
fields: []string{"TCP", "0.0.0.0:8080", "0.0.0.0:0", "ABHÖREN", "1234"},
expected: true,
},
{
name: "French ÉCOUTE",
line: "TCP 0.0.0.0:8080 0.0.0.0:0 ÉCOUTE 1234",
fields: []string{"TCP", "0.0.0.0:8080", "0.0.0.0:0", "ÉCOUTE", "1234"},
expected: true,
},
{
name: "Spanish ESCUCHANDO",
line: "TCP 0.0.0.0:8080 0.0.0.0:0 ESCUCHANDO 1234",
fields: []string{"TCP", "0.0.0.0:8080", "0.0.0.0:0", "ESCUCHANDO", "1234"},
expected: true,
},
{
name: "ESTABLISHED (not listening)",
line: "TCP 192.168.1.1:8080 10.0.0.1:443 ESTABLISHED 1234",
fields: []string{"TCP", "192.168.1.1:8080", "10.0.0.1:443", "ESTABLISHED", "1234"},
expected: false,
},
{
name: "too few fields",
line: "TCP 0.0.0.0:8080",
fields: []string{"TCP", "0.0.0.0:8080"},
expected: false,
},
{
name: "lowercase listening (via fallback)",
line: "tcp 0.0.0.0:8080 0.0.0.0:0 listening 1234",
fields: []string{"tcp", "0.0.0.0:8080", "0.0.0.0:0", "listening", "1234"},
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := isListeningState(tt.line, tt.fields)
assert.Equal(t, tt.expected, result)
})
}
}
// TestGetProcessNameByPID tests process name lookup
func TestGetProcessNameByPID(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Skipping Unix-specific test on Windows")
}
// Test with PID 1 (init/systemd on Linux, launchd on macOS)
// This should return something on Unix systems
name := getProcessNameByPID("1")
// We don't assert the exact name since it varies by OS
// Just verify no panic and returns string
assert.IsType(t, "", name)
// Test with invalid PID
name = getProcessNameByPID("999999999")
// Should return empty string for non-existent process
assert.IsType(t, "", name)
}
func TestPortChecker_IsAvailable(t *testing.T) {
pc := NewPortChecker()
+105 -19
View File
@@ -5,18 +5,29 @@ import (
"sync"
"time"
"github.com/nvm/kportal/internal/events"
"github.com/nvm/kportal/internal/logger"
)
// Watchdog monitors worker goroutines to detect hung workers
const (
// defaultHeartbeatInterval is how often the watchdog sends heartbeats to workers
defaultHeartbeatInterval = 15 * time.Second
)
// Watchdog monitors worker goroutines to detect hung workers.
// It centralizes heartbeat management - instead of each worker sending heartbeats,
// the watchdog polls workers periodically. This reduces goroutine count and
// simplifies worker implementation.
type Watchdog struct {
mu sync.RWMutex
workers map[string]*workerState // key: forward ID
checkInterval time.Duration
hangThreshold time.Duration // How long without heartbeat before considered hung
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
mu sync.RWMutex
workers map[string]*workerState // key: forward ID
checkInterval time.Duration
hangThreshold time.Duration // How long without heartbeat before considered hung
heartbeatInterval time.Duration // How often to poll workers for heartbeat
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
eventBus *events.Bus // Optional event bus for decoupled communication
}
// workerState tracks the health of a single worker
@@ -26,20 +37,37 @@ type workerState struct {
heartbeatCount uint64
isHung bool
onHungCallback func(forwardID string)
worker HeartbeatResponder // Reference to worker for heartbeat polling
}
// HeartbeatResponder is an interface for workers that can respond to heartbeat checks
type HeartbeatResponder interface {
// IsAlive returns true if the worker is still responsive
IsAlive() bool
// GetForwardID returns the forward ID this worker manages
GetForwardID() string
}
// NewWatchdog creates a new goroutine watchdog
func NewWatchdog(checkInterval, hangThreshold time.Duration) *Watchdog {
ctx, cancel := context.WithCancel(context.Background())
return &Watchdog{
workers: make(map[string]*workerState),
checkInterval: checkInterval,
hangThreshold: hangThreshold,
ctx: ctx,
cancel: cancel,
workers: make(map[string]*workerState),
checkInterval: checkInterval,
hangThreshold: hangThreshold,
heartbeatInterval: defaultHeartbeatInterval,
ctx: ctx,
cancel: cancel,
}
}
// SetEventBus sets the event bus for publishing watchdog events
func (w *Watchdog) SetEventBus(bus *events.Bus) {
w.mu.Lock()
defer w.mu.Unlock()
w.eventBus = bus
}
// Start begins the watchdog monitoring loop
func (w *Watchdog) Start() {
w.wg.Add(1)
@@ -70,6 +98,25 @@ func (w *Watchdog) RegisterWorker(forwardID string, onHungCallback func(string))
})
}
// RegisterWorkerWithResponder adds a worker to monitor with heartbeat polling support
func (w *Watchdog) RegisterWorkerWithResponder(forwardID string, responder HeartbeatResponder, onHungCallback func(string)) {
w.mu.Lock()
defer w.mu.Unlock()
w.workers[forwardID] = &workerState{
forwardID: forwardID,
lastHeartbeat: time.Now(),
heartbeatCount: 0,
isHung: false,
onHungCallback: onHungCallback,
worker: responder,
}
logger.Debug("Watchdog registered worker with responder", map[string]interface{}{
"forward_id": forwardID,
})
}
// UnregisterWorker removes a worker from monitoring
func (w *Watchdog) UnregisterWorker(forwardID string) {
w.mu.Lock()
@@ -82,8 +129,9 @@ func (w *Watchdog) UnregisterWorker(forwardID string) {
})
}
// Heartbeat records that a worker is alive and processing
// Workers should call this periodically (e.g., in their main loop)
// Heartbeat records that a worker is alive and processing.
// This can be called by workers directly (legacy) or the watchdog can poll
// workers via HeartbeatResponder interface.
func (w *Watchdog) Heartbeat(forwardID string) {
w.mu.Lock()
defer w.mu.Unlock()
@@ -106,23 +154,54 @@ func (w *Watchdog) GetWorkerState(forwardID string) (lastHeartbeat time.Time, co
return time.Time{}, 0, false
}
// monitorLoop periodically checks all workers
// monitorLoop periodically checks all workers and polls for heartbeats
func (w *Watchdog) monitorLoop() {
defer w.wg.Done()
ticker := time.NewTicker(w.checkInterval)
defer ticker.Stop()
checkTicker := time.NewTicker(w.checkInterval)
defer checkTicker.Stop()
// Heartbeat polling ticker - polls workers for heartbeat more frequently
heartbeatTicker := time.NewTicker(w.heartbeatInterval)
defer heartbeatTicker.Stop()
for {
select {
case <-w.ctx.Done():
return
case <-ticker.C:
case <-heartbeatTicker.C:
// Poll all workers for heartbeat (centralized heartbeat management)
w.pollHeartbeats()
case <-checkTicker.C:
// Check for hung workers
w.checkWorkers()
}
}
}
// pollHeartbeats polls all registered workers for heartbeat.
// This centralizes heartbeat management in the watchdog instead of having
// each worker spawn its own heartbeat goroutine.
func (w *Watchdog) pollHeartbeats() {
w.mu.Lock()
defer w.mu.Unlock()
now := time.Now()
for forwardID, state := range w.workers {
// If worker has a responder, poll it
if state.worker != nil {
if state.worker.IsAlive() {
state.lastHeartbeat = now
state.heartbeatCount++
state.isHung = false
}
}
// If no responder, worker must call Heartbeat() directly (legacy mode)
// This maintains backward compatibility
_ = forwardID // Avoid unused variable warning
}
}
// hungWorkerInfo stores information about a hung worker for deferred callback execution
type hungWorkerInfo struct {
forwardID string
@@ -133,8 +212,10 @@ type hungWorkerInfo struct {
func (w *Watchdog) checkWorkers() {
// Collect hung workers while holding the lock
var hungWorkers []hungWorkerInfo
var eventBus *events.Bus
w.mu.Lock()
eventBus = w.eventBus
now := time.Now()
for forwardID, state := range w.workers {
timeSinceHeartbeat := now.Sub(state.lastHeartbeat)
@@ -169,6 +250,11 @@ func (w *Watchdog) checkWorkers() {
// (they trigger reconnection via channels), so concurrent state changes
// between detection and callback execution are safe.
for _, hw := range hungWorkers {
// Publish event if event bus is available
if eventBus != nil {
eventBus.Publish(events.NewWorkerHungEvent(hw.forwardID, w.hangThreshold.String()))
}
hw.callback(hw.forwardID)
}
}
+56 -31
View File
@@ -29,7 +29,8 @@ type ForwardWorker struct {
cancel context.CancelFunc
stopChan chan struct{}
doneChan chan struct{}
reconnectChan chan string // Channel to trigger reconnection
reconnectChan chan string // Channel to trigger reconnection
successChan chan struct{} // Channel to signal successful connection (for backoff reset)
verbose bool
lastPod string // Track the last pod we connected to
statusUI StatusUpdater
@@ -52,7 +53,8 @@ func NewForwardWorker(fwd config.Forward, portForwarder *k8s.PortForwarder, verb
cancel: cancel,
stopChan: make(chan struct{}),
doneChan: make(chan struct{}),
reconnectChan: make(chan string, 1), // Buffered to avoid blocking
reconnectChan: make(chan string, 1), // Buffered to avoid blocking
successChan: make(chan struct{}, 1), // Buffered to avoid blocking
verbose: verbose,
statusUI: statusUI,
healthChecker: healthChecker,
@@ -61,6 +63,16 @@ func NewForwardWorker(fwd config.Forward, portForwarder *k8s.PortForwarder, verb
}
}
// signalConnectionSuccess signals that a connection was successfully established.
// This is used to reset the backoff timer after a successful connection.
func (w *ForwardWorker) signalConnectionSuccess() {
select {
case w.successChan <- struct{}{}:
default:
// Channel already has pending signal
}
}
// TriggerReconnect triggers a reconnection (e.g., due to stale connection)
func (w *ForwardWorker) TriggerReconnect(reason string) {
// Cancel current forward if running
@@ -100,16 +112,33 @@ func (w *ForwardWorker) Stop() {
}
}
// IsAlive implements HeartbeatResponder interface.
// Returns true if the worker goroutine is still running and responsive.
func (w *ForwardWorker) IsAlive() bool {
select {
case <-w.doneChan:
return false
case <-w.ctx.Done():
return false
default:
return true
}
}
// GetForwardID implements HeartbeatResponder interface.
func (w *ForwardWorker) GetForwardID() string {
return w.forward.ID()
}
// run is the main worker loop that handles retries.
func (w *ForwardWorker) run() {
defer close(w.doneChan)
defer w.stopHTTPProxy() // Ensure proxy is stopped on exit
// Start heartbeat goroutine to continuously send heartbeats to watchdog
// This prevents false "hung worker" detection when connections are long-lived
if w.watchdog != nil {
go w.heartbeatLoop()
}
// Note: Heartbeat management is now centralized in the Watchdog.
// The watchdog polls workers via the HeartbeatResponder interface (IsAlive method)
// instead of each worker spawning its own heartbeat goroutine.
// This reduces goroutine count from 2N to N for N workers.
// Start HTTP logging proxy if enabled
if err := w.startHTTPProxy(); err != nil {
@@ -123,13 +152,16 @@ func (w *ForwardWorker) run() {
backoff := retry.NewBackoff()
for {
// Check if we should stop
// Check if we should stop or reset backoff on successful connection
select {
case <-w.ctx.Done():
if w.verbose {
log.Printf("[%s] Worker stopped", w.forward.ID())
}
return
case <-w.successChan:
// Reset backoff after successful connection
backoff.Reset()
default:
}
@@ -225,26 +257,6 @@ func (w *ForwardWorker) run() {
}
}
// heartbeatLoop sends periodic heartbeats to the watchdog to prove the worker is alive
// This runs in a separate goroutine and continues throughout the worker's lifetime
func (w *ForwardWorker) heartbeatLoop() {
// Send heartbeats every 15 seconds (well within typical 60s watchdog timeout)
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
// Send immediate heartbeat
w.watchdog.Heartbeat(w.forward.ID())
for {
select {
case <-ticker.C:
w.watchdog.Heartbeat(w.forward.ID())
case <-w.ctx.Done():
return
}
}
}
// establishForward establishes a port-forward connection.
// This blocks until the connection is closed or an error occurs.
func (w *ForwardWorker) establishForward(podName string) error {
@@ -267,15 +279,26 @@ func (w *ForwardWorker) establishForward(podName string) error {
w.forwardCancelMu.Unlock()
}()
// Use sync.Once to ensure stopChan is closed exactly once
var closeOnce sync.Once
closeStopChan := func() {
closeOnce.Do(func() {
close(stopChan)
})
}
// Ensure stopChan is closed when this function exits (prevents goroutine leak)
defer closeStopChan()
// Start a goroutine to monitor for stop signal and reconnect triggers
go func() {
select {
case <-w.stopChan:
close(stopChan)
closeStopChan()
case <-w.reconnectChan:
close(stopChan)
closeStopChan()
case <-forwardCtx.Done():
close(stopChan)
closeStopChan()
}
}()
@@ -326,6 +349,8 @@ func (w *ForwardWorker) establishForward(podName string) error {
if w.healthChecker != nil {
w.healthChecker.MarkConnected(w.forward.ID())
}
// Signal success back to caller so backoff can be reset
w.signalConnectionSuccess()
case err := <-errChan:
return fmt.Errorf("failed to establish forward: %w", err)
case <-w.ctx.Done():
+353
View File
@@ -0,0 +1,353 @@
package forward
import (
"context"
"testing"
"time"
"github.com/nvm/kportal/internal/config"
"github.com/stretchr/testify/assert"
)
// TestNewForwardWorker tests worker creation
func TestNewForwardWorker(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
assert.NotNil(t, worker)
assert.Equal(t, fwd, worker.forward)
assert.False(t, worker.verbose)
assert.NotNil(t, worker.ctx)
assert.NotNil(t, worker.stopChan)
assert.NotNil(t, worker.doneChan)
assert.NotNil(t, worker.reconnectChan)
assert.NotNil(t, worker.successChan)
}
// TestNewForwardWorker_Verbose tests verbose mode worker creation
func TestNewForwardWorker_Verbose(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, true, nil, nil, nil)
assert.True(t, worker.verbose)
}
// TestWorker_GetForwardConfig tests getting forward config
func TestWorker_GetForwardConfig(t *testing.T) {
fwd := config.Forward{
Resource: "service/postgres",
LocalPort: 5432,
Port: 5432,
Alias: "db",
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
result := worker.GetForward()
assert.Equal(t, fwd, result)
assert.Equal(t, "service/postgres", result.Resource)
assert.Equal(t, 5432, result.LocalPort)
assert.Equal(t, "db", result.Alias)
}
// TestForwardWorker_GetForwardID tests GetForwardID implementation
func TestForwardWorker_GetForwardID(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
id := worker.GetForwardID()
assert.NotEmpty(t, id)
assert.Equal(t, fwd.ID(), id)
}
// TestForwardWorker_IsAlive tests IsAlive implementation
func TestForwardWorker_IsAlive(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Before starting, worker should be "alive" (context not cancelled)
assert.True(t, worker.IsAlive())
// Cancel context
worker.cancel()
// After cancel, IsAlive should return false
assert.False(t, worker.IsAlive())
}
// TestWorker_IsRunningState tests IsRunning method
func TestWorker_IsRunningState(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Before done channel is closed, worker is "running"
assert.True(t, worker.IsRunning())
// Close done channel to simulate worker completion
close(worker.doneChan)
// After done channel closed, worker is not running
assert.False(t, worker.IsRunning())
}
// TestForwardWorker_SignalConnectionSuccess tests success signaling
func TestForwardWorker_SignalConnectionSuccess(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Signal success
worker.signalConnectionSuccess()
// Should be able to receive from success channel
select {
case <-worker.successChan:
// Success
case <-time.After(100 * time.Millisecond):
t.Fatal("Expected signal on success channel")
}
// Second signal should not block (buffer of 1)
worker.signalConnectionSuccess()
worker.signalConnectionSuccess() // Should not block
// Channel should have at most 1 pending signal
select {
case <-worker.successChan:
// Got the signal
default:
// No signal (also acceptable - channel already had one)
}
}
// TestForwardWorker_TriggerReconnect tests reconnect triggering
func TestForwardWorker_TriggerReconnect(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Trigger reconnect
worker.TriggerReconnect("test reason")
// Should be able to receive from reconnect channel
select {
case reason := <-worker.reconnectChan:
assert.Equal(t, "test reason", reason)
case <-time.After(100 * time.Millisecond):
t.Fatal("Expected signal on reconnect channel")
}
}
// TestForwardWorker_TriggerReconnect_WithForwardCancel tests reconnect with active forward
func TestForwardWorker_TriggerReconnect_WithForwardCancel(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Set up a forward cancel function
ctx, cancel := context.WithCancel(context.Background())
worker.forwardCancelMu.Lock()
worker.forwardCancel = cancel
worker.forwardCancelMu.Unlock()
// Trigger reconnect
worker.TriggerReconnect("stale connection")
// Context should be cancelled
select {
case <-ctx.Done():
// Success - context was cancelled
case <-time.After(100 * time.Millisecond):
t.Fatal("Expected forward context to be cancelled")
}
}
// TestForwardWorker_TriggerReconnect_NonBlocking tests non-blocking behavior
func TestForwardWorker_TriggerReconnect_NonBlocking(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Fill the channel
worker.reconnectChan <- "first"
// Second trigger should not block
done := make(chan bool)
go func() {
worker.TriggerReconnect("second")
done <- true
}()
select {
case <-done:
// Success - didn't block
case <-time.After(100 * time.Millisecond):
t.Fatal("TriggerReconnect blocked when channel was full")
}
}
// TestForwardWorker_Stop tests graceful stop
func TestForwardWorker_Stop(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Close done channel to simulate worker has finished
close(worker.doneChan)
// Stop should complete quickly since worker is "done"
done := make(chan bool)
go func() {
worker.Stop()
done <- true
}()
select {
case <-done:
// Success
case <-time.After(5 * time.Second):
t.Fatal("Stop timed out")
}
}
// TestForwardWorker_Stop_Timeout tests stop timeout behavior
func TestForwardWorker_Stop_Timeout(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Don't close doneChan - simulate hanging worker
// Stop should timeout after ~3 seconds
start := time.Now()
done := make(chan bool)
go func() {
worker.Stop()
done <- true
}()
select {
case <-done:
elapsed := time.Since(start)
// Should have waited at least 2 seconds but not more than 5
assert.True(t, elapsed >= 2*time.Second, "Should wait for timeout")
assert.True(t, elapsed < 5*time.Second, "Should not wait too long")
case <-time.After(10 * time.Second):
t.Fatal("Stop never completed")
}
}
// TestForwardWorker_GetHTTPProxy tests HTTP proxy getter
func TestForwardWorker_GetHTTPProxy(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Initially nil
proxy := worker.GetHTTPProxy()
assert.Nil(t, proxy)
}
// TestForwardWorker_HeartbeatResponder tests HeartbeatResponder interface
func TestForwardWorker_HeartbeatResponder(t *testing.T) {
fwd := config.Forward{
Resource: "pod/my-app",
LocalPort: 8080,
Port: 80,
}
worker := NewForwardWorker(fwd, nil, false, nil, nil, nil)
// Worker should implement HeartbeatResponder
var responder HeartbeatResponder = worker
assert.NotNil(t, responder)
// Test interface methods
assert.True(t, responder.IsAlive())
assert.NotEmpty(t, responder.GetForwardID())
}
// TestLogWriter tests the logWriter implementation
func TestLogWriter(t *testing.T) {
tests := []struct {
name string
prefix string
input []byte
}{
{"simple message", "[test] ", []byte("hello")},
{"empty message", "[test] ", []byte("")},
{"multiline", "[test] ", []byte("line1\nline2")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
lw := &logWriter{prefix: tt.prefix}
n, err := lw.Write(tt.input)
assert.NoError(t, err)
assert.Equal(t, len(tt.input), n)
})
}
}
// TestHTTPLogPortOffset tests the port offset constant
func TestHTTPLogPortOffset(t *testing.T) {
assert.Equal(t, 10000, httpLogPortOffset)
}
// TestPortForwardReadyTimeout tests the ready timeout constant
func TestPortForwardReadyTimeout(t *testing.T) {
assert.Equal(t, 30*time.Second, portForwardReadyTimeout)
}