fix(health): release health checks when startup probe times out

StartHealthChecking's goroutine captures startupProbe=true and blocks on
readinessChan until the readiness probe completes. WaitForBackendReady only
closed readinessChan on SUCCESS; on timeout it returned an error without
closing it.

main proceeds anyway after a timeout ('Staring proxy anyway'), so the
health-check goroutine stayed blocked forever on readinessChan: periodic
checks never began and bhm.IsHealthy() stayed false permanently, so the
backend could never be marked healthy again even after it recovered.

Release readinessChan on the timeout path too (via sync.Once for safety),
so the health-check goroutine proceeds to periodic checking.

Test: TestHealthChecksStartAfterStartupTimeout (after WaitForBackendReady
times out, readinessChan is closed).
This commit is contained in:
2026-08-13 03:32:29 +01:00
parent 546ded1d9c
commit 9bb9db441f
2 changed files with 25 additions and 1 deletions
+7 -1
View File
@@ -28,6 +28,7 @@ type BackendHealthManager struct {
consecutiveFails atomic.Int32
isHealthy atomic.Bool
startupProbe bool
readyOnce sync.Once
}
// NewBackendHealthManager creates a new backend health manager
@@ -75,7 +76,7 @@ func (bhm *BackendHealthManager) WaitForBackendReady(timeout time.Duration) erro
"time_taken": time.Since(deadline.Add(-timeout)).String(),
},
})
close(bhm.readinessChan)
bhm.readyOnce.Do(func() { close(bhm.readinessChan) })
return nil
}
@@ -98,6 +99,11 @@ func (bhm *BackendHealthManager) WaitForBackendReady(timeout time.Duration) erro
}
}
// Startup probe timed out. Main continues and starts the proxy anyway, and
// StartHealthChecking's goroutine is blocked on readinessChan — so release
// it here too, otherwise periodic health checks never begin and the
// backend can never be marked healthy again even after it recovers.
bhm.readyOnce.Do(func() { close(bhm.readinessChan) })
return fmt.Errorf("GraphQL backend did not become ready within %v", timeout)
}
+18
View File
@@ -265,3 +265,21 @@ func (suite *ConnectionResilienceTestSuite) TestRetryLogic() {
func TestConnectionResilienceSuite(t *testing.T) {
suite.Run(t, new(ConnectionResilienceTestSuite))
}
// TestHealthChecksStartAfterStartupTimeout verifies that when the startup
// readiness probe times out, readinessChan is still released so the
// health-check goroutine (started by StartHealthChecking) proceeds and begins
// periodic checking — otherwise the backend could never be marked healthy
// again even after it recovers.
func (suite *ConnectionResilienceTestSuite) TestHealthChecksStartAfterStartupTimeout() {
bhm := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", cfg.Logger)
err := bhm.WaitForBackendReady(200 * time.Millisecond)
suite.Error(err, "expected timeout for unreachable backend")
select {
case <-bhm.readinessChan:
// Released -> periodic health checks will begin.
case <-time.After(1 * time.Second):
suite.Fail("readinessChan not closed after startup timeout; health checks would never begin")
}
}