mirror of
https://github.com/lukaszraczylo/traefikoidc.git
synced 2026-10-19 04:47:48 +00:00
Fix dead CircuitBreaker.ResetTimeout config (round 39)
error_recovery.go: ResetTimeout was copied into the CircuitBreaker but never consulted by the state machine (used only for a metric), so the documented "how long to wait before fully closing the circuit" (reset_timeout, default 30s) had no effect: Open->HalfOpen used Timeout (60s), and a single half-open success closed the circuit immediately, losing the intended open -> half-open -> (reset cooldown) -> closed hysteresis and letting a stray success immediately reopen (bounce). Now the circuit remains half-open after entering it until resetTimeout has elapsed; only then can a successful request fully close it. Half-open requests are still allowed as probes. Default behavior is more conservative (up to Timeout+ResetTimeout before full close). Tests: updated TestCircuitBreakerHalfOpenTransition to the corrected semantics; added TestCircuitBreakerResetTimeoutEnforced (fails on the old dead-field code, where resetTimeout was ignored). Verified: full source-order suite, full -race -count=1 -shuffle=on ./... (all packages), golangci-lint run ./... (0 issues).
This commit is contained in:
+15
-1
@@ -191,12 +191,16 @@ type CircuitBreaker struct {
|
|||||||
maxFailures int
|
maxFailures int
|
||||||
// timeout is how long to wait before allowing requests in half-open state
|
// timeout is how long to wait before allowing requests in half-open state
|
||||||
timeout time.Duration
|
timeout time.Duration
|
||||||
// resetTimeout is how long to wait before transitioning from open to half-open
|
// resetTimeout is how long to stay in half-open (after the open timeout)
|
||||||
|
// before a successful response fully closes the circuit
|
||||||
resetTimeout time.Duration
|
resetTimeout time.Duration
|
||||||
// state tracks the current circuit breaker state
|
// state tracks the current circuit breaker state
|
||||||
state CircuitBreakerState
|
state CircuitBreakerState
|
||||||
// failures counts consecutive failures
|
// failures counts consecutive failures
|
||||||
failures int64
|
failures int64
|
||||||
|
// halfOpenSince is when the circuit entered half-open; used to enforce
|
||||||
|
// resetTimeout before a success can fully close it
|
||||||
|
halfOpenSince time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// CircuitBreakerConfig holds configuration parameters for circuit breakers.
|
// CircuitBreakerConfig holds configuration parameters for circuit breakers.
|
||||||
@@ -275,6 +279,7 @@ func (cb *CircuitBreaker) allowRequest() bool {
|
|||||||
case CircuitBreakerOpen:
|
case CircuitBreakerOpen:
|
||||||
if now.Sub(cb.lastFailureTime) > cb.timeout {
|
if now.Sub(cb.lastFailureTime) > cb.timeout {
|
||||||
cb.state = CircuitBreakerHalfOpen
|
cb.state = CircuitBreakerHalfOpen
|
||||||
|
cb.halfOpenSince = now
|
||||||
cb.logger.Infof("Circuit breaker transitioning to half-open state")
|
cb.logger.Infof("Circuit breaker transitioning to half-open state")
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -317,6 +322,15 @@ func (cb *CircuitBreaker) recordSuccess() {
|
|||||||
|
|
||||||
switch cb.state {
|
switch cb.state {
|
||||||
case CircuitBreakerHalfOpen:
|
case CircuitBreakerHalfOpen:
|
||||||
|
// Honor resetTimeout: do not fully close until the circuit has been
|
||||||
|
// in half-open for at least resetTimeout. This provides the
|
||||||
|
// documented open -> half-open -> (reset cooldown) -> closed
|
||||||
|
// hysteresis instead of closing on the very first half-open
|
||||||
|
// success, which would let a stray success immediately reopen
|
||||||
|
// (bounce). While in half-open requests are still allowed as probes.
|
||||||
|
if time.Since(cb.halfOpenSince) < cb.resetTimeout {
|
||||||
|
return
|
||||||
|
}
|
||||||
cb.failures = 0
|
cb.failures = 0
|
||||||
cb.state = CircuitBreakerClosed
|
cb.state = CircuitBreakerClosed
|
||||||
cb.LogInfo("Circuit breaker closed after successful request in half-open state")
|
cb.LogInfo("Circuit breaker closed after successful request in half-open state")
|
||||||
|
|||||||
+41
-9
@@ -89,18 +89,50 @@ func TestCircuitBreakerHalfOpenTransition(t *testing.T) {
|
|||||||
|
|
||||||
time.Sleep(150 * time.Millisecond)
|
time.Sleep(150 * time.Millisecond)
|
||||||
|
|
||||||
allowed := false
|
// First request after the open timeout probes half-open. A single success
|
||||||
_ = cb.Execute(func() error {
|
// does NOT immediately close: the circuit stays half-open until
|
||||||
allowed = true
|
// resetTimeout has elapsed (reset-cooldown hysteresis).
|
||||||
return nil
|
if err := cb.Execute(func() error { return nil }); err != nil {
|
||||||
})
|
t.Errorf("Request should be allowed in half-open state, got: %v", err)
|
||||||
|
}
|
||||||
if !allowed {
|
if cb.GetState() != CircuitBreakerHalfOpen {
|
||||||
t.Error("Request should be allowed in half-open state")
|
t.Errorf("Circuit should stay half-open during reset cooldown, got %v", circuitBreakerStateToString(cb.GetState()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// After resetTimeout elapses, a successful request fully closes it.
|
||||||
|
time.Sleep(60 * time.Millisecond)
|
||||||
|
if err := cb.Execute(func() error { return nil }); err != nil {
|
||||||
|
t.Errorf("Request should be allowed, got: %v", err)
|
||||||
|
}
|
||||||
if cb.GetState() != CircuitBreakerClosed {
|
if cb.GetState() != CircuitBreakerClosed {
|
||||||
t.Errorf("Circuit should be closed after successful half-open request, got %v", cb.GetState())
|
t.Errorf("Circuit should be closed after successful request following reset cooldown, got %v", circuitBreakerStateToString(cb.GetState()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCircuitBreakerResetTimeoutEnforced is a regression test for the dead
|
||||||
|
// resetTimeout config field: it must be consulted before a half-open success
|
||||||
|
// can fully close the circuit. This failed on the old code where resetTimeout
|
||||||
|
// was set but never read, so a single half-open success closed the circuit
|
||||||
|
// immediately regardless of the configured reset cooldown.
|
||||||
|
func TestCircuitBreakerResetTimeoutEnforced(t *testing.T) {
|
||||||
|
cb := NewCircuitBreaker(CircuitBreakerConfig{
|
||||||
|
MaxFailures: 1,
|
||||||
|
Timeout: 50 * time.Millisecond,
|
||||||
|
ResetTimeout: time.Minute,
|
||||||
|
}, nil)
|
||||||
|
|
||||||
|
_ = cb.Execute(func() error { return errors.New("fail") })
|
||||||
|
if cb.GetState() != CircuitBreakerOpen {
|
||||||
|
t.Fatal("circuit should be open after failure")
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(70 * time.Millisecond) // past the open timeout -> half-open
|
||||||
|
|
||||||
|
if err := cb.Execute(func() error { return nil }); err != nil {
|
||||||
|
t.Fatalf("request should be allowed in half-open state, got: %v", err)
|
||||||
|
}
|
||||||
|
if cb.GetState() != CircuitBreakerHalfOpen {
|
||||||
|
t.Fatalf("circuit must remain half-open until resetTimeout elapses, got %v", circuitBreakerStateToString(cb.GetState()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user