feat(circuit): progressive backoff and configurable trip conditions

The circuit breaker ignored the trip-on-timeout, trip-on-4xx, and
trip-on-5xx flags and never cleared its counts, so the failure ratio
became lifetime-cumulative. The backoff-multiplier and max-backoff
settings did nothing. Circuit-open surfaced as HTTP 500.

Failure classification now honours the trip flags, the breaker uses a
rolling count window, and repeated trips extend the open state up to the
configured maximum. A helper maps circuit-open to HTTP 503. The request
body is copied once per request instead of once per retry. Defaults
reproduce the prior behaviour: the breaker is off by default and a
backoff multiplier of one keeps a constant open window.
This commit is contained in:
2026-08-14 11:10:37 +01:00
parent 6751346796
commit 10e51daab3
5 changed files with 1070 additions and 44 deletions
+60
View File
@@ -1,6 +1,7 @@
package main
import (
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
@@ -344,3 +345,62 @@ func (suite *Tests) Test_proxyRetryPolicyByStatusCode() {
suite.Greater(hits.Load(), int32(1), "5xx should trigger retries")
})
}
// Test_MultiAttemptRetryReusesRequestBody locks in the P3 fix: the outbound
// fasthttp.Request (headers + body) is built once, before the retry loop in
// performProxyRequestWithEnhancedRetries, and reused unmodified across every
// attempt -- instead of being re-copied from c.Request() up to 7x per
// proxied request (doProxyRequestWithTimeout previously ran
// c.Request().CopyTo(req) on every attempt). This verifies the hoist didn't
// corrupt, truncate, or drop the body on any attempt: every retried hit
// must still observe the exact original request body.
func (suite *Tests) Test_MultiAttemptRetryReusesRequestBody() {
originalCoalescing := cfg.RequestCoalescing.Enable
originalCB := cfg.CircuitBreaker.Enable
cfg.RequestCoalescing.Enable = false
cfg.CircuitBreaker.Enable = false
defer func() {
cfg.RequestCoalescing.Enable = originalCoalescing
cfg.CircuitBreaker.Enable = originalCB
}()
const wantBody = `{"query":"query { test }"}`
var hits atomic.Int32
var bodyMismatch atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := hits.Add(1)
body, _ := io.ReadAll(r.Body)
if string(body) != wantBody {
bodyMismatch.Store(true)
}
if n < 3 {
// Fail the first two attempts so the request is retried at
// least twice, exercising the reused request across attempts.
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte(`{"errors":[{"message":"error"}]}`))
return
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"data":{"ok":true}}`))
}))
defer server.Close()
cfg.Server.HostGraphQL = server.URL
cfg.Client.ClientTimeout = 5
cfg.Client.FastProxyClient = createFasthttpClient(cfg)
reqCtx := &fasthttp.RequestCtx{}
reqCtx.Request.SetRequestURI("/graphql")
reqCtx.Request.Header.SetMethod("POST")
reqCtx.Request.Header.Set("Content-Type", "application/json")
reqCtx.Request.SetBody([]byte(wantBody))
ctx := suite.app.AcquireCtx(reqCtx)
err := proxyTheRequest(ctx, cfg.Server.HostGraphQL)
suite.app.ReleaseCtx(ctx)
suite.Nil(err, "request should eventually succeed after retries")
suite.GreaterOrEqual(hits.Load(), int32(3), "must have retried at least twice before succeeding")
suite.False(bodyMismatch.Load(), "every retry attempt must see the exact original request body, not a stale or corrupted copy")
}