package traefikllmgateway import ( "context" "errors" "sync" "testing" "time" ) // --- validateBreakerConfig: defaults and validation table --- func TestValidateBreakerConfig_Table(t *testing.T) { tests := []struct { name string bc BreakerConfig wantThresh int wantOpen time.Duration wantMaxOpen time.Duration wantErr bool }{ { name: "zero value uses every default", bc: BreakerConfig{}, wantThresh: defaultBreakerFailureThreshold, wantOpen: defaultBreakerOpenDuration, wantMaxOpen: defaultBreakerMaxOpenDuration, }, { name: "explicit valid overrides used verbatim", bc: BreakerConfig{FailureThreshold: 5, OpenDuration: "10s", MaxOpenDuration: "2m"}, wantThresh: 5, wantOpen: 10 * time.Second, wantMaxOpen: 2 * time.Minute, }, { name: "threshold at the max boundary is accepted", bc: BreakerConfig{FailureThreshold: maxBreakerFailureThreshold}, wantThresh: maxBreakerFailureThreshold, wantOpen: defaultBreakerOpenDuration, wantMaxOpen: defaultBreakerMaxOpenDuration, }, { name: "threshold above max is an error", bc: BreakerConfig{FailureThreshold: maxBreakerFailureThreshold + 1}, wantErr: true, }, { name: "negative threshold is an error", bc: BreakerConfig{FailureThreshold: -1}, wantErr: true, }, { name: "unparseable openDuration is an error", bc: BreakerConfig{OpenDuration: "not-a-duration"}, wantErr: true, }, { name: "zero-valued openDuration string is an error", bc: BreakerConfig{OpenDuration: "0s"}, wantErr: true, }, { name: "negative openDuration is an error", bc: BreakerConfig{OpenDuration: "-1s"}, wantErr: true, }, { name: "unparseable maxOpenDuration is an error", bc: BreakerConfig{MaxOpenDuration: "not-a-duration"}, wantErr: true, }, { name: "maxOpenDuration below openDuration is an error", bc: BreakerConfig{OpenDuration: "5m", MaxOpenDuration: "1m"}, wantErr: true, }, { name: "maxOpenDuration equal to openDuration is accepted", bc: BreakerConfig{OpenDuration: "5m", MaxOpenDuration: "5m"}, wantThresh: defaultBreakerFailureThreshold, wantOpen: 5 * time.Minute, wantMaxOpen: 5 * time.Minute, }, { name: "maxOpenDuration at the ceiling is accepted", bc: BreakerConfig{MaxOpenDuration: "24h"}, wantThresh: defaultBreakerFailureThreshold, wantOpen: defaultBreakerOpenDuration, wantMaxOpen: 24 * time.Hour, }, { name: "maxOpenDuration above the ceiling is an error", bc: BreakerConfig{MaxOpenDuration: "25h"}, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := validateBreakerConfig(tt.bc) if (err != nil) != tt.wantErr { t.Fatalf("validateBreakerConfig(%+v): err = %v, wantErr %v", tt.bc, err, tt.wantErr) } if tt.wantErr { return } if got.failureThreshold != tt.wantThresh { t.Errorf("failureThreshold = %d, want %d", got.failureThreshold, tt.wantThresh) } if got.openBase != tt.wantOpen { t.Errorf("openBase = %v, want %v", got.openBase, tt.wantOpen) } if got.openMax != tt.wantMaxOpen { t.Errorf("openMax = %v, want %v", got.openMax, tt.wantMaxOpen) } }) } } // newHealthTestState builds a providerState with the breaker fields set // directly (no newModelRegistry round trip needed — the state machine // under test is pure providerState methods), and every other field at a // harmless zero/empty value. interval is set deliberately SHORT relative // to openBase in the tests below so a passing assertion can only be // explained by the breaker's own backoff window, never by the ordinary // per-interval throttle it lives alongside. func newHealthTestState(interval time.Duration, threshold int, openBase, openMax time.Duration) *providerState { return &providerState{ explicit: make(map[string]bool), discovered: make(map[string]bool), interval: interval, breakerThreshold: threshold, breakerOpenBase: openBase, breakerOpenMax: openMax, } } // --- breaker: opens after the configured threshold --- func TestProviderState_Breaker_OpensAfterThreshold(t *testing.T) { t.Parallel() st := newHealthTestState(time.Hour, 3, time.Minute, 10*time.Minute) now := time.Now() boom := errors.New("boom") for i := 0; i < 2; i++ { st.finishRefresh(now, nil, boom) if !st.discoveryHealthy() { t.Fatalf("after %d failure(s) (threshold 3): discoveryHealthy() = false, want true — must not open early", i+1) } } st.finishRefresh(now, nil, boom) // 3rd consecutive failure reaches the threshold if st.discoveryHealthy() { t.Fatal("after reaching the failure threshold: discoveryHealthy() = true, want false (breaker must be open)") } } // TestProviderState_Breaker_ASuccessResetsTheFailureCounter proves the // counter is CONSECUTIVE, not cumulative: two failures, a success, then two // more failures must not open a threshold-3 breaker, because the success // in between reset the count back to zero. func TestProviderState_Breaker_ASuccessResetsTheFailureCounter(t *testing.T) { t.Parallel() st := newHealthTestState(time.Hour, 3, time.Minute, 10*time.Minute) now := time.Now() boom := errors.New("boom") st.finishRefresh(now, nil, boom) st.finishRefresh(now, nil, boom) st.finishRefresh(now, []string{"m1"}, nil) // success: resets the streak st.finishRefresh(now, nil, boom) st.finishRefresh(now, nil, boom) if !st.discoveryHealthy() { t.Error("discoveryHealthy() = false, want true — the intervening success must have reset the consecutive-failure streak") } } // TestProviderState_Breaker_ErrRequestBuildFailed_CountsAsFailure is the // adversarial-review IMPORTANT-2 regression: an earlier version of // recordHealthLocked reused isTransient's RETRY-shaped classification // wholesale (retry.go) — which deliberately excludes errRequestBuildFailed, // since retrying a malformed request build within the same attempt would // never help. That is correct for retrying, but wrong for HEALTH: a // provider whose baseURL is malformed (nothing validates it at // construction) produces this exact error on every discovery cycle, // forever. The old code routed that straight to a "success" outcome, // resetting the breaker every time — it could never open despite // continuous, permanent failure. It must now count as a plain failure. func TestProviderState_Breaker_ErrRequestBuildFailed_CountsAsFailure(t *testing.T) { t.Parallel() st := newHealthTestState(time.Hour, 3, time.Minute, 10*time.Minute) now := time.Now() for i := 0; i < 3; i++ { st.finishRefresh(now, nil, errRequestBuildFailed) } if st.discoveryHealthy() { t.Fatal("discoveryHealthy() = true after 3 consecutive errRequestBuildFailed outcomes, want false — a persistent request-build failure is a provider fault the breaker must open on") } } // TestProviderState_Breaker_ContextDeadlineExceeded_CountsAsFailure proves // context.DeadlineExceeded is a plain failure, NOT given Canceled's // neutral treatment — deliberate, matching recordProviderAttempt's own // SHOULD-1 ruling (limits.go): a deadline the GATEWAY set and the upstream // never answered inside is a genuine provider-health signal, unlike a // caller-canceled context. Three consecutive backgroundRefreshTimeout // (30s) overruns against a merely slow provider trip the breaker exactly // like three plain errors would. func TestProviderState_Breaker_ContextDeadlineExceeded_CountsAsFailure(t *testing.T) { t.Parallel() st := newHealthTestState(time.Hour, 3, time.Minute, 10*time.Minute) now := time.Now() for i := 0; i < 3; i++ { st.finishRefresh(now, nil, context.DeadlineExceeded) } if st.discoveryHealthy() { t.Fatal("discoveryHealthy() = true after 3 consecutive context.DeadlineExceeded outcomes, want false — a deadline the gateway set and the upstream never answered inside is a provider fault, not a neutral cancellation") } } // --- breaker: the zero value never trips (adversarial-review finding, // round 2 — providerState's own doc comment claims "zero value behaves // exactly as before this feature existed"; without this guard that claim // was false: a zero-value breaker opened on the FIRST failure with a // ZERO backoff, after which the interval throttle was gone too) --- func TestProviderState_Breaker_ZeroThreshold_NeverOpens(t *testing.T) { t.Parallel() // A providerState assembled directly (bypassing newModelRegistry, // which always resolves breakerThreshold to >= 1 via // validateBreakerConfig — every test in this file up to here does // exactly that, through newHealthTestState) leaves every breaker // field at its Go zero value: this is that case, deliberately. st := &providerState{ explicit: make(map[string]bool), discovered: make(map[string]bool), interval: time.Millisecond, } now := time.Now() boom := errors.New("boom") for i := 0; i < 50; i++ { if !st.tryBeginRefresh(now) { t.Fatalf("iteration %d: tryBeginRefresh = false, want true — a zero-value breaker must behave like plain interval-only gating, never blocked by its own (disabled) backoff", i) } st.finishRefresh(now, nil, boom) if !st.discoveryHealthy() { t.Fatalf("iteration %d: discoveryHealthy() = false, want true — a zero-value breaker must never open at all", i) } now = now.Add(time.Millisecond) } } // --- breaker: discovery backoff must actually suppress re-probing under // DEFAULT config, strictly, not merely stay no-worse-than plain interval // throttling (BLOCKING adversarial-review findings, rounds 2 and 3) --- // TestProviderState_Breaker_DefaultCadence_StrictlyFewerThanPreFeatureBaseline // simulates 24 hours of a permanently-failing provider under every DEFAULT // setting (the shipped configuration, the xiaomi case this feature exists // for) and counts actual ADMITTED refresh attempts. // // Round 2's earlier version of tryBeginRefresh let an open breaker's own // backoff window (openUntil) REPLACE the plain interval gate instead of // adding to it. Since that round's defaultBreakerMaxOpenDuration (30m) was // shorter than defaultDiscoveryInterval (1h), that measured 50 admitted // attempts here — MORE than the 24 a plain, breaker-less interval gate // would ever produce for the same 24h window: the exact inverse of this // feature's purpose. The round-2 fix (interval gate applies // unconditionally) closed that regression, but left the feature CADENCE- // NEUTRAL: with defaultBreakerMaxOpenDuration still shorter than // defaultDiscoveryInterval, the interval alone remained the sole binding // constraint and this test measured EXACTLY 24 — no worse than baseline, // but no better either, doing nothing a plain interval gate would not // already do. // // Round 3's design ruling raised defaultBreakerMaxOpenDuration to 6h // (longer than the 1h interval) specifically so the breaker's backoff can // eventually become the binding constraint instead of the interval — see // that constant's own doc comment for the resulting cadence (roughly 5 // attempts a day). This test now asserts the STRONGER, actually-intended // property: strictly fewer admits than the pre-feature baseline. Reverting // EITHER the round-2 interval-gate fix OR the round-3 default bump makes // this fail — the first by exceeding the baseline, the second by merely // equaling it. func TestProviderState_Breaker_DefaultCadence_StrictlyFewerThanPreFeatureBaseline(t *testing.T) { t.Parallel() st := newHealthTestState(defaultDiscoveryInterval, defaultBreakerFailureThreshold, defaultBreakerOpenDuration, defaultBreakerMaxOpenDuration) base := time.Now() boom := errors.New("boom: persistent 401") var admits int for minute := 0; minute < 24*60; minute++ { now := base.Add(time.Duration(minute) * time.Minute) if st.tryBeginRefresh(now) { admits++ st.finishRefresh(now, nil, boom) } } // The pre-feature baseline: a plain interval-only gate (no breaker at // all) admits exactly one attempt per defaultDiscoveryInterval (1h) // over a 24h window for a provider that never succeeds. const preFeatureBaseline = 24 if admits >= preFeatureBaseline { t.Errorf("admitted %d discovery attempts over 24h with default config and a permanently failing provider, want STRICTLY FEWER than %d (the pre-feature, interval-only baseline) — the breaker must actually suppress re-probing under stock config, not merely match plain interval throttling", admits, preFeatureBaseline) } } // --- breaker: the admin-facing openUntil accounts for the interval floor // (MEDIUM adversarial-review finding, round 3) --- // TestProviderState_Snapshot_OpenUntil_ReflectsIntervalFloor proves // snapshot()'s emitted openUntil is the REAL next probe time — // max(raw openUntil, lastRefresh+interval) — not the raw backoff-only // value. Since tryBeginRefresh's interval gate applies unconditionally, // a fresh trip's raw openUntil (lastRefresh + breakerOpenBase) can be far // EARLIER than the actual next admitted attempt whenever breakerOpenBase // is shorter than interval — measured on pure defaults, a 59-minute skew // (raw +1m versus the real +1h). An operator watching the dashboard would // otherwise see "open (in 60s)", nothing happen, and then the countdown // silently vanish once formatUntil (webui) reads the now-past raw value. func TestProviderState_Snapshot_OpenUntil_ReflectsIntervalFloor(t *testing.T) { t.Parallel() // interval (1h) far longer than the backoff base (1m) — the exact // shape of the shipped defaults on a fresh trip. st := newHealthTestState(time.Hour, 1, time.Minute, 10*time.Minute) base := time.Now() st.tryBeginRefresh(base) st.finishRefresh(base, nil, errors.New("boom")) // opens: raw openUntil = base+1m _, lastRefresh, _, health, openUntil := st.snapshot() if health != breakerOpen { t.Fatalf("health = %v, want breakerOpen", health) } want := lastRefresh.Add(time.Hour) if !openUntil.Equal(want) { t.Errorf("snapshot openUntil = %v, want %v (lastRefresh + interval, not the raw +1m backoff — tryBeginRefresh will not actually admit a probe before then)", openUntil, want) } } // TestProviderState_Snapshot_OpenUntil_UnchangedWhenBackoffExceedsInterval // is the complementary case: once backoff is the LONGER of the two (the // steady state this feature is designed to reach — see // defaultBreakerMaxOpenDuration's own doc comment), snapshot's openUntil // is exactly the raw value, with no floor adjustment needed. func TestProviderState_Snapshot_OpenUntil_UnchangedWhenBackoffExceedsInterval(t *testing.T) { t.Parallel() // interval (1m) far shorter than the backoff base (1h) — backoff is // already the binding constraint from the very first trip. st := newHealthTestState(time.Minute, 1, time.Hour, 10*time.Hour) base := time.Now() st.tryBeginRefresh(base) st.finishRefresh(base, nil, errors.New("boom")) // opens: raw openUntil = base+1h _, _, _, health, openUntil := st.snapshot() if health != breakerOpen { t.Fatalf("health = %v, want breakerOpen", health) } want := base.Add(time.Hour) if !openUntil.Equal(want) { t.Errorf("snapshot openUntil = %v, want %v (the raw backoff value, unchanged — it already exceeds the interval floor)", openUntil, want) } } // --- breaker: stays open for the backoff window, superseding interval --- func TestProviderState_Breaker_StaysOpenForBackoffWindow(t *testing.T) { t.Parallel() // interval is deliberately much shorter than openBase: a bare pass // through tryBeginRefresh would allow a retry after 5s. Only the // breaker's own backoff should be blocking it past that point. st := newHealthTestState(5*time.Second, 1, time.Minute, 10*time.Minute) base := time.Now() if ok := st.tryBeginRefresh(base); !ok { t.Fatal("first tryBeginRefresh on a fresh provider = false, want true") } st.finishRefresh(base, nil, errors.New("boom")) // trips the breaker (threshold 1) if st.discoveryHealthy() { t.Fatal("discoveryHealthy() = true after the trip, want false") } if ok := st.tryBeginRefresh(base.Add(6 * time.Second)); ok { t.Error("tryBeginRefresh past the plain interval (5s) but well inside the 1m backoff = true, want false — backoff must supersede interval") } if ok := st.tryBeginRefresh(base.Add(59 * time.Second)); ok { t.Error("tryBeginRefresh just before the backoff window elapses = true, want false") } if ok := st.tryBeginRefresh(base.Add(61 * time.Second)); !ok { t.Fatal("tryBeginRefresh once the backoff window has elapsed = false, want true (half-open probe should start)") } if !st.discoveryHealthy() { t.Error("discoveryHealthy() during the half-open probe = false, want true — a provider being re-probed must stay routable") } } // --- breaker: half-open probe success closes it again --- func TestProviderState_Breaker_HalfOpenProbe_SuccessCloses(t *testing.T) { t.Parallel() st := newHealthTestState(5*time.Second, 1, time.Minute, 10*time.Minute) base := time.Now() st.tryBeginRefresh(base) st.finishRefresh(base, nil, errors.New("boom")) // open probeAt := base.Add(61 * time.Second) if ok := st.tryBeginRefresh(probeAt); !ok { t.Fatal("tryBeginRefresh after the backoff window = false, want true") } st.finishRefresh(probeAt, []string{"m1"}, nil) // probe succeeds if !st.discoveryHealthy() { t.Error("discoveryHealthy() after a successful half-open probe = false, want true — a recovered provider must not stay dead") } if !st.hasModel("m1") { t.Error("hasModel(m1) = false after a successful probe, want true — discovery still updates the model set normally") } // Closed again means ORDINARY interval gating resumed, not a fresh // backoff window: immediately after the probe, another attempt must // still wait out the plain 5s interval, and no longer than that. if ok := st.tryBeginRefresh(probeAt); ok { t.Error("tryBeginRefresh at the same instant as the closing probe = true, want false (interval not yet elapsed)") } if ok := st.tryBeginRefresh(probeAt.Add(6 * time.Second)); !ok { t.Error("tryBeginRefresh past the plain interval after closing = false, want true") } } // --- breaker: half-open probe failure reopens it, with escalating backoff --- func TestProviderState_Breaker_HalfOpenProbe_FailureReopens_Escalates(t *testing.T) { t.Parallel() st := newHealthTestState(5*time.Second, 1, time.Minute, 10*time.Minute) base := time.Now() st.tryBeginRefresh(base) st.finishRefresh(base, nil, errors.New("boom")) // opens, backoff = 1m probeAt := base.Add(61 * time.Second) if ok := st.tryBeginRefresh(probeAt); !ok { t.Fatal("tryBeginRefresh after the first backoff window = false, want true") } st.finishRefresh(probeAt, nil, errors.New("still broken")) // probe fails: reopen, backoff doubles to 2m if st.discoveryHealthy() { t.Fatal("discoveryHealthy() after a failed probe = true, want false") } if ok := st.tryBeginRefresh(probeAt.Add(61 * time.Second)); ok { t.Error("tryBeginRefresh after only the ORIGINAL (1m) backoff = true, want false — a failed probe must double it to 2m") } if ok := st.tryBeginRefresh(probeAt.Add(121 * time.Second)); !ok { t.Fatal("tryBeginRefresh after the doubled (2m) backoff = false, want true") } } // TestProviderState_Breaker_BackoffCapsAtOpenMax proves repeated failed // probes double the backoff only up to breakerOpenMax, never past it. func TestProviderState_Breaker_BackoffCapsAtOpenMax(t *testing.T) { t.Parallel() st := newHealthTestState(time.Second, 1, time.Minute, 3*time.Minute) // base 1m, cap 3m: 1m -> 2m -> capped at 3m (not 4m) base := time.Now() st.tryBeginRefresh(base) st.finishRefresh(base, nil, errors.New("boom")) // open, backoff 1m at := base.Add(61 * time.Second) st.tryBeginRefresh(at) st.finishRefresh(at, nil, errors.New("boom")) // reopen, backoff doubles to 2m at = at.Add(121 * time.Second) st.tryBeginRefresh(at) st.finishRefresh(at, nil, errors.New("boom")) // reopen, doubling to 4m would exceed the 3m cap if ok := st.tryBeginRefresh(at.Add(2*time.Minute + 59*time.Second)); ok { t.Error("tryBeginRefresh just before the capped 3m backoff = true, want false") } if ok := st.tryBeginRefresh(at.Add(3*time.Minute + 1*time.Second)); !ok { t.Error("tryBeginRefresh just past the capped 3m backoff = false, want true — backoff must not exceed breakerOpenMax") } } // --- breaker: context.Canceled is NEUTRAL, neither failure nor success // (IMPORTANT adversarial-review finding, round 2) --- // TestProviderState_Breaker_ContextCanceled_IsNeutral_NotSuccess replaces // a vacuous predecessor that only asserted discoveryHealthy()==true after // ONE Canceled outcome on a threshold-1 breaker — true identically whether // Canceled is ignored (correct) or treated as a success (the bug). This // sequence distinguishes the two: fail, fail, Canceled, fail on a // threshold-3 breaker opens it only if Canceled did NOT reset the // consecutive-failure streak. func TestProviderState_Breaker_ContextCanceled_IsNeutral_NotSuccess(t *testing.T) { t.Parallel() st := newHealthTestState(time.Second, 3, time.Minute, 10*time.Minute) now := time.Now() boom := errors.New("boom") st.finishRefresh(now, nil, boom) // failure 1 st.finishRefresh(now, nil, boom) // failure 2 st.finishRefresh(now, nil, context.Canceled) // neutral: must NOT reset the streak st.finishRefresh(now, nil, boom) // failure 3: reaches the threshold only if Canceled was neutral if st.discoveryHealthy() { t.Fatal("discoveryHealthy() = true after fail,fail,Canceled,fail on a threshold-3 breaker, want false — Canceled must not reset the consecutive-failure streak (that would treat cancellation as a success)") } } // TestProviderState_Breaker_ContextCanceled_IsNeutral_NotFailure is the // other direction of the bidirectional check (round 3 finding): its // _NotSuccess sibling above proves Canceled does not reset the streak, // but says nothing about whether Canceled ITSELF is wrongly counted as a // failure — a mutant that classified Canceled as a failure would still // pass that test unchanged, since either way the threshold is reached by // the trailing plain failure. A LONE Canceled call on a threshold-1 // breaker settles it directly: if Canceled counted as a failure, one call // would be enough to trip a threshold of 1. func TestProviderState_Breaker_ContextCanceled_IsNeutral_NotFailure(t *testing.T) { t.Parallel() st := newHealthTestState(time.Hour, 1, time.Minute, 10*time.Minute) now := time.Now() st.finishRefresh(now, nil, context.Canceled) if !st.discoveryHealthy() { t.Fatal("discoveryHealthy() = false after a single context.Canceled outcome on a threshold-1 breaker, want true — Canceled must not be counted as a failure") } } // TestProviderState_Breaker_ContextCanceled_DuringHalfOpen_PreservesBackoff // is the sharper half of the same finding: an earlier version treated a // Canceled half-open probe as a SUCCESS, which CLOSED an already-open // breaker and wiped its backoff — reporting a still-401ing provider // healthy purely because a probe was interrupted at shutdown. This proves // two things: discoveryHealthy reports the provider unhealthy immediately // (round 3 finding: a canceled probe reverts to OPEN rather than being // left stuck at halfOpen, which would report healthy), and the backoff // survives — the NEXT real failure must escalate from the ORIGINAL // backoff, not restart fresh at the base. func TestProviderState_Breaker_ContextCanceled_DuringHalfOpen_PreservesBackoff(t *testing.T) { t.Parallel() st := newHealthTestState(time.Second, 1, time.Minute, 10*time.Minute) base := time.Now() st.tryBeginRefresh(base) st.finishRefresh(base, nil, errors.New("boom")) // opens, backoff = 1m, openUntil = base+1m probeAt := base.Add(61 * time.Second) if !st.tryBeginRefresh(probeAt) { t.Fatal("tryBeginRefresh after the backoff window = false, want true") } st.finishRefresh(probeAt, nil, context.Canceled) // probe interrupted at shutdown: must be neutral if st.discoveryHealthy() { t.Fatal("discoveryHealthy() = true immediately after a canceled half-open probe, want false — a still-broken provider must not read healthy just because its re-probe was interrupted") } // The next real attempt must see the ORIGINAL backoff still in // force, not a wiped/reset one: if Canceled had (incorrectly) closed // the breaker, this next failure would re-open it FRESH at the 1m // base instead of escalating to 2m. retryAt := probeAt.Add(2 * time.Second) if !st.tryBeginRefresh(retryAt) { t.Fatal("tryBeginRefresh right after a canceled probe = false, want true (interval elapsed, not blocked by the cancel)") } st.finishRefresh(retryAt, nil, errors.New("still broken")) if ok := st.tryBeginRefresh(retryAt.Add(119 * time.Second)); ok { t.Error("tryBeginRefresh before the escalated 2m backoff = true, want false — a canceled probe must not have reset the backoff to its 1m base") } if ok := st.tryBeginRefresh(retryAt.Add(121 * time.Second)); !ok { t.Error("tryBeginRefresh after the escalated 2m backoff = false, want true") } } // --- breaker: a provider that never fails is unaffected (default- // preserving: today's interval-only gating, unchanged) --- func TestProviderState_Breaker_HealthyProviderNeverAffected(t *testing.T) { t.Parallel() const interval = time.Hour st := newHealthTestState(interval, defaultBreakerFailureThreshold, defaultBreakerOpenDuration, defaultBreakerMaxOpenDuration) now := time.Now() for i := 0; i < 5; i++ { if ok := st.tryBeginRefresh(now); !ok { t.Fatalf("cycle %d: tryBeginRefresh = false, want true (interval already elapsed)", i) } st.finishRefresh(now, []string{"m1"}, nil) if !st.discoveryHealthy() { t.Fatalf("cycle %d: discoveryHealthy() = false, want true — a never-failing provider must never trip its breaker", i) } // Behavioral proxy for "the breaker never opened" (assert // behavior, not the internal openUntil field, per the brief): // an opened breaker would admit again as soon as its own // (likely shorter) backoff elapsed. Checking one instant BEFORE // the plain interval elapses proves nothing shorter is gating // this provider. if ok := st.tryBeginRefresh(now.Add(interval - time.Nanosecond)); ok { t.Fatalf("cycle %d: tryBeginRefresh one nanosecond before the interval elapses = true, want false", i) } if ok := st.tryBeginRefresh(now); ok { t.Fatalf("cycle %d: tryBeginRefresh immediately after a fresh refresh = true, want false (interval not elapsed)", i) } now = now.Add(2 * interval) // past the interval for the next cycle } } // --- breaker: concurrent access is race-free (IMPORTANT adversarial- // review finding, round 2 — no prior test in this file ever exercised the // mutex-guarded breaker fields from more than one goroutine, so -race had // nothing to catch a locking bug with; the locking was correct, but // unproven) --- // TestProviderState_Breaker_ConcurrentRefreshes_RaceFree is primarily a // -race and panic net (round 2), but round 3 found it carried ZERO // assertions and so survived every one of the seven fix-reversion // mutations tried against this file — useful for what it catches, but not // a behavioral test of its own. It now asserts two invariants on every // iteration, under the SAME lock discoveryHealthy/tryBeginRefresh already // take, so the read is race-free too: backoff never exceeds breakerOpenMax // (the overflow-ceiling property, round 2 finding 7, checked here under // real concurrent mutation rather than only sequentially), and health is // always one of the three declared states — breakerState is a bare int32 // under the hood, so nothing enforces that at compile time. func TestProviderState_Breaker_ConcurrentRefreshes_RaceFree(t *testing.T) { t.Parallel() st := newHealthTestState(time.Millisecond, 3, 2*time.Millisecond, 20*time.Millisecond) base := time.Now() const workers = 8 const iterations = 500 var wg sync.WaitGroup for w := 0; w < workers; w++ { wg.Add(1) go func(worker int) { defer wg.Done() for i := 0; i < iterations; i++ { now := base.Add(time.Duration(worker*iterations+i) * time.Microsecond) if st.tryBeginRefresh(now) { var err error if (worker+i)%2 == 0 { err = errors.New("boom") } st.finishRefresh(now, []string{"m1"}, err) } _ = st.discoveryHealthy() st.mu.Lock() gotBackoff, gotMax, gotHealth := st.backoff, st.breakerOpenMax, st.health st.mu.Unlock() if gotBackoff > gotMax { t.Errorf("worker %d iteration %d: backoff = %v, want <= breakerOpenMax %v", worker, i, gotBackoff, gotMax) } switch gotHealth { case breakerClosed, breakerOpen, breakerHalfOpen: default: t.Errorf("worker %d iteration %d: health = %v, want one of closed/open/half-open", worker, i, gotHealth) } } }(w) } wg.Wait() } // --- modelRegistry.discoveryHealthy: the exported predicate feat/failover // consumes --- func TestModelRegistry_DiscoveryHealthy(t *testing.T) { t.Parallel() adapters := map[string]providerAdapter{ "flaky": newFakeAdapter("flaky"), "healthy": newFakeAdapter("healthy"), } cfg := &Config{ Providers: map[string]*ProviderConfig{ "flaky": {Models: []string{"m1"}}, "healthy": {Models: []string{"m2"}}, }, Breaker: BreakerConfig{FailureThreshold: 2, OpenDuration: "1m", MaxOpenDuration: "10m"}, } reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) if err != nil { t.Fatalf("newModelRegistry: %v", err) } now := time.Now() reg.nowFn = func() time.Time { return now } if !reg.discoveryHealthy("flaky") || !reg.discoveryHealthy("healthy") { t.Fatal("a freshly constructed registry's providers must both be discovery-healthy") } if !reg.discoveryHealthy("does-not-exist") { t.Error(`discoveryHealthy("does-not-exist") = false, want true — an unconfigured name is not a reason to block routing`) } boom := errors.New("boom") reg.states["flaky"].finishRefresh(now, nil, boom) reg.states["flaky"].finishRefresh(now, nil, boom) // reaches FailureThreshold: 2 if reg.discoveryHealthy("flaky") { t.Error(`discoveryHealthy("flaky") = true after tripping its breaker, want false`) } if !reg.discoveryHealthy("healthy") { t.Error(`discoveryHealthy("healthy") = false, want true — a sibling provider's breaker must not affect it`) } }