package traefikllmgateway import ( "context" "errors" "net/http" "net/http/httptest" "strings" "sync" "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // stateOf reports (kind, name)'s current targetHealthState — see // targetHealthState's own doc comment (target_health.go) for the exact // unknown/healthy/unhealthy rule. Test-only (finding 18 fix, review- // routes.md): it has zero production call sites — every production // reader goes through snapshot (admin.go's targets view, metrics.go), // which already applies the identical stateForLocked rule this helper // reuses — so it was moved here from target_health.go rather than kept // as unused production API surface. Every test in this package that // wants a bare state value (not snapshot's fuller struct) uses this, // including mcp_a2a_test.go and mcp_federation_test.go, which reach it // as an ordinary package-scoped method the same way they would if it // still lived in the non-test file. func (t *targetHealthTracker) stateOf(kind, name string) targetHealthState { if t == nil { return targetHealthUnknown } t.mu.Lock() defer t.mu.Unlock() e, ok := t.entries[targetHealthKey{kind: kind, name: name}] if !ok { return targetHealthUnknown } return t.stateForLocked(e) } // --- targetHealthTracker: state machine --- func TestTargetHealthTracker_NeverRecorded_ReadsUnknown(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) assert.Equal(t, targetHealthUnknown, tr.stateOf(targetKindMCP, "alpha")) } func TestTargetHealthTracker_NilReceiver_Safe(t *testing.T) { t.Parallel() var tr *targetHealthTracker assert.NotPanics(t, func() { tr.record(targetKindMCP, "alpha", "", true, nil, time.Millisecond, targetHealthSourceTraffic) }) assert.Equal(t, targetHealthUnknown, tr.stateOf(targetKindMCP, "alpha")) snap := tr.snapshot(targetKindMCP, "alpha") assert.Equal(t, targetHealthUnknown, snap.state) } func TestTargetHealthTracker_SingleSuccess_ReadsHealthy(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) tr.record(targetKindMCP, "alpha", "", true, nil, 5*time.Millisecond, targetHealthSourceTraffic) assert.Equal(t, targetHealthHealthy, tr.stateOf(targetKindMCP, "alpha")) } // TestTargetHealthTracker_BelowThreshold_StaysHealthy proves the // documented exception: 1 or 2 recent failures after a success still // reads "healthy" until consecutiveFailures actually reaches the // configured threshold (targetHealthTracker's own doc comment). func TestTargetHealthTracker_BelowThreshold_StaysHealthy(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) tr.record(targetKindMCP, "alpha", "", true, nil, 0, targetHealthSourceTraffic) tr.record(targetKindMCP, "alpha", "", false, errors.New("boom"), 0, targetHealthSourceTraffic) assert.Equal(t, targetHealthHealthy, tr.stateOf(targetKindMCP, "alpha"), "1 failure of threshold 3 must still read healthy") tr.record(targetKindMCP, "alpha", "", false, errors.New("boom again"), 0, targetHealthSourceTraffic) assert.Equal(t, targetHealthHealthy, tr.stateOf(targetKindMCP, "alpha"), "2 failures of threshold 3 must still read healthy") } func TestTargetHealthTracker_ReachesThreshold_ReadsUnhealthy(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) for i := 0; i < 3; i++ { tr.record(targetKindMCP, "alpha", "", false, errors.New("boom"), 0, targetHealthSourceTraffic) } assert.Equal(t, targetHealthUnhealthy, tr.stateOf(targetKindMCP, "alpha")) } // TestTargetHealthTracker_FirstObservationFailure_ReadsUnknown proves F4: // a target whose only observations so far are failures below // failureThreshold reads "unknown", not "healthy" — the tracker has // never actually seen it succeed, so "healthy" would be a false claim // even though the failure-threshold gate has not fired. The threshold // still protects against a transient first failure: this stays // "unknown", it does not jump straight to "unhealthy". func TestTargetHealthTracker_FirstObservationFailure_ReadsUnknown(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) tr.record(targetKindMCP, "alpha", "", false, errors.New("boom"), 0, targetHealthSourceTraffic) assert.Equal(t, targetHealthUnknown, tr.stateOf(targetKindMCP, "alpha"), "never having succeeded must read unknown, not healthy, below threshold") tr.record(targetKindMCP, "alpha", "", false, errors.New("boom again"), 0, targetHealthSourceTraffic) assert.Equal(t, targetHealthUnknown, tr.stateOf(targetKindMCP, "alpha"), "still unknown after a second failure, still below threshold") } // TestTargetHealthTracker_SuccessThenFewFailures_ReadsHealthy re-proves // TestTargetHealthTracker_BelowThreshold_StaysHealthy's claim in F4 // terms: once a target HAS succeeded at least once, 1-2 subsequent // failures below threshold still read "healthy" — F4 only changes the // NEVER-succeeded case. func TestTargetHealthTracker_SuccessThenFewFailures_ReadsHealthy(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) tr.record(targetKindMCP, "alpha", "", true, nil, 0, targetHealthSourceTraffic) tr.record(targetKindMCP, "alpha", "", false, errors.New("boom"), 0, targetHealthSourceTraffic) tr.record(targetKindMCP, "alpha", "", false, errors.New("boom again"), 0, targetHealthSourceTraffic) assert.Equal(t, targetHealthHealthy, tr.stateOf(targetKindMCP, "alpha")) } func TestTargetHealthTracker_SuccessAfterUnhealthy_RecoversToHealthy(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(2) tr.record(targetKindMCP, "alpha", "", false, errors.New("boom"), 0, targetHealthSourceTraffic) tr.record(targetKindMCP, "alpha", "", false, errors.New("boom"), 0, targetHealthSourceTraffic) require.Equal(t, targetHealthUnhealthy, tr.stateOf(targetKindMCP, "alpha")) tr.record(targetKindMCP, "alpha", "", true, nil, 0, targetHealthSourceTraffic) assert.Equal(t, targetHealthHealthy, tr.stateOf(targetKindMCP, "alpha")) snap := tr.snapshot(targetKindMCP, "alpha") assert.Equal(t, 0, snap.consecutiveFailures, "a success must reset consecutiveFailures to 0") assert.Empty(t, snap.lastError, "a success must clear lastError") } func TestTargetHealthTracker_ErrorString_BoundedLength(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) long := strings.Repeat("x", targetHealthMaxErrorRunes+500) tr.record(targetKindMCP, "alpha", "", false, errors.New(long), 0, targetHealthSourceProbe) snap := tr.snapshot(targetKindMCP, "alpha") assert.LessOrEqual(t, len([]rune(snap.lastError)), targetHealthMaxErrorRunes) assert.Equal(t, strings.Repeat("x", targetHealthMaxErrorRunes), snap.lastError) } func TestTargetHealthTracker_ShortError_Unchanged(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) tr.record(targetKindMCP, "alpha", "", false, errors.New("dial tcp: connection refused"), 0, targetHealthSourceProbe) snap := tr.snapshot(targetKindMCP, "alpha") assert.Equal(t, "dial tcp: connection refused", snap.lastError) } // TestTargetHealthTracker_Record_ScrubsCredentialsFromLastError proves F1: // record scrubs an error's embedded target URL through sanitizeProviderErr // before storing lastError, so a target URL configured with an embedded // credential (an "api-key" query parameter, or userinfo) never survives // into GET /admin/api/targets. func TestTargetHealthTracker_Record_ScrubsCredentialsFromLastError(t *testing.T) { t.Parallel() t.Run("query parameter credential", func(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) rawURL := "https://svc.internal/mcp?api-key=SECRETVALUE" err := errors.New(`Get "` + rawURL + `": dial tcp: connection refused`) tr.record(targetKindMCP, "alpha", rawURL, false, err, 0, targetHealthSourceProbe) snap := tr.snapshot(targetKindMCP, "alpha") assert.NotContains(t, snap.lastError, "SECRETVALUE") assert.NotContains(t, snap.lastError, "api-key=") }) t.Run("userinfo credential", func(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) rawURL := "https://tok3n@svc.internal/mcp" err := errors.New(`Get "` + rawURL + `": dial tcp: connection refused`) tr.record(targetKindMCP, "alpha", rawURL, false, err, 0, targetHealthSourceProbe) snap := tr.snapshot(targetKindMCP, "alpha") assert.NotContains(t, snap.lastError, "tok3n") }) } // TestTargetHealthTracker_Record_ScrubsBeforeTruncate proves F1: the URL // scrub runs against err.Error() BEFORE truncateRunes bounds it to // targetHealthMaxErrorRunes — even when the embedded URL straddles the // truncation point, no fragment of the secret survives. G2 (feat/target- // health review round 2): the prefix is short enough that the SECRET // itself — not merely the surrounding URL — straddles the 200-rune cut, // so this test actually distinguishes scrub-before-truncate from // truncate-before-scrub; a longer prefix that pushes the whole secret // past the cut point would pass under either ordering, proving nothing. func TestTargetHealthTracker_Record_ScrubsBeforeTruncate(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) rawURL := "https://svc.internal/mcp?api-key=" + strings.Repeat("S", 60) prefix := strings.Repeat("p", 150) require.Less(t, len([]rune(prefix))+strings.Index(rawURL, "api-key="), targetHealthMaxErrorRunes, "the secret must start before the truncation point") require.Greater(t, len([]rune(prefix))+len([]rune(rawURL)), targetHealthMaxErrorRunes, "the url must straddle the truncation point") errText := prefix + " " + rawURL + " connection refused" tr.record(targetKindMCP, "alpha", rawURL, false, errors.New(errText), 0, targetHealthSourceProbe) snap := tr.snapshot(targetKindMCP, "alpha") assert.LessOrEqual(t, len([]rune(snap.lastError)), targetHealthMaxErrorRunes) assert.NotContains(t, snap.lastError, "SSSS", "no fragment of the secret must survive truncation") assert.NotContains(t, snap.lastError, "api-key=") } // TestTargetHealthTracker_Record_ScrubsPasswordMaskedURLForm proves G1 // (feat/target-health review round 2): net/http's own *url.Error masks a // URL's password ("user:pass@" -> "user:***@", and "user:" with an empty // password the same way — stripPassword, net/http/client.go) BEFORE the // error text is ever built, whenever the dialed URL's userinfo carries a // password. sanitizeProviderErr (admin.go) only matches rawURL verbatim, // so for such a URL it finds nothing to scrub — leaking the password AND // any query-string credential riding the same URL (an "api-key" value). // record must scrub against BOTH the raw URL and net/http's own masked // form. Every case uses a REAL error from http.Client.Do against // 127.0.0.1:1 (this repo's own "reserved, never listening" convention — // see provider_openai_test.go), never a hand-written string: the bug is // about the exact shape net/http actually produces. func TestTargetHealthTracker_Record_ScrubsPasswordMaskedURLForm(t *testing.T) { t.Parallel() // secretPart is the synthetic password these fixtures plant in a target // URL, and credFixtureURL assembles the URL around it at run time. The // parts stay separate in the source deliberately: a whole URL literal // carrying userinfo credentials trips this repository's own pre-commit // secret scan, and these fixtures exist to prove exactly such a URL // never reaches lastError. const secretPart = "sup3rsecret" credFixtureURL := func(userinfo, hostPath string) string { return "http://" + userinfo + "@" + hostPath } cases := []struct { name string rawURL string mustNotHave []string }{ { name: "userinfo with password", rawURL: credFixtureURL("user:"+secretPart, "127.0.0.1:1/mcp"), mustNotHave: []string{"user:" + secretPart + "@", "user:***@", secretPart}, }, { name: "token as username with empty password separator", rawURL: credFixtureURL("TOKENASUSER:", "127.0.0.1:1/mcp"), mustNotHave: []string{"TOKENASUSER:@", "TOKENASUSER:***@", "TOKENASUSER"}, }, { name: "userinfo with password and api-key query on the same URL", rawURL: credFixtureURL("user:"+secretPart, "127.0.0.1:1/mcp?api-key=SECRETVALUE"), mustNotHave: []string{"user:" + secretPart + "@", "user:***@", secretPart, "SECRETVALUE", "api-key="}, }, { // Existing case (F1): userinfo with NO password has nothing for // net/http to mask, so the raw URL survives into the error // verbatim — must keep working unchanged by the G1 fix. name: "token as username, no password separator", rawURL: "http://tok3n@127.0.0.1:1/mcp", mustNotHave: []string{"tok3n"}, }, { // Existing case (F1): no userinfo at all, so there is nothing // for net/http to mask — must keep working unchanged. name: "bare api-key query, no userinfo", rawURL: "http://127.0.0.1:1/mcp?api-key=SECRETVALUE", mustNotHave: []string{"SECRETVALUE", "api-key="}, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { t.Parallel() req, err := http.NewRequest(http.MethodGet, tc.rawURL, nil) require.NoError(t, err) _, doErr := http.DefaultClient.Do(req) //nolint:bodyclose // Do returns a nil body alongside a non-nil error require.Error(t, doErr, "127.0.0.1:1 must refuse the connection") tr := newTargetHealthTracker(3) tr.record(targetKindMCP, "alpha", tc.rawURL, false, doErr, 0, targetHealthSourceProbe) snap := tr.snapshot(targetKindMCP, "alpha") for _, secret := range tc.mustNotHave { assert.NotContains(t, snap.lastError, secret) } }) } } func TestTargetHealthTracker_KeyedByKindAndName_Independent(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(1) tr.record(targetKindMCP, "shared", "", false, errors.New("boom"), 0, targetHealthSourceTraffic) assert.Equal(t, targetHealthUnhealthy, tr.stateOf(targetKindMCP, "shared")) assert.Equal(t, targetHealthUnknown, tr.stateOf(targetKindAgent, "shared"), "an mcp server and an agent sharing a name must track independently") } func TestTargetHealthTracker_Snapshot_FieldsPopulated(t *testing.T) { t.Parallel() now := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) tr := newTargetHealthTracker(3) tr.nowFn = func() time.Time { return now } tr.record(targetKindAgent, "beta", "", true, nil, 42*time.Millisecond, targetHealthSourceProbe) snap := tr.snapshot(targetKindAgent, "beta") assert.Equal(t, targetHealthHealthy, snap.state) assert.Equal(t, now, snap.lastCheck) assert.Equal(t, 42*time.Millisecond, snap.latency) assert.Equal(t, targetHealthSourceProbe, snap.source) assert.Equal(t, 0, snap.consecutiveFailures) } func TestTargetHealthTracker_ConcurrentRecord_NoRace(t *testing.T) { t.Parallel() tr := newTargetHealthTracker(3) var wg sync.WaitGroup for i := 0; i < 50; i++ { wg.Add(1) go func(i int) { defer wg.Done() tr.record(targetKindMCP, "alpha", "", i%2 == 0, errors.New("x"), time.Millisecond, targetHealthSourceTraffic) }(i) } wg.Wait() _ = tr.stateOf(targetKindMCP, "alpha") } // --- TargetHealthConfig validation --- func TestValidateTargetHealthConfig_Defaults(t *testing.T) { t.Parallel() thc, err := validateTargetHealthConfig(TargetHealthConfig{}) require.NoError(t, err) assert.False(t, thc.enabled) assert.Equal(t, defaultTargetHealthProbeInterval, thc.probeInterval) assert.Equal(t, defaultTargetHealthFailureThreshold, thc.failureThreshold) } func TestValidateTargetHealthConfig_ExplicitEnable(t *testing.T) { t.Parallel() thc, err := validateTargetHealthConfig(TargetHealthConfig{Enabled: true}) require.NoError(t, err) assert.True(t, thc.enabled) } func TestValidateTargetHealthConfig_ProbeIntervalBounds(t *testing.T) { t.Parallel() _, err := validateTargetHealthConfig(TargetHealthConfig{ProbeInterval: "1s"}) assert.Error(t, err, "below the 10s floor must be rejected") _, err = validateTargetHealthConfig(TargetHealthConfig{ProbeInterval: "2h"}) assert.Error(t, err, "above the 1h ceiling must be rejected") _, err = validateTargetHealthConfig(TargetHealthConfig{ProbeInterval: "not-a-duration"}) assert.Error(t, err) thc, err := validateTargetHealthConfig(TargetHealthConfig{ProbeInterval: "30s"}) require.NoError(t, err) assert.Equal(t, 30*time.Second, thc.probeInterval) } func TestValidateTargetHealthConfig_FailureThresholdBounds(t *testing.T) { t.Parallel() _, err := validateTargetHealthConfig(TargetHealthConfig{FailureThreshold: -1}) assert.Error(t, err) _, err = validateTargetHealthConfig(TargetHealthConfig{FailureThreshold: maxTargetHealthFailureThreshold + 1}) assert.Error(t, err) thc, err := validateTargetHealthConfig(TargetHealthConfig{FailureThreshold: 10}) require.NoError(t, err) assert.Equal(t, 10, thc.failureThreshold) } func TestNewGateway_TargetHealth_InvalidConfig_ConstructionError(t *testing.T) { t.Parallel() cfg := CreateConfig() cfg.Providers = map[string]*ProviderConfig{"openai": {Type: "openai", APIKey: "k"}} cfg.TargetHealth = TargetHealthConfig{ProbeInterval: "1s"} _, err := New(context.Background(), http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), cfg, "llmgw") assert.Error(t, err) } func TestNewGateway_TargetHealth_DisabledByDefault_TrackerStillConstructed(t *testing.T) { t.Parallel() cfg := CreateConfig() cfg.Providers = map[string]*ProviderConfig{"openai": {Type: "openai", APIKey: "k"}} h, err := New(context.Background(), http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), cfg, "llmgw") require.NoError(t, err) gw, ok := h.(*Gateway) require.True(t, ok) require.NotNil(t, gw.targetHealth, "the tracker must always be constructed regardless of Enabled — passive recording/exposure stay on unconditionally") assert.False(t, gw.targetHealthCfg.enabled) } // --- maybeSweepTargetHealth: disabled / enabled / single-flight --- func newTargetHealthSweepGateway(t *testing.T, enabled bool, probeInterval time.Duration, mcpURL string) *Gateway { t.Helper() adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")} cfg := &Config{ Providers: map[string]*ProviderConfig{"openai": {}}, MCPServers: map[string]*TargetConfig{"alpha": {URL: mcpURL}}, } reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) require.NoError(t, err) return &Gateway{ cfg: cfg, registry: reg, targetHealth: newTargetHealthTracker(3), targetHealthCfg: targetHealthConfig{enabled: enabled, probeInterval: probeInterval, failureThreshold: 3}, targetClient: http.DefaultClient, targetTimeout: 5 * time.Second, name: "test", } } func TestMaybeSweepTargetHealth_Disabled_NoOutboundRequests(t *testing.T) { t.Parallel() // F6, feat/target-health review: atomic even though a disabled sweep // must never actually invoke this handler — a regression that DID // invoke it would otherwise race this counter's read below under // -race, obscuring the real bug (an unexpected outbound probe) behind // a data-race report instead of the intended assert.Equal failure. var hits int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&hits, 1) w.WriteHeader(http.StatusOK) })) defer srv.Close() gw := newTargetHealthSweepGateway(t, false, 10*time.Millisecond, srv.URL) gw.maybeSweepTargetHealth() // maybeSweepTargetHealth spawns a goroutine when it decides to sweep; // since it must return immediately when disabled, there is nothing to // wait on — a brief sleep would only prove absence of a signal we // never expect to see. time.Sleep(20 * time.Millisecond) assert.Equal(t, int32(0), atomic.LoadInt32(&hits), "a disabled targetHealth must never send an outbound probe") assert.Equal(t, targetHealthUnknown, gw.targetHealth.stateOf(targetKindMCP, "alpha")) } func TestMaybeSweepTargetHealth_Enabled_ProbesConfiguredServer(t *testing.T) { t.Parallel() var hits int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&hits, 1) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":"federated","result":{"protocolVersion":"2025-11-25","capabilities":{}}}`)) })) defer srv.Close() gw := newTargetHealthSweepGateway(t, true, time.Hour, srv.URL) gw.maybeSweepTargetHealth() waitForCondition(t, func() bool { return atomic.LoadInt32(&hits) > 0 }, time.Second) waitForCondition(t, func() bool { return gw.targetHealth.stateOf(targetKindMCP, "alpha") == targetHealthHealthy }, time.Second) snap := gw.targetHealth.snapshot(targetKindMCP, "alpha") assert.Equal(t, targetHealthSourceProbe, snap.source) } func TestMaybeSweepTargetHealth_ConcurrentTriggers_SingleFlight(t *testing.T) { t.Parallel() var hits int32 release := make(chan struct{}) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&hits, 1) <-release w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":"federated","result":{}}`)) })) // F6, feat/target-health review: registered in this order so // t.Cleanup's LIFO order unblocks the handler (closing release) // BEFORE srv.Close runs — httptest.Server.Close blocks until // outstanding requests complete, and the assertions below can // t.Fatalf/t.Errorf before ever reaching the explicit close(release) // further down; without this a failing assertion left the handler // permanently blocked and srv.Close hung until the whole test // binary's 10-minute timeout, rather than failing fast. unblock is // idempotent (sync.Once) since the happy path below also calls it // directly, before waiting on the sweep's resulting state. var unblockOnce sync.Once unblock := func() { unblockOnce.Do(func() { close(release) }) } t.Cleanup(srv.Close) t.Cleanup(unblock) gw := newTargetHealthSweepGateway(t, true, time.Hour, srv.URL) var wg sync.WaitGroup for i := 0; i < 20; i++ { wg.Add(1) go func() { defer wg.Done() gw.maybeSweepTargetHealth() }() } wg.Wait() // Give the single in-flight probe time to actually reach the server, // then release it — regardless of how many goroutines called // maybeSweepTargetHealth concurrently, at most one outbound request // must have been sent. waitForCondition(t, func() bool { return atomic.LoadInt32(&hits) > 0 }, time.Second) unblock() waitForCondition(t, func() bool { return gw.targetHealth.stateOf(targetKindMCP, "alpha") != targetHealthUnknown }, time.Second) assert.Equal(t, int32(1), atomic.LoadInt32(&hits), "concurrent triggers within one interval must produce exactly one sweep") } // TestMaybeSweepTargetHealth_IntervalEnforcedAcrossRapidCalls proves F3: // the interval load (top of maybeSweepTargetHealth) and the CAS that wins // the single-flight flag are not atomic with each other. Without a // re-check under the CAS, a caller whose own interval read predates an // entirely separate sweep's start-and-finish could still win the CAS // once that sweep clears the flag, and spawn a second sweep inside the // same interval — worst at startup, when every caller's first read sees // last == 0. A fast-completing first sweep, immediately followed by a // burst of concurrent callers, must still yield exactly one sweep within // probeInterval; a call after the interval has genuinely elapsed must // start a second one. G5, feat/target-health review round 2: this test // proves single-flight ORDERING, not real elapsed time, so it must not // depend on wall-clock sleeps at all — probeInterval is a full hour (so // the burst phase's real, sub-millisecond elapsed time can never // accidentally read as elapsed) and the "interval has passed" half is // forced directly via targetHealthLastSweepUnixNano, never by sleeping // for probeInterval. func TestMaybeSweepTargetHealth_IntervalEnforcedAcrossRapidCalls(t *testing.T) { t.Parallel() var hits int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&hits, 1) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":"federated","result":{}}`)) })) defer srv.Close() gw := newTargetHealthSweepGateway(t, true, time.Hour, srv.URL) // First sweep: the probe answers immediately, so this completes fast // and clears targetHealthSweeping well inside probeInterval. gw.maybeSweepTargetHealth() waitForCondition(t, func() bool { return atomic.LoadInt32(&hits) > 0 }, time.Second) waitForCondition(t, func() bool { return gw.targetHealth.stateOf(targetKindMCP, "alpha") != targetHealthUnknown }, time.Second) // A burst of concurrent callers immediately after — real elapsed time // is milliseconds against a one-hour probeInterval — must not start a // second sweep. var wg sync.WaitGroup for i := 0; i < 50; i++ { wg.Add(1) go func() { defer wg.Done() gw.maybeSweepTargetHealth() }() } wg.Wait() if got := atomic.LoadInt32(&hits); got != 1 { t.Errorf("hits = %d, want exactly 1 (a burst of concurrent callers inside probeInterval must not start a second sweep)", got) } // Force the interval to read as elapsed directly, rather than // sleeping for real: a call after that must start a second sweep. atomic.StoreInt64(&gw.targetHealthLastSweepUnixNano, time.Now().Add(-2*time.Hour).UnixNano()) gw.maybeSweepTargetHealth() waitForCondition(t, func() bool { return atomic.LoadInt32(&hits) == 2 }, time.Second) } // TestMaybeSweepTargetHealth_LegacySSE_ClosesBodyWithoutReading proves a // legacy "/sse" MCP server's probe uses a plain GET and closes the body // immediately: a test server that never closes its stream must not hang // the probe. func TestMaybeSweepTargetHealth_LegacySSE_ClosesBodyWithoutReading(t *testing.T) { t.Parallel() headersSent := make(chan struct{}) release := make(chan struct{}) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/event-stream") w.WriteHeader(http.StatusOK) close(headersSent) flusher, ok := w.(http.Flusher) if ok { flusher.Flush() } <-release // never closes on its own; relies on the probe not reading })) // F6, feat/target-health review: previously relied on the PROBE'S OWN // 5s context timeout (targetTimeout below) to unblock this handler, // so srv.Close (deferred) silently added a real ~5s stall to every // run of this test. An explicit release channel, unblocked via // t.Cleanup BEFORE srv.Close (LIFO — see // TestMaybeSweepTargetHealth_ConcurrentTriggers_SingleFlight's own // comment for the identical ordering reasoning), makes this test // fast AND immune to a failing assertion ever leaving it blocked. t.Cleanup(srv.Close) t.Cleanup(func() { close(release) }) adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")} cfg := &Config{ Providers: map[string]*ProviderConfig{"openai": {}}, MCPServers: map[string]*TargetConfig{"alpha": {URL: srv.URL + "/sse"}}, } reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) require.NoError(t, err) gw := &Gateway{ cfg: cfg, registry: reg, targetHealth: newTargetHealthTracker(3), targetHealthCfg: targetHealthConfig{enabled: true, probeInterval: time.Hour, failureThreshold: 3}, targetClient: http.DefaultClient, targetTimeout: 5 * time.Second, name: "test", } gw.maybeSweepTargetHealth() waitForCondition(t, func() bool { return gw.targetHealth.stateOf(targetKindMCP, "alpha") != targetHealthUnknown }, 2*time.Second) assert.Equal(t, targetHealthHealthy, gw.targetHealth.stateOf(targetKindMCP, "alpha")) } func TestMaybeSweepTargetHealth_AgentReturns404_ReadsHealthy(t *testing.T) { t.Parallel() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) })) defer srv.Close() adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")} cfg := &Config{ Providers: map[string]*ProviderConfig{"openai": {}}, Agents: map[string]*AgentConfig{"beta": {URL: srv.URL}}, } reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) require.NoError(t, err) gw := &Gateway{ cfg: cfg, registry: reg, targetHealth: newTargetHealthTracker(3), targetHealthCfg: targetHealthConfig{enabled: true, probeInterval: time.Hour, failureThreshold: 3}, targetClient: http.DefaultClient, targetTimeout: 5 * time.Second, name: "test", } gw.maybeSweepTargetHealth() waitForCondition(t, func() bool { return gw.targetHealth.stateOf(targetKindAgent, "beta") != targetHealthUnknown }, time.Second) assert.Equal(t, targetHealthHealthy, gw.targetHealth.stateOf(targetKindAgent, "beta"), "a 404 agent-card fetch counts as reachable") } func TestMaybeSweepTargetHealth_AgentReturns503_RecordsFailure(t *testing.T) { t.Parallel() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) })) defer srv.Close() adapters := map[string]providerAdapter{"openai": newFakeAdapter("openai")} cfg := &Config{ Providers: map[string]*ProviderConfig{"openai": {}}, Agents: map[string]*AgentConfig{"beta": {URL: srv.URL}}, } reg, err := newModelRegistry(adapters, cfg, func(string, ...any) {}) require.NoError(t, err) gw := &Gateway{ cfg: cfg, registry: reg, targetHealth: newTargetHealthTracker(1), targetHealthCfg: targetHealthConfig{enabled: true, probeInterval: time.Hour, failureThreshold: 1}, targetClient: http.DefaultClient, targetTimeout: 5 * time.Second, name: "test", } gw.maybeSweepTargetHealth() waitForCondition(t, func() bool { return gw.targetHealth.stateOf(targetKindAgent, "beta") != targetHealthUnknown }, time.Second) assert.Equal(t, targetHealthUnhealthy, gw.targetHealth.stateOf(targetKindAgent, "beta")) } // waitForCondition polls cond until it returns true or deadline elapses, // failing the test on timeout — used throughout this file's sweep tests // since the sweep itself runs in a background goroutine with no // synchronous completion signal (maybeSweepTargetHealth's own doc // comment: no timers, no long-lived goroutines, fire-and-forget). func waitForCondition(t *testing.T, cond func() bool, timeout time.Duration) { t.Helper() deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { if cond() { return } time.Sleep(2 * time.Millisecond) } if !cond() { t.Fatalf("condition not met within %s", timeout) } }