test(yaegi-check): exercise the discovery circuit breaker interpreted

Add a breaker probe to the yaegi-check gate, alongside the existing
handler coverage rather than replacing it. Drives a provider whose
/v1/models 401s, the xiaomi production case, through the full closed to
open to half-open to closed cycle against the real interpreter.

This closes a gap adversarial review found: registry.go added a new
interpreted named int32 type with a String method and a new
classification call in recordHealthLocked, and nothing exercised either
under Yaegi. limits.go's matchesSentinel already documents that this
class of divergence between compiled and interpreted behavior is
invisible to go test and only an interpreted probe can catch it.

The probe configures a short breaker window (2 failures, 3 to 6 second
backoff) and a 1 millisecond discovery interval on a dedicated "brk"
provider, so the interval floor never binds and the breaker's own
timing is what the assertions observe. It proves the breaker opens
after sustained failures, suppresses discovery traffic for the whole
open window, and closes again once the provider recovers.
This commit is contained in:
2026-08-22 22:56:32 +01:00
parent 24167b16f6
commit 92b1af0705
+150 -2
View File
@@ -45,6 +45,7 @@ import (
"regexp"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/traefik/yaegi/interp"
@@ -293,7 +294,36 @@ func run() error {
return fmt.Errorf("read builtin context tokens for %q from pricing_data_gen.go: %w", builtinLookupModelID, err)
}
attemptAccountingOverride := `{"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `","` + builtinLookupModelID + `"]},"` +
// --- breaker probe (feat/provider-health): a provider whose
// /v1/models 401s, exactly the xiaomi production case, driven under
// the REAL interpreter. brkHealthy flips the upstream from always-401
// to serving real data once the probe wants to observe recovery;
// brkModelsHits counts every /v1/models hit so exerciseBreaker can
// prove the open window actually suppresses discovery traffic (a
// count staying flat), not merely that healthState reads "open".
brkModelsHits := new(int64)
brkHealthy := new(int64)
brkUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/v1/models") {
atomic.AddInt64(brkModelsHits, 1)
if atomic.LoadInt64(brkHealthy) == 0 {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"error":{"message":"invalid api key","type":"invalid_request_error"}}`))
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"brk-model"}]}`))
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"c3","object":"chat.completion","model":"brk-model","choices":[],"usage":{}}`))
}))
defer brkUpstream.Close()
attemptAccountingOverride := `{"breaker":{"failureThreshold":2,"openDuration":"3s","maxOpenDuration":"6s"},` +
`"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `","` + builtinLookupModelID + `"]},` +
`"brk":{"type":"openai","baseUrl":"` + brkUpstream.URL + `","apiKey":"sk-up","discovery":true,"discoveryInterval":"1ms"},"` +
slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]}},` +
// modelMeta (feature v0.23): a config-override entry for
// testDataWantModel, so exerciseHandler's GET /v1/models
@@ -345,7 +375,20 @@ func run() error {
return fmt.Errorf("%s.New's returned value does not implement http.Handler", pkgName)
}
return exerciseHandler(handler, builtinLookupContextTokens)
if err := exerciseHandler(handler, builtinLookupContextTokens); err != nil {
return err
}
// exerciseBreaker (feat/provider-health, adversarial-review round 2):
// runs after exerciseHandler, against the SAME interpreted handler —
// proves the discovery circuit breaker's open/half-open/closed state
// machine, its new interpreted breakerState (a named int32 with a
// String() method), and recordHealthLocked's classification call all
// run correctly under Yaegi, not merely compiled. limits.go's own
// matchesSentinel doc comment (Trap-4/reverse-direction history) is
// exactly why this class of divergence needs its own interpreted
// probe: a compiled `go test` pass here would prove nothing about the
// interpreter.
return exerciseBreaker(handler, brkModelsHits, brkHealthy)
}
// builtinLookupModelID is a real, stable entry in the generated
@@ -447,6 +490,111 @@ func readBuiltinContextTokens(repoRoot, modelID string) (int, error) {
return n, nil
}
// readHealth returns provider name's healthState string from the admin
// overview, decoded generically like readProviderAttemptCounters does
// (feat/provider-health).
func readHealth(handler http.Handler, name string) (string, error) {
req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
return "", fmt.Errorf("GET /admin/api/overview: status = %d, body=%s", rec.Code, rec.Body.String())
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
return "", err
}
providers, _ := body["providers"].([]any)
for _, raw := range providers {
p, ok := raw.(map[string]any)
if !ok {
continue
}
if p["name"] == name {
s, _ := p["healthState"].(string)
return s, nil
}
}
return "", fmt.Errorf("provider %q not in overview: %s", name, rec.Body.String())
}
// drive fires n authenticated GET /v1/models requests, each of which runs
// maybeRefresh at ServeHTTP entry, with a small gap so the 1ms discovery
// interval always permits a fresh attempt when the breaker is closed
// (feat/provider-health).
func drive(handler http.Handler, n int) {
for i := 0; i < n; i++ {
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
req.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
handler.ServeHTTP(httptest.NewRecorder(), req)
time.Sleep(20 * time.Millisecond)
}
}
// exerciseBreaker drives the "brk" provider (attemptAccountingOverride,
// run()) through the discovery circuit breaker's full state cycle —
// closed -> open -> half-open -> closed again — against the REAL
// interpreter, proving under Yaegi what registry_health_test.go already
// proves compiled: tryBeginRefresh/finishRefresh/recordHealthLocked, the
// interpreted breakerState named-int32 type and its String() method, and
// the classification call recordHealthLocked makes all behave correctly
// interpreted, not merely compiled (feat/provider-health, adversarial-
// review round 2 — limits.go's matchesSentinel doc comment is exactly why
// this class of divergence needs its own interpreted probe).
func exerciseBreaker(handler http.Handler, hits, healthy *int64) error {
// Phase 1: hammer with the upstream 401ing. The breaker must open.
deadline := time.Now().Add(10 * time.Second)
state := ""
for time.Now().Before(deadline) {
drive(handler, 5)
s, err := readHealth(handler, "brk")
if err != nil {
return err
}
state = s
if s == "open" {
break
}
}
if state != "open" {
return fmt.Errorf("BREAKER-1: healthState = %q after sustained 401s, want %q (upstream /v1/models hits=%d)", state, "open", atomic.LoadInt64(hits))
}
openedAtHits := atomic.LoadInt64(hits)
fmt.Printf("yaegi-check: breaker opened after %d upstream /v1/models hits\n", openedAtHits)
// Phase 2: while open, further traffic must NOT reach the upstream.
// openDuration is 3s; hammer for ~1s and require zero new hits.
drive(handler, 30)
duringOpen := atomic.LoadInt64(hits) - openedAtHits
if duringOpen != 0 {
return fmt.Errorf("BREAKER-2: %d upstream /v1/models hits during the open window, want 0 — backoff did not suppress discovery", duringOpen)
}
fmt.Println("yaegi-check: breaker open window suppressed all discovery hits")
// Phase 3: provider recovers. After the backoff window elapses the
// half-open probe must run and CLOSE the breaker again.
atomic.StoreInt64(healthy, 1)
recovered := false
recoverDeadline := time.Now().Add(20 * time.Second)
for time.Now().Before(recoverDeadline) {
drive(handler, 5)
s, err := readHealth(handler, "brk")
if err != nil {
return err
}
if s == "closed" {
recovered = true
break
}
}
if !recovered {
return fmt.Errorf("BREAKER-3: breaker never returned to closed after the provider recovered (hits=%d)", atomic.LoadInt64(hits))
}
fmt.Println("yaegi-check: recovered provider closed the breaker again")
return nil
}
// exerciseHandler runs two real requests against the interpreted
// handler, drawn from the plugin's own .traefik.yml testData: an
// authenticated GET /v1/models, which must return the configured